You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ha...@apache.org on 2026/08/08 23:52:02 UTC

(skywalking-banyandb) 03/06: feat(trace): back off merge dispatch after consecutive failures

This is an automated email from the ASF dual-hosted git repository.

hanahmily pushed a commit to branch trace-merge-resilience
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git

commit 84c6aba875f6cdafbf43f11c29aa1a8398c8d2c3
Author: owenwillison-vajra <30...@qq.com>
AuthorDate: Sat Aug 8 13:45:27 2026 +0000

    feat(trace): back off merge dispatch after consecutive failures
    
    A persistently failing merge was retried at flush/sync notification
    cadence (~4/sec) forever. Track consecutive merge failures in
    mergeLoopControl and delay the next dispatch exponentially from 1s up
    to a 60s cap, reset on any success. The dispatcher sleeps before
    beginDispatch so wave waiters are not delayed, wakes early on any
    control-state change, and wave mode bypasses backoff entirely so
    operator-driven controlled merges stay prompt. New metric:
    total_merge_backoff_seconds.
---
 banyand/trace/merge_backoff_test.go |  98 ++++++++++++++++++++++++++++++++++
 banyand/trace/merge_control.go      | 102 +++++++++++++++++++++++++++++-------
 banyand/trace/merger.go             |  21 ++++++++
 banyand/trace/metrics.go            |  13 +++++
 4 files changed, 216 insertions(+), 18 deletions(-)

diff --git a/banyand/trace/merge_backoff_test.go b/banyand/trace/merge_backoff_test.go
new file mode 100644
index 000000000..a13985795
--- /dev/null
+++ b/banyand/trace/merge_backoff_test.go
@@ -0,0 +1,98 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with this work for
+// additional information regarding copyright ownership. The ASF licenses this
+// file to you under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package trace
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+// TestRecordOutcomeBackoffProgression asserts the exponential schedule doubles from
+// mergeBackoffBase up to mergeBackoffCap, and that a success clears it immediately.
+func TestRecordOutcomeBackoffProgression(t *testing.T) {
+	mc := newMergeLoopControl()
+	fixedNow := time.Date(2026, 8, 8, 0, 0, 0, 0, time.UTC)
+	mc.nowFunc = func() time.Time { return fixedNow }
+
+	expected := []time.Duration{
+		time.Second, 2 * time.Second, 4 * time.Second, 8 * time.Second,
+		16 * time.Second, 32 * time.Second, 60 * time.Second, 60 * time.Second,
+		60 * time.Second, 60 * time.Second,
+	}
+	for i, want := range expected {
+		mc.recordOutcome(false)
+		got := mc.backoffUntil.Sub(fixedNow)
+		require.Equalf(t, want, got, "failure %d: expected backoff delay %s, got %s", i+1, want, got)
+	}
+	require.Equal(t, len(expected), mc.consecutiveFailures)
+
+	remaining := mc.backoffRemaining()
+	require.Greater(t, remaining, time.Duration(0), "backoff should still be active before a success is recorded")
+
+	mc.recordOutcome(true)
+	require.Zero(t, mc.consecutiveFailures)
+	require.True(t, mc.backoffUntil.IsZero())
+	require.Zero(t, mc.backoffRemaining())
+}
+
+// TestBackoffWaveBypass asserts wave mode bypasses an active backoff, and that the
+// backoff reasserts itself once the wave releases.
+func TestBackoffWaveBypass(t *testing.T) {
+	mc := newMergeLoopControl()
+	fixedNow := time.Date(2026, 8, 8, 0, 0, 0, 0, time.UTC)
+	mc.nowFunc = func() time.Time { return fixedNow }
+
+	mc.recordOutcome(false)
+	mc.recordOutcome(false)
+	require.Greater(t, mc.backoffRemaining(), time.Duration(0))
+
+	mc.startWave(0, 0)
+	require.Zero(t, mc.backoffRemaining(), "wave mode must bypass an active backoff")
+
+	mc.releaseWave()
+	require.Greater(t, mc.backoffRemaining(), time.Duration(0), "backoff resumes once the wave releases")
+}
+
+// TestDispatcherHonorsBackoff drives mergeControl through two failing dispatch cycles the
+// way dispatcherLoop and mergeLaneWorker do (beginDispatch/endDispatch bracketing a
+// recordOutcome(false)) and asserts the control state — rather than wall-clock timing —
+// reflects an active, growing backoff that a subsequent trigger must honor.
+func TestDispatcherHonorsBackoff(t *testing.T) {
+	mc := newMergeLoopControl()
+	fixedNow := time.Date(2026, 8, 8, 0, 0, 0, 0, time.UTC)
+	mc.nowFunc = func() time.Time { return fixedNow }
+
+	require.NoError(t, mc.enqueue(nil))
+	<-mc.trigger
+	dispatch, _ := mc.beginDispatch()
+	require.True(t, dispatch)
+	mc.recordOutcome(false)
+	mc.endDispatch()
+	require.Equal(t, 1, mc.consecutiveFailures)
+	require.Equal(t, time.Second, mc.backoffUntil.Sub(fixedNow))
+
+	require.NoError(t, mc.enqueue(nil))
+	<-mc.trigger
+	dispatch, _ = mc.beginDispatch()
+	require.True(t, dispatch)
+	mc.recordOutcome(false)
+	mc.endDispatch()
+	require.Equal(t, 2, mc.consecutiveFailures)
+	require.Equal(t, 2*time.Second, mc.backoffUntil.Sub(fixedNow))
+	require.Greater(t, mc.backoffRemaining(), time.Duration(0))
+}
diff --git a/banyand/trace/merge_control.go b/banyand/trace/merge_control.go
index 01210f36e..ced1c6a85 100644
--- a/banyand/trace/merge_control.go
+++ b/banyand/trace/merge_control.go
@@ -21,29 +21,43 @@ import (
 	"fmt"
 	"sync"
 	"sync/atomic"
+	"time"
 )
 
 var errMergeLoopUnavailable = errors.New("merge loop is unavailable")
 
+// mergeBackoffBase is the initial backoff delay applied after the first consecutive
+// merge failure; mergeBackoffCap is the maximum delay the exponential schedule reaches.
+// mergeBackoffMaxShift bounds the doubling exponent so the shift never overflows;
+// consecutive failures beyond it are clamped directly to mergeBackoffCap.
+const (
+	mergeBackoffBase     = time.Second
+	mergeBackoffCap      = 60 * time.Second
+	mergeBackoffMaxShift = 7
+)
+
 type mergeLoopControl struct {
-	changed            chan struct{}
-	trigger            chan struct{}
-	version            uint64
-	workVersion        uint64
-	emptyEpoch         uint64
-	emptyWorkVersion   uint64
-	waveMaxPartID      uint64
-	queued             int
-	running            int
-	triggerPending     bool
-	dispatcherActive   bool
-	emptyEpochObserved bool
-	stopped            bool
-	waveMode           bool
-	waveBlocked        bool
-	waveDispatchLimit  int
-	waveDispatches     int
-	mu                 sync.Mutex
+	changed             chan struct{}
+	trigger             chan struct{}
+	nowFunc             func() time.Time
+	version             uint64
+	workVersion         uint64
+	emptyEpoch          uint64
+	emptyWorkVersion    uint64
+	waveMaxPartID       uint64
+	backoffUntil        time.Time
+	queued              int
+	running             int
+	consecutiveFailures int
+	triggerPending      bool
+	dispatcherActive    bool
+	emptyEpochObserved  bool
+	stopped             bool
+	waveMode            bool
+	waveBlocked         bool
+	waveDispatchLimit   int
+	waveDispatches      int
+	mu                  sync.Mutex
 }
 
 type mergeLoopState struct {
@@ -75,6 +89,58 @@ func (mc *mergeLoopControl) notifyLocked() {
 	mc.changed = make(chan struct{})
 }
 
+// now returns nowFunc's value, or time.Now if nowFunc is unset; a testability seam for
+// deterministic backoff assertions.
+func (mc *mergeLoopControl) now() time.Time {
+	if mc.nowFunc != nil {
+		return mc.nowFunc()
+	}
+	return time.Now()
+}
+
+// recordOutcome reports the result of one merge attempt. A success clears the backoff
+// state; a failure extends it exponentially, doubling from mergeBackoffBase up to
+// mergeBackoffCap.
+func (mc *mergeLoopControl) recordOutcome(success bool) {
+	mc.mu.Lock()
+	defer mc.mu.Unlock()
+	if success {
+		mc.consecutiveFailures = 0
+		mc.backoffUntil = time.Time{}
+		mc.notifyLocked()
+		return
+	}
+	mc.consecutiveFailures++
+	delay := mergeBackoffCap
+	if mc.consecutiveFailures <= mergeBackoffMaxShift {
+		delay = mergeBackoffBase << uint(mc.consecutiveFailures-1)
+		if delay > mergeBackoffCap {
+			delay = mergeBackoffCap
+		}
+	}
+	mc.backoffUntil = mc.now().Add(delay)
+	mc.notifyLocked()
+}
+
+// backoffRemaining reports how long the dispatcher should still sleep before dispatching
+// another selection cycle. Wave mode always bypasses backoff: waves are operator-driven
+// and must fail visibly and immediately.
+func (mc *mergeLoopControl) backoffRemaining() time.Duration {
+	mc.mu.Lock()
+	defer mc.mu.Unlock()
+	if mc.waveMode {
+		return 0
+	}
+	if mc.backoffUntil.IsZero() {
+		return 0
+	}
+	now := mc.now()
+	if !mc.backoffUntil.After(now) {
+		return 0
+	}
+	return mc.backoffUntil.Sub(now)
+}
+
 func (mc *mergeLoopControl) enqueue(closeCh <-chan struct{}) error {
 	mc.mu.Lock()
 	if mc.stopped {
diff --git a/banyand/trace/merger.go b/banyand/trace/merger.go
index 9c81522eb..c97a2a858 100644
--- a/banyand/trace/merger.go
+++ b/banyand/trace/merger.go
@@ -325,6 +325,22 @@ func (tst *tsTable) dispatcherLoop(threshold uint64, fastCh, slowCh chan *mergeD
 		case <-tst.loopCloser.CloseNotify():
 			return
 		case <-tst.mergeControl.trigger:
+			for {
+				delay := tst.mergeControl.backoffRemaining()
+				if delay <= 0 {
+					break
+				}
+				state := tst.mergeControl.state()
+				sleepStart := time.Now()
+				select {
+				case <-time.After(delay):
+				case <-state.changed:
+				case <-tst.loopCloser.CloseNotify():
+					tst.incTotalMergeBackoffSeconds(time.Since(sleepStart).Seconds())
+					return
+				}
+				tst.incTotalMergeBackoffSeconds(time.Since(sleepStart).Seconds())
+			}
 			dispatch, maxPartID := tst.mergeControl.beginDispatch()
 			if !dispatch {
 				continue
@@ -490,7 +506,12 @@ func (tst *tsTable) mergeLaneWorker(ch chan *mergeDispatchRequest, merges chan *
 				tst.l.Logger.Warn().Err(mergeErr).Str("typ", req.typ).Str("lane", req.lane).Msg("merge lane worker error")
 				tst.incTotalMergeLoopErr(1)
 				tst.recordUnreadablePart(mergeErr)
+				if tst.mergeControl != nil {
+					tst.mergeControl.recordOutcome(false)
+				}
 			}
+		} else if tst.mergeControl != nil {
+			tst.mergeControl.recordOutcome(true)
 		}
 	}
 }
diff --git a/banyand/trace/metrics.go b/banyand/trace/metrics.go
index 0a7157ece..90a2478f6 100644
--- a/banyand/trace/metrics.go
+++ b/banyand/trace/metrics.go
@@ -67,6 +67,7 @@ type metrics struct {
 	totalMerged               meter.Counter
 	totalMergeQueueLatency    meter.Counter
 	totalMergePartQuarantined meter.Counter
+	totalMergeBackoffSeconds  meter.Counter
 
 	pipelineTracesEvaluated          meter.Counter
 	pipelineTracesDropped            meter.Counter
@@ -283,6 +284,15 @@ func (tst *tsTable) incTotalMergePartQuarantined(delta int) {
 	tst.metrics.totalMergePartQuarantined.Inc(float64(delta))
 }
 
+// incTotalMergeBackoffSeconds accumulates the time the dispatcher spent sleeping due to
+// repeated merge failures (Fix C).
+func (tst *tsTable) incTotalMergeBackoffSeconds(delta float64) {
+	if tst == nil || tst.metrics == nil {
+		return
+	}
+	tst.metrics.totalMergeBackoffSeconds.Inc(delta)
+}
+
 // The pipeline metric increment helpers below are wired into the merge filter by
 // the config-driven activation story.
 
@@ -452,6 +462,7 @@ func (m *metrics) DeleteAll() {
 	m.totalMergeQueueLatency.Delete("file", "fast")
 	m.totalMergeQueueLatency.Delete("file", "slow")
 	m.totalMergePartQuarantined.Delete()
+	m.totalMergeBackoffSeconds.Delete()
 
 	m.pipelineTracesEvaluated.Delete()
 	m.pipelineTracesDropped.Delete()
@@ -499,6 +510,7 @@ func (s *supplier) newMetrics(p common.Position) storage.Metrics {
 		totalMerged:                      factory.NewCounter("total_merged", "type", "lane"),
 		totalMergeQueueLatency:           factory.NewCounter("total_merge_queue_latency", "type", "lane"),
 		totalMergePartQuarantined:        factory.NewCounter("total_merge_part_quarantined"),
+		totalMergeBackoffSeconds:         factory.NewCounter("total_merge_backoff_seconds"),
 		pipelineTracesEvaluated:          factory.NewCounter("pipeline_traces_evaluated"),
 		pipelineTracesDropped:            factory.NewCounter("pipeline_traces_dropped"),
 		pipelineTracesRetained:           factory.NewCounter("pipeline_traces_retained"),
@@ -562,6 +574,7 @@ func (qs *queueSupplier) newMetrics(p common.Position) (storage.Metrics, observa
 		totalMerged:                      factory.NewCounter("total_merged", "type", "lane"),
 		totalMergeQueueLatency:           factory.NewCounter("total_merge_queue_latency", "type", "lane"),
 		totalMergePartQuarantined:        factory.NewCounter("total_merge_part_quarantined"),
+		totalMergeBackoffSeconds:         factory.NewCounter("total_merge_backoff_seconds"),
 		pipelineTracesEvaluated:          factory.NewCounter("pipeline_traces_evaluated"),
 		pipelineTracesDropped:            factory.NewCounter("pipeline_traces_dropped"),
 		pipelineTracesRetained:           factory.NewCounter("pipeline_traces_retained"),