You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ha...@apache.org on 2026/08/08 23:52:02 UTC
(skywalking-banyandb) 03/06: feat(trace): back off merge dispatch after consecutive failures
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch trace-merge-resilience
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit 84c6aba875f6cdafbf43f11c29aa1a8398c8d2c3
Author: owenwillison-vajra <30...@qq.com>
AuthorDate: Sat Aug 8 13:45:27 2026 +0000
feat(trace): back off merge dispatch after consecutive failures
A persistently failing merge was retried at flush/sync notification
cadence (~4/sec) forever. Track consecutive merge failures in
mergeLoopControl and delay the next dispatch exponentially from 1s up
to a 60s cap, reset on any success. The dispatcher sleeps before
beginDispatch so wave waiters are not delayed, wakes early on any
control-state change, and wave mode bypasses backoff entirely so
operator-driven controlled merges stay prompt. New metric:
total_merge_backoff_seconds.
---
banyand/trace/merge_backoff_test.go | 98 ++++++++++++++++++++++++++++++++++
banyand/trace/merge_control.go | 102 +++++++++++++++++++++++++++++-------
banyand/trace/merger.go | 21 ++++++++
banyand/trace/metrics.go | 13 +++++
4 files changed, 216 insertions(+), 18 deletions(-)
diff --git a/banyand/trace/merge_backoff_test.go b/banyand/trace/merge_backoff_test.go
new file mode 100644
index 000000000..a13985795
--- /dev/null
+++ b/banyand/trace/merge_backoff_test.go
@@ -0,0 +1,98 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with this work for
+// additional information regarding copyright ownership. The ASF licenses this
+// file to you under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package trace
+
+import (
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+)
+
+// TestRecordOutcomeBackoffProgression asserts the exponential schedule doubles from
+// mergeBackoffBase up to mergeBackoffCap, and that a success clears it immediately.
+func TestRecordOutcomeBackoffProgression(t *testing.T) {
+ mc := newMergeLoopControl()
+ fixedNow := time.Date(2026, 8, 8, 0, 0, 0, 0, time.UTC)
+ mc.nowFunc = func() time.Time { return fixedNow }
+
+ expected := []time.Duration{
+ time.Second, 2 * time.Second, 4 * time.Second, 8 * time.Second,
+ 16 * time.Second, 32 * time.Second, 60 * time.Second, 60 * time.Second,
+ 60 * time.Second, 60 * time.Second,
+ }
+ for i, want := range expected {
+ mc.recordOutcome(false)
+ got := mc.backoffUntil.Sub(fixedNow)
+ require.Equalf(t, want, got, "failure %d: expected backoff delay %s, got %s", i+1, want, got)
+ }
+ require.Equal(t, len(expected), mc.consecutiveFailures)
+
+ remaining := mc.backoffRemaining()
+ require.Greater(t, remaining, time.Duration(0), "backoff should still be active before a success is recorded")
+
+ mc.recordOutcome(true)
+ require.Zero(t, mc.consecutiveFailures)
+ require.True(t, mc.backoffUntil.IsZero())
+ require.Zero(t, mc.backoffRemaining())
+}
+
+// TestBackoffWaveBypass asserts wave mode bypasses an active backoff, and that the
+// backoff reasserts itself once the wave releases.
+func TestBackoffWaveBypass(t *testing.T) {
+ mc := newMergeLoopControl()
+ fixedNow := time.Date(2026, 8, 8, 0, 0, 0, 0, time.UTC)
+ mc.nowFunc = func() time.Time { return fixedNow }
+
+ mc.recordOutcome(false)
+ mc.recordOutcome(false)
+ require.Greater(t, mc.backoffRemaining(), time.Duration(0))
+
+ mc.startWave(0, 0)
+ require.Zero(t, mc.backoffRemaining(), "wave mode must bypass an active backoff")
+
+ mc.releaseWave()
+ require.Greater(t, mc.backoffRemaining(), time.Duration(0), "backoff resumes once the wave releases")
+}
+
+// TestDispatcherHonorsBackoff drives mergeControl through two failing dispatch cycles the
+// way dispatcherLoop and mergeLaneWorker do (beginDispatch/endDispatch bracketing a
+// recordOutcome(false)) and asserts the control state — rather than wall-clock timing —
+// reflects an active, growing backoff that a subsequent trigger must honor.
+func TestDispatcherHonorsBackoff(t *testing.T) {
+ mc := newMergeLoopControl()
+ fixedNow := time.Date(2026, 8, 8, 0, 0, 0, 0, time.UTC)
+ mc.nowFunc = func() time.Time { return fixedNow }
+
+ require.NoError(t, mc.enqueue(nil))
+ <-mc.trigger
+ dispatch, _ := mc.beginDispatch()
+ require.True(t, dispatch)
+ mc.recordOutcome(false)
+ mc.endDispatch()
+ require.Equal(t, 1, mc.consecutiveFailures)
+ require.Equal(t, time.Second, mc.backoffUntil.Sub(fixedNow))
+
+ require.NoError(t, mc.enqueue(nil))
+ <-mc.trigger
+ dispatch, _ = mc.beginDispatch()
+ require.True(t, dispatch)
+ mc.recordOutcome(false)
+ mc.endDispatch()
+ require.Equal(t, 2, mc.consecutiveFailures)
+ require.Equal(t, 2*time.Second, mc.backoffUntil.Sub(fixedNow))
+ require.Greater(t, mc.backoffRemaining(), time.Duration(0))
+}
diff --git a/banyand/trace/merge_control.go b/banyand/trace/merge_control.go
index 01210f36e..ced1c6a85 100644
--- a/banyand/trace/merge_control.go
+++ b/banyand/trace/merge_control.go
@@ -21,29 +21,43 @@ import (
"fmt"
"sync"
"sync/atomic"
+ "time"
)
var errMergeLoopUnavailable = errors.New("merge loop is unavailable")
+// mergeBackoffBase is the initial backoff delay applied after the first consecutive
+// merge failure; mergeBackoffCap is the maximum delay the exponential schedule reaches.
+// mergeBackoffMaxShift bounds the doubling exponent so the shift never overflows;
+// consecutive failures beyond it are clamped directly to mergeBackoffCap.
+const (
+ mergeBackoffBase = time.Second
+ mergeBackoffCap = 60 * time.Second
+ mergeBackoffMaxShift = 7
+)
+
type mergeLoopControl struct {
- changed chan struct{}
- trigger chan struct{}
- version uint64
- workVersion uint64
- emptyEpoch uint64
- emptyWorkVersion uint64
- waveMaxPartID uint64
- queued int
- running int
- triggerPending bool
- dispatcherActive bool
- emptyEpochObserved bool
- stopped bool
- waveMode bool
- waveBlocked bool
- waveDispatchLimit int
- waveDispatches int
- mu sync.Mutex
+ changed chan struct{}
+ trigger chan struct{}
+ nowFunc func() time.Time
+ version uint64
+ workVersion uint64
+ emptyEpoch uint64
+ emptyWorkVersion uint64
+ waveMaxPartID uint64
+ backoffUntil time.Time
+ queued int
+ running int
+ consecutiveFailures int
+ triggerPending bool
+ dispatcherActive bool
+ emptyEpochObserved bool
+ stopped bool
+ waveMode bool
+ waveBlocked bool
+ waveDispatchLimit int
+ waveDispatches int
+ mu sync.Mutex
}
type mergeLoopState struct {
@@ -75,6 +89,58 @@ func (mc *mergeLoopControl) notifyLocked() {
mc.changed = make(chan struct{})
}
+// now returns nowFunc's value, or time.Now if nowFunc is unset; a testability seam for
+// deterministic backoff assertions.
+func (mc *mergeLoopControl) now() time.Time {
+ if mc.nowFunc != nil {
+ return mc.nowFunc()
+ }
+ return time.Now()
+}
+
+// recordOutcome reports the result of one merge attempt. A success clears the backoff
+// state; a failure extends it exponentially, doubling from mergeBackoffBase up to
+// mergeBackoffCap.
+func (mc *mergeLoopControl) recordOutcome(success bool) {
+ mc.mu.Lock()
+ defer mc.mu.Unlock()
+ if success {
+ mc.consecutiveFailures = 0
+ mc.backoffUntil = time.Time{}
+ mc.notifyLocked()
+ return
+ }
+ mc.consecutiveFailures++
+ delay := mergeBackoffCap
+ if mc.consecutiveFailures <= mergeBackoffMaxShift {
+ delay = mergeBackoffBase << uint(mc.consecutiveFailures-1)
+ if delay > mergeBackoffCap {
+ delay = mergeBackoffCap
+ }
+ }
+ mc.backoffUntil = mc.now().Add(delay)
+ mc.notifyLocked()
+}
+
+// backoffRemaining reports how long the dispatcher should still sleep before dispatching
+// another selection cycle. Wave mode always bypasses backoff: waves are operator-driven
+// and must fail visibly and immediately.
+func (mc *mergeLoopControl) backoffRemaining() time.Duration {
+ mc.mu.Lock()
+ defer mc.mu.Unlock()
+ if mc.waveMode {
+ return 0
+ }
+ if mc.backoffUntil.IsZero() {
+ return 0
+ }
+ now := mc.now()
+ if !mc.backoffUntil.After(now) {
+ return 0
+ }
+ return mc.backoffUntil.Sub(now)
+}
+
func (mc *mergeLoopControl) enqueue(closeCh <-chan struct{}) error {
mc.mu.Lock()
if mc.stopped {
diff --git a/banyand/trace/merger.go b/banyand/trace/merger.go
index 9c81522eb..c97a2a858 100644
--- a/banyand/trace/merger.go
+++ b/banyand/trace/merger.go
@@ -325,6 +325,22 @@ func (tst *tsTable) dispatcherLoop(threshold uint64, fastCh, slowCh chan *mergeD
case <-tst.loopCloser.CloseNotify():
return
case <-tst.mergeControl.trigger:
+ for {
+ delay := tst.mergeControl.backoffRemaining()
+ if delay <= 0 {
+ break
+ }
+ state := tst.mergeControl.state()
+ sleepStart := time.Now()
+ select {
+ case <-time.After(delay):
+ case <-state.changed:
+ case <-tst.loopCloser.CloseNotify():
+ tst.incTotalMergeBackoffSeconds(time.Since(sleepStart).Seconds())
+ return
+ }
+ tst.incTotalMergeBackoffSeconds(time.Since(sleepStart).Seconds())
+ }
dispatch, maxPartID := tst.mergeControl.beginDispatch()
if !dispatch {
continue
@@ -490,7 +506,12 @@ func (tst *tsTable) mergeLaneWorker(ch chan *mergeDispatchRequest, merges chan *
tst.l.Logger.Warn().Err(mergeErr).Str("typ", req.typ).Str("lane", req.lane).Msg("merge lane worker error")
tst.incTotalMergeLoopErr(1)
tst.recordUnreadablePart(mergeErr)
+ if tst.mergeControl != nil {
+ tst.mergeControl.recordOutcome(false)
+ }
}
+ } else if tst.mergeControl != nil {
+ tst.mergeControl.recordOutcome(true)
}
}
}
diff --git a/banyand/trace/metrics.go b/banyand/trace/metrics.go
index 0a7157ece..90a2478f6 100644
--- a/banyand/trace/metrics.go
+++ b/banyand/trace/metrics.go
@@ -67,6 +67,7 @@ type metrics struct {
totalMerged meter.Counter
totalMergeQueueLatency meter.Counter
totalMergePartQuarantined meter.Counter
+ totalMergeBackoffSeconds meter.Counter
pipelineTracesEvaluated meter.Counter
pipelineTracesDropped meter.Counter
@@ -283,6 +284,15 @@ func (tst *tsTable) incTotalMergePartQuarantined(delta int) {
tst.metrics.totalMergePartQuarantined.Inc(float64(delta))
}
+// incTotalMergeBackoffSeconds accumulates the time the dispatcher spent sleeping due to
+// repeated merge failures (Fix C).
+func (tst *tsTable) incTotalMergeBackoffSeconds(delta float64) {
+ if tst == nil || tst.metrics == nil {
+ return
+ }
+ tst.metrics.totalMergeBackoffSeconds.Inc(delta)
+}
+
// The pipeline metric increment helpers below are wired into the merge filter by
// the config-driven activation story.
@@ -452,6 +462,7 @@ func (m *metrics) DeleteAll() {
m.totalMergeQueueLatency.Delete("file", "fast")
m.totalMergeQueueLatency.Delete("file", "slow")
m.totalMergePartQuarantined.Delete()
+ m.totalMergeBackoffSeconds.Delete()
m.pipelineTracesEvaluated.Delete()
m.pipelineTracesDropped.Delete()
@@ -499,6 +510,7 @@ func (s *supplier) newMetrics(p common.Position) storage.Metrics {
totalMerged: factory.NewCounter("total_merged", "type", "lane"),
totalMergeQueueLatency: factory.NewCounter("total_merge_queue_latency", "type", "lane"),
totalMergePartQuarantined: factory.NewCounter("total_merge_part_quarantined"),
+ totalMergeBackoffSeconds: factory.NewCounter("total_merge_backoff_seconds"),
pipelineTracesEvaluated: factory.NewCounter("pipeline_traces_evaluated"),
pipelineTracesDropped: factory.NewCounter("pipeline_traces_dropped"),
pipelineTracesRetained: factory.NewCounter("pipeline_traces_retained"),
@@ -562,6 +574,7 @@ func (qs *queueSupplier) newMetrics(p common.Position) (storage.Metrics, observa
totalMerged: factory.NewCounter("total_merged", "type", "lane"),
totalMergeQueueLatency: factory.NewCounter("total_merge_queue_latency", "type", "lane"),
totalMergePartQuarantined: factory.NewCounter("total_merge_part_quarantined"),
+ totalMergeBackoffSeconds: factory.NewCounter("total_merge_backoff_seconds"),
pipelineTracesEvaluated: factory.NewCounter("pipeline_traces_evaluated"),
pipelineTracesDropped: factory.NewCounter("pipeline_traces_dropped"),
pipelineTracesRetained: factory.NewCounter("pipeline_traces_retained"),