You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ha...@apache.org on 2026/04/09 11:47:05 UTC
(skywalking-banyandb) branch main updated: fix(storage): disable rotation task on warm and cold lifecycle nodes (#1055)
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
The following commit(s) were added to refs/heads/main by this push:
new 1c954da7c fix(storage): disable rotation task on warm and cold lifecycle nodes (#1055)
1c954da7c is described below
commit 1c954da7c5bd2b0dfe559d88c741c6bce45d98e5
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Thu Apr 9 19:47:00 2026 +0800
fix(storage): disable rotation task on warm and cold lifecycle nodes (#1055)
* fix(storage): disable rotation task on warm and cold lifecycle nodes
The rotation task uses NextTime(eventTime) which produces incorrect
segment boundaries for multi-day intervals (e.g., 3-day warm stage
interval). Warm and cold nodes receive data via lifecycle migration
which creates segments on demand via CreateSegmentIfNotExist with
correct boundaries. Disabling rotation on non-hot nodes prevents
the issue.
---
CHANGES.md | 1 +
banyand/internal/storage/rotation.go | 2 +-
banyand/internal/storage/rotation_test.go | 69 +++++++++++++++++++++++++++++++
banyand/internal/storage/tsdb.go | 3 ++
banyand/measure/metadata.go | 4 ++
banyand/stream/metadata.go | 4 ++
banyand/trace/metadata.go | 4 ++
docs/concept/rotation.md | 9 ++++
docs/operation/lifecycle.md | 9 ++++
9 files changed, 104 insertions(+), 1 deletion(-)
diff --git a/CHANGES.md b/CHANGES.md
index d7959c781..3bbc2e08c 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -22,6 +22,7 @@ Release Notes.
- Fix flaky on-disk integration tests caused by Ginkgo v2 random container shuffling closing gRPC connections prematurely.
- Fix snapshot error when there is no data in a segment.
- ui: fix query editor refresh/reset behavior and BydbQL keyword highlighting.
+- Disable the rotation task on warm and cold nodes to prevent incorrect segment boundaries during lifecycle migration.
## 0.10.0
diff --git a/banyand/internal/storage/rotation.go b/banyand/internal/storage/rotation.go
index 7d1a737aa..fd30f3edd 100644
--- a/banyand/internal/storage/rotation.go
+++ b/banyand/internal/storage/rotation.go
@@ -100,7 +100,7 @@ func (d *database[T, O]) startRotationTask() error {
gap := latest.End.UnixNano() - ts
// gap <=0 means the event is from the future
// the segment will be created by a written event directly
- if gap <= 0 || gap > newSegmentTimeGap {
+ if gap <= 0 || gap > newSegmentTimeGap || d.disableRotation {
return
}
d.incTotalRotationStarted(1)
diff --git a/banyand/internal/storage/rotation_test.go b/banyand/internal/storage/rotation_test.go
index e9e003916..b9251a357 100644
--- a/banyand/internal/storage/rotation_test.go
+++ b/banyand/internal/storage/rotation_test.go
@@ -238,3 +238,72 @@ func (m *MockMetrics) Factory() observability.Factory {
}
var MockMetricsCreator = func(_ common.Position) Metrics { return &MockMetrics{} }
+
+func TestRotationDisabled(t *testing.T) {
+ t.Run("rotation should not create segments when disabled", func(t *testing.T) {
+ dir, defFn := test.Space(require.New(t))
+ defer defFn()
+
+ TSDBOpts := TSDBOpts[*MockTSTable, any]{
+ Location: dir,
+ SegmentInterval: IntervalRule{Unit: DAY, Num: 3},
+ TTL: IntervalRule{Unit: DAY, Num: 30},
+ ShardNum: 1,
+ TSTableCreator: MockTSTableCreator,
+ DisableRotation: true,
+ }
+ ctx := context.Background()
+ mc := timestamp.NewMockClock()
+ ts, err := time.ParseInLocation("2006-01-02 15:04:05", "2024-05-01 00:00:00", time.Local)
+ require.NoError(t, err)
+ mc.Set(ts)
+ ctx = timestamp.SetClock(ctx, mc)
+
+ sc := NewServiceCache()
+ tsdb, err := OpenTSDB(ctx, TSDBOpts, sc, group)
+ require.NoError(t, err)
+ defer tsdb.Close()
+
+ // Create initial segment for day 1
+ seg, err := tsdb.CreateSegmentIfNotExist(ts)
+ require.NoError(t, err)
+ seg.DecRef()
+
+ db := tsdb.(*database[*MockTSTable, any])
+ segCtrl := db.segmentController
+
+ // Simulate data arriving day by day for 6 days (mimicking lifecycle migration)
+ // Day 1 (05-01): already in segment [05-01, 05-04)
+ // Day 2 (05-02): still in [05-01, 05-04)
+ // Day 3 (05-03): still in [05-01, 05-04)
+ // Day 4 (05-04): need new segment [05-04, 05-07)
+ for day := 0; day < 6; day++ {
+ dayTime := ts.AddDate(0, 0, day)
+ mc.Set(dayTime)
+ // Simulate what migration does: CreateSegmentIfNotExist then Tick
+ seg, segErr := tsdb.CreateSegmentIfNotExist(dayTime)
+ require.NoError(t, segErr)
+ seg.DecRef()
+ // Tick with max timestamp near end of day (like migration does with ctx.MaxTimestamp)
+ maxTS := dayTime.Add(23*time.Hour + 59*time.Minute + 59*time.Second)
+ tsdb.Tick(maxTS.UnixNano())
+ }
+
+ // Verify: only 2 segments created, both with correct 3-day boundaries
+ segments, _ := segCtrl.segments(false)
+ defer func() {
+ for i := range segments {
+ segments[i].DecRef()
+ }
+ }()
+ require.Equal(t, 2, len(segments), "expected 2 segments for 6 days with 3-day interval")
+
+ // First segment: [05-01, 05-04)
+ assert.Equal(t, "2024-05-01 00:00:00", segments[0].Start.Format("2006-01-02 15:04:05"))
+ assert.Equal(t, "2024-05-04 00:00:00", segments[0].End.Format("2006-01-02 15:04:05"))
+
+ // Second segment: [05-04, 05-07)
+ assert.Equal(t, "2024-05-04 00:00:00", segments[1].Start.Format("2006-01-02 15:04:05"))
+ assert.Equal(t, "2024-05-07 00:00:00", segments[1].End.Format("2006-01-02 15:04:05"))
+ })
+}
diff --git a/banyand/internal/storage/tsdb.go b/banyand/internal/storage/tsdb.go
index ca6b74ba9..7e7993085 100644
--- a/banyand/internal/storage/tsdb.go
+++ b/banyand/internal/storage/tsdb.go
@@ -63,6 +63,7 @@ type TSDBOpts[T TSTable, O any] struct {
SeriesIndexCacheMaxBytes int
ShardNum uint32
DisableRetention bool
+ DisableRotation bool
SegmentIdleTimeout time.Duration
MemoryLimit uint64
}
@@ -157,6 +158,7 @@ type database[T TSTable, O any] struct {
rotationProcessOn atomic.Bool
closed atomic.Bool
disableRetention bool
+ disableRotation bool
}
func (d *database[T, O]) Close() error {
@@ -244,6 +246,7 @@ func OpenTSDB[T TSTable, O any](ctx context.Context, opts TSDBOpts[T, O], cache
return nil, err
}
obsservice.MetricsCollector.Register(location, db.collect)
+ db.disableRotation = opts.DisableRotation
return db, db.startRotationTask()
}
diff --git a/banyand/measure/metadata.go b/banyand/measure/metadata.go
index 524286c41..51bc324a7 100644
--- a/banyand/measure/metadata.go
+++ b/banyand/measure/metadata.go
@@ -682,6 +682,7 @@ func (s *supplier) OpenDB(groupSchema *commonv1.Group) (resourceSchema.DB, error
segInterval := ro.SegmentInterval
segmentIdleTimeout := time.Duration(0)
disableRetention := false
+ disableRotation := false
if len(ro.Stages) > 0 && len(s.nodeLabels) > 0 {
var ttlNum uint32
foundMatched := false
@@ -705,10 +706,12 @@ func (s *supplier) OpenDB(groupSchema *commonv1.Group) (resourceSchema.DB, error
segmentIdleTimeout = 5 * time.Minute
}
disableRetention = i+1 < len(ro.Stages)
+ disableRotation = true
break
}
if !foundMatched {
disableRetention = true
+ disableRotation = true
}
}
group := groupSchema.Metadata.Name
@@ -725,6 +728,7 @@ func (s *supplier) OpenDB(groupSchema *commonv1.Group) (resourceSchema.DB, error
StorageMetricsFactory: factory,
SegmentIdleTimeout: segmentIdleTimeout,
DisableRetention: disableRetention,
+ DisableRotation: disableRotation,
MemoryLimit: s.pm.GetLimit(),
}
return storage.OpenTSDB(
diff --git a/banyand/stream/metadata.go b/banyand/stream/metadata.go
index 50bfd08aa..e7bbe3c82 100644
--- a/banyand/stream/metadata.go
+++ b/banyand/stream/metadata.go
@@ -537,6 +537,7 @@ func (s *supplier) OpenDB(groupSchema *commonv1.Group) (resourceSchema.DB, error
segInterval := ro.SegmentInterval
segmentIdleTimeout := time.Duration(0)
disableRetention := false
+ disableRotation := false
if len(ro.Stages) > 0 && len(s.nodeLabels) > 0 {
var ttlNum uint32
foundMatched := false
@@ -560,10 +561,12 @@ func (s *supplier) OpenDB(groupSchema *commonv1.Group) (resourceSchema.DB, error
segmentIdleTimeout = 5 * time.Minute
}
disableRetention = i+1 < len(ro.Stages)
+ disableRotation = true
break
}
if !foundMatched {
disableRetention = true
+ disableRotation = true
}
}
group := groupSchema.Metadata.Name
@@ -580,6 +583,7 @@ func (s *supplier) OpenDB(groupSchema *commonv1.Group) (resourceSchema.DB, error
StorageMetricsFactory: s.omr.With(storageScope.ConstLabels(meter.ToLabelPairs(common.DBLabelNames(), p.DBLabelValues()))),
SegmentIdleTimeout: segmentIdleTimeout,
DisableRetention: disableRetention,
+ DisableRotation: disableRotation,
MemoryLimit: s.pm.GetLimit(),
}
return storage.OpenTSDB(
diff --git a/banyand/trace/metadata.go b/banyand/trace/metadata.go
index 51e63b1d4..5f4e0fc17 100644
--- a/banyand/trace/metadata.go
+++ b/banyand/trace/metadata.go
@@ -560,6 +560,7 @@ func (s *supplier) OpenDB(groupSchema *commonv1.Group) (resourceSchema.DB, error
segInterval := ro.SegmentInterval
segmentIdleTimeout := time.Duration(0)
disableRetention := false
+ disableRotation := false
if len(ro.Stages) > 0 && len(s.nodeLabels) > 0 {
var ttlNum uint32
foundMatched := false
@@ -583,10 +584,12 @@ func (s *supplier) OpenDB(groupSchema *commonv1.Group) (resourceSchema.DB, error
segmentIdleTimeout = 5 * time.Minute
}
disableRetention = i+1 < len(ro.Stages)
+ disableRotation = true
break
}
if !foundMatched {
disableRetention = true
+ disableRotation = true
}
}
group := groupSchema.Metadata.Name
@@ -603,6 +606,7 @@ func (s *supplier) OpenDB(groupSchema *commonv1.Group) (resourceSchema.DB, error
StorageMetricsFactory: s.omr.With(traceScope.ConstLabels(meter.ToLabelPairs(common.DBLabelNames(), p.DBLabelValues()))),
SegmentIdleTimeout: segmentIdleTimeout,
DisableRetention: disableRetention,
+ DisableRotation: disableRotation,
MemoryLimit: s.pm.GetLimit(),
}
return storage.OpenTSDB(
diff --git a/docs/concept/rotation.md b/docs/concept/rotation.md
index 7d0a7ab80..0675ff87f 100644
--- a/docs/concept/rotation.md
+++ b/docs/concept/rotation.md
@@ -136,6 +136,15 @@ S = 2
So, **2 segments** are required to retain data for 7 days with an 8-day segment interval. 2 segments are the minimum number whatever the TTL and segment interval are. When the TTL is less than the segment interval, you can have the minimum number of segments.
+## Rotation in Lifecycle Deployments
+
+In a hot-warm-cold lifecycle deployment, data rotation behaves differently depending on the node's role:
+
+- **Hot nodes**: Rotation is enabled. The rotation task pre-creates segments based on the hot stage's `segment_interval`.
+- **Warm and cold nodes**: Rotation is disabled. Segments are created on demand by the lifecycle migration process via `CreateSegmentIfNotExist`, using the target stage's `segment_interval`. Since the rotation task uses `NextTime(eventTime)` which produces incorrect boundaries for multi-day intervals, disabling it on warm/cold nodes ensures segments are always created with correct boundaries.
+
+Additionally, data retention (TTL-based segment deletion) is disabled on non-last-stage nodes, since the lifecycle agent manages data migration between stages instead.
+
## Conclusion
Data rotation is a critical aspect of managing data in BanyanDB. By understanding the relationship between the number of segments, segment interval, and TTL, you can effectively manage data retention and query performance in the database. The formula provided here offers a simple way to calculate the number of segments required based on the chosen segment interval and TTL.
diff --git a/docs/operation/lifecycle.md b/docs/operation/lifecycle.md
index 0da3edd4c..686d4bb02 100644
--- a/docs/operation/lifecycle.md
+++ b/docs/operation/lifecycle.md
@@ -132,6 +132,15 @@ lifecycle \
| `--etcd-endpoints` | Endpoints for etcd connections | `""` |
| `--schedule` | Schedule for periodic backup (e.g., @yearly, @monthly, @weekly, @daily, etc.) | `""` |
+## Automatic Behavior on Warm and Cold Nodes
+
+When a data node matches a lifecycle stage (i.e., its labels match a stage's `node_selector`), two behaviors are automatically configured:
+
+- **Rotation is disabled**: The rotation task (which pre-creates segments on hot nodes) is not started on warm or cold nodes. Segments are created on demand by the lifecycle migration process, ensuring correct segment boundaries for multi-day intervals.
+- **Retention is disabled on non-last stages**: TTL-based segment deletion is disabled on all stages except the last one. Data removal on intermediate stages is managed by the lifecycle migration process, not by the built-in retention mechanism.
+
+If a node's labels do not match any stage, both rotation and retention are disabled as a safety measure.
+
## Best Practices
1. **Node Labeling:**