You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ha...@apache.org on 2025/06/24 13:46:59 UTC
(skywalking-banyandb) branch main updated: Implement primary block cache for measure (#670)
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
The following commit(s) were added to refs/heads/main by this push:
new eb28d4d8 Implement primary block cache for measure (#670)
eb28d4d8 is described below
commit eb28d4d8c9507fda72244d641ba0b0213a98827c
Author: Huang Youliang <bu...@gmail.com>
AuthorDate: Tue Jun 24 21:46:53 2025 +0800
Implement primary block cache for measure (#670)
* Implement primary block cache for measure
---------
Co-authored-by: Gao Hongtao <ha...@gmail.com>
---
CHANGES.md | 1 +
banyand/internal/storage/cache.go | 265 ++++++++++++++++++
banyand/internal/storage/cache_test.go | 187 +++++++++++++
banyand/internal/storage/rotation_test.go | 3 +-
banyand/internal/storage/segment.go | 60 +++--
banyand/internal/storage/segment_test.go | 27 +-
banyand/internal/storage/shard.go | 47 +++-
banyand/internal/storage/storage.go | 2 +-
banyand/internal/storage/tsdb.go | 28 +-
banyand/internal/storage/tsdb_test.go | 22 +-
banyand/measure/cache_benchmark_test.go | 429 ++++++++++++++++++++++++++++++
banyand/measure/measure.go | 31 ++-
banyand/measure/metadata.go | 11 +-
banyand/measure/metrics.go | 23 ++
banyand/measure/part.go | 1 +
banyand/measure/part_iter.go | 18 ++
banyand/measure/part_iter_test.go | 4 +
banyand/measure/query.go | 50 ++--
banyand/measure/query_test.go | 7 +-
banyand/measure/service.go | 47 ++++
banyand/measure/snapshot.go | 3 +-
banyand/measure/snapshot_test.go | 2 +-
banyand/measure/tstable_test.go | 4 +-
banyand/stream/benchmark_test.go | 5 +-
banyand/stream/block_scanner.go | 2 +-
banyand/stream/metadata.go | 6 +-
banyand/stream/query.go | 6 +-
27 files changed, 1197 insertions(+), 94 deletions(-)
diff --git a/CHANGES.md b/CHANGES.md
index 582827ea..4b644d56 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -14,6 +14,7 @@ Release Notes.
- Replica: Support configurable replica count on Group.
- Replica: Move the TopN pre-calculation flow from the Data Node to the Liaison Node.
- Add a wait and retry to write handlers to avoid the local metadata cache being loaded.
+- Implement primary block cache for measure.
- Implement versioning properties and replace physical deletion with the tombstone mechanism for the property database.
- Add Load Balancer Feature to Liaison.
diff --git a/banyand/internal/storage/cache.go b/banyand/internal/storage/cache.go
new file mode 100644
index 00000000..3e2d265d
--- /dev/null
+++ b/banyand/internal/storage/cache.go
@@ -0,0 +1,265 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package storage
+
+import (
+ "container/heap"
+ "sync"
+ "sync/atomic"
+ "time"
+ "unsafe"
+
+ "github.com/apache/skywalking-banyandb/api/common"
+ "github.com/apache/skywalking-banyandb/pkg/run"
+)
+
+// Cache encapsulates the cache operations.
+type Cache interface {
+ Get(key EntryKey) any
+ Put(key EntryKey, value any)
+ Close()
+ Requests() uint64
+ Misses() uint64
+ Entries() uint64
+ Size() uint64
+}
+
+// CacheConfig holds configuration parameters for the cache.
+type CacheConfig struct {
+ MaxCacheSize run.Bytes
+ CleanupInterval time.Duration
+ IdleTimeout time.Duration
+}
+
+// DefaultCacheConfig returns the default cache configuration.
+func DefaultCacheConfig() CacheConfig {
+ return CacheConfig{
+ MaxCacheSize: run.Bytes(100 * 1024 * 1024),
+ CleanupInterval: 30 * time.Second,
+ IdleTimeout: 2 * time.Minute,
+ }
+}
+
+type entry struct {
+ value any
+ lastAccess uint64
+}
+
+// EntryKey is the key of an entry in the cache.
+type EntryKey struct {
+ group string
+ partID uint64
+ offset uint64
+ segmentID segmentID
+ shardID common.ShardID
+}
+
+// NewEntryKey creates an entry key with partID and offset.
+func NewEntryKey(partID uint64, offset uint64) EntryKey {
+ return EntryKey{
+ partID: partID,
+ offset: offset,
+ }
+}
+
+type entryIndex struct {
+ *entry
+ key EntryKey
+ index int
+}
+
+type entryIndexHeap []*entryIndex
+
+func (h entryIndexHeap) Len() int { return len(h) }
+
+func (h entryIndexHeap) Less(i, j int) bool {
+ return atomic.LoadUint64(&h[i].entry.lastAccess) < atomic.LoadUint64(&h[j].entry.lastAccess)
+}
+
+func (h entryIndexHeap) Swap(i, j int) {
+ h[i], h[j] = h[j], h[i]
+ h[i].index = i
+ h[j].index = j
+}
+
+func (h *entryIndexHeap) Push(x interface{}) {
+ n := len(*h)
+ ei := x.(*entryIndex)
+ ei.index = n
+ *h = append(*h, ei)
+}
+
+func (h *entryIndexHeap) Pop() interface{} {
+ old := *h
+ n := len(old)
+ x := old[n-1]
+ *h = old[:n-1]
+ return x
+}
+
+var _ Cache = (*serviceCache)(nil)
+
+type serviceCache struct {
+ entry map[EntryKey]*entry
+ entryIndex map[EntryKey]*entryIndex
+ entryIndexHeap *entryIndexHeap
+ stopCh chan struct{}
+ requests uint64
+ misses uint64
+ mu sync.RWMutex
+ wg sync.WaitGroup
+ maxCacheSize uint64
+ cleanupInterval time.Duration
+ idleTimeout time.Duration
+}
+
+// NewServiceCache creates a cache for service with default configuration.
+func NewServiceCache() Cache {
+ return NewServiceCacheWithConfig(DefaultCacheConfig())
+}
+
+// NewServiceCacheWithConfig creates a cache for service with custom configuration.
+func NewServiceCacheWithConfig(config CacheConfig) Cache {
+ h := &entryIndexHeap{}
+ heap.Init(h)
+ sc := &serviceCache{
+ entry: make(map[EntryKey]*entry),
+ entryIndexHeap: h,
+ entryIndex: make(map[EntryKey]*entryIndex),
+ stopCh: make(chan struct{}),
+ wg: sync.WaitGroup{},
+ maxCacheSize: uint64(config.MaxCacheSize),
+ cleanupInterval: config.CleanupInterval,
+ idleTimeout: config.IdleTimeout,
+ }
+ sc.wg.Add(1)
+ sc.startCleaner()
+ return sc
+}
+
+func (sc *serviceCache) startCleaner() {
+ go func() {
+ defer sc.wg.Done()
+ sc.clean()
+ }()
+}
+
+func (sc *serviceCache) clean() {
+ ticker := time.NewTicker(sc.cleanupInterval)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-ticker.C:
+ now := uint64(time.Now().UnixNano())
+ sc.mu.Lock()
+ for key, entry := range sc.entry {
+ if now-atomic.LoadUint64(&entry.lastAccess) > uint64(sc.idleTimeout.Nanoseconds()) {
+ delete(sc.entry, key)
+ heap.Remove(sc.entryIndexHeap, sc.entryIndex[key].index)
+ }
+ }
+ sc.mu.Unlock()
+ case <-sc.stopCh:
+ return
+ }
+ }
+}
+
+func (sc *serviceCache) Close() {
+ close(sc.stopCh)
+ sc.wg.Wait()
+ sc.entry = nil
+ sc.entryIndex = nil
+ sc.entryIndexHeap = nil
+}
+
+func (sc *serviceCache) Get(key EntryKey) any {
+ atomic.AddUint64(&sc.requests, 1)
+
+ sc.mu.RLock()
+ entry := sc.entry[key]
+ sc.mu.RUnlock()
+
+ if entry != nil {
+ now := uint64(time.Now().UnixNano())
+ if atomic.LoadUint64(&entry.lastAccess) != now {
+ sc.mu.Lock()
+ atomic.StoreUint64(&entry.lastAccess, now)
+ if ei := sc.entryIndex[key]; ei != nil {
+ heap.Fix(sc.entryIndexHeap, ei.index)
+ }
+ sc.mu.Unlock()
+ }
+ return entry.value
+ }
+
+ atomic.AddUint64(&sc.misses, 1)
+ return nil
+}
+
+func (sc *serviceCache) Put(key EntryKey, value any) {
+ sc.mu.Lock()
+ defer sc.mu.Unlock()
+
+ for sc.size() > sc.maxCacheSize && sc.len() > 0 {
+ ei := heap.Pop(sc.entryIndexHeap).(*entryIndex)
+ delete(sc.entry, ei.key)
+ delete(sc.entryIndex, ei.key)
+ }
+
+ now := uint64(time.Now().UnixNano())
+ e := &entry{
+ value: value,
+ lastAccess: now,
+ }
+ ei := &entryIndex{
+ key: key,
+ entry: e,
+ }
+ sc.entry[key] = e
+ sc.entryIndex[key] = ei
+ heap.Push(sc.entryIndexHeap, ei)
+}
+
+func (sc *serviceCache) Requests() uint64 {
+ return atomic.LoadUint64(&sc.requests)
+}
+
+func (sc *serviceCache) Misses() uint64 {
+ return atomic.LoadUint64(&sc.misses)
+}
+
+func (sc *serviceCache) Entries() uint64 {
+ sc.mu.RLock()
+ defer sc.mu.RUnlock()
+ return sc.len()
+}
+
+func (sc *serviceCache) len() uint64 {
+ return uint64(len(sc.entry))
+}
+
+func (sc *serviceCache) Size() uint64 {
+ sc.mu.RLock()
+ defer sc.mu.RUnlock()
+ return sc.size()
+}
+
+func (sc *serviceCache) size() uint64 {
+ return uint64(unsafe.Sizeof(*sc))
+}
diff --git a/banyand/internal/storage/cache_test.go b/banyand/internal/storage/cache_test.go
new file mode 100644
index 00000000..0bf4eb3d
--- /dev/null
+++ b/banyand/internal/storage/cache_test.go
@@ -0,0 +1,187 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package storage
+
+import (
+ "strconv"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+
+ "github.com/apache/skywalking-banyandb/api/common"
+ "github.com/apache/skywalking-banyandb/pkg/run"
+)
+
+func TestCachePutAndGet(t *testing.T) {
+ serviceCache := NewServiceCache()
+
+ key := EntryKey{
+ group: "test-group",
+ partID: 0,
+ offset: 0,
+ segmentID: segmentID(0),
+ shardID: common.ShardID(0),
+ }
+ value := "test-value"
+
+ serviceCache.Put(key, value)
+ assert.Equal(t, uint64(1), serviceCache.Entries())
+ result := serviceCache.Get(key)
+ assert.Equal(t, value, result)
+ assert.Equal(t, uint64(1), serviceCache.Requests())
+ assert.Equal(t, uint64(0), serviceCache.Misses())
+
+ nonExistentKey := EntryKey{
+ group: "test-group",
+ partID: 1,
+ offset: 100,
+ segmentID: segmentID(1),
+ shardID: common.ShardID(1),
+ }
+
+ result = serviceCache.Get(nonExistentKey)
+ assert.Nil(t, result)
+ assert.Equal(t, uint64(2), serviceCache.Requests())
+ assert.Equal(t, uint64(1), serviceCache.Misses())
+}
+
+func TestCacheEvict(t *testing.T) {
+ serviceCache := NewServiceCache().(*serviceCache)
+ serviceCache.maxCacheSize = 50
+
+ var expectedKey EntryKey
+ var expectedValue string
+ for i := 0; i < 10; i++ {
+ key := EntryKey{
+ group: "test-group",
+ partID: uint64(i),
+ offset: uint64(i * 100),
+ segmentID: segmentID(i),
+ shardID: common.ShardID(i),
+ }
+ value := "test-value" + strconv.Itoa(i)
+ serviceCache.Put(key, value)
+ if i == 9 {
+ expectedKey, expectedValue = key, value
+ }
+ }
+
+ assert.Equal(t, uint64(1), serviceCache.Entries())
+ result := serviceCache.Get(expectedKey)
+ assert.Equal(t, expectedValue, result)
+}
+
+func TestCacheClean(t *testing.T) {
+ config := CacheConfig{
+ MaxCacheSize: run.Bytes(100 * 1024 * 1024),
+ CleanupInterval: 10 * time.Millisecond,
+ IdleTimeout: 10 * time.Millisecond,
+ }
+ serviceCache := NewServiceCacheWithConfig(config).(*serviceCache)
+ defer serviceCache.Close()
+
+ key := EntryKey{
+ group: "test-group",
+ partID: 0,
+ offset: 0,
+ segmentID: segmentID(0),
+ shardID: common.ShardID(0),
+ }
+ value := "test-value"
+
+ serviceCache.Put(key, value)
+ assert.Equal(t, uint64(1), serviceCache.Entries())
+ time.Sleep(50 * time.Millisecond)
+ assert.Equal(t, uint64(0), serviceCache.Entries())
+ assert.Nil(t, serviceCache.Get(key))
+}
+
+func TestCacheClose(t *testing.T) {
+ serviceCache := NewServiceCache().(*serviceCache)
+
+ key := EntryKey{
+ group: "test-group",
+ partID: 0,
+ offset: 0,
+ segmentID: segmentID(0),
+ shardID: common.ShardID(0),
+ }
+ value := "test-value"
+
+ serviceCache.Put(key, value)
+ assert.Equal(t, uint64(1), serviceCache.Entries())
+ serviceCache.Close()
+ assert.Nil(t, serviceCache.entry)
+ assert.Nil(t, serviceCache.entryIndex)
+ assert.Nil(t, serviceCache.entryIndexHeap)
+}
+
+func TestCacheConcurrency(t *testing.T) {
+ serviceCache := NewServiceCache()
+
+ const numGoroutines = 10
+ const numOperations = 100
+ var wg sync.WaitGroup
+ var expectedKey EntryKey
+ var expectedValue string
+ wg.Add(numGoroutines * 2)
+ for i := 0; i < numGoroutines; i++ {
+ go func(id int) {
+ defer wg.Done()
+ for j := 0; j < numOperations; j++ {
+ num := id*numOperations + j
+ key := EntryKey{
+ group: "test-group",
+ partID: uint64(num),
+ offset: uint64(num * 100),
+ segmentID: segmentID(num),
+ shardID: common.ShardID(num),
+ }
+ value := "test-value" + strconv.Itoa(num)
+ if id == 0 && j == 0 {
+ expectedKey, expectedValue = key, value
+ }
+ serviceCache.Put(key, value)
+ }
+ }(i)
+ }
+ for i := 0; i < numGoroutines; i++ {
+ go func(id int) {
+ defer wg.Done()
+ for j := 0; j < numOperations; j++ {
+ num := id*numOperations + j
+ key := EntryKey{
+ group: "test-group",
+ partID: uint64(num),
+ offset: uint64(num * 100),
+ segmentID: segmentID(num),
+ shardID: common.ShardID(num),
+ }
+ serviceCache.Get(key)
+ }
+ }(i)
+ }
+ wg.Wait()
+
+ assert.Equal(t, uint64(numGoroutines*numOperations), serviceCache.Entries())
+ assert.Equal(t, uint64(numGoroutines*numOperations), serviceCache.Requests())
+ result := serviceCache.Get(expectedKey)
+ assert.Equal(t, expectedValue, result)
+}
diff --git a/banyand/internal/storage/rotation_test.go b/banyand/internal/storage/rotation_test.go
index 26e8bd5d..3b1667a5 100644
--- a/banyand/internal/storage/rotation_test.go
+++ b/banyand/internal/storage/rotation_test.go
@@ -174,7 +174,8 @@ func setUpDB(t *testing.T, ttlDays ...int) (*database[*MockTSTable, any], timest
mc.Set(ts)
ctx = timestamp.SetClock(ctx, mc)
- tsdb, err := OpenTSDB(ctx, TSDBOpts)
+ sc := NewServiceCache()
+ tsdb, err := OpenTSDB(ctx, TSDBOpts, sc, group)
require.NoError(t, err)
seg, err := tsdb.CreateSegmentIfNotExist(ts)
require.NoError(t, err)
diff --git a/banyand/internal/storage/segment.go b/banyand/internal/storage/segment.go
index 3fed9c49..e7b686ed 100644
--- a/banyand/internal/storage/segment.go
+++ b/banyand/internal/storage/segment.go
@@ -46,12 +46,30 @@ var ErrExpiredData = errors.New("expired data")
// ErrSegmentClosed is returned when trying to access a closed segment.
var ErrSegmentClosed = errors.New("segment closed")
+var _ Cache = (*segmentCache)(nil)
+
+type segmentCache struct {
+ *groupCache
+ segmentID segmentID
+}
+
+func (sc *segmentCache) get(key EntryKey) any {
+ key.segmentID = sc.segmentID
+ return sc.groupCache.get(key)
+}
+
+func (sc *segmentCache) put(key EntryKey, value any) {
+ key.segmentID = sc.segmentID
+ sc.groupCache.put(key, value)
+}
+
type segment[T TSTable, O any] struct {
- metrics any
- tsdbOpts *TSDBOpts[T, O]
- l *logger.Logger
- index *seriesIndex
- sLst atomic.Pointer[[]*shard[T]]
+ metrics any
+ tsdbOpts *TSDBOpts[T, O]
+ l *logger.Logger
+ index *seriesIndex
+ sLst atomic.Pointer[[]*shard[T]]
+ *segmentCache
indexMetrics *inverted.Metrics
lfs banyanfs.FileSystem
position common.Position
@@ -65,7 +83,7 @@ type segment[T TSTable, O any] struct {
id segmentID
}
-func (sc *segmentController[T, O]) openSegment(ctx context.Context, startTime, endTime time.Time, path, suffix string,
+func (sc *segmentController[T, O]) openSegment(ctx context.Context, startTime, endTime time.Time, path, suffix string, groupCache *groupCache,
) (s *segment[T, O], err error) {
suffixInteger, err := strconv.Atoi(suffix)
if err != nil {
@@ -89,6 +107,7 @@ func (sc *segmentController[T, O]) openSegment(ctx context.Context, startTime, e
indexMetrics: sc.indexMetrics,
tsdbOpts: options,
lfs: sc.lfs,
+ segmentCache: &segmentCache{groupCache: groupCache, segmentID: id},
}
s.l = logger.Fetch(ctx, s.String())
s.lastAccessed.Store(time.Now().UnixNano())
@@ -114,14 +133,15 @@ func (s *segment[T, O]) GetTimeRange() timestamp.TimeRange {
return s.TimeRange
}
-func (s *segment[T, O]) Tables() (tt []T) {
+func (s *segment[T, O]) Tables() (tt []T, cc []Cache) {
sLst := s.sLst.Load()
if sLst != nil {
for _, s := range *sLst {
tt = append(tt, s.table)
+ cc = append(cc, s.shardCache)
}
}
- return tt
+ return tt, cc
}
func (s *segment[T, O]) incRef(ctx context.Context) error {
@@ -284,21 +304,22 @@ type segmentController[T TSTable, O any] struct {
opts *TSDBOpts[T, O]
l *logger.Logger
indexMetrics *inverted.Metrics
- lfs banyanfs.FileSystem
- position common.Position
- db string
- stage string
- location string
- lst []*segment[T, O]
- deadline atomic.Int64
- idleTimeout time.Duration
- optsMutex sync.RWMutex
+ *groupCache
+ lfs banyanfs.FileSystem
+ position common.Position
+ db string
+ stage string
+ location string
+ lst []*segment[T, O]
+ deadline atomic.Int64
+ idleTimeout time.Duration
+ optsMutex sync.RWMutex
sync.RWMutex
}
func newSegmentController[T TSTable, O any](ctx context.Context, location string,
l *logger.Logger, opts TSDBOpts[T, O], indexMetrics *inverted.Metrics, metrics Metrics,
- idleTimeout time.Duration, lfs banyanfs.FileSystem,
+ idleTimeout time.Duration, lfs banyanfs.FileSystem, serviceCache *serviceCache, group string,
) *segmentController[T, O] {
clock, _ := timestamp.GetClock(ctx)
p := common.GetPosition(ctx)
@@ -314,6 +335,7 @@ func newSegmentController[T TSTable, O any](ctx context.Context, location string
db: p.Database,
idleTimeout: idleTimeout,
lfs: lfs,
+ groupCache: &groupCache{serviceCache, group},
}
}
@@ -526,7 +548,7 @@ func (sc *segmentController[T, O]) load(start, end time.Time, root string) (seg
ctx := common.SetPosition(context.WithValue(context.Background(), logger.ContextKey, sc.l), func(_ common.Position) common.Position {
return sc.position
})
- seg, err = sc.openSegment(ctx, start, end, segPath, suffix)
+ seg, err = sc.openSegment(ctx, start, end, segPath, suffix, sc.groupCache)
if err != nil {
return nil, err
}
diff --git a/banyand/internal/storage/segment_test.go b/banyand/internal/storage/segment_test.go
index b409dbd3..b97d34fd 100644
--- a/banyand/internal/storage/segment_test.go
+++ b/banyand/internal/storage/segment_test.go
@@ -101,6 +101,7 @@ func TestSegmentOpenAndReopen(t *testing.T) {
SeriesIndexCacheMaxBytes: 1024 * 1024,
}
+ serviceCache := NewServiceCache().(*serviceCache)
sc := newSegmentController[mockTSTable, mockTSTableOpener](
ctx,
tempDir,
@@ -110,6 +111,8 @@ func TestSegmentOpenAndReopen(t *testing.T) {
nil, // metrics
5*time.Minute, // idleTimeout
fs.NewLocalFileSystemWithLoggerAndLimit(logger.GetLogger("storage"), opts.MemoryLimit),
+ serviceCache,
+ group,
)
now := time.Now().UTC()
@@ -128,7 +131,7 @@ func TestSegmentOpenAndReopen(t *testing.T) {
// Open segment
suffix := startTime.Format(dayFormat)
- segment, err := sc.openSegment(ctx, startTime, endTime, segmentPath, suffix)
+ segment, err := sc.openSegment(ctx, startTime, endTime, segmentPath, suffix, sc.groupCache)
require.NoError(t, err)
require.NotNil(t, segment)
@@ -192,6 +195,7 @@ func TestSegmentCloseIfIdle(t *testing.T) {
SeriesIndexCacheMaxBytes: 1024 * 1024,
}
+ serviceCache := NewServiceCache().(*serviceCache)
sc := newSegmentController[mockTSTable, mockTSTableOpener](
ctx,
tempDir,
@@ -201,6 +205,8 @@ func TestSegmentCloseIfIdle(t *testing.T) {
nil, // metrics
time.Second, // Set short idle timeout for testing,
fs.NewLocalFileSystemWithLoggerAndLimit(logger.GetLogger("storage"), opts.MemoryLimit),
+ serviceCache,
+ group,
)
// Test time parameters
@@ -220,7 +226,7 @@ func TestSegmentCloseIfIdle(t *testing.T) {
// Open segment
suffix := startTime.Format(dayFormat)
- segment, err := sc.openSegment(ctx, startTime, endTime, segmentPath, suffix)
+ segment, err := sc.openSegment(ctx, startTime, endTime, segmentPath, suffix, sc.groupCache)
require.NoError(t, err)
// Force last access time to be in the past
@@ -275,6 +281,7 @@ func TestCloseIdleAndSelectSegments(t *testing.T) {
// Create segment controller with a short idle timeout (100ms)
idleTimeout := 100 * time.Millisecond
+ serviceCache := NewServiceCache().(*serviceCache)
sc := newSegmentController[mockTSTable, mockTSTableOpener](
ctx,
tempDir,
@@ -284,6 +291,8 @@ func TestCloseIdleAndSelectSegments(t *testing.T) {
nil, // metrics
idleTimeout, // short idle timeout,
fs.NewLocalFileSystemWithLoggerAndLimit(logger.GetLogger("storage"), opts.MemoryLimit),
+ serviceCache,
+ group,
)
// Test time parameters
@@ -305,7 +314,7 @@ func TestCloseIdleAndSelectSegments(t *testing.T) {
// Open segment
suffix := startTime.Format(dayFormat)
- segment, err := sc.openSegment(ctx, startTime, startTime.Add(24*time.Hour), segmentPath, suffix)
+ segment, err := sc.openSegment(ctx, startTime, startTime.Add(24*time.Hour), segmentPath, suffix, sc.groupCache)
require.NoError(t, err)
// Add segment to controller's list
@@ -407,6 +416,7 @@ func TestOpenExistingSegmentWithShards(t *testing.T) {
SeriesIndexCacheMaxBytes: 1024 * 1024,
}
+ serviceCache := NewServiceCache().(*serviceCache)
sc := newSegmentController[mockTSTable, mockTSTableOpener](
ctx,
tempDir,
@@ -416,6 +426,8 @@ func TestOpenExistingSegmentWithShards(t *testing.T) {
nil, // metrics
5*time.Minute, // idleTimeout,
fs.NewLocalFileSystemWithLoggerAndLimit(logger.GetLogger("storage"), opts.MemoryLimit),
+ serviceCache,
+ group,
)
// Test time parameters
@@ -447,7 +459,7 @@ func TestOpenExistingSegmentWithShards(t *testing.T) {
}
// Open the segment
- segment, err := sc.openSegment(ctx, startTime, endTime, segmentPath, suffix)
+ segment, err := sc.openSegment(ctx, startTime, endTime, segmentPath, suffix, sc.groupCache)
require.NoError(t, err)
require.NotNil(t, segment)
@@ -465,7 +477,7 @@ func TestOpenExistingSegmentWithShards(t *testing.T) {
assert.Contains(t, shardIDs, common.ShardID(1), "Shard 1 should be loaded")
// Verify tables can be retrieved
- tables := segment.Tables()
+ tables, _ := segment.Tables()
require.Len(t, tables, 2, "Should have 2 tables")
// Make sure each table has the correct ID
@@ -516,6 +528,7 @@ func TestDeleteExpiredSegmentsWithClosedSegments(t *testing.T) {
// Create segment controller with a short idle timeout
idleTimeout := 100 * time.Millisecond
+ serviceCache := NewServiceCache().(*serviceCache)
sc := newSegmentController[mockTSTable, mockTSTableOpener](
ctx,
tempDir,
@@ -525,6 +538,8 @@ func TestDeleteExpiredSegmentsWithClosedSegments(t *testing.T) {
nil, // metrics
idleTimeout, // short idle timeout
fs.NewLocalFileSystemWithLoggerAndLimit(logger.GetLogger("storage"), opts.MemoryLimit),
+ serviceCache,
+ group,
)
// Create segments spanning 6 days - some will be expired, some won't
@@ -558,7 +573,7 @@ func TestDeleteExpiredSegmentsWithClosedSegments(t *testing.T) {
// Open segment
suffix := date.Format(dayFormat)
- segment, err := sc.openSegment(ctx, date, date.Add(24*time.Hour), segmentPath, suffix)
+ segment, err := sc.openSegment(ctx, date, date.Add(24*time.Hour), segmentPath, suffix, sc.groupCache)
require.NoError(t, err)
// Add segment to controller
diff --git a/banyand/internal/storage/shard.go b/banyand/internal/storage/shard.go
index dda56abf..2f892280 100644
--- a/banyand/internal/storage/shard.go
+++ b/banyand/internal/storage/shard.go
@@ -28,8 +28,43 @@ import (
"github.com/apache/skywalking-banyandb/pkg/timestamp"
)
+var _ Cache = (*shardCache)(nil)
+
+type shardCache struct {
+ *segmentCache
+ shardID common.ShardID
+}
+
+// NewShardCache creates a new shard cache.
+func NewShardCache(group string, segmentID segmentID, shardID common.ShardID) Cache {
+ serviceCache := NewServiceCache().(*serviceCache)
+ groupCache := &groupCache{
+ serviceCache: serviceCache,
+ group: group,
+ }
+ segmentCache := &segmentCache{
+ groupCache: groupCache,
+ segmentID: segmentID,
+ }
+ return &shardCache{
+ segmentCache: segmentCache,
+ shardID: shardID,
+ }
+}
+
+func (sc *shardCache) Get(key EntryKey) any {
+ key.shardID = sc.shardID
+ return sc.segmentCache.get(key)
+}
+
+func (sc *shardCache) Put(key EntryKey, value any) {
+ key.shardID = sc.shardID
+ sc.segmentCache.put(key, value)
+}
+
type shard[T TSTable] struct {
- table T
+ table T
+ *shardCache
l *logger.Logger
timeRange timestamp.TimeRange
location string
@@ -49,9 +84,13 @@ func (s *segment[T, O]) openShard(ctx context.Context, id common.ShardID) (*shar
}
return &shard[T]{
- id: id,
- l: l,
- table: t,
+ id: id,
+ l: l,
+ table: t,
+ shardCache: &shardCache{
+ segmentCache: s.segmentCache,
+ shardID: id,
+ },
timeRange: s.TimeRange,
location: location,
}, nil
diff --git a/banyand/internal/storage/storage.go b/banyand/internal/storage/storage.go
index 1367e367..185f6e74 100644
--- a/banyand/internal/storage/storage.go
+++ b/banyand/internal/storage/storage.go
@@ -118,7 +118,7 @@ type Segment[T TSTable, O any] interface {
DecRef()
GetTimeRange() timestamp.TimeRange
CreateTSTableIfNotExist(shardID common.ShardID) (T, error)
- Tables() []T
+ Tables() ([]T, []Cache)
Lookup(ctx context.Context, series []*pbv1.Series) (pbv1.SeriesList, error)
IndexDB() IndexDB
}
diff --git a/banyand/internal/storage/tsdb.go b/banyand/internal/storage/tsdb.go
index d3612ef4..4332be83 100644
--- a/banyand/internal/storage/tsdb.go
+++ b/banyand/internal/storage/tsdb.go
@@ -74,6 +74,23 @@ func generateSegID(unit IntervalUnit, suffix int) segmentID {
return segmentID(unit)<<31 | ((segmentID(suffix) << 1) >> 1)
}
+var _ Cache = (*groupCache)(nil)
+
+type groupCache struct {
+ *serviceCache
+ group string
+}
+
+func (gc *groupCache) get(key EntryKey) any {
+ key.group = gc.group
+ return gc.Get(key)
+}
+
+func (gc *groupCache) put(key EntryKey, value any) {
+ key.group = gc.group
+ gc.Put(key, value)
+}
+
type database[T TSTable, O any] struct {
lock fs.File
logger *logger.Logger
@@ -110,7 +127,7 @@ func (d *database[T, O]) Close() error {
// OpenTSDB returns a new tsdb runtime. This constructor will create a new database if it's absent,
// or load an existing one.
-func OpenTSDB[T TSTable, O any](ctx context.Context, opts TSDBOpts[T, O]) (TSDB[T, O], error) {
+func OpenTSDB[T TSTable, O any](ctx context.Context, opts TSDBOpts[T, O], cache Cache, group string) (TSDB[T, O], error) {
if opts.SegmentInterval.Num == 0 {
return nil, errors.Wrap(errOpenDatabase, "segment interval is absent")
}
@@ -129,6 +146,10 @@ func OpenTSDB[T TSTable, O any](ctx context.Context, opts TSDBOpts[T, O]) (TSDB[
if opts.StorageMetricsFactory != nil {
indexMetrics = inverted.NewMetrics(opts.StorageMetricsFactory, common.SegLabelNames()...)
}
+ var sc *serviceCache
+ if cache != nil {
+ sc = cache.(*serviceCache)
+ }
db := &database[T, O]{
location: location,
scheduler: scheduler,
@@ -136,7 +157,7 @@ func OpenTSDB[T TSTable, O any](ctx context.Context, opts TSDBOpts[T, O]) (TSDB[
tsEventCh: make(chan int64),
p: p,
segmentController: newSegmentController(ctx, location,
- l, opts, indexMetrics, opts.TableMetrics, opts.SegmentIdleTimeout, tsdbLfs),
+ l, opts, indexMetrics, opts.TableMetrics, opts.SegmentIdleTimeout, tsdbLfs, sc, group),
metrics: newMetrics(opts.StorageMetricsFactory),
disableRetention: opts.DisableRetention,
lfs: tsdbLfs,
@@ -247,7 +268,8 @@ func (d *database[T, O]) collect() {
if atomic.LoadInt32(&s.refCount) <= 0 {
continue
}
- for _, t := range s.Tables() {
+ tables, _ := s.Tables()
+ for _, t := range tables {
t.Collect(d.segmentController.metrics)
}
s.index.store.CollectMetrics(s.index.p.SegLabelValues()...)
diff --git a/banyand/internal/storage/tsdb_test.go b/banyand/internal/storage/tsdb_test.go
index d4d7b611..57b7e4c4 100644
--- a/banyand/internal/storage/tsdb_test.go
+++ b/banyand/internal/storage/tsdb_test.go
@@ -34,6 +34,8 @@ import (
"github.com/apache/skywalking-banyandb/pkg/timestamp"
)
+const group = "test"
+
// MockGaugeWithValue wraps MockGauge to track the value.
type MockGaugeWithValue struct {
*meter.MockGauge
@@ -107,7 +109,8 @@ func TestOpenTSDB(t *testing.T) {
mc.Set(ts)
ctx = timestamp.SetClock(ctx, mc)
- tsdb, err := OpenTSDB(ctx, opts)
+ serviceCache := NewServiceCache()
+ tsdb, err := OpenTSDB(ctx, opts, serviceCache, group)
require.NoError(t, err)
require.NotNil(t, tsdb)
@@ -141,7 +144,8 @@ func TestOpenTSDB(t *testing.T) {
ctx = timestamp.SetClock(ctx, mc)
// Create new TSDB
- tsdb, err := OpenTSDB(ctx, opts)
+ serviceCache := NewServiceCache()
+ tsdb, err := OpenTSDB(ctx, opts, serviceCache, group)
require.NoError(t, err)
require.NotNil(t, tsdb)
@@ -158,7 +162,7 @@ func TestOpenTSDB(t *testing.T) {
tsdb.Close()
// Reopen existing TSDB
- tsdb, err = OpenTSDB(ctx, opts)
+ tsdb, err = OpenTSDB(ctx, opts, serviceCache, group)
require.NoError(t, err)
require.NotNil(t, tsdb)
@@ -191,7 +195,8 @@ func TestOpenTSDB(t *testing.T) {
ctx = timestamp.SetClock(ctx, mc)
// Create new TSDB
- tsdb, err := OpenTSDB(ctx, opts)
+ serviceCache := NewServiceCache()
+ tsdb, err := OpenTSDB(ctx, opts, serviceCache, group)
require.NoError(t, err)
require.NotNil(t, tsdb)
@@ -243,7 +248,8 @@ func TestTakeFileSnapshot(t *testing.T) {
mc.Set(ts)
ctx = timestamp.SetClock(ctx, mc)
- tsdb, err := OpenTSDB(ctx, opts)
+ serviceCache := NewServiceCache()
+ tsdb, err := OpenTSDB(ctx, opts, serviceCache, group)
require.NoError(t, err)
require.NotNil(t, tsdb)
@@ -314,7 +320,8 @@ func TestTSDBCollect(t *testing.T) {
mc.Set(ts)
ctx = timestamp.SetClock(ctx, mc)
- tsdb, err := OpenTSDB(ctx, opts)
+ serviceCache := NewServiceCache()
+ tsdb, err := OpenTSDB(ctx, opts, serviceCache, group)
require.NoError(t, err)
require.NotNil(t, tsdb)
@@ -393,7 +400,8 @@ func TestCollectWithPartialClosedSegments(t *testing.T) {
mc.Set(baseDate)
ctx = timestamp.SetClock(ctx, mc)
- tsdb, err := OpenTSDB(ctx, opts)
+ serviceCache := NewServiceCache()
+ tsdb, err := OpenTSDB(ctx, opts, serviceCache, group)
require.NoError(t, err)
require.NotNil(t, tsdb)
diff --git a/banyand/measure/cache_benchmark_test.go b/banyand/measure/cache_benchmark_test.go
new file mode 100644
index 00000000..abd40e81
--- /dev/null
+++ b/banyand/measure/cache_benchmark_test.go
@@ -0,0 +1,429 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package measure
+
+import (
+ "context"
+ "errors"
+ "io"
+ "math"
+ "strconv"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/apache/skywalking-banyandb/api/common"
+ databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+ "github.com/apache/skywalking-banyandb/banyand/internal/storage"
+ "github.com/apache/skywalking-banyandb/banyand/metadata/embeddedserver"
+ "github.com/apache/skywalking-banyandb/banyand/observability"
+ "github.com/apache/skywalking-banyandb/banyand/protector"
+ "github.com/apache/skywalking-banyandb/banyand/queue"
+ "github.com/apache/skywalking-banyandb/pkg/convert"
+ "github.com/apache/skywalking-banyandb/pkg/index"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
+ "github.com/apache/skywalking-banyandb/pkg/query/model"
+ "github.com/apache/skywalking-banyandb/pkg/run"
+ "github.com/apache/skywalking-banyandb/pkg/test"
+ "github.com/apache/skywalking-banyandb/pkg/timestamp"
+)
+
+const (
+ version = "1.0.0"
+ entityTagValuePrefix = "entity"
+)
+
+type parameter struct {
+ batchCount int
+ timestampCount int
+ seriesCount int
+ seriesToQuery int
+ startTimestamp int
+ endTimestamp int
+}
+
+var pList = [1]parameter{
+ {
+ batchCount: 10,
+ timestampCount: 144,
+ seriesCount: 100,
+ seriesToQuery: 10,
+ startTimestamp: 1410,
+ endTimestamp: 1440,
+ },
+}
+
+type databaseSupplier struct {
+ database atomic.Value
+}
+
+func (dbs *databaseSupplier) SupplyTSDB() io.Closer {
+ if v := dbs.database.Load(); v != nil {
+ return v.(io.Closer)
+ }
+ return nil
+}
+
+func generateData(p parameter) []*dataPoints {
+ dpsList := make([]*dataPoints, 0)
+ for i := 0; i < p.batchCount; i++ {
+ dps := &dataPoints{
+ seriesIDs: []common.SeriesID{},
+ timestamps: []int64{},
+ versions: []int64{},
+ tagFamilies: [][]nameValues{},
+ fields: []nameValues{},
+ }
+ for j := 1; j <= p.timestampCount; j++ {
+ timestamp := i*p.timestampCount + j
+ unixTimestamp := time.Unix(int64(timestamp), 0).UnixNano()
+ for k := 1; k <= p.seriesCount; k++ {
+ dps.seriesIDs = append(dps.seriesIDs, common.SeriesID(k))
+ dps.timestamps = append(dps.timestamps, unixTimestamp)
+ dps.versions = append(dps.versions, int64(timestamp))
+
+ tagFamilies := []nameValues{
+ {
+ name: "benchmark-family",
+ values: []*nameValue{
+ {
+ name: "entity-tag",
+ valueType: pbv1.ValueTypeStr,
+ value: []byte(entityTagValuePrefix + strconv.Itoa(k)),
+ },
+ },
+ },
+ }
+ dps.tagFamilies = append(dps.tagFamilies, tagFamilies)
+
+ fields := nameValues{
+ name: "benchmark-fields",
+ values: []*nameValue{
+ {
+ name: "value",
+ valueType: pbv1.ValueTypeFloat64,
+ value: convert.Float64ToBytes(float64(k)*10.5 + float64(j)*0.1),
+ },
+ },
+ }
+ dps.fields = append(dps.fields, fields)
+ }
+ }
+ dpsList = append(dpsList, dps)
+ }
+ return dpsList
+}
+
+func openDatabase(b *testing.B, path string, cache storage.Cache) storage.TSDB[*tsTable, option] {
+ ir := storage.IntervalRule{
+ Unit: storage.DAY,
+ Num: 1,
+ }
+ opts := storage.TSDBOpts[*tsTable, option]{
+ ShardNum: 1,
+ Location: path,
+ TSTableCreator: newTSTable,
+ SegmentInterval: ir,
+ TTL: ir,
+ Option: option{
+ mergePolicy: newMergePolicy(math.MaxInt32, math.MaxFloat64, run.Bytes(math.MaxInt64)),
+ flushTimeout: time.Hour,
+ seriesCacheMaxSize: 0,
+ },
+ }
+
+ group := "benchmark"
+ db, err := storage.OpenTSDB(
+ common.SetPosition(context.Background(), func(p common.Position) common.Position {
+ p.Module = "measure"
+ p.Database = "benchmark"
+ return p
+ }),
+ opts, cache, group,
+ )
+ require.NoError(b, err)
+ return db
+}
+
+func write(b *testing.B, p parameter, dpsList []*dataPoints) (storage.TSDB[*tsTable, option], storage.Cache, func()) {
+ tmpPath, defFn := test.Space(require.New(b))
+ cache := storage.NewServiceCache()
+ db := openDatabase(b, tmpPath, cache)
+
+ var docs index.Documents
+ for i := 1; i <= p.seriesCount; i++ {
+ entity := []*modelv1.TagValue{
+ {
+ Value: &modelv1.TagValue_Str{
+ Str: &modelv1.Str{
+ Value: entityTagValuePrefix + strconv.Itoa(i),
+ },
+ },
+ },
+ }
+ series := &pbv1.Series{
+ Subject: "benchmark",
+ EntityValues: entity,
+ }
+ err := series.Marshal()
+ require.NoError(b, err)
+ docs = append(docs, index.Document{
+ DocID: uint64(i),
+ EntityValues: series.Buffer,
+ })
+ }
+
+ seg, err := db.CreateSegmentIfNotExist(time.Unix(0, dpsList[0].timestamps[0]))
+ require.NoError(b, err)
+ seg.IndexDB().Insert(docs)
+ tst, err := seg.CreateTSTableIfNotExist(common.ShardID(0))
+ require.NoError(b, err)
+ for i := range dpsList {
+ tst.mustAddDataPoints(dpsList[i])
+ }
+ return db, cache, func() {
+ db.Close()
+ defFn()
+ }
+}
+
+func generateMeasure(db storage.TSDB[*tsTable, option], cache storage.Cache) *measure {
+ dbSupplier := &databaseSupplier{
+ database: atomic.Value{},
+ }
+ dbSupplier.database.Store(db)
+
+ entity := &databasev1.Entity{
+ TagNames: []string{"entity-tag"},
+ }
+ tagFamily := &databasev1.TagFamilySpec{
+ Name: "benchmark-family",
+ Tags: []*databasev1.TagSpec{
+ {
+ Name: "entity-tag",
+ Type: databasev1.TagType_TAG_TYPE_STRING,
+ },
+ },
+ }
+ fields := []*databasev1.FieldSpec{
+ {
+ Name: "value",
+ FieldType: databasev1.FieldType_FIELD_TYPE_FLOAT,
+ EncodingMethod: databasev1.EncodingMethod_ENCODING_METHOD_GORILLA,
+ CompressionMethod: databasev1.CompressionMethod_COMPRESSION_METHOD_ZSTD,
+ },
+ }
+ schema := &databasev1.Measure{
+ Entity: entity,
+ TagFamilies: []*databasev1.TagFamilySpec{tagFamily},
+ Fields: fields,
+ }
+ l := logger.GetLogger("bootstrap")
+ ctx := context.Background()
+ pipeline := queue.Local()
+ metaSvc, err := embeddedserver.NewService(ctx)
+ if err != nil {
+ l.Fatal().Err(err).Msg("failed to initiate metadata service")
+ }
+ metricSvc := observability.NewMetricService(metaSvc, pipeline, "standalone", nil)
+ pm := protector.NewMemory(metricSvc)
+
+ m := &measure{
+ schema: schema,
+ c: cache,
+ pm: pm,
+ }
+ if err := m.parseSpec(); err != nil {
+ panic(err)
+ }
+ m.tsdb.Store(db)
+ return m
+}
+
+func generateMeasureQueryOptions(p parameter) model.MeasureQueryOptions {
+ timeRange := timestamp.TimeRange{
+ Start: time.Unix(int64(p.startTimestamp), 0),
+ End: time.Unix(int64(p.endTimestamp), 0),
+ IncludeStart: true,
+ IncludeEnd: true,
+ }
+ entities := make([][]*modelv1.TagValue, 0)
+ for i := 1; i <= p.seriesToQuery; i++ {
+ entity := []*modelv1.TagValue{
+ {
+ Value: &modelv1.TagValue_Str{
+ Str: &modelv1.Str{
+ Value: entityTagValuePrefix + strconv.Itoa(i),
+ },
+ },
+ },
+ }
+ entities = append(entities, entity)
+ }
+ return model.MeasureQueryOptions{
+ Name: "benchmark",
+ TimeRange: &timeRange,
+ Entities: entities,
+ FieldProjection: []string{"value"},
+ }
+}
+
+func (m *measure) QueryWithCache(ctx context.Context, mqo model.MeasureQueryOptions, cache storage.Cache) (mqr model.MeasureQueryResult, err error) {
+ if mqo.TimeRange == nil {
+ return nil, errors.New("invalid query options: timeRange are required")
+ }
+ if len(mqo.TagProjection) == 0 && len(mqo.FieldProjection) == 0 {
+ return nil, errors.New("invalid query options: tagProjection or fieldProjection is required")
+ }
+ var tsdb storage.TSDB[*tsTable, option]
+ db := m.tsdb.Load()
+ if db == nil {
+ tsdb, err = m.schemaRepo.loadTSDB(m.group)
+ if err != nil {
+ return nil, err
+ }
+ m.tsdb.Store(tsdb)
+ } else {
+ tsdb = db.(storage.TSDB[*tsTable, option])
+ }
+
+ segments, err := tsdb.SelectSegments(*mqo.TimeRange)
+ if err != nil {
+ return nil, err
+ }
+ if len(segments) < 1 {
+ return nilResult, nil
+ }
+
+ if m.schema.IndexMode {
+ return m.buildIndexQueryResult(ctx, mqo, segments)
+ }
+
+ if len(mqo.Entities) < 1 {
+ return nil, errors.New("invalid query options: series is required")
+ }
+
+ series := make([]*pbv1.Series, len(mqo.Entities))
+ for i := range mqo.Entities {
+ series[i] = &pbv1.Series{
+ Subject: mqo.Name,
+ EntityValues: mqo.Entities[i],
+ }
+ }
+
+ sids, tables, _, storedIndexValue, newTagProjection, err := m.searchSeriesList(ctx, series, mqo, segments)
+ if err != nil {
+ return nil, err
+ }
+ if len(sids) < 1 {
+ for i := range segments {
+ segments[i].DecRef()
+ }
+ return nilResult, nil
+ }
+ result := queryResult{
+ ctx: ctx,
+ segments: segments,
+ tagProjection: mqo.TagProjection,
+ storedIndexValue: storedIndexValue,
+ }
+ defer func() {
+ if err != nil {
+ result.Release()
+ }
+ }()
+ mqo.TagProjection = newTagProjection
+ var parts []*part
+ qo := queryOptions{
+ MeasureQueryOptions: mqo,
+ minTimestamp: mqo.TimeRange.Start.UnixNano(),
+ maxTimestamp: mqo.TimeRange.End.UnixNano(),
+ }
+ var n int
+ for i := range tables {
+ s := tables[i].currentSnapshot()
+ if s == nil {
+ continue
+ }
+ parts, n = s.getParts(parts, cache, qo.minTimestamp, qo.maxTimestamp)
+ if n < 1 {
+ s.decRef()
+ continue
+ }
+ result.snapshots = append(result.snapshots, s)
+ }
+
+ if err = m.searchBlocks(ctx, &result, sids, parts, qo); err != nil {
+ return nil, err
+ }
+
+ if mqo.Order == nil {
+ result.ascTS = true
+ result.orderByTS = true
+ } else {
+ if mqo.Order.Sort == modelv1.Sort_SORT_ASC || mqo.Order.Sort == modelv1.Sort_SORT_UNSPECIFIED {
+ result.ascTS = true
+ }
+ switch mqo.Order.Type {
+ case index.OrderByTypeTime:
+ result.orderByTS = true
+ case index.OrderByTypeIndex:
+ result.orderByTS = false
+ case index.OrderByTypeSeries:
+ result.orderByTS = false
+ }
+ }
+
+ return &result, nil
+}
+
+func BenchmarkMeasureQuery(b *testing.B) {
+ b.ReportAllocs()
+ ctx := context.TODO()
+
+ for _, p := range pList {
+ dpsList := generateData(p)
+ db, cache, cleanup := write(b, p, dpsList)
+ defer cleanup()
+ m := generateMeasure(db, cache)
+ mqo := generateMeasureQueryOptions(p)
+
+ result, err := m.QueryWithCache(ctx, mqo, cache)
+ require.NoError(b, err)
+ if result != nil {
+ result.Release()
+ }
+
+ b.Run("query-with-cache", func(b *testing.B) {
+ b.ResetTimer()
+ b.RunParallel(func(pb *testing.PB) {
+ for pb.Next() {
+ result, err := m.QueryWithCache(ctx, mqo, cache)
+ require.NoError(b, err)
+ if result != nil {
+ result.Release()
+ }
+ }
+ })
+ })
+ }
+}
diff --git a/banyand/measure/measure.go b/banyand/measure/measure.go
index 56aedaaa..90bf41fc 100644
--- a/banyand/measure/measure.go
+++ b/banyand/measure/measure.go
@@ -25,6 +25,7 @@ import (
"time"
databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ "github.com/apache/skywalking-banyandb/banyand/internal/storage"
"github.com/apache/skywalking-banyandb/banyand/protector"
"github.com/apache/skywalking-banyandb/pkg/logger"
"github.com/apache/skywalking-banyandb/pkg/partition"
@@ -69,6 +70,7 @@ func (i *indexSchema) parse(schema *databasev1.Measure) {
type measure struct {
indexSchema atomic.Value
tsdb atomic.Value
+ c storage.Cache
pm protector.Memory
l *logger.Logger
schema *databasev1.Measure
@@ -78,33 +80,33 @@ type measure struct {
interval time.Duration
}
-func (s *measure) GetSchema() *databasev1.Measure {
- return s.schema
+func (m *measure) GetSchema() *databasev1.Measure {
+ return m.schema
}
-func (s *measure) GetIndexRules() []*databasev1.IndexRule {
- is := s.indexSchema.Load()
+func (m *measure) GetIndexRules() []*databasev1.IndexRule {
+ is := m.indexSchema.Load()
if is == nil {
return nil
}
return is.(indexSchema).indexRules
}
-func (s *measure) OnIndexUpdate(index []*databasev1.IndexRule) {
+func (m *measure) OnIndexUpdate(index []*databasev1.IndexRule) {
var is indexSchema
is.indexRules = index
- is.parse(s.schema)
- s.indexSchema.Store(is)
+ is.parse(m.schema)
+ m.indexSchema.Store(is)
}
-func (s *measure) parseSpec() (err error) {
- s.name, s.group = s.schema.GetMetadata().GetName(), s.schema.GetMetadata().GetGroup()
- if s.schema.Interval != "" {
- s.interval, err = timestamp.ParseDuration(s.schema.Interval)
+func (m *measure) parseSpec() (err error) {
+ m.name, m.group = m.schema.GetMetadata().GetName(), m.schema.GetMetadata().GetGroup()
+ if m.schema.Interval != "" {
+ m.interval, err = timestamp.ParseDuration(m.schema.Interval)
}
var is indexSchema
- is.parse(s.schema)
- s.indexSchema.Store(is)
+ is.parse(m.schema)
+ m.indexSchema.Store(is)
return err
}
@@ -113,11 +115,12 @@ type measureSpec struct {
}
func openMeasure(spec measureSpec,
- l *logger.Logger, pm protector.Memory, schemaRepo *schemaRepo,
+ l *logger.Logger, c storage.Cache, pm protector.Memory, schemaRepo *schemaRepo,
) (*measure, error) {
m := &measure{
schema: spec.schema,
l: l,
+ c: c,
pm: pm,
schemaRepo: schemaRepo,
}
diff --git a/banyand/measure/metadata.go b/banyand/measure/metadata.go
index 69f28f59..f7ec24a4 100644
--- a/banyand/measure/metadata.go
+++ b/banyand/measure/metadata.go
@@ -344,6 +344,7 @@ type supplier struct {
metadata metadata.Repo
omr observability.MetricsRegistry
l *logger.Logger
+ c storage.Cache
pm protector.Memory
schemaRepo *schemaRepo
nodeLabels map[string]string
@@ -356,6 +357,7 @@ func newSupplier(path string, svc *service, sr *schemaRepo, nodeLabels map[strin
path: path,
metadata: svc.metadata,
l: svc.l,
+ c: svc.c,
option: svc.option,
omr: svc.omr,
pm: svc.pm,
@@ -368,7 +370,7 @@ func (s *supplier) OpenResource(spec resourceSchema.Resource) (resourceSchema.In
measureSchema := spec.Schema().(*databasev1.Measure)
return openMeasure(measureSpec{
schema: measureSchema,
- }, s.l, s.pm, s.schemaRepo)
+ }, s.l, s.c, s.pm, s.schemaRepo)
}
func (s *supplier) ResourceSchema(md *commonv1.Metadata) (resourceSchema.ResourceSchema, error) {
@@ -415,9 +417,10 @@ func (s *supplier) OpenDB(groupSchema *commonv1.Group) (resourceSchema.DB, error
break
}
}
+ group := groupSchema.Metadata.Name
opts := storage.TSDBOpts[*tsTable, option]{
ShardNum: shardNum,
- Location: path.Join(s.path, groupSchema.Metadata.Name),
+ Location: path.Join(s.path, group),
TSTableCreator: newTSTable,
TableMetrics: metrics,
SegmentInterval: storage.MustToIntervalRule(segInterval),
@@ -433,7 +436,7 @@ func (s *supplier) OpenDB(groupSchema *commonv1.Group) (resourceSchema.DB, error
common.SetPosition(context.Background(), func(_ common.Position) common.Position {
return p
}),
- opts)
+ opts, s.c, group)
}
type portableSupplier struct {
@@ -462,7 +465,7 @@ func (s *portableSupplier) OpenResource(spec resourceSchema.Resource) (resourceS
measureSchema := spec.Schema().(*databasev1.Measure)
return openMeasure(measureSpec{
schema: measureSchema,
- }, s.l, nil, nil)
+ }, s.l, nil, nil, nil)
}
// GetTopNSchema returns the schema of the topN result measure.
diff --git a/banyand/measure/metrics.go b/banyand/measure/metrics.go
index 96e8ff22..6ab3cb47 100644
--- a/banyand/measure/metrics.go
+++ b/banyand/measure/metrics.go
@@ -384,3 +384,26 @@ func (s *service) createNativeObservabilityGroup(ctx context.Context) error {
}
return nil
}
+
+type cacheMetrics struct {
+ requests meter.Gauge
+ misses meter.Gauge
+ hitRatio meter.Gauge
+ entries meter.Gauge
+ size meter.Gauge
+}
+
+func newCacheMetrics(omr observability.MetricsRegistry) *cacheMetrics {
+ if omr == nil {
+ return nil
+ }
+
+ factory := omr.With(measureScope.SubScope("cache"))
+ return &cacheMetrics{
+ requests: factory.NewGauge("requests_total"),
+ misses: factory.NewGauge("misses_total"),
+ hitRatio: factory.NewGauge("hit_ratio"),
+ entries: factory.NewGauge("entries_count"),
+ size: factory.NewGauge("size_bytes"),
+ }
+}
diff --git a/banyand/measure/part.go b/banyand/measure/part.go
index da47baf9..05836dc4 100644
--- a/banyand/measure/part.go
+++ b/banyand/measure/part.go
@@ -49,6 +49,7 @@ type part struct {
fileSystem fs.FileSystem
tagFamilyMetadata map[string]fs.Reader
tagFamilies map[string]fs.Reader
+ cache storage.Cache
path string
primaryBlockMetadata []primaryBlockMetadata
partMetadata partMetadata
diff --git a/banyand/measure/part_iter.go b/banyand/measure/part_iter.go
index cf67d320..3089258b 100644
--- a/banyand/measure/part_iter.go
+++ b/banyand/measure/part_iter.go
@@ -24,6 +24,7 @@ import (
"sort"
"github.com/apache/skywalking-banyandb/api/common"
+ "github.com/apache/skywalking-banyandb/banyand/internal/storage"
"github.com/apache/skywalking-banyandb/pkg/bytes"
"github.com/apache/skywalking-banyandb/pkg/compress/zstd"
"github.com/apache/skywalking-banyandb/pkg/encoding"
@@ -35,6 +36,7 @@ import (
type partIter struct {
err error
p *part
+ c storage.Cache
curBlock *blockMetadata
sids []common.SeriesID
primaryBlockMetadata []primaryBlockMetadata
@@ -49,6 +51,7 @@ type partIter struct {
func (pi *partIter) reset() {
pi.curBlock = nil
pi.p = nil
+ pi.c = nil
pi.sids = nil
pi.sidIdx = 0
pi.primaryBlockMetadata = nil
@@ -62,6 +65,7 @@ func (pi *partIter) init(bma *blockMetadataArray, p *part, sids []common.SeriesI
pi.reset()
pi.curBlock = &blockMetadata{}
pi.p = p
+ pi.c = p.cache
pi.bms = bma.arr
pi.sids = sids
@@ -180,6 +184,15 @@ func searchPBM(pbmIndex []primaryBlockMetadata, sid common.SeriesID) []primaryBl
}
func (pi *partIter) readPrimaryBlock(bms []blockMetadata, mr *primaryBlockMetadata) ([]blockMetadata, error) {
+ value := pi.c.Get(storage.NewEntryKey(pi.p.partMetadata.ID, mr.offset))
+ if value != nil {
+ bmPtrs := value.([]*blockMetadata)
+ for _, bmsPtr := range bmPtrs {
+ bms = append(bms, *bmsPtr)
+ }
+ return bms, nil
+ }
+
pi.compressedPrimaryBuf = bytes.ResizeOver(pi.compressedPrimaryBuf, int(mr.size))
fs.MustReadData(pi.p.primary, int64(mr.offset), pi.compressedPrimaryBuf)
@@ -192,6 +205,11 @@ func (pi *partIter) readPrimaryBlock(bms []blockMetadata, mr *primaryBlockMetada
if err != nil {
return nil, fmt.Errorf("cannot unmarshal index block: %w", err)
}
+ bmPtrs := make([]*blockMetadata, 0, len(bms))
+ for _, bm := range bms {
+ bmPtrs = append(bmPtrs, &bm)
+ }
+ pi.c.Put(storage.NewEntryKey(pi.p.partMetadata.ID, mr.offset), bmPtrs)
return bms, nil
}
diff --git a/banyand/measure/part_iter_test.go b/banyand/measure/part_iter_test.go
index 11dae20c..45e6e7de 100644
--- a/banyand/measure/part_iter_test.go
+++ b/banyand/measure/part_iter_test.go
@@ -26,6 +26,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/apache/skywalking-banyandb/api/common"
+ "github.com/apache/skywalking-banyandb/banyand/internal/storage"
"github.com/apache/skywalking-banyandb/pkg/encoding"
"github.com/apache/skywalking-banyandb/pkg/fs"
"github.com/apache/skywalking-banyandb/pkg/test"
@@ -147,6 +148,8 @@ func Test_partIter_nextBlock(t *testing.T) {
mp.mustInitFromDataPoints(dps)
p := openMemPart(mp)
+ shardCache := storage.NewShardCache("test-group", 0, 0)
+ p.cache = shardCache
verifyPart(p)
tmpDir, defFn := test.Space(require.New(t))
defer defFn()
@@ -155,6 +158,7 @@ func Test_partIter_nextBlock(t *testing.T) {
fileSystem := fs.NewLocalFileSystem()
mp.mustFlush(fileSystem, partPath)
p = mustOpenFilePart(epoch, tmpDir, fileSystem)
+ p.cache = shardCache
verifyPart(p)
})
}
diff --git a/banyand/measure/query.go b/banyand/measure/query.go
index 4b30e1c9..44a47927 100644
--- a/banyand/measure/query.go
+++ b/banyand/measure/query.go
@@ -70,7 +70,7 @@ type queryOptions struct {
maxTimestamp int64
}
-func (s *measure) Query(ctx context.Context, mqo model.MeasureQueryOptions) (mqr model.MeasureQueryResult, err error) {
+func (m *measure) Query(ctx context.Context, mqo model.MeasureQueryOptions) (mqr model.MeasureQueryResult, err error) {
if mqo.TimeRange == nil {
return nil, errors.New("invalid query options: timeRange are required")
}
@@ -78,13 +78,13 @@ func (s *measure) Query(ctx context.Context, mqo model.MeasureQueryOptions) (mqr
return nil, errors.New("invalid query options: tagProjection or fieldProjection is required")
}
var tsdb storage.TSDB[*tsTable, option]
- db := s.tsdb.Load()
+ db := m.tsdb.Load()
if db == nil {
- tsdb, err = s.schemaRepo.loadTSDB(s.group)
+ tsdb, err = m.schemaRepo.loadTSDB(m.group)
if err != nil {
return nil, err
}
- s.tsdb.Store(tsdb)
+ m.tsdb.Store(tsdb)
} else {
tsdb = db.(storage.TSDB[*tsTable, option])
}
@@ -97,8 +97,8 @@ func (s *measure) Query(ctx context.Context, mqo model.MeasureQueryOptions) (mqr
return nilResult, nil
}
- if s.schema.IndexMode {
- return s.buildIndexQueryResult(ctx, mqo, segments)
+ if m.schema.IndexMode {
+ return m.buildIndexQueryResult(ctx, mqo, segments)
}
if len(mqo.Entities) < 1 {
@@ -113,7 +113,7 @@ func (s *measure) Query(ctx context.Context, mqo model.MeasureQueryOptions) (mqr
}
}
- sids, tables, storedIndexValue, newTagProjection, err := s.searchSeriesList(ctx, series, mqo, segments)
+ sids, tables, caches, storedIndexValue, newTagProjection, err := m.searchSeriesList(ctx, series, mqo, segments)
if err != nil {
return nil, err
}
@@ -147,7 +147,7 @@ func (s *measure) Query(ctx context.Context, mqo model.MeasureQueryOptions) (mqr
if s == nil {
continue
}
- parts, n = s.getParts(parts, qo.minTimestamp, qo.maxTimestamp)
+ parts, n = s.getParts(parts, caches[i], qo.minTimestamp, qo.maxTimestamp)
if n < 1 {
s.decRef()
continue
@@ -155,7 +155,7 @@ func (s *measure) Query(ctx context.Context, mqo model.MeasureQueryOptions) (mqr
result.snapshots = append(result.snapshots, s)
}
- if err = s.searchBlocks(ctx, &result, sids, parts, qo); err != nil {
+ if err = m.searchBlocks(ctx, &result, sids, parts, qo); err != nil {
return nil, err
}
@@ -184,22 +184,22 @@ type tagNameWithType struct {
typ pbv1.ValueType
}
-func (s *measure) searchSeriesList(ctx context.Context, series []*pbv1.Series, mqo model.MeasureQueryOptions,
+func (m *measure) searchSeriesList(ctx context.Context, series []*pbv1.Series, mqo model.MeasureQueryOptions,
segments []storage.Segment[*tsTable, option],
-) (sl []common.SeriesID, tables []*tsTable, storedIndexValue map[common.SeriesID]map[string]*modelv1.TagValue,
+) (sl []common.SeriesID, tables []*tsTable, caches []storage.Cache, storedIndexValue map[common.SeriesID]map[string]*modelv1.TagValue,
newTagProjection []model.TagProjection, err error,
) {
var indexProjection []index.FieldKey
fieldToValueType := make(map[string]tagNameWithType)
var projectedEntityOffsets map[string]int
newTagProjection = make([]model.TagProjection, 0)
- is := s.indexSchema.Load().(indexSchema)
+ is := m.indexSchema.Load().(indexSchema)
for _, tp := range mqo.TagProjection {
var tagProjection model.TagProjection
TAG:
for _, n := range tp.Names {
- for i := range s.schema.GetEntity().GetTagNames() {
- if n == s.schema.GetEntity().GetTagNames()[i] {
+ for i := range m.schema.GetEntity().GetTagNames() {
+ if n == m.schema.GetEntity().GetTagNames()[i] {
if projectedEntityOffsets == nil {
projectedEntityOffsets = make(map[string]int)
}
@@ -235,10 +235,12 @@ func (s *measure) searchSeriesList(ctx context.Context, series []*pbv1.Series, m
Projection: indexProjection,
})
if err != nil {
- return nil, nil, nil, nil, err
+ return nil, nil, nil, nil, nil, err
}
if len(sd.SeriesList) > 0 {
- tables = append(tables, segments[i].Tables()...)
+ tt, cc := segments[i].Tables()
+ tables = append(tables, tt...)
+ caches = append(caches, cc...)
}
for j := range sd.SeriesList {
if seriesFilter.Contains(uint64(sd.SeriesList[j].ID)) {
@@ -274,10 +276,10 @@ func (s *measure) searchSeriesList(ctx context.Context, series []*pbv1.Series, m
}
}
}
- return sl, tables, storedIndexValue, newTagProjection, nil
+ return sl, tables, caches, storedIndexValue, newTagProjection, nil
}
-func (s *measure) buildIndexQueryResult(ctx context.Context, mqo model.MeasureQueryOptions,
+func (m *measure) buildIndexQueryResult(ctx context.Context, mqo model.MeasureQueryOptions,
segments []storage.Segment[*tsTable, option],
) (model.MeasureQueryResult, error) {
defer func() {
@@ -285,7 +287,7 @@ func (s *measure) buildIndexQueryResult(ctx context.Context, mqo model.MeasureQu
segments[i].DecRef()
}
}()
- is := s.indexSchema.Load().(indexSchema)
+ is := m.indexSchema.Load().(indexSchema)
r := &indexSortResult{}
var indexProjection []index.FieldKey
for _, tp := range mqo.TagProjection {
@@ -297,8 +299,8 @@ func (s *measure) buildIndexQueryResult(ctx context.Context, mqo model.MeasureQu
TAG:
for _, n := range tp.Names {
tagFamilyLocation.tagNames = append(tagFamilyLocation.tagNames, n)
- for i := range s.schema.GetEntity().GetTagNames() {
- if n == s.schema.GetEntity().GetTagNames()[i] {
+ for i := range m.schema.GetEntity().GetTagNames() {
+ if n == m.schema.GetEntity().GetTagNames()[i] {
tagFamilyLocation.projectedEntityOffsets[n] = i
continue TAG
}
@@ -358,7 +360,7 @@ func (s *measure) buildIndexQueryResult(ctx context.Context, mqo model.MeasureQu
return r, nil
}
-func (s *measure) searchBlocks(ctx context.Context, result *queryResult, sids []common.SeriesID, parts []*part, qo queryOptions) error {
+func (m *measure) searchBlocks(ctx context.Context, result *queryResult, sids []common.SeriesID, parts []*part, qo queryOptions) error {
bma := generateBlockMetadataArray()
defer releaseBlockMetadataArray(bma)
defFn := startBlockScanSpan(ctx, len(sids), parts, result)
@@ -374,7 +376,7 @@ func (s *measure) searchBlocks(ctx context.Context, result *queryResult, sids []
}
var hit int
var totalBlockBytes uint64
- quota := s.pm.AvailableBytes()
+ quota := m.pm.AvailableBytes()
for tstIter.nextBlock() {
if hit%checkDoneEvery == 0 {
select {
@@ -396,7 +398,7 @@ func (s *measure) searchBlocks(ctx context.Context, result *queryResult, sids []
if tstIter.Error() != nil {
return fmt.Errorf("cannot iterate tstIter: %w", tstIter.Error())
}
- if err := s.pm.AcquireResource(ctx, totalBlockBytes); err != nil {
+ if err := m.pm.AcquireResource(ctx, totalBlockBytes); err != nil {
return err
}
result.sidToIndex = make(map[common.SeriesID]int)
diff --git a/banyand/measure/query_test.go b/banyand/measure/query_test.go
index 268f0671..256a6bac 100644
--- a/banyand/measure/query_test.go
+++ b/banyand/measure/query_test.go
@@ -30,6 +30,7 @@ import (
"github.com/apache/skywalking-banyandb/api/common"
modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+ "github.com/apache/skywalking-banyandb/banyand/internal/storage"
itest "github.com/apache/skywalking-banyandb/banyand/internal/test"
"github.com/apache/skywalking-banyandb/pkg/fs"
"github.com/apache/skywalking-banyandb/pkg/logger"
@@ -1243,7 +1244,8 @@ func TestQueryResult(t *testing.T) {
s := tst.currentSnapshot()
require.NotNil(t, s)
defer s.decRef()
- pp, _ := s.getParts(nil, queryOpts.minTimestamp, queryOpts.maxTimestamp)
+ shardCache := storage.NewShardCache("test-group", 0, 0)
+ pp, _ := s.getParts(nil, shardCache, queryOpts.minTimestamp, queryOpts.maxTimestamp)
sids := make([]common.SeriesID, len(tt.sids))
copy(sids, tt.sids)
sort.Slice(sids, func(i, j int) bool {
@@ -1467,7 +1469,8 @@ func TestQueryResult_QuotaExceeded(t *testing.T) {
s := tst.currentSnapshot()
require.NotNil(t, s)
defer s.decRef()
- pp, _ := s.getParts(nil, queryOpts.minTimestamp, queryOpts.maxTimestamp)
+ shardCache := storage.NewShardCache("test-group", 0, 0)
+ pp, _ := s.getParts(nil, shardCache, queryOpts.minTimestamp, queryOpts.maxTimestamp)
var result queryResult
result.ctx = context.TODO()
// Query all tags
diff --git a/banyand/measure/service.go b/banyand/measure/service.go
index 2106fd95..ef8ac056 100644
--- a/banyand/measure/service.go
+++ b/banyand/measure/service.go
@@ -73,10 +73,13 @@ type service struct {
pm protector.Memory
schemaRepo *schemaRepo
l *logger.Logger
+ c storage.Cache
+ cm *cacheMetrics
root string
snapshotDir string
dataPath string
option option
+ cc storage.CacheConfig
maxDiskUsagePercent int
maxFileSnapshotNum int
}
@@ -108,6 +111,10 @@ func (s *service) FlagSet() *run.FlagSet {
flagS.VarP(&s.option.seriesCacheMaxSize, "measure-series-cache-max-size", "", "the max size of series cache in each group")
flagS.IntVar(&s.maxDiskUsagePercent, "measure-max-disk-usage-percent", 95, "the maximum disk usage percentage allowed")
flagS.IntVar(&s.maxFileSnapshotNum, "measure-max-file-snapshot-num", 10, "the maximum number of file snapshots allowed")
+ s.cc.MaxCacheSize = run.Bytes(100 * 1024 * 1024)
+ flagS.VarP(&s.cc.MaxCacheSize, "service-cache-max-size", "", "maximum service cache size (e.g., 100M)")
+ flagS.DurationVar(&s.cc.CleanupInterval, "service-cache-cleanup-interval", 30*time.Second, "service cache cleanup interval")
+ flagS.DurationVar(&s.cc.IdleTimeout, "service-cache-idle-timeout", 2*time.Minute, "service cache entry idle timeout")
return flagS
}
@@ -121,6 +128,15 @@ func (s *service) Validate() error {
if s.maxDiskUsagePercent > 100 {
return errors.New("measure-max-disk-usage-percen must be less than or equal to 100")
}
+ if s.cc.MaxCacheSize < 0 {
+ return errors.New("service-cache-max-size must be greater than or equal to 0")
+ }
+ if s.cc.CleanupInterval <= 0 {
+ return errors.New("service-cache-cleanup-interval must be greater than 0")
+ }
+ if s.cc.IdleTimeout <= 0 {
+ return errors.New("service-cache-idle-timeout must be greater than 0")
+ }
return nil
}
@@ -149,8 +165,13 @@ func (s *service) PreRun(ctx context.Context) error {
if val == nil {
return errors.New("node id is empty")
}
+ s.c = storage.NewServiceCacheWithConfig(s.cc)
node := val.(common.Node)
s.schemaRepo = newSchemaRepo(s.dataPath, s, node.Labels)
+
+ s.cm = newCacheMetrics(s.omr)
+ observability.MetricsCollector.Register("measure_cache", s.collectCacheMetrics)
+
if s.pipeline == nil {
return nil
}
@@ -187,7 +208,12 @@ func (s *service) Serve() run.StopNotify {
}
func (s *service) GracefulStop() {
+ observability.MetricsCollector.Unregister("measure_cache")
s.schemaRepo.Close()
+ s.c.Close()
+ if s.localPipeline != nil {
+ s.localPipeline.GracefulStop()
+ }
}
func (s *service) InFlow(stm *databasev1.Measure, seriesID uint64, shardID uint32, entityValues []*modelv1.TagValue, dp *measurev1.DataPointValue) {
@@ -198,6 +224,27 @@ func (s *service) InFlow(stm *databasev1.Measure, seriesID uint64, shardID uint3
s.schemaRepo.InFlow(stm, seriesID, shardID, entityValues, dp)
}
+func (s *service) collectCacheMetrics() {
+ if s.cm == nil || s.c == nil {
+ return
+ }
+
+ requests := s.c.Requests()
+ misses := s.c.Misses()
+ length := s.c.Entries()
+ size := s.c.Size()
+ var hitRatio float64
+ if requests > 0 {
+ hitRatio = float64(requests-misses) / float64(requests)
+ }
+
+ s.cm.requests.Set(float64(requests))
+ s.cm.misses.Set(float64(misses))
+ s.cm.hitRatio.Set(hitRatio)
+ s.cm.entries.Set(float64(length))
+ s.cm.size.Set(float64(size))
+}
+
// NewService returns a new service.
func NewService(metadata metadata.Repo, pipeline queue.Server, metricPipeline queue.Server, omr observability.MetricsRegistry, pm protector.Memory) (Service, error) {
return &service{
diff --git a/banyand/measure/snapshot.go b/banyand/measure/snapshot.go
index fe57c0fe..4151d6de 100644
--- a/banyand/measure/snapshot.go
+++ b/banyand/measure/snapshot.go
@@ -65,13 +65,14 @@ type snapshot struct {
ref int32
}
-func (s *snapshot) getParts(dst []*part, minTimestamp, maxTimestamp int64) ([]*part, int) {
+func (s *snapshot) getParts(dst []*part, cache storage.Cache, minTimestamp, maxTimestamp int64) ([]*part, int) {
var count int
for _, p := range s.parts {
pm := p.p.partMetadata
if maxTimestamp < pm.MinTimestamp || minTimestamp > pm.MaxTimestamp {
continue
}
+ p.p.cache = cache
dst = append(dst, p.p)
count++
}
diff --git a/banyand/measure/snapshot_test.go b/banyand/measure/snapshot_test.go
index b224f4f8..dad0c925 100644
--- a/banyand/measure/snapshot_test.go
+++ b/banyand/measure/snapshot_test.go
@@ -122,7 +122,7 @@ func TestSnapshotGetParts(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- result, count := tt.snapshot.getParts(tt.dst, tt.opts.minTimestamp, tt.opts.maxTimestamp)
+ result, count := tt.snapshot.getParts(tt.dst, nil, tt.opts.minTimestamp, tt.opts.maxTimestamp)
assert.Equal(t, tt.expected, result)
assert.Equal(t, tt.count, count)
})
diff --git a/banyand/measure/tstable_test.go b/banyand/measure/tstable_test.go
index 314bdcf7..75ca5b2d 100644
--- a/banyand/measure/tstable_test.go
+++ b/banyand/measure/tstable_test.go
@@ -28,6 +28,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/apache/skywalking-banyandb/api/common"
+ "github.com/apache/skywalking-banyandb/banyand/internal/storage"
"github.com/apache/skywalking-banyandb/pkg/convert"
"github.com/apache/skywalking-banyandb/pkg/fs"
pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
@@ -147,7 +148,8 @@ func Test_tstIter(t *testing.T) {
s = new(snapshot)
}
defer s.decRef()
- pp, n := s.getParts(nil, tt.minTimestamp, tt.maxTimestamp)
+ shardCache := storage.NewShardCache("test-group", 0, 0)
+ pp, n := s.getParts(nil, shardCache, tt.minTimestamp, tt.maxTimestamp)
require.Equal(t, len(s.parts), n)
ti := &tstIter{}
ti.init(bma, pp, tt.sids, tt.minTimestamp, tt.maxTimestamp)
diff --git a/banyand/stream/benchmark_test.go b/banyand/stream/benchmark_test.go
index 97945f2f..9c7c68db 100644
--- a/banyand/stream/benchmark_test.go
+++ b/banyand/stream/benchmark_test.go
@@ -177,13 +177,16 @@ func openDatabase(b *testing.B, path string) storage.TSDB[*tsTable, option] {
SegmentInterval: ir,
TTL: ir,
}
+
+ group := "test"
db, err := storage.OpenTSDB(
common.SetPosition(context.Background(), func(p common.Position) common.Position {
p.Module = "stream"
p.Database = "benchmark"
return p
}),
- opts)
+ opts, nil, group,
+ )
require.NoError(b, err)
return db
}
diff --git a/banyand/stream/block_scanner.go b/banyand/stream/block_scanner.go
index 5510dac6..889adb64 100644
--- a/banyand/stream/block_scanner.go
+++ b/banyand/stream/block_scanner.go
@@ -107,7 +107,7 @@ func searchSeries(ctx context.Context, qo queryOptions, segment storage.Segment[
func getBlockScanner(ctx context.Context, segment storage.Segment[*tsTable, *option], qo queryOptions,
l *logger.Logger, pm protector.Memory, tr *index.RangeOpts,
) (bc *blockScanner, err error) {
- tabs := segment.Tables()
+ tabs, _ := segment.Tables()
finalizers := make([]scanFinalizer, 0, len(tabs)+1)
finalizers = append(finalizers, segment.DecRef)
defer func() {
diff --git a/banyand/stream/metadata.go b/banyand/stream/metadata.go
index 69e65b15..d8526bbe 100644
--- a/banyand/stream/metadata.go
+++ b/banyand/stream/metadata.go
@@ -320,9 +320,10 @@ func (s *supplier) OpenDB(groupSchema *commonv1.Group) (resourceSchema.DB, error
break
}
}
+ group := groupSchema.Metadata.Name
opts := storage.TSDBOpts[*tsTable, option]{
ShardNum: shardNum,
- Location: path.Join(s.path, groupSchema.Metadata.Name),
+ Location: path.Join(s.path, group),
TSTableCreator: newTSTable,
TableMetrics: s.newMetrics(p),
SegmentInterval: storage.MustToIntervalRule(segInterval),
@@ -338,7 +339,8 @@ func (s *supplier) OpenDB(groupSchema *commonv1.Group) (resourceSchema.DB, error
common.SetPosition(context.Background(), func(_ common.Position) common.Position {
return p
}),
- opts)
+ opts, nil, group,
+ )
}
type portableSupplier struct {
diff --git a/banyand/stream/query.go b/banyand/stream/query.go
index 8d195e62..0bb1a7e9 100644
--- a/banyand/stream/query.go
+++ b/banyand/stream/query.go
@@ -216,7 +216,8 @@ func (s *stream) processSegmentsAndBuildFilters(
}
var filter, filterTS posting.List
- if filter, filterTS, err = indexSearch(ctx, sqo, segments[i].Tables(), sl.ToList().ToSlice(), tr); err != nil {
+ tables, _ := segments[i].Tables()
+ if filter, filterTS, err = indexSearch(ctx, sqo, tables, sl.ToList().ToSlice(), tr); err != nil {
return result, nil, nil, err
}
@@ -244,7 +245,8 @@ func (s *stream) processSegmentsAndBuildFilters(
result.qo.seriesToEntity[sl[j].ID] = sl[j].EntityValues
}
- result.tabs = append(result.tabs, result.segments[i].Tables()...)
+ tables, _ = result.segments[i].Tables()
+ result.tabs = append(result.tabs, tables...)
}
return result, seriesFilter, resultTS, nil