You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ha...@apache.org on 2023/05/30 03:33:57 UTC
[skywalking-banyandb] 01/03: Add units to memory flags
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch load-tet
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit f5811a77b952693343e84dd6d8e9c6d9d1e1fc12
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Fri May 12 07:05:55 2023 +0000
Add units to memory flags
Signed-off-by: Gao Hongtao <ha...@gmail.com>
---
banyand/kv/badger.go | 4 ++
banyand/kv/kv.go | 9 +++-
banyand/liaison/grpc/server.go | 9 ++--
banyand/measure/measure_query.go | 7 +++-
banyand/measure/measure_write.go | 2 +-
banyand/measure/service.go | 6 ++-
banyand/observability/collector.go | 7 ++++
banyand/stream/service.go | 9 ++--
banyand/stream/stream_query.go | 3 ++
banyand/stream/stream_write.go | 2 +-
banyand/tsdb/block.go | 62 ++++++++++++++++++++++++++-
banyand/tsdb/buffer.go | 6 +--
banyand/tsdb/index/writer.go | 2 +-
banyand/tsdb/segment.go | 2 +-
banyand/tsdb/seriesdb.go | 2 +-
banyand/tsdb/shard.go | 4 +-
banyand/tsdb/tsdb.go | 85 ++++++++++++++++++++++++--------------
banyand/tsdb/tsdb_test.go | 4 ++
go.mod | 2 +-
go.sum | 4 +-
pkg/convert/size.go | 70 +++++++++++++++++++++++++++++++
pkg/convert/size_test.go | 34 +++++++++++++++
pkg/encoding/encoder.go | 29 ++-----------
pkg/run/flags.go | 40 ++++++++++++++++++
24 files changed, 319 insertions(+), 85 deletions(-)
diff --git a/banyand/kv/badger.go b/banyand/kv/badger.go
index 48a0a367..f26f2f57 100644
--- a/banyand/kv/badger.go
+++ b/banyand/kv/badger.go
@@ -65,6 +65,10 @@ func (b *badgerTSS) SizeOnDisk() int64 {
return lsmSize + vlogSize
}
+func (b *badgerTSS) CollectStats() *badger.Statistics {
+ return b.db.CollectStats()
+}
+
type mergedIter struct {
delegated Iterator
data []byte
diff --git a/banyand/kv/kv.go b/banyand/kv/kv.go
index 5c8bcb18..cc816211 100644
--- a/banyand/kv/kv.go
+++ b/banyand/kv/kv.go
@@ -41,7 +41,7 @@ var (
PrefetchValues: true,
}
- defaultKVMemorySize = 8 << 20
+ defaultKVMemorySize = 4 << 20
)
type writer interface {
@@ -90,6 +90,7 @@ type TimeSeriesStore interface {
Handover(skl *skl.Skiplist) error
TimeSeriesReader
SizeOnDisk() int64
+ CollectStats() *badger.Statistics
}
// TimeSeriesOptions sets an options for creating a TimeSeriesStore.
@@ -184,6 +185,9 @@ func OpenTimeSeriesStore(path string, options ...TimeSeriesOptions) (TimeSeriesS
if btss.dbOpts.MemTableSize < int64(defaultKVMemorySize) {
btss.dbOpts.MemTableSize = int64(defaultKVMemorySize)
}
+ if btss.dbOpts.MemTableSize < 8<<20 {
+ btss.dbOpts = btss.dbOpts.WithValueThreshold(1 << 10)
+ }
btss.dbOpts = btss.dbOpts.WithInTable()
var err error
btss.db, err = badger.Open(btss.dbOpts)
@@ -236,6 +240,9 @@ func OpenStore(path string, options ...StoreOptions) (Store, error) {
if bdb.dbOpts.MemTableSize < int64(defaultKVMemorySize) {
bdb.dbOpts.MemTableSize = int64(defaultKVMemorySize)
}
+ if bdb.dbOpts.MemTableSize < 8<<20 {
+ bdb.dbOpts = bdb.dbOpts.WithValueThreshold(1 << 10)
+ }
var err error
bdb.db, err = badger.Open(bdb.dbOpts)
diff --git a/banyand/liaison/grpc/server.go b/banyand/liaison/grpc/server.go
index 5bcf3596..01b1d9f5 100644
--- a/banyand/liaison/grpc/server.go
+++ b/banyand/liaison/grpc/server.go
@@ -48,7 +48,7 @@ import (
"github.com/apache/skywalking-banyandb/pkg/run"
)
-const defaultRecvSize = 1024 * 1024 * 10
+const defaultRecvSize = 10 << 20
var (
errServerCert = errors.New("invalid server cert file")
@@ -76,7 +76,7 @@ type server struct {
addr string
keyFile string
certFile string
- maxRecvMsgSize int
+ maxRecvMsgSize run.Bytes
tls bool
}
@@ -155,7 +155,8 @@ func (s *server) Name() string {
func (s *server) FlagSet() *run.FlagSet {
fs := run.NewFlagSet("grpc")
- fs.IntVarP(&s.maxRecvMsgSize, "max-recv-msg-size", "", defaultRecvSize, "the size of max receiving message")
+ s.maxRecvMsgSize = defaultRecvSize
+ fs.VarP(&s.maxRecvMsgSize, "max-recv-msg-size", "", "the size of max receiving message")
fs.BoolVarP(&s.tls, "tls", "", false, "connection uses TLS if true, else plain TCP")
fs.StringVarP(&s.certFile, "cert-file", "", "", "the TLS cert file")
fs.StringVarP(&s.keyFile, "key-file", "", "", "the TLS key file")
@@ -211,7 +212,7 @@ func (s *server) Serve() run.StopNotify {
unaryChain = append(unaryChain, unaryMetrics)
}
- opts = append(opts, grpclib.MaxRecvMsgSize(s.maxRecvMsgSize),
+ opts = append(opts, grpclib.MaxRecvMsgSize(int(s.maxRecvMsgSize)),
grpclib.ChainUnaryInterceptor(unaryChain...),
grpclib.ChainStreamInterceptor(streamChain...),
)
diff --git a/banyand/measure/measure_query.go b/banyand/measure/measure_query.go
index 5cf77005..bd4b9e38 100644
--- a/banyand/measure/measure_query.go
+++ b/banyand/measure/measure_query.go
@@ -83,11 +83,14 @@ func (s *measure) Shards(entity tsdb.Entity) ([]tsdb.Shard, error) {
if err != nil {
return nil, err
}
- shard, err := s.Shard(common.ShardID(shardID))
+ shard, err := s.databaseSupplier.SupplyTSDB().Shard(common.ShardID(shardID))
if err != nil {
+ if errors.Is(err, tsdb.ErrUnknownShard) {
+ return []tsdb.Shard{}, nil
+ }
return nil, err
}
- return []tsdb.Shard{shard}, nil
+ return []tsdb.Shard{tsdb.NewScopedShard(tsdb.Entry(s.name), shard)}, nil
}
func (s *measure) CompanionShards(metadata *commonv1.Metadata) ([]tsdb.Shard, error) {
diff --git a/banyand/measure/measure_write.go b/banyand/measure/measure_write.go
index 8b9c220e..605bea42 100644
--- a/banyand/measure/measure_write.go
+++ b/banyand/measure/measure_write.go
@@ -52,7 +52,7 @@ func (s *measure) write(shardID common.ShardID, entity []byte, entityValues tsdb
if fLen > len(s.schema.GetTagFamilies()) {
return errors.Wrap(errMalformedElement, "tag family number is more than expected")
}
- shard, err := s.databaseSupplier.SupplyTSDB().Shard(shardID)
+ shard, err := s.databaseSupplier.SupplyTSDB().CreateShardsAndGetByID(shardID)
if err != nil {
return err
}
diff --git a/banyand/measure/service.go b/banyand/measure/service.go
index 46eacd30..fa7c0f6b 100644
--- a/banyand/measure/service.go
+++ b/banyand/measure/service.go
@@ -82,8 +82,10 @@ func (s *service) LoadGroup(name string) (resourceSchema.Group, bool) {
func (s *service) FlagSet() *run.FlagSet {
flagS := run.NewFlagSet("storage")
flagS.StringVar(&s.root, "measure-root-path", "/tmp", "the root path of database")
- flagS.Int64Var(&s.dbOpts.BlockMemSize, "measure-block-mem-size", 16<<20, "block memory size")
- flagS.Int64Var(&s.dbOpts.SeriesMemSize, "measure-seriesmeta-mem-size", 1<<20, "series metadata memory size")
+ s.dbOpts.BlockMemSize = 16 << 20
+ s.dbOpts.SeriesMemSize = 1 << 20
+ flagS.Var(&s.dbOpts.BlockMemSize, "measure-block-mem-size", "block memory size")
+ flagS.Var(&s.dbOpts.SeriesMemSize, "measure-seriesmeta-mem-size", "series metadata memory size")
flagS.Int64Var(&s.dbOpts.BlockInvertedIndex.BatchWaitSec, "measure-idx-batch-wait-sec", 1, "index batch wait in second")
return flagS
}
diff --git a/banyand/observability/collector.go b/banyand/observability/collector.go
index dd03482f..24f012d0 100644
--- a/banyand/observability/collector.go
+++ b/banyand/observability/collector.go
@@ -42,6 +42,13 @@ func (c *Collector) Register(name string, getter MetricsGetter) {
c.getters[name] = getter
}
+// Unregister unregisters a metrics getter.
+func (c *Collector) Unregister(name string) {
+ c.gMux.Lock()
+ defer c.gMux.Unlock()
+ delete(c.getters, name)
+}
+
func (c *Collector) collect() {
c.gMux.RLock()
defer c.gMux.RUnlock()
diff --git a/banyand/stream/service.go b/banyand/stream/service.go
index c4f2c7b5..6460f1f8 100644
--- a/banyand/stream/service.go
+++ b/banyand/stream/service.go
@@ -72,10 +72,13 @@ func (s *service) Stream(metadata *commonv1.Metadata) (Stream, error) {
func (s *service) FlagSet() *run.FlagSet {
flagS := run.NewFlagSet("storage")
+ s.dbOpts.BlockMemSize = 8 << 20
+ s.dbOpts.SeriesMemSize = 1 << 20
+ s.dbOpts.GlobalIndexMemSize = 2 << 20
flagS.StringVar(&s.root, "stream-root-path", "/tmp", "the root path of database")
- flagS.Int64Var(&s.dbOpts.BlockMemSize, "stream-block-mem-size", 8<<20, "block memory size")
- flagS.Int64Var(&s.dbOpts.SeriesMemSize, "stream-seriesmeta-mem-size", 1<<20, "series metadata memory size")
- flagS.Int64Var(&s.dbOpts.GlobalIndexMemSize, "stream-global-index-mem-size", 2<<20, "global index memory size")
+ flagS.Var(&s.dbOpts.BlockMemSize, "stream-block-mem-size", "block memory size")
+ flagS.Var(&s.dbOpts.SeriesMemSize, "stream-seriesmeta-mem-size", "series metadata memory size")
+ flagS.Var(&s.dbOpts.GlobalIndexMemSize, "stream-global-index-mem-size", "global index memory size")
flagS.Int64Var(&s.dbOpts.BlockInvertedIndex.BatchWaitSec, "stream-idx-batch-wait-sec", 1, "index batch wait in second")
return flagS
}
diff --git a/banyand/stream/stream_query.go b/banyand/stream/stream_query.go
index b2af26db..e90f7024 100644
--- a/banyand/stream/stream_query.go
+++ b/banyand/stream/stream_query.go
@@ -74,6 +74,9 @@ func (s *stream) Shards(entity tsdb.Entity) ([]tsdb.Shard, error) {
}
shard, err := db.Shard(common.ShardID(shardID))
if err != nil {
+ if errors.Is(err, tsdb.ErrUnknownShard) {
+ return []tsdb.Shard{}, nil
+ }
return nil, err
}
return []tsdb.Shard{tsdb.NewScopedShard(tsdb.Entry(s.name), shard)}, nil
diff --git a/banyand/stream/stream_write.go b/banyand/stream/stream_write.go
index 01bb6568..78b086df 100644
--- a/banyand/stream/stream_write.go
+++ b/banyand/stream/stream_write.go
@@ -64,7 +64,7 @@ func (s *stream) write(shardID common.ShardID, entity []byte, entityValues tsdb.
if fLen > len(sm.TagFamilies) {
return errors.Wrap(errMalformedElement, "tag family number is more than expected")
}
- shard, err := s.db.SupplyTSDB().Shard(shardID)
+ shard, err := s.db.SupplyTSDB().CreateShardsAndGetByID(shardID)
if err != nil {
return err
}
diff --git a/banyand/tsdb/block.go b/banyand/tsdb/block.go
index ff82c386..a5963ae6 100644
--- a/banyand/tsdb/block.go
+++ b/banyand/tsdb/block.go
@@ -25,11 +25,13 @@ import (
"path"
"runtime"
"strconv"
+ "strings"
"sync"
"sync/atomic"
"time"
"unsafe"
+ "github.com/dgraph-io/badger/v3"
"github.com/dgraph-io/badger/v3/skl"
"github.com/pkg/errors"
"go.uber.org/multierr"
@@ -64,6 +66,25 @@ var (
blockOpenedTimeSecondsGauge = blockMeterProvider.Gauge("opened_time_seconds", common.LabelNames()...)
blockReferencesGauge = blockMeterProvider.Gauge("refs", common.LabelNames()...)
+ blockCompressedSize = blockMeterProvider.Gauge("compressed_size", append(common.LabelNames(), "from", "to")...)
+ blockUncompressedSize = blockMeterProvider.Gauge("uncompressed_size", append(common.LabelNames(), "from", "to")...)
+ blockCompressedBlockNum = blockMeterProvider.Gauge("compressed_block_num", append(common.LabelNames(), "from", "to")...)
+ blockCompressedEntryNum = blockMeterProvider.Gauge("compressed_entry_num", append(common.LabelNames(), "from", "to")...)
+ blockEncodedSize = blockMeterProvider.Gauge("encoded_size", append(common.LabelNames(), "from", "to")...)
+ blockUnencodedSize = blockMeterProvider.Gauge("unencoded_size", append(common.LabelNames(), "from", "to")...)
+ blockEncodedBlockNum = blockMeterProvider.Gauge("encoded_block_num", append(common.LabelNames(), "from", "to")...)
+ blockEncodedEntryNum = blockMeterProvider.Gauge("encoded_entry_num", append(common.LabelNames(), "from", "to")...)
+ blockCompactionMap = map[badger.TableBuilderSizeKeyType]meter.Gauge{
+ badger.TableBuilderSizeKeyCompressedSize: blockCompressedSize,
+ badger.TableBuilderSizeKeyUncompressedSize: blockUncompressedSize,
+ badger.TableBuilderSizeKeyCompressedBlockNum: blockCompressedBlockNum,
+ badger.TableBuilderSizeKeyCompressedEntryNum: blockCompressedEntryNum,
+ badger.TableBuilderSizeKeyEncodedSize: blockEncodedSize,
+ badger.TableBuilderSizeKeyUnEncodedSize: blockUnencodedSize,
+ badger.TableBuilderSizeKeyEncodedBlockNum: blockEncodedBlockNum,
+ badger.TableBuilderSizeKeyEncodedEntryNum: blockEncodedEntryNum,
+ }
+
errBlockClosingInterrupted = errors.New("interrupt to close the block")
)
@@ -157,7 +178,7 @@ func (b *block) options(ctx context.Context) {
if options.BlockMemSize < 1 {
bufferSize = defaultBufferSize
} else {
- bufferSize = options.BlockMemSize
+ bufferSize = int64(options.BlockMemSize)
}
b.openOpts.bufferSize = bufferSize
b.openOpts.store = append(b.openOpts.store, kv.TSSWithMemTableSize(bufferSize), kv.TSSWithLogger(b.l.Named(componentMain)))
@@ -167,7 +188,7 @@ func (b *block) options(ctx context.Context) {
Logger: b.l.Named(componentSecondInvertedIdx),
BatchWaitSec: options.BlockInvertedIndex.BatchWaitSec,
}
- lsmMemSize := bufferSize / 2
+ lsmMemSize := bufferSize / 4
if lsmMemSize < defaultKVMemorySize {
lsmMemSize = defaultKVMemorySize
}
@@ -210,6 +231,20 @@ func (b *block) open() (err error) {
b.ref.Store(0)
b.closed.Store(false)
blockOpenedTimeSecondsGauge.Set(float64(time.Now().Unix()), b.position.LabelValues()...)
+ plv := b.position.LabelValues()
+ observability.MetricsCollector.Register(strings.Join(plv, "-"), func() {
+ stats := b.sst.CollectStats()
+ stats.TableBuilderSize.Range(func(label interface{}, value interface{}) bool {
+ key := label.(badger.TableBuilderSizeKey)
+ counter := value.(*atomic.Int64)
+ from, to := getLevelLabels(key.FromLevel, key.ToLevel)
+ labelValues := append(b.position.LabelValues(), from, to)
+ if block, ok := blockCompactionMap[key.Type]; ok {
+ block.Set(float64(counter.Load()), labelValues...)
+ }
+ return true
+ })
+ })
return nil
}
@@ -344,6 +379,18 @@ func (b *block) close(ctx context.Context) (err error) {
if b.buffer != nil {
err = multierr.Append(err, b.buffer.Close())
}
+ plv := b.position.LabelValues()
+ observability.MetricsCollector.Unregister(strings.Join(plv, "-"))
+ stats := b.sst.CollectStats()
+ stats.TableBuilderSize.Range(func(label interface{}, value interface{}) bool {
+ key := label.(badger.TableBuilderSizeKey)
+ from, to := getLevelLabels(key.FromLevel, key.ToLevel)
+ labelValues := append(b.position.LabelValues(), from, to)
+ if block, ok := blockCompactionMap[key.Type]; ok {
+ block.Delete(labelValues...)
+ }
+ return true
+ })
for _, closer := range b.closableLst {
err = multierr.Append(err, closer.Close())
}
@@ -492,3 +539,14 @@ func (d *bDelegate) Close() error {
d.delegate.Done()
return nil
}
+
+func getLevelLabels(fromLevel, toLevel int) (string, string) {
+ from := fmt.Sprintf("l%d", fromLevel)
+ to := fmt.Sprintf("l%d", toLevel)
+
+ if fromLevel < 0 {
+ from = "mem"
+ }
+
+ return from, to
+}
diff --git a/banyand/tsdb/buffer.go b/banyand/tsdb/buffer.go
index 621b5c4b..49ebdf64 100644
--- a/banyand/tsdb/buffer.go
+++ b/banyand/tsdb/buffer.go
@@ -42,7 +42,6 @@ const (
var (
bufferMeterProvider meter.Provider
maxBytes meter.Gauge
- immutableBytes meter.Gauge
mutableBytes meter.Gauge
)
@@ -51,7 +50,6 @@ func init() {
labelNames := append(common.LabelNames(), "bucket")
maxBytes = bufferMeterProvider.Gauge("max_bytes", labelNames...)
mutableBytes = bufferMeterProvider.Gauge("mutable_bytes", labelNames...)
- immutableBytes = bufferMeterProvider.Gauge("immutable_bytes", labelNames...)
}
type operation struct {
@@ -209,7 +207,7 @@ func (bsb *bufferShardBucket) getAll() ([]*skl.Skiplist, func()) {
func (bsb *bufferShardBucket) start(onFlushFn onFlush) {
go func() {
defer func() {
- for _, g := range []meter.Gauge{maxBytes, immutableBytes, mutableBytes} {
+ for _, g := range []meter.Gauge{maxBytes, mutableBytes} {
g.Delete(bsb.labelValues...)
}
}()
@@ -228,7 +226,6 @@ func (bsb *bufferShardBucket) start(onFlushFn onFlush) {
break
}
flushLatency.Observe(time.Since(t1).Seconds(), bsb.shardLabelValues...)
- immutableBytes.Add(float64(-memSize), bsb.labelValues...)
flushBytes.Inc(float64(memSize), bsb.shardLabelValues...)
flushNum.Inc(1, append(bsb.shardLabelValues, "false")...)
@@ -274,6 +271,5 @@ func (bsb *bufferShardBucket) triggerFlushing() {
func (bsb *bufferShardBucket) swap() {
bsb.immutables = append(bsb.immutables, bsb.mutable)
- immutableBytes.Add(float64(bsb.mutable.MemSize()), bsb.labelValues...)
bsb.mutable = skl.NewSkiplist(int64(bsb.capacity))
}
diff --git a/banyand/tsdb/index/writer.go b/banyand/tsdb/index/writer.go
index 2958c8f7..01afa143 100644
--- a/banyand/tsdb/index/writer.go
+++ b/banyand/tsdb/index/writer.go
@@ -161,7 +161,7 @@ func (s *Writer) writeGlobalIndex(scope tsdb.Entry, ref tsdb.GlobalItemID, value
}
}
for shardID, rules := range fields {
- shard, err := s.db.SupplyTSDB().Shard(common.ShardID(shardID))
+ shard, err := s.db.SupplyTSDB().CreateShardsAndGetByID(common.ShardID(shardID))
if err != nil {
return err
}
diff --git a/banyand/tsdb/segment.go b/banyand/tsdb/segment.go
index 1e2abaa7..b27d36a9 100644
--- a/banyand/tsdb/segment.go
+++ b/banyand/tsdb/segment.go
@@ -90,7 +90,7 @@ func openSegment(ctx context.Context, startTime, endTime time.Time, path, suffix
if s.globalIndex, err = kv.OpenStore(
indexPath,
kv.StoreWithLogger(s.l),
- kv.StoreWithMemTableSize(memSize),
+ kv.StoreWithMemTableSize(int64(memSize)),
); err != nil {
return nil, err
}
diff --git a/banyand/tsdb/seriesdb.go b/banyand/tsdb/seriesdb.go
index 679fcf43..e8b635ba 100644
--- a/banyand/tsdb/seriesdb.go
+++ b/banyand/tsdb/seriesdb.go
@@ -512,7 +512,7 @@ func newSeriesDataBase(ctx context.Context, shardID common.ShardID, path string,
if o != nil {
options := o.(DatabaseOpts)
if options.SeriesMemSize > 1 {
- memSize = options.SeriesMemSize
+ memSize = int64(options.SeriesMemSize)
} else {
memSize = defaultKVMemorySize
}
diff --git a/banyand/tsdb/shard.go b/banyand/tsdb/shard.go
index 31373b84..01ebe329 100644
--- a/banyand/tsdb/shard.go
+++ b/banyand/tsdb/shard.go
@@ -38,9 +38,9 @@ import (
)
const (
- defaultBlockQueueSize = 4
+ defaultBlockQueueSize = 2
defaultMaxBlockQueueSize = 64
- defaultKVMemorySize = 8 << 20
+ defaultKVMemorySize = 4 << 20
)
var (
diff --git a/banyand/tsdb/tsdb.go b/banyand/tsdb/tsdb.go
index fb499c4b..73787d97 100644
--- a/banyand/tsdb/tsdb.go
+++ b/banyand/tsdb/tsdb.go
@@ -31,6 +31,7 @@ import (
"strconv"
"strings"
"sync"
+ "sync/atomic"
"github.com/pkg/errors"
"go.uber.org/multierr"
@@ -40,6 +41,7 @@ import (
"github.com/apache/skywalking-banyandb/pkg/convert"
"github.com/apache/skywalking-banyandb/pkg/encoding"
"github.com/apache/skywalking-banyandb/pkg/logger"
+ "github.com/apache/skywalking-banyandb/pkg/run"
"github.com/apache/skywalking-banyandb/pkg/timestamp"
)
@@ -62,8 +64,9 @@ const (
)
var (
- errInvalidShardID = errors.New("invalid shard id")
- errOpenDatabase = errors.New("fails to open the database")
+ // ErrUnknownShard indicates that the shard is not found.
+ ErrUnknownShard = errors.New("unknown shard")
+ errOpenDatabase = errors.New("fails to open the database")
optionsKey = contextOptionsKey{}
meterStorage = observability.RootScope.SubScope("storage")
@@ -80,6 +83,7 @@ type Supplier interface {
// Database allows listing and getting shard details.
type Database interface {
io.Closer
+ CreateShardsAndGetByID(id common.ShardID) (Shard, error)
Shards() []Shard
Shard(id common.ShardID) (Shard, error)
}
@@ -105,10 +109,10 @@ type DatabaseOpts struct {
SegmentInterval IntervalRule
BlockInterval IntervalRule
TTL IntervalRule
- BlockMemSize int64
+ BlockMemSize run.Bytes
BlockInvertedIndex InvertedIndexOpts
- SeriesMemSize int64
- GlobalIndexMemSize int64
+ SeriesMemSize run.Bytes
+ GlobalIndexMemSize run.Bytes
ShardNum uint32
EnableGlobalIndex bool
}
@@ -189,28 +193,60 @@ type ShardState struct {
}
type database struct {
- logger *logger.Logger
- location string
- sLst []Shard
- segmentSize IntervalRule
- blockSize IntervalRule
- ttl IntervalRule
- sync.Mutex
- shardNum uint32
+ shardCreationCtx context.Context
+ logger *logger.Logger
+ location string
+ sLst []Shard
+ segmentSize IntervalRule
+ blockSize IntervalRule
+ ttl IntervalRule
+ sync.RWMutex
+ shardNum uint32
+ shardCreationState uint32
+}
+
+func (d *database) CreateShardsAndGetByID(id common.ShardID) (Shard, error) {
+ if atomic.LoadUint32(&d.shardCreationState) != 0 {
+ return d.shard(id)
+ }
+ d.Lock()
+ defer d.Unlock()
+ if atomic.LoadUint32(&d.shardCreationState) != 0 {
+ return d.shard(id)
+ }
+ loadedShardsNum := len(d.sLst)
+ if loadedShardsNum < int(d.shardNum) {
+ _, err := createDatabase(d, loadedShardsNum)
+ if err != nil {
+ return nil, errors.WithMessage(err, "create the database failed")
+ }
+ }
+ atomic.StoreUint32(&d.shardCreationState, 1)
+ return d.shard(id)
}
func (d *database) Shards() []Shard {
+ d.RLock()
+ defer d.RUnlock()
return d.sLst
}
func (d *database) Shard(id common.ShardID) (Shard, error) {
+ d.RLock()
+ defer d.RUnlock()
+ return d.shard(id)
+}
+
+func (d *database) shard(id common.ShardID) (Shard, error) {
if uint(id) >= uint(len(d.sLst)) {
- return nil, errInvalidShardID
+ return nil, ErrUnknownShard
}
return d.sLst[id], nil
}
func (d *database) Close() error {
+ d.Lock()
+ defer d.Unlock()
var err error
for _, s := range d.sLst {
innerErr := s.Close()
@@ -256,23 +292,18 @@ func OpenDatabase(ctx context.Context, opts DatabaseOpts) (Database, error) {
}
thisContext := context.WithValue(ctx, logger.ContextKey, db.logger)
thisContext = context.WithValue(thisContext, optionsKey, opts)
+ db.shardCreationCtx = thisContext
if len(entries) > 0 {
return loadDatabase(thisContext, db)
}
- return initDatabase(thisContext, db)
-}
-
-func initDatabase(ctx context.Context, db *database) (Database, error) {
- db.Lock()
- defer db.Unlock()
- return createDatabase(ctx, db, 0)
+ return db, nil
}
-func createDatabase(ctx context.Context, db *database, startID int) (Database, error) {
+func createDatabase(db *database, startID int) (Database, error) {
var err error
for i := startID; i < int(db.shardNum); i++ {
db.logger.Info().Int("shard_id", i).Msg("creating a shard")
- so, errNewShard := OpenShard(ctx, common.ShardID(i),
+ so, errNewShard := OpenShard(db.shardCreationCtx, common.ShardID(i),
db.location, db.segmentSize, db.blockSize, db.ttl, defaultBlockQueueSize, defaultMaxBlockQueueSize)
if errNewShard != nil {
err = multierr.Append(err, errNewShard)
@@ -316,14 +347,6 @@ func loadDatabase(ctx context.Context, db *database) (Database, error) {
if err != nil {
return nil, errors.WithMessage(err, "load the database failed")
}
-
- loadedShardsNum := len(db.sLst)
- if loadedShardsNum < int(db.shardNum) {
- _, err := createDatabase(ctx, db, loadedShardsNum)
- if err != nil {
- return nil, errors.WithMessage(err, "load the database failed")
- }
- }
return db, nil
}
diff --git a/banyand/tsdb/tsdb_test.go b/banyand/tsdb/tsdb_test.go
index ffe4ac81..01ca7eea 100644
--- a/banyand/tsdb/tsdb_test.go
+++ b/banyand/tsdb/tsdb_test.go
@@ -27,6 +27,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
+ "github.com/apache/skywalking-banyandb/api/common"
"github.com/apache/skywalking-banyandb/pkg/logger"
"github.com/apache/skywalking-banyandb/pkg/test"
"github.com/apache/skywalking-banyandb/pkg/test/flags"
@@ -116,6 +117,9 @@ func openDatabase(ctx context.Context, t *require.Assertions, path string) (db D
})
t.NoError(err)
t.NotNil(db)
+ shard, err := db.CreateShardsAndGetByID(0)
+ t.NoError(err)
+ t.Equal(common.ShardID(0), shard.ID())
return db
}
diff --git a/go.mod b/go.mod
index 881d6470..6a703647 100644
--- a/go.mod
+++ b/go.mod
@@ -148,5 +148,5 @@ replace (
github.com/blugelabs/bluge => github.com/zinclabs/bluge v1.1.5
github.com/blugelabs/bluge_segment_api => github.com/zinclabs/bluge_segment_api v1.0.0
github.com/blugelabs/ice => github.com/zinclabs/ice v1.1.3
- github.com/dgraph-io/badger/v3 v3.2011.1 => github.com/SkyAPM/badger/v3 v3.0.0-20230329010346-dfdf57f0581b
+ github.com/dgraph-io/badger/v3 v3.2011.1 => github.com/SkyAPM/badger/v3 v3.0.0-20230511223516-583c6a825854
)
diff --git a/go.sum b/go.sum
index 4029823a..d1179005 100644
--- a/go.sum
+++ b/go.sum
@@ -47,8 +47,8 @@ github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/RoaringBitmap/roaring v0.9.4 h1:ckvZSX5gwCRaJYBNe7syNawCU5oruY9gQmjXlp4riwo=
github.com/RoaringBitmap/roaring v0.9.4/go.mod h1:icnadbWcNyfEHlYdr+tDlOTih1Bf/h+rzPpv4sbomAA=
-github.com/SkyAPM/badger/v3 v3.0.0-20230329010346-dfdf57f0581b h1:ZThiWg9yHnIMYy+KaAgkNZhnFgF7kvtytsdjb24zAGU=
-github.com/SkyAPM/badger/v3 v3.0.0-20230329010346-dfdf57f0581b/go.mod h1:4WETftF8A4mEeFgIsYB/VvGo5kfTVl/neYgEqlVW9Ag=
+github.com/SkyAPM/badger/v3 v3.0.0-20230511223516-583c6a825854 h1:GlNxZHaMREV6V6+T838i4xr8lBS8ekjOfenlJF26Z3U=
+github.com/SkyAPM/badger/v3 v3.0.0-20230511223516-583c6a825854/go.mod h1:4WETftF8A4mEeFgIsYB/VvGo5kfTVl/neYgEqlVW9Ag=
github.com/SkyAPM/clock v1.3.1-0.20220809233656-dc7607c94a97 h1:FKuhJ+6n/DHspGeLleeNbziWnKr9gHKYN4q7NcoCp4s=
github.com/SkyAPM/clock v1.3.1-0.20220809233656-dc7607c94a97/go.mod h1:2xGRl9H1pllhxTbEGO1W3gDkip8P9GQaHPni/wpdR44=
github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
diff --git a/pkg/convert/size.go b/pkg/convert/size.go
new file mode 100644
index 00000000..a3383fa1
--- /dev/null
+++ b/pkg/convert/size.go
@@ -0,0 +1,70 @@
+package convert
+
+import (
+ "fmt"
+ "strconv"
+ "strings"
+)
+
+var units = map[string]int64{
+ "B": 1,
+ "K": 1000,
+ "KB": 1000,
+ "M": 1000 * 1000,
+ "MB": 1000 * 1000,
+ "G": 1000 * 1000 * 1000,
+ "GB": 1000 * 1000 * 1000,
+ "T": 1000 * 1000 * 1000 * 1000,
+ "TB": 1000 * 1000 * 1000 * 1000,
+ "P": 1000 * 1000 * 1000 * 1000 * 1000,
+ "PB": 1000 * 1000 * 1000 * 1000 * 1000,
+ "KI": 1 << 10,
+ "KIB": 1 << 10,
+ "MI": 1 << 20,
+ "MIB": 1 << 20,
+ "GI": 1 << 30,
+ "GIB": 1 << 30,
+ "TI": 1 << 40,
+ "TIB": 1 << 40,
+ "PI": 1 << 50,
+ "PIB": 1 << 50,
+}
+
+// ParseSize parses a string like "1.5GB" or "1000" and returns the
+// number of bytes. The following units are supported:
+//
+// B, K, KB, M, MB, G, GB, T, TB, P, PB
+// KI, KIB, MI, MIB, GI, GIB, TI, TIB, PI, PIB
+//
+// The units are case insensitive.
+func ParseSize(sizeStr string) (int64, error) {
+ sep := strings.LastIndexAny(sizeStr, "01234567890. ")
+ if sep == -1 {
+ return -1, fmt.Errorf("invalid size: '%s'", sizeStr)
+ }
+ var num, sfx string
+ if sizeStr[sep] != ' ' {
+ num = sizeStr[:sep+1]
+ sfx = sizeStr[sep+1:]
+ } else {
+ num = sizeStr[:sep]
+ sfx = sizeStr[sep+1:]
+ }
+ value, err := strconv.ParseFloat(num, 64)
+ if err != nil {
+ return 0, err
+ }
+ if value < 0 {
+ return -1, fmt.Errorf("invalid size: '%s'", sizeStr)
+ }
+
+ if len(sfx) == 0 {
+ return int64(value), nil
+ }
+ sfx = strings.ToUpper(sfx)
+ unit, ok := units[sfx]
+ if !ok {
+ return -1, fmt.Errorf("invalid size: '%s'", sizeStr)
+ }
+ return int64(value * float64(unit)), nil
+}
diff --git a/pkg/convert/size_test.go b/pkg/convert/size_test.go
new file mode 100644
index 00000000..b6560db5
--- /dev/null
+++ b/pkg/convert/size_test.go
@@ -0,0 +1,34 @@
+package convert_test
+
+import (
+ "testing"
+
+ "github.com/apache/skywalking-banyandb/pkg/convert"
+)
+
+func TestParseSize(t *testing.T) {
+ tests := []struct {
+ input string
+ expected int64
+ }{
+ {"0B", 0},
+ {"1KB", 1000},
+ {"1MB", 1000 * 1000},
+ {"1GB", 1000 * 1000 * 1000},
+ {"1KiB", 1 << 10},
+ {"1MiB", 1 << 20},
+ {"1GiB", 1 << 30},
+ {"1.5Gi", int64(1.5 * float64(1<<30))},
+ {"1024 M", 1024 * 1000 * 1000},
+ {"42 Ki", 42 * (1 << 10)},
+ }
+
+ for _, test := range tests {
+ result, err := convert.ParseSize(test.input)
+ if err != nil {
+ t.Errorf("Failed to parse size for input %q: %v", test.input, err)
+ } else if result != test.expected {
+ t.Errorf("Incorrect result for input %q: expected %d, got %d", test.input, test.expected, result)
+ }
+ }
+}
diff --git a/pkg/encoding/encoder.go b/pkg/encoding/encoder.go
index 7b0f7478..78add92c 100644
--- a/pkg/encoding/encoder.go
+++ b/pkg/encoding/encoder.go
@@ -26,9 +26,7 @@ import (
"github.com/pkg/errors"
- "github.com/apache/skywalking-banyandb/banyand/observability"
"github.com/apache/skywalking-banyandb/pkg/convert"
- "github.com/apache/skywalking-banyandb/pkg/meter"
)
var (
@@ -45,33 +43,20 @@ var (
errNoData = errors.New("there is no data")
)
-type metrics struct {
- rawSizeCounter meter.Counter
- encodingSizeCounter meter.Counter
- itemsNumCounter meter.Counter
-}
-
type encoderPoolDelegator struct {
- metrics metrics
- pool *sync.Pool
- fn ParseInterval
- name string
- size int
+ pool *sync.Pool
+ fn ParseInterval
+ name string
+ size int
}
// NewEncoderPool returns a SeriesEncoderPool which provides int-based xor encoders.
func NewEncoderPool(name string, size int, fn ParseInterval) SeriesEncoderPool {
- provider := observability.NewMeterProvider(encodingScope.ConstLabels(meter.LabelPairs{"name": name}))
return &encoderPoolDelegator{
name: name,
pool: &encoderPool,
size: size,
fn: fn,
- metrics: metrics{
- rawSizeCounter: provider.Counter("raw_size"),
- encodingSizeCounter: provider.Counter("encoding_size"),
- itemsNumCounter: provider.Counter("items_num"),
- },
}
}
@@ -80,7 +65,6 @@ func (b *encoderPoolDelegator) Get(metadata []byte, buffer BufferWriter) SeriesE
encoder.name = b.name
encoder.size = b.size
encoder.fn = b.fn
- encoder.metrics = &b.metrics
encoder.Reset(metadata, buffer)
return encoder
}
@@ -134,7 +118,6 @@ type encoder struct {
bw *Writer
values *XOREncoder
fn ParseInterval
- metrics *metrics
name string
interval time.Duration
startTime uint64
@@ -175,8 +158,6 @@ func (ie *encoder) Append(ts uint64, value []byte) {
ie.bw.WriteBool(l > 0)
ie.values.Write(convert.BytesToUint64(value))
ie.num++
- ie.metrics.itemsNumCounter.Inc(1)
- ie.metrics.rawSizeCounter.Inc(float64(l + 8))
}
func (ie *encoder) IsFull() bool {
@@ -198,8 +179,6 @@ func (ie *encoder) Encode() error {
buffWriter := NewPacker(ie.buff)
buffWriter.PutUint64(ie.startTime)
buffWriter.PutUint16(uint16(ie.num))
- bb := buffWriter.Bytes()
- ie.metrics.encodingSizeCounter.Inc(float64(len(bb)))
return nil
}
diff --git a/pkg/run/flags.go b/pkg/run/flags.go
new file mode 100644
index 00000000..72420d06
--- /dev/null
+++ b/pkg/run/flags.go
@@ -0,0 +1,40 @@
+package run
+
+import (
+ "fmt"
+ "math"
+
+ "github.com/apache/skywalking-banyandb/pkg/convert"
+)
+
+// Bytes is a custom type to store memory size in bytes.
+type Bytes int64
+
+// String returns a string representation of the Bytes value.
+func (b Bytes) String() string {
+ suffixes := []string{"B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB"}
+ size := float64(b)
+ base := 1024.0
+ if size < base {
+ return fmt.Sprintf("%.0f%s", size, suffixes[0])
+ }
+
+ exp := int(math.Log(size) / math.Log(base))
+ result := size / math.Pow(base, float64(exp))
+ return fmt.Sprintf("%.2f%s", result, suffixes[exp])
+}
+
+// Set sets the Bytes value from the input string.
+func (b *Bytes) Set(s string) error {
+ size, err := convert.ParseSize(s)
+ if err != nil {
+ return err
+ }
+ *b = Bytes(size)
+ return nil
+}
+
+// Type returns the type name of the Bytes custom type.
+func (b *Bytes) Type() string {
+ return "bytes"
+}