You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ha...@apache.org on 2024/04/08 00:31:01 UTC
(skywalking-banyandb) 01/03: Clean deprecated modules:
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch clean-up
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit cf7702590411eba47710009b61118a8f58e33b37
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Sun Apr 7 05:00:34 2024 +0000
Clean deprecated modules:
- API: remove lsm type, remove location
- Remove kv module
- Remove tsdb module
Signed-off-by: Gao Hongtao <ha...@gmail.com>
---
api/proto/banyandb/common/v1/common.proto | 10 +-
api/proto/banyandb/database/v1/schema.proto | 14 +-
banyand/dquery/topn.go | 4 +-
banyand/{tsdb => internal}/bucket/bucket.go | 0
.../{tsdb => internal}/bucket/bucket_suite_test.go | 0
banyand/{tsdb => internal}/bucket/queue.go | 0
banyand/{tsdb => internal}/bucket/queue_test.go | 2 +-
banyand/{tsdb => internal}/bucket/strategy.go | 0
banyand/{tsdb => internal}/bucket/strategy_test.go | 2 +-
banyand/internal/storage/segment.go | 2 +-
banyand/internal/storage/shard.go | 2 +-
banyand/kv/badger.go | 396 -----------
banyand/kv/badger_test.go | 72 --
banyand/kv/kv.go | 306 --------
banyand/liaison/grpc/discovery.go | 4 +-
banyand/liaison/grpc/measure.go | 4 +-
banyand/liaison/grpc/stream.go | 4 +-
banyand/metadata/schema/etcd_test.go | 4 +-
banyand/query/processor_topn.go | 12 +-
banyand/tsdb/block.go | 484 -------------
banyand/tsdb/block_ctrl.go | 347 ---------
banyand/tsdb/buffer.go | 565 ---------------
banyand/tsdb/buffer_test.go | 367 ----------
banyand/tsdb/index/writer.go | 267 -------
banyand/tsdb/indexdb.go | 193 -----
banyand/tsdb/retention.go | 61 --
banyand/tsdb/scope.go | 107 ---
banyand/tsdb/segment.go | 140 ----
banyand/tsdb/segment_ctrl.go | 283 --------
banyand/tsdb/series.go | 234 -------
banyand/tsdb/series_seek.go | 136 ----
banyand/tsdb/series_seek_filter.go | 58 --
banyand/tsdb/series_seek_sort.go | 269 -------
banyand/tsdb/series_test.go | 130 ----
banyand/tsdb/series_write.go | 232 ------
banyand/tsdb/seriesdb.go | 779 ---------------------
banyand/tsdb/seriesdb_test.go | 579 ---------------
banyand/tsdb/shard.go | 333 ---------
banyand/tsdb/shard_test.go | 605 ----------------
banyand/tsdb/tsdb.go | 404 -----------
banyand/tsdb/tsdb_suite_test.go | 39 --
banyand/tsdb/tsdb_test.go | 177 -----
banyand/tsdb/tstable.go | 49 --
docs/api-reference.md | 21 +-
go.mod | 8 +-
go.sum | 38 +-
pkg/bytes/buffer.go | 7 +
pkg/index/inverted/inverted.go | 4 +-
pkg/index/lsm/iterator.go | 219 ------
pkg/index/lsm/lsm.go | 89 ---
pkg/index/lsm/lsm_test.go | 72 --
pkg/index/lsm/search.go | 101 ---
pkg/partition/entity.go | 9 +-
pkg/pb/v1/entity.go | 101 +++
pkg/pb/v1/tsdb/options.go | 42 --
pkg/query/logical/common.go | 7 -
pkg/query/logical/index_filter.go | 261 -------
pkg/query/logical/measure/schema.go | 5 -
pkg/query/logical/schema.go | 2 -
pkg/query/logical/stream/schema.go | 5 -
60 files changed, 146 insertions(+), 8521 deletions(-)
diff --git a/api/proto/banyandb/common/v1/common.proto b/api/proto/banyandb/common/v1/common.proto
index 16fe8e67..22d4a8d9 100644
--- a/api/proto/banyandb/common/v1/common.proto
+++ b/api/proto/banyandb/common/v1/common.proto
@@ -37,6 +37,8 @@ message Metadata {
string group = 1;
// name of the entity
string name = 2;
+ // id is the unique identifier of the entity
+ // if id is not set, the system will generate a unique id
uint32 id = 3;
// readonly. create_revision is the revision of last creation on this key.
int64 create_revision = 4;
@@ -59,14 +61,10 @@ message IntervalRule {
message ResourceOpts {
// shard_num is the number of shards
uint32 shard_num = 1 [(validate.rules).uint32.gt = 0];
- // block_interval indicates the length of a block
- // block_interval should be less than or equal to segment_interval
- IntervalRule block_interval = 2 [(validate.rules).message.required = true];
// segment_interval indicates the length of a segment
- IntervalRule segment_interval = 3 [(validate.rules).message.required = true];
-
+ IntervalRule segment_interval = 2 [(validate.rules).message.required = true];
// ttl indicates time to live, how long the data will be cached
- IntervalRule ttl = 4 [(validate.rules).message.required = true];
+ IntervalRule ttl = 3 [(validate.rules).message.required = true];
}
// Group is an internal object for Group management
diff --git a/api/proto/banyandb/database/v1/schema.proto b/api/proto/banyandb/database/v1/schema.proto
index dc8812c6..a7d391fd 100644
--- a/api/proto/banyandb/database/v1/schema.proto
+++ b/api/proto/banyandb/database/v1/schema.proto
@@ -152,20 +152,12 @@ message IndexRule {
// Type determine the index structure under the hood
enum Type {
TYPE_UNSPECIFIED = 0;
- TYPE_TREE = 1;
- TYPE_INVERTED = 2;
+ TYPE_INVERTED = 1;
}
// type is the IndexType of this IndexObject.
Type type = 3 [(validate.rules).enum.defined_only = true];
- enum Location {
- LOCATION_UNSPECIFIED = 0;
- LOCATION_SERIES = 1;
- LOCATION_GLOBAL = 2;
- }
- // location indicates where to store index.
- Location location = 4 [(validate.rules).enum.defined_only = true];
// updated_at indicates when the IndexRule is updated
- google.protobuf.Timestamp updated_at = 5;
+ google.protobuf.Timestamp updated_at = 4;
enum Analyzer {
ANALYZER_UNSPECIFIED = 0;
// Keyword analyzer is a “noop” analyzer which returns the entire input string as a single token.
@@ -178,7 +170,7 @@ message IndexRule {
ANALYZER_SIMPLE = 3;
}
// analyzer analyzes tag value to support the full-text searching for TYPE_INVERTED indices.
- Analyzer analyzer = 6;
+ Analyzer analyzer = 5;
}
// Subject defines which stream or measure would generate indices
diff --git a/banyand/dquery/topn.go b/banyand/dquery/topn.go
index a885f829..f5d7cf1f 100644
--- a/banyand/dquery/topn.go
+++ b/banyand/dquery/topn.go
@@ -25,10 +25,10 @@ import (
measurev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/measure/v1"
modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
"github.com/apache/skywalking-banyandb/banyand/query"
- "github.com/apache/skywalking-banyandb/banyand/tsdb"
"github.com/apache/skywalking-banyandb/pkg/bus"
"github.com/apache/skywalking-banyandb/pkg/convert"
"github.com/apache/skywalking-banyandb/pkg/iter/sort"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
)
type topNQueryProcessor struct {
@@ -78,7 +78,7 @@ func (t *topNQueryProcessor) Rev(message bus.Message) (resp bus.Message) {
tags = append(tags, e.Key)
}
}
- entityValues := make(tsdb.EntityValues, 0, len(tn.Entity))
+ entityValues := make(pbv1.EntityValues, 0, len(tn.Entity))
for _, e := range tn.Entity {
entityValues = append(entityValues, e.Value)
}
diff --git a/banyand/tsdb/bucket/bucket.go b/banyand/internal/bucket/bucket.go
similarity index 100%
rename from banyand/tsdb/bucket/bucket.go
rename to banyand/internal/bucket/bucket.go
diff --git a/banyand/tsdb/bucket/bucket_suite_test.go b/banyand/internal/bucket/bucket_suite_test.go
similarity index 100%
rename from banyand/tsdb/bucket/bucket_suite_test.go
rename to banyand/internal/bucket/bucket_suite_test.go
diff --git a/banyand/tsdb/bucket/queue.go b/banyand/internal/bucket/queue.go
similarity index 100%
rename from banyand/tsdb/bucket/queue.go
rename to banyand/internal/bucket/queue.go
diff --git a/banyand/tsdb/bucket/queue_test.go b/banyand/internal/bucket/queue_test.go
similarity index 98%
rename from banyand/tsdb/bucket/queue_test.go
rename to banyand/internal/bucket/queue_test.go
index eb780723..c0defbfd 100644
--- a/banyand/tsdb/bucket/queue_test.go
+++ b/banyand/internal/bucket/queue_test.go
@@ -26,7 +26,7 @@ import (
. "github.com/onsi/gomega"
"github.com/onsi/gomega/gleak"
- "github.com/apache/skywalking-banyandb/banyand/tsdb/bucket"
+ "github.com/apache/skywalking-banyandb/banyand/internal/bucket"
"github.com/apache/skywalking-banyandb/pkg/logger"
"github.com/apache/skywalking-banyandb/pkg/test/flags"
"github.com/apache/skywalking-banyandb/pkg/timestamp"
diff --git a/banyand/tsdb/bucket/strategy.go b/banyand/internal/bucket/strategy.go
similarity index 100%
rename from banyand/tsdb/bucket/strategy.go
rename to banyand/internal/bucket/strategy.go
diff --git a/banyand/tsdb/bucket/strategy_test.go b/banyand/internal/bucket/strategy_test.go
similarity index 98%
rename from banyand/tsdb/bucket/strategy_test.go
rename to banyand/internal/bucket/strategy_test.go
index b3659059..0ca374c0 100644
--- a/banyand/tsdb/bucket/strategy_test.go
+++ b/banyand/internal/bucket/strategy_test.go
@@ -23,7 +23,7 @@ import (
. "github.com/onsi/gomega"
"github.com/onsi/gomega/gleak"
- "github.com/apache/skywalking-banyandb/banyand/tsdb/bucket"
+ "github.com/apache/skywalking-banyandb/banyand/internal/bucket"
"github.com/apache/skywalking-banyandb/pkg/test/flags"
)
diff --git a/banyand/internal/storage/segment.go b/banyand/internal/storage/segment.go
index 22890ddc..2df03813 100644
--- a/banyand/internal/storage/segment.go
+++ b/banyand/internal/storage/segment.go
@@ -30,7 +30,7 @@ import (
"time"
"github.com/apache/skywalking-banyandb/api/common"
- "github.com/apache/skywalking-banyandb/banyand/tsdb/bucket"
+ "github.com/apache/skywalking-banyandb/banyand/internal/bucket"
"github.com/apache/skywalking-banyandb/pkg/logger"
"github.com/apache/skywalking-banyandb/pkg/timestamp"
)
diff --git a/banyand/internal/storage/shard.go b/banyand/internal/storage/shard.go
index 38145f38..ebe7209f 100644
--- a/banyand/internal/storage/shard.go
+++ b/banyand/internal/storage/shard.go
@@ -25,7 +25,7 @@ import (
"sync"
"github.com/apache/skywalking-banyandb/api/common"
- "github.com/apache/skywalking-banyandb/banyand/tsdb/bucket"
+ "github.com/apache/skywalking-banyandb/banyand/internal/bucket"
"github.com/apache/skywalking-banyandb/pkg/logger"
)
diff --git a/banyand/kv/badger.go b/banyand/kv/badger.go
deleted file mode 100644
index 09f1d385..00000000
--- a/banyand/kv/badger.go
+++ /dev/null
@@ -1,396 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package kv
-
-import (
- "bytes"
- "errors"
- "log"
- "math"
- "time"
-
- "github.com/dgraph-io/badger/v3"
- "github.com/dgraph-io/badger/v3/banyandb"
- "github.com/dgraph-io/badger/v3/skl"
- "github.com/dgraph-io/badger/v3/y"
-
- "github.com/apache/skywalking-banyandb/pkg/encoding"
- "github.com/apache/skywalking-banyandb/pkg/logger"
- "github.com/apache/skywalking-banyandb/pkg/timestamp"
-)
-
-var (
- _ Store = (*badgerDB)(nil)
- _ IndexStore = (*badgerDB)(nil)
- _ y.Iterator = (*mergedIter)(nil)
- _ TimeSeriesStore = (*badgerTSS)(nil)
- bitMergeEntry byte = 1 << 3
- // ErrKeyNotFound denotes the expected key can not be got from the kv service.
- ErrKeyNotFound = badger.ErrKeyNotFound
-)
-
-type badgerTSS struct {
- badger.TSet
- timeRange timestamp.TimeRange
- db *badger.DB
- dbOpts badger.Options
-}
-
-func (b *badgerTSS) Handover(skl *skl.Skiplist) error {
- return b.db.HandoverIterator(&timeRangeIterator{
- timeRange: b.timeRange,
- UniIterator: skl.NewUniIterator(false),
- })
-}
-
-func (b *badgerTSS) Close() error {
- if b.db != nil && !b.db.IsClosed() {
- return b.db.Close()
- }
- return nil
-}
-
-func (b *badgerTSS) SizeOnDisk() int64 {
- lsmSize, vlogSize := b.db.Size()
- return lsmSize + vlogSize
-}
-
-func (b *badgerTSS) CollectStats() *badger.Statistics {
- return b.db.CollectStats()
-}
-
-type mergedIter struct {
- delegated Iterator
- data []byte
- valid bool
-}
-
-func (i *mergedIter) Next() {
- i.delegated.Next()
- i.parseData()
-}
-
-func (i *mergedIter) Rewind() {
- i.delegated.Rewind()
- i.parseData()
-}
-
-func (i *mergedIter) Seek(key []byte) {
- i.delegated.Seek(y.KeyWithTs(key, math.MaxInt64))
-}
-
-func (i *mergedIter) Key() []byte {
- return y.KeyWithTs(i.delegated.Key(), uint64(time.Now().UnixNano()))
-}
-
-func (i *mergedIter) Valid() bool {
- return i.valid
-}
-
-func (i *mergedIter) parseData() {
- i.data = nil
- i.valid = i.delegated.Valid()
- if !i.valid {
- return
- }
- i.data = i.delegated.Val()
-}
-
-func (i *mergedIter) Close() error {
- i.data = nil
- i.valid = false
- return i.delegated.Close()
-}
-
-func (i mergedIter) Value() y.ValueStruct {
- return y.ValueStruct{
- Value: i.data,
- Meta: bitMergeEntry,
- }
-}
-
-type timeRangeIterator struct {
- *skl.UniIterator
- timeRange timestamp.TimeRange
-}
-
-func (i *timeRangeIterator) Next() {
- i.UniIterator.Next()
- for !i.validTime() {
- i.UniIterator.Next()
- }
-}
-
-func (i *timeRangeIterator) Rewind() {
- i.UniIterator.Rewind()
- if !i.validTime() {
- i.Next()
- }
-}
-
-func (i *timeRangeIterator) Seek(key []byte) {
- i.UniIterator.Seek(key)
- if !i.validTime() {
- i.Next()
- }
-}
-
-func (i *timeRangeIterator) validTime() bool {
- if !i.Valid() {
- // If the underlying iterator is invalid, we should return true to stop iterating.
- return true
- }
- ts := y.ParseTs(i.Key())
- return i.timeRange.Contains(ts)
-}
-
-type badgerDB struct {
- db *badger.DB
- dbOpts badger.Options
-}
-
-func (b *badgerDB) Scan(prefix, seekKey []byte, opt ScanOpts, f ScanFunc) error {
- opts := badger.DefaultIteratorOptions
- opts.PrefetchSize = opt.PrefetchSize
- opts.PrefetchValues = opt.PrefetchValues
- opts.Reverse = opt.Reverse
- it := b.db.NewIterator(opts)
- defer func() {
- _ = it.Close()
- }()
- for it.Seek(seekKey); it.Valid(); it.Next() {
- k := y.ParseKey(it.Key())
- if len(k) < len(seekKey) {
- continue
- }
- if !bytes.Equal(prefix, k[0:len(prefix)]) {
- continue
- }
- err := f(k, func() ([]byte, error) {
- return y.Copy(it.Value().Value), nil
- })
- if errors.Is(err, errStopScan) {
- break
- }
- if err != nil {
- return err
- }
- }
- return nil
-}
-
-func (b *badgerDB) SizeOnDisk() int64 {
- lsmSize, vlogSize := b.db.Size()
- return lsmSize + vlogSize
-}
-
-var _ Iterator = (*iterator)(nil)
-
-type iterator struct {
- delegated y.Iterator
- reverse bool
-}
-
-func (i *iterator) Next() {
- i.delegated.Next()
-}
-
-func (i *iterator) Rewind() {
- i.delegated.Rewind()
-}
-
-func (i *iterator) Seek(key []byte) {
- if i.reverse {
- i.delegated.Seek(y.KeyWithTs(key, 0))
- } else {
- i.delegated.Seek(y.KeyWithTs(key, math.MaxInt64))
- }
-}
-
-func (i *iterator) Key() []byte {
- return y.ParseKey(i.delegated.Key())
-}
-
-func (i *iterator) RawKey() []byte {
- return i.delegated.Key()
-}
-
-func (i *iterator) Val() []byte {
- return y.Copy(i.delegated.Value().Value)
-}
-
-func (i *iterator) Valid() bool {
- return i.delegated.Valid()
-}
-
-func (i *iterator) Close() error {
- return i.delegated.Close()
-}
-
-func (b *badgerDB) NewIterator(opt ScanOpts) Iterator {
- opts := badger.DefaultIteratorOptions
- opts.PrefetchSize = opt.PrefetchSize
- opts.PrefetchValues = opt.PrefetchValues
- opts.Reverse = opt.Reverse
- opts.Prefix = opt.Prefix
- it := b.db.NewIterator(opts)
- return &iterator{
- delegated: it,
- reverse: opts.Reverse,
- }
-}
-
-func (b *badgerDB) Close() error {
- if b.db != nil && !b.db.IsClosed() {
- return b.db.Close()
- }
- return nil
-}
-
-func (b *badgerDB) Put(key, val []byte) error {
- return b.db.Put(y.KeyWithTs(key, math.MaxInt64), val)
-}
-
-func (b *badgerDB) PutWithVersion(key, val []byte, version uint64) error {
- return b.db.Put(y.KeyWithTs(key, version), val)
-}
-
-func (b *badgerDB) Get(key []byte) ([]byte, error) {
- v, err := b.db.Get(y.KeyWithTs(key, math.MaxInt64))
- if errors.Is(err, badger.ErrKeyNotFound) {
- return nil, ErrKeyNotFound
- }
- if err != nil {
- return nil, err
- }
- return v.Value, nil
-}
-
-func (b *badgerDB) GetAll(key []byte, applyFn func([]byte) error) error {
- iter := b.db.NewIterator(badger.DefaultIteratorOptions)
- var count int
- for iter.Seek(y.KeyWithTs(key, math.MaxUint64)); iter.Valid(); iter.Next() {
- if !bytes.Equal(y.ParseKey(iter.Key()), key) {
- break
- }
- count++
- err := applyFn(y.Copy(iter.Value().Value))
- if err != nil {
- return err
- }
- }
- if count > 0 {
- return nil
- }
- return ErrKeyNotFound
-}
-
-// badgerLog delegates the zap log to the badger logger.
-type badgerLog struct {
- *log.Logger
- delegated *logger.Logger
-}
-
-func (l *badgerLog) Errorf(f string, v ...interface{}) {
- l.delegated.Error().Msgf(f, v...)
-}
-
-func (l *badgerLog) Warningf(f string, v ...interface{}) {
- l.delegated.Warn().Msgf(f, v...)
-}
-
-func (l *badgerLog) Infof(f string, v ...interface{}) {
- l.delegated.Info().Msgf(f, v...)
-}
-
-func (l *badgerLog) Debugf(f string, v ...interface{}) {
- l.delegated.Debug().Msgf(f, v...)
-}
-
-var _ banyandb.SeriesEncoderPool = (*encoderPoolDelegate)(nil)
-
-type encoderPoolDelegate struct {
- encoding.SeriesEncoderPool
-}
-
-func (e *encoderPoolDelegate) Get(metadata []byte, buffer banyandb.Buffer) banyandb.SeriesEncoder {
- encoder := e.SeriesEncoderPool.Get(metadata, buffer)
- if encoder == nil {
- return nil
- }
- return &encoderDelegate{SeriesEncoder: encoder}
-}
-
-func (e *encoderPoolDelegate) Put(encoder banyandb.SeriesEncoder) {
- ee := encoder.(*encoderDelegate)
- e.SeriesEncoderPool.Put(ee.SeriesEncoder)
-}
-
-var _ banyandb.SeriesEncoder = (*encoderDelegate)(nil)
-
-type encoderDelegate struct {
- encoding.SeriesEncoder
-}
-
-func (e *encoderDelegate) Reset(key []byte, buffer banyandb.Buffer) {
- e.SeriesEncoder.Reset(key, &bufferDelegate{BufferWriter: buffer})
-}
-
-var _ encoding.BufferWriter = (*bufferDelegate)(nil)
-
-type bufferDelegate struct {
- encoding.BufferWriter
-}
-
-var _ banyandb.SeriesDecoderPool = (*decoderPoolDelegate)(nil)
-
-type decoderPoolDelegate struct {
- encoding.SeriesDecoderPool
-}
-
-func (e *decoderPoolDelegate) Get(metadata []byte) banyandb.SeriesDecoder {
- if decoder := e.SeriesDecoderPool.Get(metadata); decoder != nil {
- return &decoderDelegate{
- e.SeriesDecoderPool.Get(metadata),
- }
- }
- return nil
-}
-
-func (e *decoderPoolDelegate) Put(decoder banyandb.SeriesDecoder) {
- if decoder != nil {
- dd := decoder.(*decoderDelegate)
- e.SeriesDecoderPool.Put(dd.SeriesDecoder)
- }
-}
-
-var _ banyandb.SeriesDecoder = (*decoderDelegate)(nil)
-
-type decoderDelegate struct {
- encoding.SeriesDecoder
-}
-
-func (d *decoderDelegate) Iterator() banyandb.SeriesIterator {
- return &iterDelegate{
- SeriesIterator: d.SeriesDecoder.Iterator(),
- }
-}
-
-type iterDelegate struct {
- encoding.SeriesIterator
-}
diff --git a/banyand/kv/badger_test.go b/banyand/kv/badger_test.go
deleted file mode 100644
index 5313bbfa..00000000
--- a/banyand/kv/badger_test.go
+++ /dev/null
@@ -1,72 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package kv
-
-import (
- "reflect"
- "testing"
- "time"
-
- "github.com/dgraph-io/badger/v3/skl"
- "github.com/dgraph-io/badger/v3/y"
-
- "github.com/apache/skywalking-banyandb/pkg/timestamp"
-)
-
-var arenaSize = int64(4 << 20)
-
-func TestTimeRangeIterator(t *testing.T) {
- // Create a new skiplist and insert some data
- sl := skl.NewSkiplist(arenaSize)
- sl.Put(y.KeyWithTs([]byte("key"), 1), y.ValueStruct{Value: []byte("value1"), Meta: 0})
- sl.Put(y.KeyWithTs([]byte("key"), 2), y.ValueStruct{Value: []byte("value2"), Meta: 0})
- sl.Put(y.KeyWithTs([]byte("key"), 3), y.ValueStruct{Value: []byte("value3"), Meta: 0})
- sl.Put(y.KeyWithTs([]byte("key"), 4), y.ValueStruct{Value: []byte("value4"), Meta: 0})
-
- // Create a new time range iterator for the skiplist
- iter := &timeRangeIterator{
- timeRange: timestamp.NewInclusiveTimeRange(
- timestamp.DefaultTimeRange.Begin.AsTime(),
- timestamp.DefaultTimeRange.End.AsTime()),
- UniIterator: sl.NewUniIterator(false),
- }
-
- // Test Next() and Value() methods
- var values []string
- for iter.Rewind(); iter.Valid(); iter.Next() {
- values = append(values, string(iter.Value().Value))
- }
- expectedValues := []string{"value4", "value3", "value2", "value1"}
- if !reflect.DeepEqual(values, expectedValues) {
- t.Errorf("unexpected values: %v, expected: %v", values, expectedValues)
- }
-
- // Test Next() method with time range filtering
- iter = &timeRangeIterator{
- timeRange: timestamp.NewSectionTimeRange(time.Unix(0, 2), time.Unix(0, 3)),
- UniIterator: sl.NewUniIterator(false),
- }
- values = nil
- for iter.Rewind(); iter.Valid(); iter.Next() {
- values = append(values, string(iter.Value().Value))
- }
- expectedValues = []string{"value2"}
- if !reflect.DeepEqual(values, expectedValues) {
- t.Errorf("unexpected values: %v, expected: %v", values, expectedValues)
- }
-}
diff --git a/banyand/kv/kv.go b/banyand/kv/kv.go
deleted file mode 100644
index e6180683..00000000
--- a/banyand/kv/kv.go
+++ /dev/null
@@ -1,306 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-// Package kv implements a key-value engine.
-package kv
-
-import (
- "fmt"
- "io"
- "math"
-
- "github.com/dgraph-io/badger/v3"
- "github.com/dgraph-io/badger/v3/options"
- "github.com/dgraph-io/badger/v3/skl"
- "github.com/pkg/errors"
-
- "github.com/apache/skywalking-banyandb/pkg/encoding"
- "github.com/apache/skywalking-banyandb/pkg/logger"
- "github.com/apache/skywalking-banyandb/pkg/timestamp"
-)
-
-var (
- errStopScan = errors.New("stop scanning")
-
- // DefaultScanOpts is a helper to provides canonical options for scanning.
- DefaultScanOpts = ScanOpts{
- PrefetchSize: 100,
- PrefetchValues: true,
- }
-
- defaultKVMemorySize = 4 << 20
-)
-
-type writer interface {
- // Put a value
- Put(key, val []byte) error
- PutWithVersion(key, val []byte, version uint64) error
-}
-
-// ScanFunc is the closure executed on scanning out a pair of key-value.
-type ScanFunc func(key []byte, getVal func() ([]byte, error)) error
-
-// ScanOpts wraps options for scanning the kv storage.
-type ScanOpts struct {
- Prefix []byte
- PrefetchSize int
- PrefetchValues bool
- Reverse bool
-}
-
-// Reader allows retrieving data from kv.
-type Reader interface {
- Iterable
- // Get a value by its key
- Get(key []byte) ([]byte, error)
- GetAll(key []byte, applyFn func([]byte) error) error
- Scan(prefix, seekKey []byte, opt ScanOpts, f ScanFunc) error
-}
-
-// Store is a common kv storage with auto-generated key.
-type Store interface {
- io.Closer
- writer
- Reader
- SizeOnDisk() int64
-}
-
-// TimeSeriesReader allows retrieving data from a time-series storage.
-type TimeSeriesReader interface {
- // Get a value by its key and timestamp/version
- Get(key []byte, ts uint64) ([]byte, error)
-}
-
-// TimeSeriesStore is time series storage.
-type TimeSeriesStore interface {
- io.Closer
- Handover(skl *skl.Skiplist) error
- TimeSeriesReader
- SizeOnDisk() int64
- CollectStats() *badger.Statistics
-}
-
-// TimeSeriesOptions sets an options for creating a TimeSeriesStore.
-type TimeSeriesOptions func(TimeSeriesStore)
-
-// TSSWithLogger sets a external logger into underlying TimeSeriesStore.
-func TSSWithLogger(l *logger.Logger) TimeSeriesOptions {
- return func(store TimeSeriesStore) {
- if btss, ok := store.(*badgerTSS); ok {
- btss.dbOpts = btss.dbOpts.WithLogger(&badgerLog{
- delegated: l.Named("ts-kv"),
- })
- }
- }
-}
-
-// TSSWithEncoding sets encoding and decoding pools for building chunks.
-func TSSWithEncoding(encoderPool encoding.SeriesEncoderPool, decoderPool encoding.SeriesDecoderPool, chunkSize int) TimeSeriesOptions {
- return func(store TimeSeriesStore) {
- if btss, ok := store.(*badgerTSS); ok {
- btss.dbOpts = btss.dbOpts.
- WithKeyBasedEncoder(
- &encoderPoolDelegate{
- encoderPool,
- }, &decoderPoolDelegate{
- decoderPool,
- }, chunkSize).
- WithSameKeyBlock()
- }
- }
-}
-
-// TSSWithZSTDCompression sets a ZSTD based compression method.
-func TSSWithZSTDCompression(chunkSize int) TimeSeriesOptions {
- return func(store TimeSeriesStore) {
- if btss, ok := store.(*badgerTSS); ok {
- btss.dbOpts = btss.dbOpts.
- WithCompression(options.ZSTD).
- WithBlockSize(chunkSize).
- WithZSTDCompressionLevel(3)
- }
- }
-}
-
-// TSSWithMemTableSize sets the size of memory table in bytes.
-func TSSWithMemTableSize(sizeInBytes int64) TimeSeriesOptions {
- return func(store TimeSeriesStore) {
- if sizeInBytes < 1 {
- return
- }
- if btss, ok := store.(*badgerTSS); ok {
- btss.dbOpts.MemTableSize = sizeInBytes
- }
- }
-}
-
-// Iterator allows iterating the kv tables.
-// TODO: use generic to provide a unique iterator.
-type Iterator interface {
- Next()
- Rewind()
- Seek(key []byte)
- Key() []byte
- RawKey() []byte
- Val() []byte
- Valid() bool
- Close() error
-}
-
-// Iterable allows creating a Iterator.
-type Iterable interface {
- NewIterator(opt ScanOpts) Iterator
-}
-
-// IndexStore allows writing and reading index format data.
-type IndexStore interface {
- Iterable
- Reader
- Close() error
- SizeOnDisk() int64
-}
-
-// OpenTimeSeriesStore creates a new TimeSeriesStore.
-// nolint: contextcheck
-func OpenTimeSeriesStore(path string, timeRange timestamp.TimeRange, options ...TimeSeriesOptions) (TimeSeriesStore, error) {
- btss := new(badgerTSS)
- btss.dbOpts = badger.DefaultOptions(path)
- for _, opt := range options {
- opt(btss)
- }
- // Put all values into LSM
- btss.dbOpts = btss.dbOpts.
- WithNumVersionsToKeep(math.MaxUint32).
- WithVLogPercentile(1.0).
- WithInTable().
- WithMaxLevels(2).
- WithBaseTableSize(10 << 20).
- WithBaseLevelSize(math.MaxInt64).
- WithBlockCacheSize(10 << 20)
- if btss.dbOpts.MemTableSize < int64(defaultKVMemorySize) {
- btss.dbOpts.MemTableSize = int64(defaultKVMemorySize)
- }
- if btss.dbOpts.MemTableSize < 8<<20 {
- btss.dbOpts = btss.dbOpts.WithValueThreshold(1 << 10)
- }
- btss.dbOpts.LmaxCompaction = true
- var err error
- btss.db, err = badger.Open(btss.dbOpts)
- if err != nil {
- return nil, fmt.Errorf("failed to open time series store: %w", err)
- }
- btss.TSet = *badger.NewTSet(btss.db)
- btss.timeRange = timeRange
- return btss, nil
-}
-
-// StoreOptions sets options for creating Store.
-type StoreOptions func(Store)
-
-// StoreWithLogger sets a external logger into underlying Store.
-func StoreWithLogger(l *logger.Logger) StoreOptions {
- return StoreWithNamedLogger("normal-kv", l)
-}
-
-// StoreWithNamedLogger sets a external logger with a name into underlying Store.
-func StoreWithNamedLogger(name string, l *logger.Logger) StoreOptions {
- return func(store Store) {
- if bdb, ok := store.(*badgerDB); ok {
- bdb.dbOpts = bdb.dbOpts.WithLogger(&badgerLog{
- delegated: l.Named(name),
- })
- }
- }
-}
-
-// StoreWithMemTableSize sets MemTable size.
-func StoreWithMemTableSize(size int64) StoreOptions {
- return func(store Store) {
- if size < 1 {
- return
- }
- if bdb, ok := store.(*badgerDB); ok {
- bdb.dbOpts = bdb.dbOpts.WithMemTableSize(size)
- }
- }
-}
-
-// OpenStore creates a new Store.
-// nolint: contextcheck
-func OpenStore(path string, opts ...StoreOptions) (Store, error) {
- bdb := new(badgerDB)
- bdb.dbOpts = badger.DefaultOptions(path)
- for _, opt := range opts {
- opt(bdb)
- }
- if bdb.dbOpts.MemTableSize < int64(defaultKVMemorySize) {
- bdb.dbOpts.MemTableSize = int64(defaultKVMemorySize)
- }
- if bdb.dbOpts.MemTableSize < 8<<20 {
- bdb.dbOpts = bdb.dbOpts.WithValueThreshold(1 << 10)
- }
- bdb.dbOpts = bdb.dbOpts.
- WithBaseTableSize(5 << 20).
- WithBaseLevelSize(25 << 20).
- WithCompression(options.ZSTD).
- WithZSTDCompressionLevel(1).
- WithBlockCacheSize(10 << 20)
-
- var err error
- bdb.db, err = badger.Open(bdb.dbOpts)
- if err != nil {
- return nil, fmt.Errorf("failed to open normal store: %w", err)
- }
- return bdb, nil
-}
-
-// IndexOptions sets options for creating the index store.
-type IndexOptions func(store IndexStore)
-
-// IndexWithLogger sets a external logger into underlying IndexStore.
-func IndexWithLogger(l *logger.Logger) IndexOptions {
- return func(store IndexStore) {
- if bdb, ok := store.(*badgerDB); ok {
- bdb.dbOpts = bdb.dbOpts.WithLogger(&badgerLog{
- delegated: l.Named("index-kv"),
- })
- }
- }
-}
-
-// OpenIndexStore creates a new IndexStore.
-func OpenIndexStore(path string, options ...IndexOptions) (IndexStore, error) {
- bdb := new(badgerDB)
- bdb.dbOpts = badger.DefaultOptions(path)
- for _, opt := range options {
- opt(bdb)
- }
- bdb.dbOpts = bdb.dbOpts.WithNumVersionsToKeep(math.MaxUint32).
- WithNumCompactors(2).
- WithMemTableSize(2 << 20).
- WithMaxLevels(2).
- WithBaseTableSize(2 << 20).
- WithBaseLevelSize(math.MaxInt64).
- WithValueThreshold(1 << 10)
-
- var err error
- bdb.db, err = badger.Open(bdb.dbOpts)
- if err != nil {
- return nil, fmt.Errorf("failed to index store: %w", err)
- }
- return bdb, nil
-}
diff --git a/banyand/liaison/grpc/discovery.go b/banyand/liaison/grpc/discovery.go
index 07f308d5..661deb01 100644
--- a/banyand/liaison/grpc/discovery.go
+++ b/banyand/liaison/grpc/discovery.go
@@ -29,9 +29,9 @@ import (
modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
"github.com/apache/skywalking-banyandb/banyand/metadata"
"github.com/apache/skywalking-banyandb/banyand/metadata/schema"
- "github.com/apache/skywalking-banyandb/banyand/tsdb"
"github.com/apache/skywalking-banyandb/pkg/logger"
"github.com/apache/skywalking-banyandb/pkg/partition"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
)
var errNotExist = errors.New("the object doesn't exist")
@@ -69,7 +69,7 @@ func (ds *discoveryService) SetLogger(log *logger.Logger) {
ds.entityRepo.log = log
}
-func (ds *discoveryService) navigate(metadata *commonv1.Metadata, tagFamilies []*modelv1.TagFamilyForWrite) (tsdb.Entity, tsdb.EntityValues, common.ShardID, error) {
+func (ds *discoveryService) navigate(metadata *commonv1.Metadata, tagFamilies []*modelv1.TagFamilyForWrite) (pbv1.Entity, pbv1.EntityValues, common.ShardID, error) {
shardNum, existed := ds.shardRepo.shardNum(getID(&commonv1.Metadata{
Name: metadata.Group,
}))
diff --git a/banyand/liaison/grpc/measure.go b/banyand/liaison/grpc/measure.go
index 9517b6cb..930cec03 100644
--- a/banyand/liaison/grpc/measure.go
+++ b/banyand/liaison/grpc/measure.go
@@ -32,10 +32,10 @@ import (
measurev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/measure/v1"
modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
"github.com/apache/skywalking-banyandb/banyand/queue"
- "github.com/apache/skywalking-banyandb/banyand/tsdb"
"github.com/apache/skywalking-banyandb/pkg/accesslog"
"github.com/apache/skywalking-banyandb/pkg/bus"
"github.com/apache/skywalking-banyandb/pkg/logger"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
"github.com/apache/skywalking-banyandb/pkg/timestamp"
)
@@ -115,7 +115,7 @@ func (ms *measureService) Write(measure measurev1.MeasureService_WriteServer) er
iwr := &measurev1.InternalWriteRequest{
Request: writeRequest,
ShardId: uint32(shardID),
- SeriesHash: tsdb.HashEntity(entity),
+ SeriesHash: pbv1.HashEntity(entity),
// TODO: remove the first value (measure name) of tagValues
EntityValues: tagValues[1:].Encode(),
}
diff --git a/banyand/liaison/grpc/stream.go b/banyand/liaison/grpc/stream.go
index 9ad52102..3e530ff6 100644
--- a/banyand/liaison/grpc/stream.go
+++ b/banyand/liaison/grpc/stream.go
@@ -32,10 +32,10 @@ import (
modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
streamv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/stream/v1"
"github.com/apache/skywalking-banyandb/banyand/queue"
- "github.com/apache/skywalking-banyandb/banyand/tsdb"
"github.com/apache/skywalking-banyandb/pkg/accesslog"
"github.com/apache/skywalking-banyandb/pkg/bus"
"github.com/apache/skywalking-banyandb/pkg/logger"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
"github.com/apache/skywalking-banyandb/pkg/timestamp"
)
@@ -115,7 +115,7 @@ func (s *streamService) Write(stream streamv1.StreamService_WriteServer) error {
iwr := &streamv1.InternalWriteRequest{
Request: writeEntity,
ShardId: uint32(shardID),
- SeriesHash: tsdb.HashEntity(entity),
+ SeriesHash: pbv1.HashEntity(entity),
// TODO: remove the first value (stream name) of tagValues
EntityValues: tagValues[1:].Encode(),
}
diff --git a/banyand/metadata/schema/etcd_test.go b/banyand/metadata/schema/etcd_test.go
index 45aa7fd8..1ec32cf5 100644
--- a/banyand/metadata/schema/etcd_test.go
+++ b/banyand/metadata/schema/etcd_test.go
@@ -387,7 +387,7 @@ func Test_Etcd_Entity_Update(t *testing.T) {
}
// reset
ir.Metadata.Id = 0
- ir.Type = databasev1.IndexRule_TYPE_TREE
+ ir.Type = databasev1.IndexRule_TYPE_INVERTED
return r.UpdateIndexRule(ctx, ir)
},
validationFunc: func(ctx context.Context, r Registry) bool {
@@ -401,7 +401,7 @@ func Test_Etcd_Entity_Update(t *testing.T) {
if ir.Metadata.Id != 1 {
return false
}
- if ir.Type != databasev1.IndexRule_TYPE_TREE {
+ if ir.Type != databasev1.IndexRule_TYPE_INVERTED {
return false
}
return true
diff --git a/banyand/query/processor_topn.go b/banyand/query/processor_topn.go
index dc40753e..e40339f9 100644
--- a/banyand/query/processor_topn.go
+++ b/banyand/query/processor_topn.go
@@ -29,10 +29,10 @@ import (
measurev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/measure/v1"
modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
"github.com/apache/skywalking-banyandb/banyand/measure"
- "github.com/apache/skywalking-banyandb/banyand/tsdb"
"github.com/apache/skywalking-banyandb/pkg/bus"
"github.com/apache/skywalking-banyandb/pkg/flow"
"github.com/apache/skywalking-banyandb/pkg/logger"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
"github.com/apache/skywalking-banyandb/pkg/query/aggregation"
"github.com/apache/skywalking-banyandb/pkg/query/executor"
logical_measure "github.com/apache/skywalking-banyandb/pkg/query/logical/measure"
@@ -152,7 +152,7 @@ var _ heap.Interface = (*postAggregationProcessor)(nil)
type aggregatorItem struct {
int64Func aggregation.Func[int64]
key string
- values tsdb.EntityValues
+ values pbv1.EntityValues
index int
}
@@ -169,7 +169,7 @@ func (n *aggregatorItem) GetTags(tagNames []string) []*modelv1.Tag {
// PostProcessor defines necessary methods for Top-N post processor with or without aggregation.
type PostProcessor interface {
- Put(entityValues tsdb.EntityValues, val int64, timestampMillis uint64) error
+ Put(entityValues pbv1.EntityValues, val int64, timestampMillis uint64) error
Val([]string) []*measurev1.TopNList
}
@@ -241,7 +241,7 @@ func (aggr *postAggregationProcessor) Pop() any {
return item
}
-func (aggr *postAggregationProcessor) Put(entityValues tsdb.EntityValues, val int64, timestampMillis uint64) error {
+func (aggr *postAggregationProcessor) Put(entityValues pbv1.EntityValues, val int64, timestampMillis uint64) error {
// update latest ts
if aggr.latestTimestamp < timestampMillis {
aggr.latestTimestamp = timestampMillis
@@ -314,7 +314,7 @@ var _ flow.Element = (*nonAggregatorItem)(nil)
type nonAggregatorItem struct {
key string
- values tsdb.EntityValues
+ values pbv1.EntityValues
val int64
index int
}
@@ -375,7 +375,7 @@ func (naggr *postNonAggregationProcessor) Val(tagNames []string) []*measurev1.To
return topNLists
}
-func (naggr *postNonAggregationProcessor) Put(entityValues tsdb.EntityValues, val int64, timestampMillis uint64) error {
+func (naggr *postNonAggregationProcessor) Put(entityValues pbv1.EntityValues, val int64, timestampMillis uint64) error {
key := entityValues.String()
if timeline, ok := naggr.timelines[timestampMillis]; ok {
if timeline.Len() < int(naggr.topN) {
diff --git a/banyand/tsdb/block.go b/banyand/tsdb/block.go
deleted file mode 100644
index a8596508..00000000
--- a/banyand/tsdb/block.go
+++ /dev/null
@@ -1,484 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package tsdb
-
-import (
- "context"
- "fmt"
- "io"
- "os"
- "path"
- "runtime"
- "strconv"
- "strings"
- "sync"
- "sync/atomic"
- "time"
- "unsafe"
-
- "github.com/dgraph-io/badger/v3"
- "github.com/pkg/errors"
- "go.uber.org/multierr"
-
- "github.com/apache/skywalking-banyandb/api/common"
- "github.com/apache/skywalking-banyandb/banyand/kv"
- "github.com/apache/skywalking-banyandb/banyand/observability"
- "github.com/apache/skywalking-banyandb/banyand/tsdb/bucket"
- "github.com/apache/skywalking-banyandb/pkg/index"
- "github.com/apache/skywalking-banyandb/pkg/index/inverted"
- "github.com/apache/skywalking-banyandb/pkg/index/lsm"
- "github.com/apache/skywalking-banyandb/pkg/logger"
- "github.com/apache/skywalking-banyandb/pkg/meter"
- "github.com/apache/skywalking-banyandb/pkg/timestamp"
-)
-
-const (
- componentSecondInvertedIdx = "inverted"
- componentSecondLSMIdx = "lsm"
-
- defaultEnqueueTimeout = 500 * time.Millisecond
- itemIDLength = unsafe.Sizeof(common.ItemID(0))
-)
-
-var (
- blockMeterProvider = observability.NewMeterProvider(meterTSDB.SubScope("block"))
- blockOpenedTimeSecondsGauge = blockMeterProvider.Gauge("opened_time_seconds", common.LabelNames()...)
- blockReferencesGauge = blockMeterProvider.Gauge("refs", common.LabelNames()...)
-
- blockCompressedSize = blockMeterProvider.Gauge("compressed_size", append(common.LabelNames(), "from", "to")...)
- blockUncompressedSize = blockMeterProvider.Gauge("uncompressed_size", append(common.LabelNames(), "from", "to")...)
- blockCompressedBlockNum = blockMeterProvider.Gauge("compressed_block_num", append(common.LabelNames(), "from", "to")...)
- blockCompressedEntryNum = blockMeterProvider.Gauge("compressed_entry_num", append(common.LabelNames(), "from", "to")...)
- blockEncodedSize = blockMeterProvider.Gauge("encoded_size", append(common.LabelNames(), "from", "to")...)
- blockUnencodedSize = blockMeterProvider.Gauge("unencoded_size", append(common.LabelNames(), "from", "to")...)
- blockEncodedBlockNum = blockMeterProvider.Gauge("encoded_block_num", append(common.LabelNames(), "from", "to")...)
- blockEncodedEntryNum = blockMeterProvider.Gauge("encoded_entry_num", append(common.LabelNames(), "from", "to")...)
- blockCompactionMap = map[badger.TableBuilderSizeKeyType]meter.Gauge{
- badger.TableBuilderSizeKeyCompressedSize: blockCompressedSize,
- badger.TableBuilderSizeKeyUncompressedSize: blockUncompressedSize,
- badger.TableBuilderSizeKeyCompressedBlockNum: blockCompressedBlockNum,
- badger.TableBuilderSizeKeyCompressedEntryNum: blockCompressedEntryNum,
- badger.TableBuilderSizeKeyEncodedSize: blockEncodedSize,
- badger.TableBuilderSizeKeyUnEncodedSize: blockUnencodedSize,
- badger.TableBuilderSizeKeyEncodedBlockNum: blockEncodedBlockNum,
- badger.TableBuilderSizeKeyEncodedEntryNum: blockEncodedEntryNum,
- }
-
- errBlockClosingInterrupted = errors.New("interrupt to close the block")
-)
-
-type block struct {
- openOpts openOpts
- invertedIndex index.Store
- tsTable TSTable
- queue bucket.Queue
- bucket.Reporter
- clock timestamp.Clock
- lsmIndex index.Store
- closeBufferTimer *time.Timer
- closed *atomic.Bool
- ref *atomic.Int32
- l *logger.Logger
- deleted *atomic.Bool
- position common.Position
- timestamp.TimeRange
- segSuffix string
- suffix string
- path string
- closableLst []io.Closer
- lock sync.RWMutex
- segID SectionID
- blockID SectionID
-}
-
-type openOpts struct {
- tsTableFactory TSTableFactory
- bufferSupplier *BufferSupplier
- inverted *inverted.StoreOpts
- lsm lsm.StoreOpts
-}
-
-type blockOpts struct {
- queue bucket.Queue
- scheduler *timestamp.Scheduler
- timeRange timestamp.TimeRange
- segSuffix string
- suffix string
- path string
- blockSize IntervalRule
- segID SectionID
-}
-
-func newBlock(ctx context.Context, opts blockOpts) (b *block, err error) {
- suffixInteger, err := strconv.Atoi(opts.suffix)
- if err != nil {
- return nil, err
- }
- id := GenerateInternalID(opts.blockSize.Unit, suffixInteger)
- clock, _ := timestamp.GetClock(ctx)
- b = &block{
- segID: opts.segID,
- segSuffix: opts.segSuffix,
- suffix: opts.suffix,
- blockID: id,
- path: opts.path,
- TimeRange: opts.timeRange,
- clock: clock,
- ref: &atomic.Int32{},
- closed: &atomic.Bool{},
- deleted: &atomic.Bool{},
- queue: opts.queue,
- position: common.GetPosition(ctx),
- }
- l := logger.Fetch(ctx, b.String())
- b.openOpts, err = options(ctx, opts.path, l)
- if err != nil {
- return nil, err
- }
- b.l = l
- b.Reporter = bucket.NewTimeBasedReporter(b.String(), opts.timeRange, clock, opts.scheduler)
- b.closed.Store(true)
- return b, nil
-}
-
-func options(ctx context.Context, root string, l *logger.Logger) (openOpts, error) {
- var options DatabaseOpts
- o := ctx.Value(OptionsKey)
- if o == nil {
- return openOpts{}, errors.New("database options not found")
- }
- options = o.(DatabaseOpts)
- var opts openOpts
- if options.IndexGranularity == IndexGranularityBlock {
- opts.inverted = &inverted.StoreOpts{
- Path: path.Join(root, componentSecondInvertedIdx),
- Logger: l.Named(componentSecondInvertedIdx),
- BatchWaitSec: options.BlockInvertedIndex.BatchWaitSec,
- }
- }
- opts.lsm = lsm.StoreOpts{
- Path: path.Join(root, componentSecondLSMIdx),
- Logger: l.Named(componentSecondLSMIdx),
- MemTableSize: defaultKVMemorySize,
- }
- opts.tsTableFactory = options.TSTableFactory
- if opts.tsTableFactory == nil {
- return opts, errors.New("ts table factory is nil")
- }
- bs := ctx.Value(bufferSupplierKey)
- if bs == nil {
- return opts, errors.New("buffer supplier not found")
- }
- opts.bufferSupplier = bs.(*BufferSupplier)
- return opts, nil
-}
-
-func (b *block) openSafely() (err error) {
- if b.deleted.Load() || !b.Closed() {
- return nil
- }
- b.lock.Lock()
- defer b.lock.Unlock()
- if !b.Closed() {
- return
- }
- return b.open()
-}
-
-func (b *block) open() (err error) {
- if b.tsTable, err = b.openOpts.tsTableFactory.NewTSTable(b.openOpts.bufferSupplier,
- b.path, b.position, b.l, b.TimeRange); err != nil {
- return err
- }
- b.closableLst = append(b.closableLst, b.tsTable)
- if b.openOpts.inverted != nil {
- if b.invertedIndex, err = inverted.NewStore(*b.openOpts.inverted); err != nil {
- return err
- }
- b.closableLst = append(b.closableLst, b.invertedIndex)
- }
- if b.lsmIndex, err = lsm.NewStore(b.openOpts.lsm); err != nil {
- return err
- }
- b.closableLst = append(b.closableLst, b.lsmIndex)
- b.ref.Store(0)
- b.closed.Store(false)
- blockOpenedTimeSecondsGauge.Set(float64(time.Now().Unix()), b.position.LabelValues()...)
- plv := b.position.LabelValues()
- observability.MetricsCollector.Register(strings.Join(plv, "-"), func() {
- stats := b.tsTable.CollectStats()
- stats.TableBuilderSize.Range(func(label interface{}, value interface{}) bool {
- key := label.(badger.TableBuilderSizeKey)
- counter := value.(*atomic.Int64)
- from, to := getLevelLabels(key.FromLevel, key.ToLevel)
- labelValues := append(b.position.LabelValues(), from, to)
- if block, ok := blockCompactionMap[key.Type]; ok {
- block.Set(float64(counter.Load()), labelValues...)
- }
- return true
- })
- })
- return nil
-}
-
-func (b *block) delegate(ctx context.Context) (blockDelegate, error) {
- if b.deleted.Load() {
- return nil, errors.WithMessagef(errBlockAbsent, "block %s is deleted", b)
- }
- blockID := BlockID{
- BlockID: b.blockID,
- SegID: b.segID,
- }
- if b.incRef() {
- b.queue.Touch(blockID)
- return &bDelegate{
- delegate: b,
- }, nil
- }
- b.lock.Lock()
- defer b.lock.Unlock()
- if err := b.queue.Push(ctx, blockID, func() error {
- if b.deleted.Load() || !b.Closed() {
- return nil
- }
- return b.open()
- }); err != nil {
- b.l.Error().Err(err).Stringer("block", b).Msg("fail to open block")
- return nil, err
- }
- b.incRef()
- return &bDelegate{
- delegate: b,
- }, nil
-}
-
-func (b *block) incRef() bool {
- if b.Closed() {
- return false
- }
- b.ref.Add(1)
- blockReferencesGauge.Set(float64(b.ref.Load()), b.position.LabelValues()...)
- return true
-}
-
-func (b *block) Done() {
- b.ref.Add(-1)
- blockReferencesGauge.Set(float64(b.ref.Load()), b.position.LabelValues()...)
-}
-
-func (b *block) waitDone(stopped *atomic.Bool) <-chan struct{} {
- ch := make(chan struct{})
- go func() {
- loop:
- if b.ref.Load() < 1 {
- b.ref.Store(0)
- close(ch)
- return
- }
- if stopped.Load() {
- close(ch)
- return
- }
- runtime.Gosched()
- goto loop
- }()
- return ch
-}
-
-func (b *block) close(ctx context.Context) (err error) {
- b.lock.Lock()
- defer b.lock.Unlock()
- if b.closed.Load() {
- return nil
- }
- stopWaiting := &atomic.Bool{}
- ch := b.waitDone(stopWaiting)
- select {
- case <-ctx.Done():
- stopWaiting.Store(true)
- return errors.Wrapf(errBlockClosingInterrupted, "block:%s", b)
- case <-ch:
- }
- b.closed.Store(true)
- if b.closeBufferTimer != nil {
- b.closeBufferTimer.Stop()
- }
- plv := b.position.LabelValues()
- observability.MetricsCollector.Unregister(strings.Join(plv, "-"))
- stats := b.tsTable.CollectStats()
- stats.TableBuilderSize.Range(func(label interface{}, value interface{}) bool {
- key := label.(badger.TableBuilderSizeKey)
- from, to := getLevelLabels(key.FromLevel, key.ToLevel)
- labelValues := append(b.position.LabelValues(), from, to)
- if block, ok := blockCompactionMap[key.Type]; ok {
- block.Delete(labelValues...)
- }
- return true
- })
- for _, closer := range b.closableLst {
- err = multierr.Append(err, closer.Close())
- }
- for _, g := range []meter.Gauge{blockOpenedTimeSecondsGauge, blockReferencesGauge} {
- g.Delete(b.position.LabelValues()...)
- }
- return err
-}
-
-func (b *block) delete(ctx context.Context) error {
- if b.deleted.Load() {
- return nil
- }
- b.deleted.Store(true)
- b.close(ctx)
- return os.RemoveAll(b.path)
-}
-
-func (b *block) Closed() bool {
- return b.closed.Load()
-}
-
-func (b *block) String() string {
- return fmt.Sprintf("BlockID-%s-%s", b.segSuffix, b.suffix)
-}
-
-func (b *block) Get(key []byte, ts uint64) ([]byte, error) {
- return b.tsTable.Get(key, time.Unix(0, int64(ts)))
-}
-
-type blockDelegate interface {
- io.Closer
- contains(ts time.Time) bool
- write(key []byte, val []byte, ts time.Time) error
- writePrimaryIndex(field index.Field, id common.ItemID) error
- writeLSMIndex(fields []index.Field, id common.ItemID) error
- writeInvertedIndex(fields []index.Field, id common.ItemID) error
- dataReader() kv.TimeSeriesReader
- lsmIndexReader() index.Searcher
- invertedIndexReader() index.Searcher
- primaryIndexReader() index.FieldIterable
- identity() (segID SectionID, blockID SectionID)
- startTime() time.Time
- String() string
-}
-
-var _ blockDelegate = (*bDelegate)(nil)
-
-type bDelegate struct {
- delegate *block
-}
-
-func (d *bDelegate) dataReader() kv.TimeSeriesReader {
- return d.delegate
-}
-
-func (d *bDelegate) lsmIndexReader() index.Searcher {
- return d.delegate.lsmIndex
-}
-
-func (d *bDelegate) invertedIndexReader() index.Searcher {
- return d.delegate.invertedIndex
-}
-
-func (d *bDelegate) primaryIndexReader() index.FieldIterable {
- return d.delegate.lsmIndex
-}
-
-func (d *bDelegate) startTime() time.Time {
- return d.delegate.Start
-}
-
-func (d *bDelegate) identity() (segID SectionID, blockID SectionID) {
- return d.delegate.segID, d.delegate.blockID
-}
-
-func (d *bDelegate) write(key []byte, val []byte, ts time.Time) error {
- if err := d.delegate.tsTable.Put(key, val, ts); err != nil {
- receivedNumCounter.Inc(1, append(d.delegate.position.ShardLabelValues(), "tst", "true")...)
- return err
- }
- receivedBytesCounter.Inc(float64(len(key)+len(val)), append(d.delegate.position.ShardLabelValues(), "tst")...)
- receivedNumCounter.Inc(1, append(d.delegate.position.ShardLabelValues(), "tst", "false")...)
- return nil
-}
-
-func (d *bDelegate) writePrimaryIndex(field index.Field, id common.ItemID) error {
- if err := d.delegate.lsmIndex.Write([]index.Field{field}, uint64(id)); err != nil {
- receivedNumCounter.Inc(1, append(d.delegate.position.ShardLabelValues(), "primary", "true")...)
- return err
- }
- receivedBytesCounter.Inc(float64(len(field.Marshal())+int(itemIDLength)), append(d.delegate.position.ShardLabelValues(), "primary")...)
- receivedNumCounter.Inc(1, append(d.delegate.position.ShardLabelValues(), "primary", "false")...)
- return nil
-}
-
-func (d *bDelegate) writeLSMIndex(fields []index.Field, id common.ItemID) error {
- total := 0
- for _, f := range fields {
- total += len(f.Marshal())
- }
-
- if err := d.delegate.lsmIndex.Write(fields, uint64(id)); err != nil {
- receivedNumCounter.Inc(1, append(d.delegate.position.ShardLabelValues(), "local_lsm", "true")...)
- return err
- }
- receivedBytesCounter.Inc(float64(total+int(itemIDLength)), append(d.delegate.position.ShardLabelValues(), "local_lsm")...)
- receivedNumCounter.Inc(1, append(d.delegate.position.ShardLabelValues(), "local_lsm", "false")...)
- return nil
-}
-
-func (d *bDelegate) writeInvertedIndex(fields []index.Field, id common.ItemID) error {
- if d.delegate.invertedIndex == nil {
- return errors.New("inverted index is not enabled")
- }
- total := 0
- for _, f := range fields {
- total += len(f.Marshal())
- }
-
- if err := d.delegate.invertedIndex.Write(fields, uint64(id)); err != nil {
- receivedNumCounter.Inc(1, append(d.delegate.position.ShardLabelValues(), "local_inverted", "true")...)
- return err
- }
- receivedBytesCounter.Inc(float64(total+int(itemIDLength)), append(d.delegate.position.ShardLabelValues(), "local_inverted")...)
- receivedNumCounter.Inc(1, append(d.delegate.position.ShardLabelValues(), "local_inverted", "false")...)
- return nil
-}
-
-func (d *bDelegate) contains(ts time.Time) bool {
- return d.delegate.Contains(uint64(ts.UnixNano()))
-}
-
-func (d *bDelegate) String() string {
- return d.delegate.String()
-}
-
-func (d *bDelegate) Close() error {
- d.delegate.Done()
- return nil
-}
-
-func getLevelLabels(fromLevel, toLevel int) (string, string) {
- from := fmt.Sprintf("l%d", fromLevel)
- to := fmt.Sprintf("l%d", toLevel)
-
- if fromLevel < 0 {
- from = "mem"
- }
-
- return from, to
-}
diff --git a/banyand/tsdb/block_ctrl.go b/banyand/tsdb/block_ctrl.go
deleted file mode 100644
index d834a51d..00000000
--- a/banyand/tsdb/block_ctrl.go
+++ /dev/null
@@ -1,347 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package tsdb
-
-import (
- "context"
- "fmt"
- "sort"
- "sync"
- "time"
-
- "go.uber.org/multierr"
-
- "github.com/apache/skywalking-banyandb/api/common"
- "github.com/apache/skywalking-banyandb/banyand/tsdb/bucket"
- "github.com/apache/skywalking-banyandb/pkg/logger"
- "github.com/apache/skywalking-banyandb/pkg/timestamp"
-)
-
-type blockController struct {
- segCtx context.Context
- blockQueue bucket.Queue
- clock timestamp.Clock
- scheduler *timestamp.Scheduler
- l *logger.Logger
- segTimeRange timestamp.TimeRange
- segSuffix string
- location string
- lst []*block
- blockSize IntervalRule
- sync.RWMutex
- segID SectionID
-}
-
-func newBlockController(segCtx context.Context, segID SectionID, segSuffix, location string, segTimeRange timestamp.TimeRange,
- blockSize IntervalRule, l *logger.Logger, blockQueue bucket.Queue, scheduler *timestamp.Scheduler,
-) *blockController {
- clock, _ := timestamp.GetClock(segCtx)
- return &blockController{
- segCtx: segCtx,
- segID: segID,
- segSuffix: segSuffix,
- location: location,
- blockSize: blockSize,
- segTimeRange: segTimeRange,
- blockQueue: blockQueue,
- l: l,
- clock: clock,
- scheduler: scheduler,
- }
-}
-
-func (bc *blockController) Current() (bucket.Reporter, error) {
- now := bc.Standard(bc.clock.Now())
- ns := uint64(now.UnixNano())
- if b := func() *block {
- bc.RLock()
- defer bc.RUnlock()
- for _, s := range bc.lst {
- if s.Contains(ns) {
- return s
- }
- }
- return nil
- }(); b != nil {
- if err := b.openSafely(); err != nil {
- return nil, err
- }
- return b, nil
- }
- return bc.newHeadBlock(now)
-}
-
-func (bc *blockController) Next() (bucket.Reporter, error) {
- c, err := bc.Current()
- if err != nil {
- return nil, err
- }
- b := c.(*block)
-
- return bc.newHeadBlock(bc.blockSize.nextTime(b.Start))
-}
-
-func (bc *blockController) newHeadBlock(now time.Time) (*block, error) {
- b, err := bc.create(now)
- if err != nil {
- return nil, err
- }
- return b, nil
-}
-
-func (bc *blockController) OnMove(prev bucket.Reporter, next bucket.Reporter) {
- event := bc.l.Info()
- if prev != nil {
- event.Stringer("prev", prev)
- b := prev.(*block)
- ctx, cancel := context.WithTimeout(context.Background(), defaultEnqueueTimeout)
- defer cancel()
- if err := bc.blockQueue.Push(ctx, BlockID{
- SegID: bc.segID,
- BlockID: b.blockID,
- }, nil); err != nil {
- bc.l.Error().Err(err).Msg("failed to push a expired head block to the queue")
- ctxClosing, cancelClosing := context.WithTimeout(context.Background(), defaultEnqueueTimeout)
- defer cancelClosing()
- b.close(ctxClosing)
- }
- }
- if next != nil {
- event.Stringer("next", next)
- }
- event.Msg("move to the next block")
-}
-
-func (bc *blockController) Standard(t time.Time) time.Time {
- switch bc.blockSize.Unit {
- case HOUR:
- return time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), 0, 0, 0, t.Location())
- case DAY:
- return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location())
- }
- panic("invalid interval unit")
-}
-
-func (bc *blockController) Format(tm time.Time) string {
- switch bc.blockSize.Unit {
- case HOUR:
- return tm.Format(hourFormat)
- case DAY:
- return tm.Format(dayFormat)
- }
- panic("invalid interval unit")
-}
-
-func (bc *blockController) Parse(value string) (time.Time, error) {
- switch bc.blockSize.Unit {
- case HOUR:
- return time.ParseInLocation(hourFormat, value, time.Local)
- case DAY:
- return time.ParseInLocation(dayFormat, value, time.Local)
- }
- panic("invalid interval unit")
-}
-
-func (bc *blockController) span(ctx context.Context, timeRange timestamp.TimeRange) ([]blockDelegate, error) {
- bb := bc.search(func(b *block) bool {
- return b.Overlapping(timeRange)
- })
- if bb == nil {
- return nil, nil
- }
- dd := make([]blockDelegate, len(bb))
- for i, b := range bb {
- d, err := b.delegate(ctx)
- if err != nil {
- return nil, err
- }
- dd[i] = d
- }
- return dd, nil
-}
-
-func (bc *blockController) get(ctx context.Context, blockID SectionID) (blockDelegate, error) {
- b := bc.getBlock(blockID)
- if b != nil {
- return b.delegate(ctx)
- }
- return nil, nil
-}
-
-func (bc *blockController) getBlock(blockID SectionID) *block {
- bb := bc.search(func(b *block) bool {
- return b.blockID == blockID
- })
- if len(bb) > 0 {
- return bb[0]
- }
- return nil
-}
-
-func (bc *blockController) blocks() []*block {
- bc.RLock()
- defer bc.RUnlock()
- r := make([]*block, len(bc.lst))
- copy(r, bc.lst)
- return r
-}
-
-func (bc *blockController) search(matcher func(*block) bool) (bb []*block) {
- lst := bc.blocks()
- last := len(lst) - 1
- for i := range lst {
- b := lst[last-i]
- if matcher(b) {
- bb = append(bb, b)
- }
- }
- return bb
-}
-
-func (bc *blockController) closeBlock(ctx context.Context, blockID SectionID) error {
- bc.RLock()
- b := bc.getBlock(blockID)
- bc.RUnlock()
- if b == nil {
- return nil
- }
- return b.close(ctx)
-}
-
-func (bc *blockController) open() error {
- bc.Lock()
- defer bc.Unlock()
- return loadSections(bc.location, blockPathPrefix, bc, bc.blockSize, func(start, end time.Time) error {
- _, err := bc.load(start, end, bc.location)
- return err
- })
-}
-
-func (bc *blockController) create(start time.Time) (*block, error) {
- start = bc.Standard(start)
- if start.Before(bc.segTimeRange.Start) {
- start = bc.segTimeRange.Start
- }
- if !start.Before(bc.segTimeRange.End) {
- return nil, bucket.ErrNoMoreBucket
- }
- bc.Lock()
- defer bc.Unlock()
- var next *block
- for _, s := range bc.lst {
- if s.Start.Equal(start) {
- return s, nil
- }
- if next == nil && s.Start.After(start) {
- next = s
- }
- }
- stdEnd := bc.blockSize.nextTime(start)
- var end time.Time
- if next != nil && next.Start.Before(stdEnd) {
- end = next.Start
- } else {
- end = stdEnd
- }
- if end.After(bc.segTimeRange.End) {
- end = bc.segTimeRange.End
- }
- if err := mkdirIfNotExist(blockTemplate, bc.location, bc.Format(start)); err != nil {
- return nil, err
- }
- b, err := bc.load(start, end, bc.location)
- if err != nil {
- return nil, err
- }
- if err = b.openSafely(); err != nil {
- return nil, err
- }
- return b, nil
-}
-
-// nolint: contextcheck
-func (bc *blockController) load(startTime, endTime time.Time, root string) (b *block, err error) {
- suffix := bc.Format(startTime)
- if b, err = newBlock(
- common.SetPosition(bc.segCtx, func(p common.Position) common.Position {
- p.Block = suffix
- return p
- }),
- blockOpts{
- segID: bc.segID,
- segSuffix: bc.segSuffix,
- path: fmt.Sprintf(blockTemplate, root, suffix),
- timeRange: timestamp.NewSectionTimeRange(startTime, endTime),
- suffix: suffix,
- blockSize: bc.blockSize,
- queue: bc.blockQueue,
- scheduler: bc.scheduler,
- }); err != nil {
- return nil, err
- }
- bc.lst = append(bc.lst, b)
- bc.sortLst()
- return b, nil
-}
-
-func (bc *blockController) sortLst() {
- sort.Slice(bc.lst, func(i, j int) bool {
- return bc.lst[i].blockID < bc.lst[j].blockID
- })
-}
-
-func (bc *blockController) close(ctx context.Context) (err error) {
- bc.Lock()
- defer bc.Unlock()
- for _, s := range bc.lst {
- err = multierr.Append(err, s.close(ctx))
- }
- bc.lst = bc.lst[:0]
- return err
-}
-
-func (bc *blockController) remove(ctx context.Context, deadline time.Time) (err error) {
- for _, b := range bc.blocks() {
- if b.End.Before(deadline) {
- if e := bc.l.Debug(); e.Enabled() {
- e.Stringer("block", b).Msg("start to remove data in a block")
- }
- bc.Lock()
- if errDel := b.delete(ctx); errDel != nil {
- err = multierr.Append(err, errDel)
- } else {
- b.queue.Remove(BlockID{
- BlockID: b.blockID,
- SegID: b.segID,
- })
- bc.removeBlock(b.blockID)
- }
- bc.Unlock()
- }
- }
- return err
-}
-
-func (bc *blockController) removeBlock(blockID SectionID) {
- for i, b := range bc.lst {
- if b.blockID == blockID {
- bc.lst = append(bc.lst[:i], bc.lst[i+1:]...)
- break
- }
- }
-}
diff --git a/banyand/tsdb/buffer.go b/banyand/tsdb/buffer.go
deleted file mode 100644
index 49dbf4d7..00000000
--- a/banyand/tsdb/buffer.go
+++ /dev/null
@@ -1,565 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package tsdb
-
-import (
- "fmt"
- "sync"
- "time"
- "unsafe"
-
- "github.com/dgraph-io/badger/v3/skl"
- "github.com/dgraph-io/badger/v3/y"
- "github.com/pkg/errors"
-
- "github.com/apache/skywalking-banyandb/api/common"
- "github.com/apache/skywalking-banyandb/banyand/observability"
- "github.com/apache/skywalking-banyandb/pkg/convert"
- "github.com/apache/skywalking-banyandb/pkg/logger"
- "github.com/apache/skywalking-banyandb/pkg/meter"
- "github.com/apache/skywalking-banyandb/pkg/run"
- "github.com/apache/skywalking-banyandb/pkg/wal"
-)
-
-const (
- defaultSize = 1 << 20 // 1MB
- nodeAlign = int(unsafe.Sizeof(uint64(0))) - 1
- defaultWalSyncMode = false
-)
-
-var (
- bufferMeterProvider meter.Provider
- maxBytes meter.Gauge
- mutableBytes meter.Gauge
-)
-
-func init() {
- bufferMeterProvider = observability.NewMeterProvider(meterTSDB.SubScope("buffer"))
- labelNames := append(common.LabelNames(), "bucket")
- maxBytes = bufferMeterProvider.Gauge("max_bytes", labelNames...)
- mutableBytes = bufferMeterProvider.Gauge("mutable_bytes", labelNames...)
-}
-
-type operation struct {
- recoveryDoneFn func()
- key []byte
- value []byte
- epoch uint64
-}
-
-type flushEvent struct {
- data *skl.Skiplist
- walSegmentID wal.SegmentID
-}
-
-type onFlush func(shardIndex int, skl *skl.Skiplist) error
-
-type bufferShardBucket struct {
- mutable *skl.Skiplist
- writeCh chan operation
- flushCh chan flushEvent
- writeWaitGroup *sync.WaitGroup
- flushWaitGroup *sync.WaitGroup
- log *logger.Logger
- wal wal.WAL
- immutables []*skl.Skiplist
- labelValues []string
- shardLabelValues []string
- index int
- capacity int
- mutex sync.RWMutex
- walSyncMode bool
- enableWal bool
-}
-
-// Buffer is an exported struct that represents a buffer composed of multiple shard buckets.
-type Buffer struct {
- onFlushFn sync.Map
- entryCloser *run.Closer
- log *logger.Logger
- buckets []bufferShardBucket
- writeWaitGroup sync.WaitGroup
- flushWaitGroup sync.WaitGroup
- numShards int
- enableWal bool
- closerOnce sync.Once
-}
-
-// NewBuffer creates a new Buffer instance with the given parameters.
-func NewBuffer(log *logger.Logger, position common.Position, flushSize, writeConcurrency, numShards int) (*Buffer, error) {
- return NewBufferWithWal(log, position, flushSize, writeConcurrency, numShards, false, "")
-}
-
-// NewBufferWithWal creates a new Buffer instance with the given parameters.
-func NewBufferWithWal(log *logger.Logger, position common.Position, flushSize, writeConcurrency, numShards int, enableWal bool, walPath string,
-) (*Buffer, error) {
- buckets := make([]bufferShardBucket, numShards)
- buffer := &Buffer{
- buckets: buckets,
- numShards: numShards,
- entryCloser: run.NewCloser(1),
- log: log.Named("buffer"),
- enableWal: enableWal,
- }
- buffer.writeWaitGroup.Add(numShards)
- buffer.flushWaitGroup.Add(numShards)
- for i := 0; i < numShards; i++ {
- buckets[i] = bufferShardBucket{
- index: i,
- capacity: flushSize,
- mutable: skl.NewSkiplist(int64(flushSize)),
- writeCh: make(chan operation, writeConcurrency),
- flushCh: make(chan flushEvent, 1),
- writeWaitGroup: &buffer.writeWaitGroup,
- flushWaitGroup: &buffer.flushWaitGroup,
- log: buffer.log.Named(fmt.Sprintf("shard-%d", i)),
- labelValues: append(position.LabelValues(), fmt.Sprintf("%d", i)),
- shardLabelValues: position.ShardLabelValues(),
- enableWal: enableWal,
- }
- buckets[i].start(buffer.flushers)
- if enableWal {
- if walPath == "" {
- return nil, errors.New("wal path is required")
- }
- shardWalPath := fmt.Sprintf("%s/buffer-%d", walPath, i)
- if err := buckets[i].startWal(shardWalPath, defaultWalSyncMode); err != nil {
- return nil, errors.Wrap(err, "failed to start wal")
- }
- }
- maxBytes.Set(float64(flushSize), buckets[i].labelValues...)
- }
- return buffer, nil
-}
-
-// Register registers a callback function that will be called when a shard bucket is flushed.
-func (b *Buffer) Register(id string, onFlushFn onFlush) {
- b.onFlushFn.LoadOrStore(id, onFlushFn)
-}
-
-// Unregister unregisters a callback function that will be called when a shard bucket is flushed.
-func (b *Buffer) Unregister(id string) {
- b.onFlushFn.Delete(id)
-}
-
-func (b *Buffer) flushers() []onFlush {
- var flushers []onFlush
- b.onFlushFn.Range(func(key, value interface{}) bool {
- flushers = append(flushers, value.(onFlush))
- return true
- })
- return flushers
-}
-
-func (b *Buffer) isEmpty() bool {
- return len(b.flushers()) == 0
-}
-
-// Write adds a key-value pair with a timestamp to the appropriate shard bucket in the buffer.
-func (b *Buffer) Write(key, value []byte, timestamp time.Time) error {
- if b == nil || !b.entryCloser.AddRunning() {
- return errors.New("buffer is invalid")
- }
- defer b.entryCloser.Done()
- index := b.getShardIndex(key)
- if b.log.Debug().Enabled() {
- b.log.Debug().Uint64("shard", index).Bytes("key", key).
- Time("ts", timestamp).Msg("route a shard")
- }
-
- if b.enableWal {
- if err := b.buckets[index].writeWal(key, value, timestamp); err != nil {
- return errors.Wrap(err, "failed to write wal")
- }
- }
-
- b.buckets[index].writeCh <- operation{key: key, value: value, epoch: uint64(timestamp.UnixNano())}
- return nil
-}
-
-// Read retrieves the value associated with the given key and timestamp from the appropriate shard bucket in the buffer.
-func (b *Buffer) Read(key []byte, ts time.Time) ([]byte, bool) {
- if b == nil || !b.entryCloser.AddRunning() {
- return nil, false
- }
- defer b.entryCloser.Done()
- keyWithTS := y.KeyWithTs(key, uint64(ts.UnixNano()))
- index := b.getShardIndex(key)
- epoch := uint64(ts.UnixNano())
- ll, deferFn := b.buckets[index].getAll()
- defer deferFn()
- for _, bk := range ll {
- value := bk.Get(keyWithTS)
- if value.Meta == 0 && value.Value == nil {
- continue
- }
- if value.Version == epoch {
- return value.Value, true
- }
- }
- return nil, false
-}
-
-// Close gracefully closes the Buffer and ensures that all pending operations are completed.
-func (b *Buffer) Close() error {
- if b == nil {
- return nil
- }
- b.closerOnce.Do(func() {
- b.entryCloser.Done()
- b.entryCloser.CloseThenWait()
- for i := 0; i < b.numShards; i++ {
- close(b.buckets[i].writeCh)
- }
- b.writeWaitGroup.Wait()
- for i := 0; i < b.numShards; i++ {
- ff := b.flushers()
- for _, fn := range ff {
- if err := fn(i, b.buckets[i].mutable); err != nil {
- b.buckets[i].log.Err(err).Msg("flushing mutable buffer failed")
- }
- }
- b.buckets[i].mutable.DecrRef()
- }
- for i := 0; i < b.numShards; i++ {
- close(b.buckets[i].flushCh)
- if b.enableWal {
- if err := b.buckets[i].wal.Close(); err != nil {
- b.buckets[i].log.Err(err).Msg("closing buffer shard wal failed")
- }
- }
- }
- b.flushWaitGroup.Wait()
- })
- return nil
-}
-
-func (b *Buffer) getShardIndex(key []byte) uint64 {
- return convert.Hash(key) % uint64(b.numShards)
-}
-
-func (bsb *bufferShardBucket) getAll() ([]*skl.Skiplist, func()) {
- bsb.mutex.RLock()
- defer bsb.mutex.RUnlock()
- allList := make([]*skl.Skiplist, len(bsb.immutables)+1)
- bsb.mutable.IncrRef()
- allList[0] = bsb.mutable
- last := len(bsb.immutables) - 1
- for i := range bsb.immutables {
- allList[i+1] = bsb.immutables[last-i]
- bsb.immutables[last-i].IncrRef()
- }
- return allList, func() {
- for _, l := range allList {
- l.DecrRef()
- }
- }
-}
-
-func (bsb *bufferShardBucket) start(flushers func() []onFlush) {
- go func() {
- defer func() {
- for _, g := range []meter.Gauge{maxBytes, mutableBytes} {
- g.Delete(bsb.labelValues...)
- }
- }()
- defer bsb.flushWaitGroup.Done()
- for event := range bsb.flushCh {
- oldSkipList := event.data
- memSize := oldSkipList.MemSize()
- onFlushFnDone := false
- t1 := time.Now()
- ff := flushers()
- for {
- if !onFlushFnDone {
- failedFns := make([]onFlush, 0)
- for i := 0; i < len(ff); i++ {
- fn := ff[i]
- if err := fn(bsb.index, oldSkipList); err != nil {
- bsb.log.Err(err).Msg("flushing immutable buffer failed. Retrying...")
- failedFns = append(failedFns, fn)
- }
- }
- if len(failedFns) > 0 {
- flushNum.Inc(1, append(bsb.labelValues[:2], "true")...)
- time.Sleep(time.Second)
- ff = failedFns
- continue
- }
- onFlushFnDone = true
- }
- if bsb.enableWal {
- if err := bsb.wal.Delete(event.walSegmentID); err != nil {
- bsb.log.Err(err).Msg("delete wal segment file failed. Retrying...")
- time.Sleep(time.Second)
- continue
- }
- }
- break
- }
- flushLatency.Observe(time.Since(t1).Seconds(), bsb.shardLabelValues...)
- flushBytes.Inc(float64(memSize), bsb.shardLabelValues...)
- flushNum.Inc(1, append(bsb.shardLabelValues, "false")...)
-
- bsb.mutex.Lock()
- if len(bsb.immutables) > 0 {
- bsb.immutables = bsb.immutables[1:]
- }
- bsb.mutex.Unlock()
- oldSkipList.DecrRef()
- }
- }()
- go func() {
- defer bsb.writeWaitGroup.Done()
- volume := 0
- for op := range bsb.writeCh {
- k := y.KeyWithTs(op.key, op.epoch)
- v := y.ValueStruct{Value: op.value}
- volume += len(k) + int(v.EncodedSize()) + skl.MaxNodeSize + nodeAlign
- memSize := bsb.mutable.MemSize()
- mutableBytes.Set(float64(memSize), bsb.labelValues...)
- if op.recoveryDoneFn == nil && (volume >= bsb.capacity || memSize >= int64(bsb.capacity)) {
- if err := bsb.triggerFlushing(); err != nil {
- bsb.log.Err(err).Msg("triggering flushing failed")
- } else {
- volume = 0
- }
- }
- bsb.mutable.Put(k, v)
- if bsb.enableWal && op.recoveryDoneFn != nil {
- op.recoveryDoneFn()
- }
- }
- }()
-}
-
-func (bsb *bufferShardBucket) triggerFlushing() error {
- var walSegmentID wal.SegmentID
- if bsb.enableWal {
- segment, err := bsb.wal.Rotate()
- if err != nil {
- return errors.Wrap(err, "rotating wal failed")
- }
- walSegmentID = segment.GetSegmentID()
- }
-
- for {
- select {
- case bsb.flushCh <- flushEvent{data: bsb.mutable, walSegmentID: walSegmentID}:
- bsb.mutex.Lock()
- defer bsb.mutex.Unlock()
- bsb.swap()
- return nil
- default:
- }
- time.Sleep(10 * time.Second)
- }
-}
-
-func (bsb *bufferShardBucket) swap() {
- bsb.immutables = append(bsb.immutables, bsb.mutable)
- bsb.mutable = skl.NewSkiplist(int64(bsb.capacity))
-}
-
-func (bsb *bufferShardBucket) startWal(path string, syncMode bool) error {
- wal, err := wal.New(path, wal.DefaultOptions)
- if err != nil {
- return errors.Wrap(err, fmt.Sprintf("failed to create wal: %s", path))
- }
- bsb.wal = wal
- bsb.walSyncMode = syncMode
- bsb.log.Info().Msg(fmt.Sprintf(
- "wal started with path: %s, sync mode: %v", path, syncMode))
-
- if err = bsb.recoveryWal(); err != nil {
- return errors.Wrap(err, "failed to recovery wal")
- }
- return nil
-}
-
-func (bsb *bufferShardBucket) recoveryWal() error {
- segments, err := bsb.wal.ReadAllSegments()
- if err != nil {
- return errors.Wrap(err, "failed to load wal segments")
- }
-
- recoveredRecords := 0
- for index, segment := range segments {
- recoveredRecords += len(segment.GetLogEntries())
- isWorkSegment := index == len(segments)-1
- if isWorkSegment {
- bsb.recoveryWorkSegment(segment)
- } else {
- bsb.recoveryStableSegment(segment)
- }
- bsb.log.Info().Msg(fmt.Sprintf(
- "recovered %d log records from wal segment %d",
- len(segment.GetLogEntries()),
- segment.GetSegmentID()))
- }
- bsb.log.Info().Msg(fmt.Sprintf(
- "recovered %d log records from wal", recoveredRecords))
- return nil
-}
-
-func (bsb *bufferShardBucket) recoveryWorkSegment(segment wal.Segment) {
- var wg sync.WaitGroup
- for _, logEntry := range segment.GetLogEntries() {
- timestamps := logEntry.GetTimestamps()
- values := logEntry.GetValues()
- elementIndex := 0
- for element := values.Front(); element != nil; element = element.Next() {
- timestamp := timestamps[elementIndex]
- wg.Add(1)
- bsb.writeCh <- operation{
- key: logEntry.GetSeriesID(),
- value: element.Value.([]byte),
- epoch: uint64(timestamp.UnixNano()),
- recoveryDoneFn: func() {
- wg.Done()
- if bsb.log.Trace().Enabled() {
- bsb.log.Trace().Msg(fmt.Sprintf("recovered key: %v, ts: %v",
- logEntry.GetSeriesID(), timestamp.UnixNano()))
- }
- },
- }
- elementIndex++
- }
- }
- wg.Wait()
-}
-
-func (bsb *bufferShardBucket) recoveryStableSegment(segment wal.Segment) {
- for _, logEntries := range segment.GetLogEntries() {
- timestamps := logEntries.GetTimestamps()
- values := logEntries.GetValues()
- elementIndex := 0
- for element := values.Front(); element != nil; element = element.Next() {
- timestamp := timestamps[elementIndex]
- k := y.KeyWithTs(logEntries.GetSeriesID(), uint64(timestamp.UnixNano()))
- v := y.ValueStruct{Value: element.Value.([]byte)}
- bsb.mutable.Put(k, v)
- elementIndex++
- }
- }
- bsb.flushCh <- flushEvent{data: bsb.mutable, walSegmentID: segment.GetSegmentID()}
- // Sync recover data to immutables
- bsb.swap()
-}
-
-func (bsb *bufferShardBucket) writeWal(key, value []byte, timestamp time.Time) error {
- if !bsb.walSyncMode {
- bsb.wal.Write(key, timestamp, value, nil)
- return nil
- }
-
- var walErr error
- var wg sync.WaitGroup
- wg.Add(1)
- walCallback := func(key []byte, t time.Time, value []byte, err error) {
- walErr = err
- wg.Done()
- }
- bsb.wal.Write(key, timestamp, value, walCallback)
- wg.Wait()
- return walErr
-}
-
-// BufferSupplier lends a Buffer to a caller and returns it when the caller is done with it.
-type BufferSupplier struct {
- l *logger.Logger
- p common.Position
- buffers sync.Map
- path string
- writeConcurrency int
- numShards int
- enableWAL bool
-}
-
-// NewBufferSupplier creates a new BufferSupplier instance with the given parameters.
-func NewBufferSupplier(l *logger.Logger, p common.Position, writeConcurrency, numShards int, enableWAL bool, path string) *BufferSupplier {
- return &BufferSupplier{
- l: l.Named("buffer-supplier"),
- p: p,
- writeConcurrency: writeConcurrency,
- numShards: numShards,
- enableWAL: enableWAL,
- path: path,
- }
-}
-
-// Borrow borrows a Buffer from the BufferSupplier.
-func (b *BufferSupplier) Borrow(bufferName, name string, bufferSize int, onFlushFn onFlush) (buffer *Buffer, err error) {
- if bufferName == "" || name == "" {
- return nil, errors.New("bufferName and name are required")
- }
- if onFlushFn == nil {
- return nil, errors.New("onFlushFn is required")
- }
- defer func() {
- if buffer != nil {
- buffer.Register(name, onFlushFn)
- }
- }()
- if v, ok := b.buffers.Load(bufferName); ok {
- buffer = v.(*Buffer)
- return v.(*Buffer), nil
- }
- if buffer, err = NewBufferWithWal(b.l.Named("buffer-"+bufferName), b.p,
- bufferSize, b.writeConcurrency, b.numShards, b.enableWAL, b.path); err != nil {
- return nil, err
- }
- if v, ok := b.buffers.LoadOrStore(bufferName, buffer); ok {
- _ = buffer.Close()
- buffer = v.(*Buffer)
- return buffer, nil
- }
- return buffer, nil
-}
-
-// Return returns a Buffer to the BufferSupplier.
-func (b *BufferSupplier) Return(bufferName, name string) {
- if v, ok := b.buffers.Load(bufferName); ok {
- buffer := v.(*Buffer)
- buffer.Unregister(name)
- if buffer.isEmpty() {
- b.buffers.Delete(bufferName)
- _ = buffer.Close()
- }
- }
-}
-
-// Volume returns the number of Buffers in the BufferSupplier.
-func (b *BufferSupplier) Volume() int {
- volume := 0
- b.buffers.Range(func(key, value interface{}) bool {
- volume++
- return true
- })
- return volume
-}
-
-// Close closes all Buffers in the BufferSupplier.
-func (b *BufferSupplier) Close() error {
- b.buffers.Range(func(key, value interface{}) bool {
- buffer := value.(*Buffer)
- _ = buffer.Close()
- return true
- })
- return nil
-}
diff --git a/banyand/tsdb/buffer_test.go b/banyand/tsdb/buffer_test.go
deleted file mode 100644
index 82df4251..00000000
--- a/banyand/tsdb/buffer_test.go
+++ /dev/null
@@ -1,367 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package tsdb_test
-
-import (
- "bytes"
- "crypto/rand"
- "fmt"
- "math/big"
- "os"
- "path/filepath"
- "strconv"
- "sync"
- "time"
-
- "github.com/dgraph-io/badger/v3/skl"
- . "github.com/onsi/ginkgo/v2"
- . "github.com/onsi/gomega"
- "github.com/onsi/gomega/gleak"
-
- "github.com/apache/skywalking-banyandb/api/common"
- "github.com/apache/skywalking-banyandb/banyand/tsdb"
- "github.com/apache/skywalking-banyandb/pkg/logger"
- "github.com/apache/skywalking-banyandb/pkg/test/flags"
-)
-
-var emptyFn = func(shardIndex int, skl *skl.Skiplist) error {
- return nil
-}
-
-var _ = Describe("Buffer", func() {
- var (
- buffer *tsdb.Buffer
- log = logger.GetLogger("buffer-test")
- goods []gleak.Goroutine
- )
-
- BeforeEach(func() {
- goods = gleak.Goroutines()
- })
- AfterEach(func() {
- Eventually(gleak.Goroutines, flags.EventuallyTimeout).ShouldNot(gleak.HaveLeaked(goods))
- })
- Context("Write and Read", func() {
- BeforeEach(func() {
- var err error
- buffer, err = tsdb.NewBuffer(log, common.Position{}, 1024*1024, 16, 4)
- Expect(err).ToNot(HaveOccurred())
- buffer.Register("test", emptyFn)
- })
-
- AfterEach(func() {
- buffer.Close()
- })
- It("should write and read data correctly", func() {
- var wg sync.WaitGroup
- wg.Add(100)
-
- for i := 0; i < 100; i++ {
- go func(idx int) {
- defer GinkgoRecover()
- defer wg.Done()
-
- key := []byte(fmt.Sprintf("key-%d", idx))
- value := []byte(fmt.Sprintf("value-%d", idx))
- ts := time.Now()
-
- buffer.Write(key, value, ts)
- Eventually(func(g Gomega) {
- readValue, ok := buffer.Read(key, ts)
- g.Expect(ok).To(BeTrue())
- g.Expect(bytes.Equal(value, readValue)).To(BeTrue())
- }, flags.EventuallyTimeout).Should(Succeed())
- }(i)
- }
-
- wg.Wait()
- })
- })
-
- Context("Flush", func() {
- It("should trigger flush when buffer size exceeds the limit", func() {
- numShards := 4
- doneChs := make([]chan struct{}, numShards)
- for i := 0; i < numShards; i++ {
- doneChs[i] = make(chan struct{})
- }
-
- onFlushFn := func(shardIndex int, skl *skl.Skiplist) error {
- if doneChs[shardIndex] == nil {
- return nil
- }
- close(doneChs[shardIndex])
- doneChs[shardIndex] = nil
- return nil
- }
-
- var wg sync.WaitGroup
- wg.Add(numShards)
-
- for _, ch := range doneChs {
- go func(c <-chan struct{}) {
- select {
- case res := <-c:
- GinkgoWriter.Printf("Received value: %d\n", res)
- case <-time.After(10 * time.Second):
- GinkgoWriter.Printf("Timeout")
- }
- wg.Done()
- }(ch)
- }
-
- var err error
- buffer, err = tsdb.NewBuffer(log, common.Position{}, 1024, 16, numShards)
- defer func() {
- _ = buffer.Close()
- }()
- Expect(err).ToNot(HaveOccurred())
- buffer.Register("test", onFlushFn)
-
- randInt := func() int {
- n, err := rand.Int(rand.Reader, big.NewInt(1000))
- if err != nil {
- return 0
- }
- return int(n.Int64())
- }
- for i := 0; i < 1000; i++ {
- key := fmt.Sprintf("key-%d", randInt())
- value := fmt.Sprintf("value-%d", randInt())
- ts := time.Now()
-
- buffer.Write([]byte(key), []byte(value), ts)
- }
-
- wg.Wait()
- for i, elem := range doneChs {
- if elem != nil {
- Fail(fmt.Sprintf("%d in doneChs is not nil", i))
- }
- }
- })
- })
-
- Context("Write and Recover of wal correctly", func() {
- writeConcurrency := 2
- numShards := 2
- flushSize := 1024
- baseTime := time.Now()
- var path string
-
- BeforeEach(func() {
- var err error
- path, err = os.MkdirTemp("", "banyandb-test-buffer-wal-*")
- Expect(err).ToNot(HaveOccurred())
- })
-
- AfterEach(func() {
- err := os.RemoveAll(path)
- Expect(err).ToNot(HaveOccurred())
- })
-
- It("should write and rotate wal file correctly", func() {
- var err error
- var flushMutex sync.Mutex
-
- shardWalFileHistory := make(map[int][]string)
- buffer, err = tsdb.NewBufferWithWal(
- log,
- common.Position{},
- flushSize,
- writeConcurrency,
- numShards,
- true,
- path)
- Expect(err).ToNot(HaveOccurred())
- buffer.Register("test", func(shardIndex int, skl *skl.Skiplist) error {
- flushMutex.Lock()
- defer flushMutex.Unlock()
-
- shardWalDir := filepath.Join(path, "buffer-"+strconv.Itoa(shardIndex))
- var shardWalList []os.DirEntry
- shardWalList, err = os.ReadDir(shardWalDir)
- Expect(err).ToNot(HaveOccurred())
- for _, shardWalFile := range shardWalList {
- Expect(shardWalFile.IsDir()).To(BeFalse())
- Expect(shardWalFile.Name()).To(HaveSuffix(".wal"))
- shardWalFileHistory[shardIndex] = append(shardWalFileHistory[shardIndex], shardWalFile.Name())
- }
- return nil
- })
- // Write buffer & wal
- var wg sync.WaitGroup
- wg.Add(writeConcurrency)
- for i := 0; i < writeConcurrency; i++ {
- go func(writerIndex int) {
- for j := 0; j < numShards; j++ {
- for k := 0; k < flushSize; k++ {
- buffer.Write(
- []byte(fmt.Sprintf("writer-%d-shard-%d-key-%d", writerIndex, j, k)),
- []byte(fmt.Sprintf("writer-%d-shard-%d-value-%d", writerIndex, j, k)),
- time.UnixMilli(baseTime.UnixMilli()+int64(writerIndex+j+k)))
- }
- }
- wg.Done()
- }(i)
- }
- wg.Wait()
- buffer.Close()
-
- // Check wal
- Expect(len(shardWalFileHistory) == numShards).To(BeTrue())
- for shardIndex := 0; shardIndex < numShards; shardIndex++ {
- // Check wal rotate
- Expect(len(shardWalFileHistory[shardIndex]) > 1).To(BeTrue())
-
- shardWalDir := filepath.Join(path, "buffer-"+strconv.Itoa(shardIndex))
- currentShardWalFiles, err := os.ReadDir(shardWalDir)
- Expect(err).ToNot(HaveOccurred())
- Expect(len(currentShardWalFiles) <= 2).To(BeTrue())
- // Check wal delete
- Expect(len(shardWalFileHistory[shardIndex]) > len(currentShardWalFiles)).To(BeTrue())
- }
- })
-
- It("should recover buffer from wal file correctly", func() {
- var err error
- var flushMutex sync.Mutex
- var bufferFlushed bool
-
- buffer, err = tsdb.NewBufferWithWal(
- log,
- common.Position{},
- flushSize,
- writeConcurrency,
- numShards,
- true,
- path)
- Expect(err).ToNot(HaveOccurred())
- buffer.Register("test", func(shardIndex int, skl *skl.Skiplist) error {
- flushMutex.Lock()
- defer flushMutex.Unlock()
-
- if !bufferFlushed {
- bufferFlushed = true
- }
- return nil
- })
-
- // Write buffer & wal
- for i := 0; i < numShards; i++ {
- buffer.Write(
- []byte(fmt.Sprintf("shard-%d-key-1", i)),
- []byte(fmt.Sprintf("shard-%d-value-1", i)),
- time.UnixMilli(baseTime.UnixMilli()+int64(i)))
- }
-
- flushMutex.Lock()
- Expect(bufferFlushed).To(BeFalse())
- flushMutex.Unlock()
-
- // Restart buffer
- buffer.Close()
- buffer, err = tsdb.NewBufferWithWal(
- log,
- common.Position{},
- flushSize,
- writeConcurrency,
- numShards,
- true,
- path)
- Expect(err).ToNot(HaveOccurred())
- defer buffer.Close()
- buffer.Register("test", emptyFn)
-
- // Check buffer was recovered from wal
- for i := 0; i < numShards; i++ {
- expectValue := []byte(fmt.Sprintf("shard-%d-value-1", i))
- value, exist := buffer.Read(
- []byte(fmt.Sprintf("shard-%d-key-1", i)),
- time.UnixMilli(baseTime.UnixMilli()+int64(i)))
- Expect(exist).To(BeTrue())
- Expect(bytes.Equal(expectValue, value)).To(BeTrue())
- }
- })
- })
-})
-
-var _ = Describe("bufferSupplier", func() {
- var (
- b *tsdb.BufferSupplier
- goods []gleak.Goroutine
- )
-
- BeforeEach(func() {
- goods = gleak.Goroutines()
- b = tsdb.NewBufferSupplier(logger.GetLogger("buffer-supplier-test"), common.Position{}, 16, 4, false, "")
- })
- AfterEach(func() {
- b.Close()
- Eventually(gleak.Goroutines, flags.EventuallyTimeout).ShouldNot(gleak.HaveLeaked(goods))
- })
-
- Describe("Borrow", func() {
- Context("when borrowing a buffer with a new name", func() {
- It("should return a new buffer instance", func() {
- buf, err := b.Borrow("buffer", "test", 1024*1024, emptyFn)
- Expect(err).ToNot(HaveOccurred())
- Expect(buf).ToNot(BeNil())
- Expect(b.Volume()).To(Equal(1))
- b.Return("buffer", "test")
- Expect(b.Volume()).To(Equal(0))
- })
- })
-
- Context("when borrowing a buffer with an existing name", func() {
- It("should return the same buffer instance", func() {
- buf1, err := b.Borrow("buffer", "test", 1024*1024, emptyFn)
- Expect(err).ToNot(HaveOccurred())
- Expect(buf1).ToNot(BeNil())
- Expect(b.Volume()).To(Equal(1))
-
- buf2, err := b.Borrow("buffer", "test", 1024*1024, emptyFn)
- Expect(err).ToNot(HaveOccurred())
- Expect(buf2).ToNot(BeNil())
- Expect(b.Volume()).To(Equal(1))
-
- Expect(buf2).To(Equal(buf1))
- b.Return("buffer", "test")
- Expect(b.Volume()).To(Equal(0))
- })
- })
-
- Context("when borrowing a buffer from different buffer pools", func() {
- It("should return different buffer instances", func() {
- buf1, err := b.Borrow("buffer1", "test", 1024*1024, emptyFn)
- Expect(err).ToNot(HaveOccurred())
- Expect(buf1).ToNot(BeNil())
- Expect(b.Volume()).To(Equal(1))
-
- buf2, err := b.Borrow("buffer2", "test", 1024*1024, emptyFn)
- Expect(err).ToNot(HaveOccurred())
- Expect(buf2).ToNot(BeNil())
- Expect(b.Volume()).To(Equal(2))
-
- Expect(buf2).ToNot(Equal(buf1))
- b.Return("buffer1", "test")
- b.Return("buffer2", "test")
- Expect(b.Volume()).To(Equal(0))
- })
- })
- })
-})
diff --git a/banyand/tsdb/index/writer.go b/banyand/tsdb/index/writer.go
deleted file mode 100644
index d7f24d0c..00000000
--- a/banyand/tsdb/index/writer.go
+++ /dev/null
@@ -1,267 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-// Package index implements transferring data to indices.
-package index
-
-import (
- "context"
- "io"
- "time"
-
- "github.com/pkg/errors"
- "go.uber.org/multierr"
-
- "github.com/apache/skywalking-banyandb/api/common"
- databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
- modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
- "github.com/apache/skywalking-banyandb/banyand/tsdb"
- "github.com/apache/skywalking-banyandb/pkg/index"
- "github.com/apache/skywalking-banyandb/pkg/logger"
- "github.com/apache/skywalking-banyandb/pkg/partition"
- pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
-)
-
-// Message wraps value and other info to generate relative indices.
-type Message struct {
- IndexWriter tsdb.IndexWriter
- BlockCloser io.Closer
- Value Value
- Scope tsdb.Entry
- GlobalItemID tsdb.GlobalItemID
-}
-
-// Value represents the input data for generating indices.
-type Value struct {
- Timestamp time.Time
- TagFamilies []*modelv1.TagFamilyForWrite
-}
-
-// WriterOptions wrap all options to create an index writer.
-type WriterOptions struct {
- DB tsdb.Supplier
- Families []*databasev1.TagFamilySpec
- IndexRules []*databasev1.IndexRule
- ShardNum uint32
- EnableGlobalIndex bool
-}
-
-const (
- local = 1 << iota
- global
- inverted
- tree
-)
-
-// Writer generates indices based on index rules.
-type Writer struct {
- db tsdb.Supplier
- l *logger.Logger
- invertRuleIndex map[byte][]*partition.IndexRuleLocator
- shardNum uint32
- enableGlobalIndex bool
-}
-
-// NewWriter returns a new Writer with WriterOptions.
-func NewWriter(ctx context.Context, options WriterOptions) *Writer {
- w := new(Writer)
- parentLogger := ctx.Value(logger.ContextKey)
- if parentLogger != nil {
- if pl, ok := parentLogger.(*logger.Logger); ok {
- w.l = pl.Named("index-writer")
- }
- }
- w.shardNum = options.ShardNum
- w.db = options.DB
- w.enableGlobalIndex = options.EnableGlobalIndex
- w.invertRuleIndex = make(map[byte][]*partition.IndexRuleLocator, 4)
- for _, ruleIndex := range partition.ParseIndexRuleLocators(options.Families, options.IndexRules) {
- rule := ruleIndex.Rule
- var key byte
- switch rule.GetLocation() {
- case databasev1.IndexRule_LOCATION_SERIES:
- key |= local
- case databasev1.IndexRule_LOCATION_GLOBAL:
- if !w.enableGlobalIndex {
- w.l.Warn().RawJSON("index-rule", logger.Proto(ruleIndex.Rule)).Msg("global index is disabled")
- continue
- }
- key |= global
- case databasev1.IndexRule_LOCATION_UNSPECIFIED:
- w.l.Warn().RawJSON("index-rule", logger.Proto(ruleIndex.Rule)).Msg("invalid:unspecified location")
- continue
- }
- switch rule.Type {
- case databasev1.IndexRule_TYPE_INVERTED:
- key |= inverted
- case databasev1.IndexRule_TYPE_TREE:
- key |= tree
- case databasev1.IndexRule_TYPE_UNSPECIFIED:
- w.l.Warn().RawJSON("index-rule", logger.Proto(ruleIndex.Rule)).Msg("invalid:unspecified type")
- continue
- }
- rules := w.invertRuleIndex[key]
- rules = append(rules, ruleIndex)
- w.invertRuleIndex[key] = rules
- }
- return w
-}
-
-func (s *Writer) Write(m Message) {
- err := multierr.Combine(
- s.writeLocalIndex(m.IndexWriter, m.Value),
- s.writeGlobalIndex(m.Scope, m.GlobalItemID, m.Value),
- m.BlockCloser.Close(),
- )
- if err != nil {
- s.l.Error().Err(err).Msg("encounter some errors when generating indices")
- }
-}
-
-// TODO: should listen to pipeline in a distributed cluster.
-func (s *Writer) writeGlobalIndex(scope tsdb.Entry, ref tsdb.GlobalItemID, value Value) error {
- collect := func(ruleIndexes []*partition.IndexRuleLocator, fn func(indexWriter tsdb.IndexWriter, fields []index.Field) error) error {
- fields := make(map[uint][]index.Field)
- for _, ruleIndex := range ruleIndexes {
- values, err := getIndexValue(ruleIndex, value)
- if err != nil {
- return err
- }
- if values == nil {
- continue
- }
- for _, val := range values {
- indexShardID, err := partition.ShardID(val, s.shardNum)
- if err != nil {
- return err
- }
- rule := ruleIndex.Rule
- rr := fields[indexShardID]
- rr = append(rr, index.Field{
- Key: index.FieldKey{
- IndexRuleID: rule.GetMetadata().GetId(),
- Analyzer: rule.Analyzer,
- },
- Term: val,
- })
- fields[indexShardID] = rr
- }
- }
- for shardID, rules := range fields {
- shard, err := s.db.SupplyTSDB().CreateShardsAndGetByID(common.ShardID(shardID))
- if err != nil {
- return err
- }
- builder := shard.Index().WriterBuilder()
- indexWriter, err := builder.
- Scope(scope).
- GlobalItemID(ref).
- Time(value.Timestamp).
- Build()
- if err != nil {
- return err
- }
- err = fn(indexWriter, rules)
- if err != nil {
- return err
- }
- }
- return nil
- }
- return multierr.Combine(
- collect(s.invertRuleIndex[global|inverted], func(indexWriter tsdb.IndexWriter, fields []index.Field) error {
- return indexWriter.WriteInvertedIndex(fields)
- }),
- collect(s.invertRuleIndex[global|tree], func(indexWriter tsdb.IndexWriter, fields []index.Field) error {
- return indexWriter.WriteLSMIndex(fields)
- }),
- )
-}
-
-func (s *Writer) writeLocalIndex(writer tsdb.IndexWriter, value Value) (err error) {
- collect := func(ruleIndexes []*partition.IndexRuleLocator, fn func(fields []index.Field) error) error {
- fields := make([]index.Field, 0)
- for _, ruleIndex := range ruleIndexes {
- values, err := getIndexValue(ruleIndex, value)
- if err != nil {
- return err
- }
- if values == nil {
- continue
- }
- for _, val := range values {
- rule := ruleIndex.Rule
- fields = append(fields, index.Field{
- Key: index.FieldKey{
- IndexRuleID: rule.GetMetadata().GetId(),
- Analyzer: rule.Analyzer,
- },
- Term: val,
- })
- }
- }
- if len(fields) == 0 {
- return nil
- }
- return fn(fields)
- }
- return multierr.Combine(
- collect(s.invertRuleIndex[local|inverted], func(fields []index.Field) error {
- return writer.WriteInvertedIndex(fields)
- }),
- collect(s.invertRuleIndex[local|tree], func(fields []index.Field) error {
- return writer.WriteLSMIndex(fields)
- }),
- )
-}
-
-var errUnsupportedIndexType = errors.New("unsupported index type")
-
-func getIndexValue(ruleIndex *partition.IndexRuleLocator, value Value) (val [][]byte, err error) {
- val = make([][]byte, 0)
- if len(ruleIndex.TagIndices) != 1 {
- return nil, errors.WithMessagef(errUnsupportedIndexType,
- "the index rule %s(%v) didn't support composited tags",
- ruleIndex.Rule.Metadata.Name, ruleIndex.Rule.Tags)
- }
- tIndex := ruleIndex.TagIndices[0]
- tag, err := partition.GetTagByOffset(value.TagFamilies, tIndex.FamilyOffset, tIndex.TagOffset)
-
- if errors.Is(err, partition.ErrMalformedElement) {
- return val, nil
- }
- if err != nil {
- return nil, errors.WithMessagef(err, "index rule:%v", ruleIndex.Rule.Metadata)
- }
- fv, err := pbv1.ParseTagValue(tag)
- if err != nil {
- return nil, err
- }
- if fv.GetValue() == nil && fv.GetArr() == nil {
- return nil, nil
- }
- v := fv.GetValue()
- if v != nil {
- val = append(val, v)
- return val, nil
- }
- arr := fv.GetArr()
- if arr != nil {
- val = append(val, arr...)
- }
- return val, nil
-}
diff --git a/banyand/tsdb/indexdb.go b/banyand/tsdb/indexdb.go
deleted file mode 100644
index d9e57068..00000000
--- a/banyand/tsdb/indexdb.go
+++ /dev/null
@@ -1,193 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package tsdb
-
-import (
- "context"
- "time"
-
- "github.com/pkg/errors"
- "go.uber.org/multierr"
-
- "github.com/apache/skywalking-banyandb/api/common"
- "github.com/apache/skywalking-banyandb/banyand/kv"
- "github.com/apache/skywalking-banyandb/pkg/convert"
- "github.com/apache/skywalking-banyandb/pkg/index"
-)
-
-// IndexDatabase allows stocking index data.
-type IndexDatabase interface {
- WriterBuilder() IndexWriterBuilder
- Seek(field index.Field) ([]GlobalItemID, error)
-}
-
-// IndexWriter allows ingesting index data.
-type IndexWriter interface {
- WriteLSMIndex(field []index.Field) error
- WriteInvertedIndex(field []index.Field) error
-}
-
-// IndexWriterBuilder is a helper to build IndexWriter.
-type IndexWriterBuilder interface {
- Scope(scope Entry) IndexWriterBuilder
- Time(ts time.Time) IndexWriterBuilder
- GlobalItemID(itemID GlobalItemID) IndexWriterBuilder
- Build() (IndexWriter, error)
-}
-
-var _ IndexDatabase = (*indexDB)(nil)
-
-type indexDB struct {
- segCtrl *segmentController
- shardID common.ShardID
-}
-
-func (i *indexDB) Seek(field index.Field) ([]GlobalItemID, error) {
- result := make([]GlobalItemID, 0)
- f := field.Marshal()
- for _, s := range i.segCtrl.segments() {
- err := s.globalIndex.GetAll(f, func(rawBytes []byte) error {
- id := &GlobalItemID{}
- errUnMarshal := id.unMarshal(rawBytes)
- if errUnMarshal != nil {
- return errUnMarshal
- }
- result = append(result, *id)
- return nil
- })
- if errors.Is(err, kv.ErrKeyNotFound) {
- return result, nil
- }
- if err != nil {
- return nil, err
- }
- }
- return result, nil
-}
-
-func (i *indexDB) WriterBuilder() IndexWriterBuilder {
- return newIndexWriterBuilder(i.segCtrl)
-}
-
-func newIndexDatabase(_ context.Context, id common.ShardID, segCtrl *segmentController) IndexDatabase {
- return &indexDB{
- shardID: id,
- segCtrl: segCtrl,
- }
-}
-
-var _ IndexWriterBuilder = (*indexWriterBuilder)(nil)
-
-type indexWriterBuilder struct {
- ts time.Time
- segCtrl *segmentController
- globalItemID *GlobalItemID
- scope Entry
-}
-
-func (i *indexWriterBuilder) Scope(scope Entry) IndexWriterBuilder {
- i.scope = scope
- return i
-}
-
-func (i *indexWriterBuilder) Time(ts time.Time) IndexWriterBuilder {
- i.ts = ts
- return i
-}
-
-func (i *indexWriterBuilder) GlobalItemID(itemID GlobalItemID) IndexWriterBuilder {
- i.globalItemID = &itemID
- return i
-}
-
-func (i *indexWriterBuilder) Build() (IndexWriter, error) {
- seg, err := i.segCtrl.create(i.ts)
- if err != nil {
- return nil, err
- }
- if i.globalItemID == nil {
- return nil, errors.WithStack(errNoVal)
- }
- return &indexWriter{
- scope: i.scope,
- seg: seg,
- ts: i.ts,
- itemID: i.globalItemID,
- }, nil
-}
-
-func newIndexWriterBuilder(segCtrl *segmentController) IndexWriterBuilder {
- return &indexWriterBuilder{
- segCtrl: segCtrl,
- }
-}
-
-var _ IndexWriter = (*indexWriter)(nil)
-
-type indexWriter struct {
- ts time.Time
- seg *segment
- itemID *GlobalItemID
- scope Entry
-}
-
-func (i *indexWriter) WriteLSMIndex(fields []index.Field) (err error) {
- total := 0
- for _, field := range fields {
- if i.scope != nil {
- field.Key.SeriesID = GlobalSeriesID(i.scope)
- }
- bb := field.Marshal()
- ibb := i.itemID.marshal()
- err = multierr.Append(err, i.seg.globalIndex.PutWithVersion(bb, ibb, uint64(i.ts.UnixNano())))
- total += len(bb) + len(ibb) + 8
- }
- if err != nil {
- receivedNumCounter.Inc(1, append(i.seg.position.ShardLabelValues(), "global_lsm", "true")...)
- return err
- }
- receivedNumCounter.Inc(1, append(i.seg.position.ShardLabelValues(), "global_lsm", "false")...)
- receivedBytesCounter.Inc(float64(total), append(i.seg.position.ShardLabelValues(), "global_lsm")...)
- return nil
-}
-
-func (i *indexWriter) WriteInvertedIndex(fields []index.Field) (err error) {
- total := 0
- for _, field := range fields {
- if i.scope != nil {
- field.Key.SeriesID = GlobalSeriesID(i.scope)
- }
- bb := field.Marshal()
- ibb := i.itemID.marshal()
- err = multierr.Append(err, i.seg.globalIndex.PutWithVersion(bb, ibb, uint64(i.ts.UnixNano())))
- total += len(bb) + len(ibb) + 8
- }
-
- if err != nil {
- receivedNumCounter.Inc(1, append(i.seg.position.ShardLabelValues(), "global_inverted", "true")...)
- return err
- }
- receivedNumCounter.Inc(1, append(i.seg.position.ShardLabelValues(), "global_inverted", "false")...)
- receivedBytesCounter.Inc(float64(total), append(i.seg.position.ShardLabelValues(), "global_inverted")...)
- return nil
-}
-
-// GlobalSeriesID encodes Entry to common.SeriesID.
-func GlobalSeriesID(scope Entry) common.SeriesID {
- return common.SeriesID(convert.Hash(scope))
-}
diff --git a/banyand/tsdb/retention.go b/banyand/tsdb/retention.go
deleted file mode 100644
index 43ff3f3e..00000000
--- a/banyand/tsdb/retention.go
+++ /dev/null
@@ -1,61 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package tsdb
-
-import (
- "context"
- "time"
-
- "github.com/robfig/cron/v3"
-
- "github.com/apache/skywalking-banyandb/pkg/logger"
-)
-
-type retentionTask struct {
- segment *segmentController
- expr string
- option cron.ParseOption
- duration time.Duration
-}
-
-func newRetentionTask(segment *segmentController, ttl IntervalRule) *retentionTask {
- var expr string
- switch ttl.Unit {
- case HOUR:
- // Every hour on the 5th minute
- expr = "5 *"
- case DAY:
- // Every day on 00:05
- expr = "5 0"
- }
- return &retentionTask{
- segment: segment,
- option: cron.Minute | cron.Hour,
- expr: expr,
- duration: ttl.estimatedDuration(),
- }
-}
-
-func (rc *retentionTask) run(now time.Time, l *logger.Logger) bool {
- ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
- defer cancel()
- if err := rc.segment.remove(ctx, now.Add(-rc.duration)); err != nil {
- l.Error().Err(err)
- }
- return true
-}
diff --git a/banyand/tsdb/scope.go b/banyand/tsdb/scope.go
deleted file mode 100644
index 5a157e6b..00000000
--- a/banyand/tsdb/scope.go
+++ /dev/null
@@ -1,107 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package tsdb
-
-import (
- "context"
-
- "github.com/apache/skywalking-banyandb/api/common"
- "github.com/apache/skywalking-banyandb/pkg/index"
-)
-
-var _ Shard = (*scopedShard)(nil)
-
-type scopedShard struct {
- delegated Shard
- scope Entry
-}
-
-// NewScopedShard returns a shard in a scope.
-func NewScopedShard(scope Entry, delegated Shard) Shard {
- return &scopedShard{
- scope: scope,
- delegated: delegated,
- }
-}
-
-func (sd *scopedShard) Close() error {
- // the delegate can't close the underlying shard
- return nil
-}
-
-func (sd *scopedShard) ID() common.ShardID {
- return sd.delegated.ID()
-}
-
-func (sd *scopedShard) Series() SeriesDatabase {
- return &scopedSeriesDatabase{
- scope: sd.scope,
- delegated: sd.delegated.Series(),
- }
-}
-
-func (sd *scopedShard) Index() IndexDatabase {
- return sd.delegated.Index()
-}
-
-func (sd *scopedShard) TriggerSchedule(task string) bool {
- return sd.delegated.TriggerSchedule(task)
-}
-
-func (sd *scopedShard) State() ShardState {
- return sd.delegated.State()
-}
-
-var _ SeriesDatabase = (*scopedSeriesDatabase)(nil)
-
-type scopedSeriesDatabase struct {
- delegated SeriesDatabase
- scope Entry
-}
-
-func (sdd *scopedSeriesDatabase) writeInvertedIndex(fields []index.Field, seriesID common.SeriesID) error {
- return sdd.delegated.writeInvertedIndex(fields, seriesID)
-}
-
-func (sdd *scopedSeriesDatabase) writeLSMIndex(fields []index.Field, seriesID common.SeriesID) error {
- return sdd.delegated.writeLSMIndex(fields, seriesID)
-}
-
-func (sdd *scopedSeriesDatabase) Search(ctx context.Context, path Path, filter index.Filter, order *OrderBy) (SeriesList, error) {
- return sdd.delegated.Search(ctx, path.prepend(sdd.scope), filter, order)
-}
-
-func (sdd *scopedSeriesDatabase) Close() error {
- return nil
-}
-
-func (sdd *scopedSeriesDatabase) Get(key []byte, entityValues EntityValues) (Series, error) {
- return sdd.delegated.Get(key, entityValues)
-}
-
-func (sdd *scopedSeriesDatabase) GetByID(id common.SeriesID) (Series, error) {
- return sdd.delegated.GetByID(id)
-}
-
-func (sdd *scopedSeriesDatabase) List(ctx context.Context, path Path) (SeriesList, error) {
- return sdd.delegated.List(ctx, path.prepend(sdd.scope))
-}
-
-func (sdd *scopedSeriesDatabase) SizeOnDisk() int64 {
- return sdd.delegated.SizeOnDisk()
-}
diff --git a/banyand/tsdb/segment.go b/banyand/tsdb/segment.go
deleted file mode 100644
index 6dd11d1d..00000000
--- a/banyand/tsdb/segment.go
+++ /dev/null
@@ -1,140 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package tsdb
-
-import (
- "context"
- "errors"
- "os"
- "strconv"
- "sync"
- "time"
-
- "github.com/apache/skywalking-banyandb/api/common"
- "github.com/apache/skywalking-banyandb/banyand/kv"
- "github.com/apache/skywalking-banyandb/banyand/tsdb/bucket"
- "github.com/apache/skywalking-banyandb/pkg/logger"
- "github.com/apache/skywalking-banyandb/pkg/timestamp"
-)
-
-var errEndOfSegment = errors.New("reached the end of the segment")
-
-type segment struct {
- globalIndex kv.Store
- bucket.Reporter
- l *logger.Logger
- blockController *blockController
- blockManageStrategy *bucket.Strategy
- position common.Position
- timestamp.TimeRange
- path string
- suffix string
- closeOnce sync.Once
- id SectionID
-}
-
-func openSegment(ctx context.Context, startTime, endTime time.Time, path, suffix string,
- segmentSize, blockSize IntervalRule, blockQueue bucket.Queue, scheduler *timestamp.Scheduler,
-) (s *segment, err error) {
- suffixInteger, err := strconv.Atoi(suffix)
- if err != nil {
- return nil, err
- }
- id := GenerateInternalID(segmentSize.Unit, suffixInteger)
- timeRange := timestamp.NewSectionTimeRange(startTime, endTime)
- s = &segment{
- id: id,
- path: path,
- suffix: suffix,
- TimeRange: timeRange,
- position: common.GetPosition(ctx),
- }
- l := logger.Fetch(ctx, s.String())
- s.l = l
- segCtx := context.WithValue(ctx, logger.ContextKey, l)
- clock, segCtx := timestamp.GetClock(segCtx)
- s.blockController = newBlockController(segCtx, id, suffix, path, timeRange, blockSize, l, blockQueue, scheduler)
- s.Reporter = bucket.NewTimeBasedReporter(s.String(), timeRange, clock, scheduler)
- err = s.blockController.open()
- if err != nil {
- return nil, err
- }
-
- indexPath, err := mkdir(globalIndexTemplate, path)
- if err != nil {
- return nil, err
- }
- o := ctx.Value(OptionsKey)
- if o != nil {
- options := o.(DatabaseOpts)
- if options.EnableGlobalIndex {
- memSize := options.GlobalIndexMemSize
- if memSize == 0 {
- memSize = defaultKVMemorySize
- }
- if s.globalIndex, err = kv.OpenStore(
- indexPath,
- kv.StoreWithLogger(s.l),
- kv.StoreWithMemTableSize(int64(memSize)),
- ); err != nil {
- return nil, err
- }
- }
- }
- if !s.End.After(clock.Now()) {
- return
- }
- s.blockManageStrategy, err = bucket.NewStrategy(s.blockController, bucket.WithLogger(s.l))
- if err != nil {
- return nil, err
- }
- s.blockManageStrategy.Run()
- return s, nil
-}
-
-func (s *segment) close(ctx context.Context) (err error) {
- s.closeOnce.Do(func() {
- if err = s.blockController.close(ctx); err != nil {
- return
- }
- if s.globalIndex != nil {
- if err = s.globalIndex.Close(); err != nil {
- return
- }
- }
- if s.blockManageStrategy != nil {
- s.blockManageStrategy.Close()
- }
- })
- return nil
-}
-
-func (s *segment) closeBlock(ctx context.Context, id SectionID) error {
- return s.blockController.closeBlock(ctx, id)
-}
-
-func (s *segment) delete(ctx context.Context) error {
- if err := s.close(ctx); err != nil {
- return err
- }
- return os.RemoveAll(s.path)
-}
-
-func (s *segment) String() string {
- return "SegID-" + s.suffix
-}
diff --git a/banyand/tsdb/segment_ctrl.go b/banyand/tsdb/segment_ctrl.go
deleted file mode 100644
index 4101c018..00000000
--- a/banyand/tsdb/segment_ctrl.go
+++ /dev/null
@@ -1,283 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package tsdb
-
-import (
- "context"
- "fmt"
- "sort"
- "sync"
- "time"
-
- "github.com/pkg/errors"
- "go.uber.org/multierr"
-
- "github.com/apache/skywalking-banyandb/api/common"
- "github.com/apache/skywalking-banyandb/banyand/tsdb/bucket"
- "github.com/apache/skywalking-banyandb/pkg/logger"
- "github.com/apache/skywalking-banyandb/pkg/timestamp"
-)
-
-type segmentController struct {
- shardCtx context.Context
- blockQueue bucket.Queue
- clock timestamp.Clock
- scheduler *timestamp.Scheduler
- l *logger.Logger
- location string
- lst []*segment
- segmentSize IntervalRule
- blockSize IntervalRule
- sync.RWMutex
-}
-
-func newSegmentController(shardCtx context.Context, location string, segmentSize, blockSize IntervalRule,
- openedBlockSize, maxOpenedBlockSize int, l *logger.Logger, scheduler *timestamp.Scheduler,
-) (*segmentController, error) {
- clock, _ := timestamp.GetClock(shardCtx)
- sc := &segmentController{
- shardCtx: shardCtx,
- location: location,
- segmentSize: segmentSize,
- blockSize: blockSize,
- l: l,
- clock: clock,
- scheduler: scheduler,
- }
- var err error
- sc.blockQueue, err = bucket.NewQueue(
- l.Named("block-queue"),
- openedBlockSize,
- maxOpenedBlockSize,
- scheduler,
- func(ctx context.Context, id interface{}) error {
- bsID := id.(BlockID)
- seg := sc.get(bsID.SegID)
- if seg == nil {
- l.Warn().Int("segID", parseSuffix(bsID.SegID)).Msg("segment is absent")
- return nil
- }
- l.Info().Int("blockID", parseSuffix(bsID.BlockID)).Msg("closing the block")
- return seg.closeBlock(ctx, bsID.BlockID)
- })
- return sc, err
-}
-
-func (sc *segmentController) get(segID SectionID) *segment {
- lst := sc.segments()
- last := len(lst) - 1
- for i := range lst {
- s := lst[last-i]
- if s.id == segID {
- return s
- }
- }
- return nil
-}
-
-func (sc *segmentController) span(timeRange timestamp.TimeRange) (ss []*segment) {
- lst := sc.segments()
- last := len(lst) - 1
- for i := range lst {
- s := lst[last-i]
- if s.Overlapping(timeRange) {
- ss = append(ss, s)
- }
- }
- return ss
-}
-
-func (sc *segmentController) segments() (ss []*segment) {
- sc.RLock()
- defer sc.RUnlock()
- r := make([]*segment, len(sc.lst))
- copy(r, sc.lst)
- return r
-}
-
-func (sc *segmentController) Current() (bucket.Reporter, error) {
- now := sc.Standard(sc.clock.Now())
- ns := uint64(now.UnixNano())
- if b := func() bucket.Reporter {
- sc.RLock()
- defer sc.RUnlock()
- for _, s := range sc.lst {
- if s.Contains(ns) {
- return s
- }
- }
- return nil
- }(); b != nil {
- return b, nil
- }
- return sc.create(now)
-}
-
-func (sc *segmentController) Next() (bucket.Reporter, error) {
- c, err := sc.Current()
- if err != nil {
- return nil, err
- }
- seg := c.(*segment)
- reporter, err := sc.create(sc.segmentSize.nextTime(seg.Start))
- if errors.Is(err, errEndOfSegment) {
- return nil, bucket.ErrNoMoreBucket
- }
- return reporter, err
-}
-
-func (sc *segmentController) OnMove(prev bucket.Reporter, next bucket.Reporter) {
- event := sc.l.Info()
- if prev != nil {
- event.Stringer("prev", prev)
- }
- if next != nil {
- event.Stringer("next", next)
- }
- event.Msg("move to the next segment")
-}
-
-func (sc *segmentController) Standard(t time.Time) time.Time {
- switch sc.segmentSize.Unit {
- case HOUR:
- return time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), 0, 0, 0, t.Location())
- case DAY:
- return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location())
- }
- panic("invalid interval unit")
-}
-
-func (sc *segmentController) Format(tm time.Time) string {
- switch sc.segmentSize.Unit {
- case HOUR:
- return tm.Format(hourFormat)
- case DAY:
- return tm.Format(dayFormat)
- }
- panic("invalid interval unit")
-}
-
-func (sc *segmentController) Parse(value string) (time.Time, error) {
- switch sc.segmentSize.Unit {
- case HOUR:
- return time.ParseInLocation(hourFormat, value, time.Local)
- case DAY:
- return time.ParseInLocation(dayFormat, value, time.Local)
- }
- panic("invalid interval unit")
-}
-
-func (sc *segmentController) open() error {
- sc.Lock()
- defer sc.Unlock()
- return loadSections(sc.location, segPathPrefix, sc, sc.segmentSize, func(start, end time.Time) error {
- _, err := sc.load(start, end, sc.location)
- if errors.Is(err, errEndOfSegment) {
- return nil
- }
- return err
- })
-}
-
-func (sc *segmentController) create(start time.Time) (*segment, error) {
- sc.Lock()
- defer sc.Unlock()
- start = sc.Standard(start)
- var next *segment
- for _, s := range sc.lst {
- if s.Contains(uint64(start.UnixNano())) {
- return s, nil
- }
- if next == nil && s.Start.After(start) {
- next = s
- }
- }
- stdEnd := sc.segmentSize.nextTime(start)
- var end time.Time
- if next != nil && next.Start.Before(stdEnd) {
- end = next.Start
- } else {
- end = stdEnd
- }
- if err := mkdirIfNotExist(segTemplate, sc.location, sc.Format(start)); err != nil {
- return nil, err
- }
- return sc.load(start, end, sc.location)
-}
-
-func (sc *segmentController) sortLst() {
- sort.Slice(sc.lst, func(i, j int) bool {
- return sc.lst[i].id < sc.lst[j].id
- })
-}
-
-// nolint: contextcheck
-func (sc *segmentController) load(start, end time.Time, root string) (seg *segment, err error) {
- suffix := sc.Format(start)
- seg, err = openSegment(common.SetPosition(sc.shardCtx, func(p common.Position) common.Position {
- p.Segment = suffix
- return p
- }), start, end, fmt.Sprintf(segTemplate, root, suffix), suffix, sc.segmentSize, sc.blockSize, sc.blockQueue, sc.scheduler)
- if err != nil {
- return nil, err
- }
- sc.lst = append(sc.lst, seg)
- sc.sortLst()
- return seg, nil
-}
-
-func (sc *segmentController) remove(ctx context.Context, deadline time.Time) (err error) {
- sc.l.Info().Time("deadline", deadline).Msg("start to remove before deadline")
- for _, s := range sc.segments() {
- if s.End.Before(deadline) || s.Contains(uint64(deadline.UnixNano())) {
- if e := sc.l.Debug(); e.Enabled() {
- e.Stringer("segment", s).Msg("start to remove data in a segment")
- }
- err = multierr.Append(err, s.blockController.remove(ctx, deadline))
- if s.End.Before(deadline) {
- sc.Lock()
- if errDel := s.delete(ctx); errDel != nil {
- err = multierr.Append(err, errDel)
- } else {
- sc.removeSeg(s.id)
- }
- sc.Unlock()
- }
- }
- }
- return err
-}
-
-func (sc *segmentController) removeSeg(segID SectionID) {
- for i, b := range sc.lst {
- if b.id == segID {
- sc.lst = append(sc.lst[:i], sc.lst[i+1:]...)
- break
- }
- }
-}
-
-func (sc *segmentController) close(ctx context.Context) (err error) {
- sc.Lock()
- defer sc.Unlock()
- for _, s := range sc.lst {
- err = multierr.Append(err, s.close(ctx))
- }
- sc.lst = sc.lst[:0]
- return err
-}
diff --git a/banyand/tsdb/series.go b/banyand/tsdb/series.go
deleted file mode 100644
index 0326bf66..00000000
--- a/banyand/tsdb/series.go
+++ /dev/null
@@ -1,234 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package tsdb
-
-import (
- "bytes"
- "context"
- "io"
- "time"
-
- "github.com/pkg/errors"
- "go.uber.org/multierr"
-
- "github.com/apache/skywalking-banyandb/api/common"
- "github.com/apache/skywalking-banyandb/pkg/convert"
- "github.com/apache/skywalking-banyandb/pkg/logger"
- "github.com/apache/skywalking-banyandb/pkg/timestamp"
-)
-
-var (
- // ErrEmptySeriesSpan hints there is no any data blocks based on the input time range.
- ErrEmptySeriesSpan = errors.New("there is no data in such time range")
- errItemIDMalformed = errors.New("serialized item id is malformed")
- errBlockAbsent = errors.New("block is absent")
-)
-
-// GlobalItemID is the top level identity of an item.
-// The item could be retrieved by a GlobalItemID in a tsdb.
-type GlobalItemID struct {
- ShardID common.ShardID
- segID SectionID
- blockID SectionID
- SeriesID common.SeriesID
- ID common.ItemID
-}
-
-func (i *GlobalItemID) marshal() []byte {
- return bytes.Join([][]byte{
- convert.Uint32ToBytes(uint32(i.ShardID)),
- sectionIDToBytes(i.segID),
- sectionIDToBytes(i.blockID),
- convert.Uint64ToBytes(uint64(i.SeriesID)),
- convert.Uint64ToBytes(uint64(i.ID)),
- }, nil)
-}
-
-func (i *GlobalItemID) unMarshal(data []byte) error {
- if len(data) != 4+4+4+8+8 {
- return errItemIDMalformed
- }
- var offset int
- i.ShardID = common.ShardID(convert.BytesToUint32(data[offset : offset+4]))
- offset += 4
- i.segID, offset = readSectionID(data, offset)
- i.blockID, offset = readSectionID(data, offset)
- i.SeriesID = common.SeriesID(convert.BytesToUint64(data[offset : offset+8]))
- offset += 8
- i.ID = common.ItemID(convert.BytesToUint64(data[offset:]))
- return nil
-}
-
-// Series denotes a series of data points group by a common.SeriesID
-// common.SeriesID is encoded by a entity defined by Stream or Measure.
-type Series interface {
- ID() common.SeriesID
- Span(ctx context.Context, timeRange timestamp.TimeRange) (SeriesSpan, error)
- Create(ctx context.Context, t time.Time) (SeriesSpan, error)
- Get(ctx context.Context, id GlobalItemID) (Item, io.Closer, error)
- String() string
-}
-
-// SeriesSpan is a span in a time series. It contains data blocks in such time range.
-type SeriesSpan interface {
- io.Closer
- WriterBuilder() WriterBuilder
- SeekerBuilder() SeekerBuilder
-}
-
-var _ Series = (*series)(nil)
-
-type series struct {
- blockDB blockDatabase
- l *logger.Logger
- idLiteral string
- id common.SeriesID
- shardID common.ShardID
-}
-
-func (s *series) Get(ctx context.Context, id GlobalItemID) (Item, io.Closer, error) {
- b, err := s.blockDB.block(ctx, id)
- if err != nil {
- return nil, nil, err
- }
- if b == nil {
- return nil, nil, errors.WithMessagef(errBlockAbsent, "id: %v", id)
- }
- return &item{
- data: b.dataReader(),
- itemID: id.ID,
- seriesID: s.id,
- }, b, nil
-}
-
-func (s *series) ID() common.SeriesID {
- return s.id
-}
-
-func (s *series) String() string {
- return s.idLiteral
-}
-
-func (s *series) Span(ctx context.Context, timeRange timestamp.TimeRange) (SeriesSpan, error) {
- blocks, err := s.blockDB.span(ctx, timeRange)
- if err != nil {
- return nil, err
- }
- if len(blocks) < 1 {
- return nil, ErrEmptySeriesSpan
- }
- l := logger.FetchOrDefault(ctx, "series", s.l)
- if e := l.Debug(); e.Enabled() {
- e.Times("time_range", []time.Time{timeRange.Start, timeRange.End}).
- Uint64("series_id", uint64(s.id)).
- Str("series", s.idLiteral).
- Msg("select series span")
- }
- return newSeriesSpan(context.WithValue(ctx, logger.ContextKey, l), timeRange, blocks,
- s.id, s.idLiteral, s.shardID), nil
-}
-
-func (s *series) Create(ctx context.Context, t time.Time) (SeriesSpan, error) {
- tr := timestamp.NewInclusiveTimeRange(t, t)
- blocks, err := s.blockDB.span(ctx, tr)
- if err != nil {
- return nil, err
- }
- if len(blocks) > 0 {
- if e := s.l.Debug(); e.Enabled() {
- e.Time("time", t).
- Uint64("series_id", uint64(s.id)).
- Str("series", s.idLiteral).
- Msg("load a series span")
- }
- return newSeriesSpan(context.WithValue(ctx, logger.ContextKey, s.l), tr, blocks,
- s.id, s.idLiteral, s.shardID), nil
- }
- b, err := s.blockDB.create(ctx, t)
- if err != nil {
- return nil, err
- }
- blocks = append(blocks, b)
- if e := s.l.Debug(); e.Enabled() {
- e.Time("time", t).
- Uint64("series_id", uint64(s.id)).
- Str("series", s.idLiteral).
- Msg("create a series span")
- }
- return newSeriesSpan(context.WithValue(ctx, logger.ContextKey, s.l), tr, blocks,
- s.id, s.idLiteral, s.shardID), nil
-}
-
-func newSeries(ctx context.Context, id common.SeriesID, idLiteral string, blockDB blockDatabase) *series {
- s := &series{
- id: id,
- idLiteral: idLiteral,
- blockDB: blockDB,
- shardID: blockDB.shardID(),
- }
- parentLogger := ctx.Value(logger.ContextKey)
- if pl, ok := parentLogger.(*logger.Logger); ok {
- s.l = pl.Named("series")
- } else {
- s.l = logger.GetLogger("series")
- }
- return s
-}
-
-var _ SeriesSpan = (*seriesSpan)(nil)
-
-type seriesSpan struct {
- l *logger.Logger
- timeRange timestamp.TimeRange
- series string
- blocks []blockDelegate
- seriesID common.SeriesID
- shardID common.ShardID
-}
-
-func (s *seriesSpan) Close() (err error) {
- for _, delegate := range s.blocks {
- err = multierr.Append(err, delegate.Close())
- }
- return err
-}
-
-func (s *seriesSpan) WriterBuilder() WriterBuilder {
- return newWriterBuilder(s)
-}
-
-func (s *seriesSpan) SeekerBuilder() SeekerBuilder {
- return newSeekerBuilder(s)
-}
-
-func newSeriesSpan(ctx context.Context, timeRange timestamp.TimeRange, blocks []blockDelegate, id common.SeriesID, series string, shardID common.ShardID) *seriesSpan {
- s := &seriesSpan{
- blocks: blocks,
- seriesID: id,
- series: series,
- shardID: shardID,
- timeRange: timeRange,
- }
- parentLogger := ctx.Value(logger.ContextKey)
- if pl, ok := parentLogger.(*logger.Logger); ok {
- s.l = pl.Named("series_span")
- } else {
- s.l = logger.GetLogger("series_span")
- }
- return s
-}
diff --git a/banyand/tsdb/series_seek.go b/banyand/tsdb/series_seek.go
deleted file mode 100644
index a802df32..00000000
--- a/banyand/tsdb/series_seek.go
+++ /dev/null
@@ -1,136 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package tsdb
-
-import (
- "github.com/apache/skywalking-banyandb/api/common"
- databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
- modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
- "github.com/apache/skywalking-banyandb/banyand/kv"
- "github.com/apache/skywalking-banyandb/pkg/index"
- "github.com/apache/skywalking-banyandb/pkg/logger"
-)
-
-// Iterator allows iterating a series in a time span.
-type Iterator interface {
- Next() bool
- Val() Item
- Close() error
-}
-
-// Item allows retrieving raw data from an item.
-type Item interface {
- Family(family []byte) ([]byte, error)
- Val() ([]byte, error)
- ID() common.ItemID
- SortedField() []byte
- Time() uint64
-}
-
-// SeekerBuilder a helper to build a Seeker.
-type SeekerBuilder interface {
- Filter(predicator index.Filter) SeekerBuilder
- OrderByIndex(indexRule *databasev1.IndexRule, order modelv1.Sort) SeekerBuilder
- OrderByTime(order modelv1.Sort) SeekerBuilder
- Build() (Seeker, error)
-}
-
-// Seeker allows searching data in a Database.
-type Seeker interface {
- Seek() ([]Iterator, error)
-}
-
-var _ SeekerBuilder = (*seekerBuilder)(nil)
-
-type seekerBuilder struct {
- predicator index.Filter
- seriesSpan *seriesSpan
- indexRuleForSorting *databasev1.IndexRule
- l *logger.Logger
- order modelv1.Sort
-}
-
-func (s *seekerBuilder) Build() (Seeker, error) {
- if s.order == modelv1.Sort_SORT_UNSPECIFIED {
- s.order = modelv1.Sort_SORT_ASC
- }
- se, err := s.buildSeries()
- if err != nil {
- return nil, err
- }
- return newSeeker(se), nil
-}
-
-func newSeekerBuilder(s *seriesSpan) SeekerBuilder {
- return &seekerBuilder{
- seriesSpan: s,
- l: logger.GetLogger("seeker-builder"),
- }
-}
-
-var _ Seeker = (*seeker)(nil)
-
-type seeker struct {
- series []Iterator
-}
-
-func (s *seeker) Seek() ([]Iterator, error) {
- return s.series, nil
-}
-
-func newSeeker(series []Iterator) Seeker {
- return &seeker{
- series: series,
- }
-}
-
-var _ Item = (*item)(nil)
-
-type item struct {
- data kv.TimeSeriesReader
- sortedField []byte
- itemID common.ItemID
- seriesID common.SeriesID
-}
-
-func (i *item) Time() uint64 {
- return uint64(i.itemID)
-}
-
-func (i *item) SortedField() []byte {
- return i.sortedField
-}
-
-func (i *item) Family(family []byte) ([]byte, error) {
- d := dataBucket{
- seriesID: i.seriesID,
- family: family,
- }
- return i.data.Get(d.marshal(), uint64(i.itemID))
-}
-
-func (i *item) Val() ([]byte, error) {
- d := dataBucket{
- seriesID: i.seriesID,
- }
- return i.data.Get(d.marshal(), uint64(i.itemID))
-}
-
-func (i *item) ID() common.ItemID {
- return i.itemID
-}
diff --git a/banyand/tsdb/series_seek_filter.go b/banyand/tsdb/series_seek_filter.go
deleted file mode 100644
index 60b323e7..00000000
--- a/banyand/tsdb/series_seek_filter.go
+++ /dev/null
@@ -1,58 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package tsdb
-
-import (
- "github.com/pkg/errors"
-
- databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
- "github.com/apache/skywalking-banyandb/pkg/index"
-)
-
-var errUnsupportedIndexRule = errors.New("the index rule is not supported")
-
-func (s *seekerBuilder) Filter(predicator index.Filter) SeekerBuilder {
- s.predicator = predicator
- return s
-}
-
-func (s *seekerBuilder) buildIndexFilter(block blockDelegate) (filterFn, error) {
- if s.predicator == nil {
- return nil, nil
- }
- allItemIDs, err := s.predicator.Execute(func(typ databasev1.IndexRule_Type) (index.Searcher, error) {
- switch typ {
- case databasev1.IndexRule_TYPE_INVERTED:
- return block.invertedIndexReader(), nil
- case databasev1.IndexRule_TYPE_TREE:
- return block.lsmIndexReader(), nil
- default:
- return nil, errUnsupportedIndexRule
- }
- }, s.seriesSpan.seriesID)
- if err != nil {
- return nil, err
- }
- return func(item Item) bool {
- valid := allItemIDs.Contains(uint64(item.ID()))
- s.seriesSpan.l.Trace().Int("valid_item_num", allItemIDs.Len()).Bool("valid", valid).Msg("filter item by index")
- return valid
- }, nil
-}
-
-type filterFn func(item Item) bool
diff --git a/banyand/tsdb/series_seek_sort.go b/banyand/tsdb/series_seek_sort.go
deleted file mode 100644
index 604d5dff..00000000
--- a/banyand/tsdb/series_seek_sort.go
+++ /dev/null
@@ -1,269 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package tsdb
-
-import (
- "math"
- "sort"
- "time"
-
- "github.com/pkg/errors"
- "go.uber.org/multierr"
-
- "github.com/apache/skywalking-banyandb/api/common"
- databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
- modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
- "github.com/apache/skywalking-banyandb/banyand/kv"
- "github.com/apache/skywalking-banyandb/pkg/convert"
- "github.com/apache/skywalking-banyandb/pkg/index"
- "github.com/apache/skywalking-banyandb/pkg/index/posting"
- "github.com/apache/skywalking-banyandb/pkg/logger"
-)
-
-var (
- errUnspecifiedIndexType = errors.New("Unspecified index type")
- emptyFilters = make([]filterFn, 0)
- rangeOpts = index.RangeOpts{}
-)
-
-func (s *seekerBuilder) OrderByIndex(indexRule *databasev1.IndexRule, order modelv1.Sort) SeekerBuilder {
- s.indexRuleForSorting = indexRule
- s.order = order
- return s
-}
-
-func (s *seekerBuilder) OrderByTime(order modelv1.Sort) SeekerBuilder {
- s.order = order
- s.indexRuleForSorting = nil
- return s
-}
-
-func (s *seekerBuilder) buildSeries() ([]Iterator, error) {
- if s.indexRuleForSorting == nil {
- return s.buildSeriesByTime()
- }
- return s.buildSeriesByIndex()
-}
-
-func (s *seekerBuilder) buildSeriesByIndex() (series []Iterator, err error) {
- timeFilter := func(item Item) bool {
- valid := s.seriesSpan.timeRange.Contains(item.Time())
- timeRange := s.seriesSpan.timeRange
- s.seriesSpan.l.Trace().
- Times("time_range", []time.Time{timeRange.Start, timeRange.End}).
- Bool("valid", valid).Msg("filter item by time range")
- return valid
- }
- for _, b := range s.seriesSpan.blocks {
- var inner index.FieldIterator
- var err error
- fieldKey := index.FieldKey{
- SeriesID: s.seriesSpan.seriesID,
- IndexRuleID: s.indexRuleForSorting.GetMetadata().GetId(),
- }
- filters := []filterFn{timeFilter}
- filter, err := s.buildIndexFilter(b)
- if err != nil {
- return nil, err
- }
- if filter != nil {
- filters = append(filters, filter)
- }
- switch s.indexRuleForSorting.GetType() {
- case databasev1.IndexRule_TYPE_TREE:
- inner, err = b.lsmIndexReader().Iterator(fieldKey, rangeOpts, s.order, math.MaxInt)
- case databasev1.IndexRule_TYPE_INVERTED:
- inner, err = b.invertedIndexReader().Iterator(fieldKey, rangeOpts, s.order, math.MaxInt)
- case databasev1.IndexRule_TYPE_UNSPECIFIED:
- return nil, errors.WithMessagef(errUnspecifiedIndexType, "index rule:%v", s.indexRuleForSorting)
- }
- if err != nil {
- return nil, err
- }
- if inner != nil {
- series = append(series, newSearcherIterator(s.seriesSpan.l, inner, b.dataReader(),
- s.seriesSpan.seriesID, filters))
- }
- }
- return
-}
-
-func (s *seekerBuilder) buildSeriesByTime() ([]Iterator, error) {
- bb := s.seriesSpan.blocks
- switch s.order {
- case modelv1.Sort_SORT_ASC, modelv1.Sort_SORT_UNSPECIFIED:
- sort.SliceStable(bb, func(i, j int) bool {
- return bb[i].startTime().Before(bb[j].startTime())
- })
-
- case modelv1.Sort_SORT_DESC:
- sort.SliceStable(bb, func(i, j int) bool {
- return bb[i].startTime().After(bb[j].startTime())
- })
- }
- delegated := make([]Iterator, 0, len(bb))
- bTimes := make([]time.Time, 0, len(bb))
- timeRange := s.seriesSpan.timeRange
- termRange := index.RangeOpts{
- Lower: convert.Int64ToBytes(timeRange.Start.UnixNano()),
- Upper: convert.Int64ToBytes(timeRange.End.UnixNano()),
- IncludesLower: true,
- }
- for _, b := range bb {
- bTimes = append(bTimes, b.startTime())
- inner, err := b.primaryIndexReader().
- Iterator(
- index.FieldKey{
- SeriesID: s.seriesSpan.seriesID,
- },
- termRange,
- s.order,
- math.MaxInt,
- )
- if err != nil {
- return nil, err
- }
- if inner != nil {
- filter, err := s.buildIndexFilter(b)
- if err != nil {
- return nil, err
- }
- if filter == nil {
- delegated = append(delegated, newSearcherIterator(s.seriesSpan.l, inner, b.dataReader(),
- s.seriesSpan.seriesID, emptyFilters))
- } else {
- delegated = append(delegated, newSearcherIterator(s.seriesSpan.l, inner, b.dataReader(),
- s.seriesSpan.seriesID, []filterFn{filter}))
- }
- }
- }
- if e := s.seriesSpan.l.Debug(); e.Enabled() {
- e.Str("order", modelv1.Sort_name[int32(s.order)]).
- Times("blocks", bTimes).
- Uint64("series_id", uint64(s.seriesSpan.seriesID)).
- Str("series", s.seriesSpan.series).
- Int("shard_id", int(s.seriesSpan.shardID)).
- Msg("seek series by time")
- }
- return []Iterator{newMergedIterator(delegated)}, nil
-}
-
-var _ Iterator = (*searcherIterator)(nil)
-
-type searcherIterator struct {
- fieldIterator index.FieldIterator
- cur posting.Iterator
- data kv.TimeSeriesReader
- l *logger.Logger
- curKey []byte
- filters []filterFn
- seriesID common.SeriesID
-}
-
-func (s *searcherIterator) Next() bool {
- if s.cur == nil {
- if s.fieldIterator.Next() {
- v := s.fieldIterator.Val()
- s.cur = v.Value.Iterator()
- s.curKey = v.Term
- } else {
- return false
- }
- }
- if s.cur.Next() {
- for _, filter := range s.filters {
- if !filter(s.Val()) {
- return s.Next()
- }
- }
- if e := s.l.Debug(); e.Enabled() {
- e.Uint64("series_id", uint64(s.seriesID)).Uint64("item_id", uint64(s.Val().ID())).Msg("got an item")
- }
- return true
- }
- s.cur = nil
- return s.Next()
-}
-
-func (s *searcherIterator) Val() Item {
- return &item{
- sortedField: s.curKey,
- itemID: common.ItemID(s.cur.Current()),
- data: s.data,
- seriesID: s.seriesID,
- }
-}
-
-func (s *searcherIterator) Close() error {
- return s.fieldIterator.Close()
-}
-
-func newSearcherIterator(l *logger.Logger, fieldIterator index.FieldIterator, data kv.TimeSeriesReader,
- seriesID common.SeriesID, filters []filterFn,
-) Iterator {
- return &searcherIterator{
- fieldIterator: fieldIterator,
- data: data,
- seriesID: seriesID,
- filters: filters,
- l: l,
- }
-}
-
-var _ Iterator = (*mergedIterator)(nil)
-
-type mergedIterator struct {
- curr Iterator
- delegated []Iterator
- index int
-}
-
-func (m *mergedIterator) Next() bool {
- if m.curr == nil {
- m.index++
- if m.index >= len(m.delegated) {
- return false
- }
- m.curr = m.delegated[m.index]
- }
- hasNext := m.curr.Next()
- if !hasNext {
- m.curr = nil
- return m.Next()
- }
- return true
-}
-
-func (m *mergedIterator) Val() Item {
- return m.curr.Val()
-}
-
-func (m *mergedIterator) Close() error {
- var err error
- for _, d := range m.delegated {
- err = multierr.Append(err, d.Close())
- }
- return err
-}
-
-func newMergedIterator(delegated []Iterator) Iterator {
- return &mergedIterator{
- index: -1,
- delegated: delegated,
- }
-}
diff --git a/banyand/tsdb/series_test.go b/banyand/tsdb/series_test.go
deleted file mode 100644
index da2493fe..00000000
--- a/banyand/tsdb/series_test.go
+++ /dev/null
@@ -1,130 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package tsdb_test
-
-import (
- "fmt"
- "time"
-
- . "github.com/onsi/ginkgo/v2"
- . "github.com/onsi/gomega"
-
- "github.com/apache/skywalking-banyandb/pkg/timestamp"
-)
-
-var _ = Describe("Series", func() {
- Context("TimeRange", func() {
- Context("Contains", func() {
- verifyFn := func(start, end string, includeStart, includeEnd bool, ts string, expected bool) {
- startTime, _ := time.Parse("20060202", start)
- endTime, _ := time.Parse("20060202", end)
- tsTime, _ := time.Parse("20060202", ts)
- Expect(timestamp.NewTimeRange(startTime, endTime, includeStart, includeEnd).Contains(uint64(tsTime.UnixNano()))).To(Equal(expected))
- }
- DescribeTable("It's a exclusive range",
- func(start, end, ts string, expected bool) {
- verifyFn(start, end, false, false, ts, expected)
- },
- Entry("is in the middle", "20220205", "20220107", "20220106", true),
- Entry("is at the lower", "20220205", "20220107", "20220105", false),
- Entry("is at the upper", "20220205", "20220107", "20220107", false),
- Entry("is before the lower", "20220205", "20220107", "20220104", false),
- Entry("is after the upper", "20220205", "20220107", "20220108", false),
- )
- DescribeTable("It's a inclusive range",
- func(start, end, ts string, expected bool) {
- verifyFn(start, end, true, true, ts, expected)
- },
- Entry("is in the middle", "20220205", "20220107", "20220106", true),
- Entry("is at the lower", "20220205", "20220107", "20220105", true),
- Entry("is at the upper", "20220205", "20220107", "20220107", true),
- Entry("is before the lower", "20220205", "20220107", "20220104", false),
- Entry("is after the upper", "20220205", "20220107", "20220108", false),
- )
- DescribeTable("It's a inclusive lower and exclusive upper range",
- func(start, end, ts string, expected bool) {
- verifyFn(start, end, true, false, ts, expected)
- },
- Entry("is in the middle", "20220205", "20220107", "20220106", true),
- Entry("is at the lower", "20220205", "20220107", "20220105", true),
- Entry("is at the upper", "20220205", "20220107", "20220107", false),
- Entry("is before the lower", "20220205", "20220107", "20220104", false),
- Entry("is after the upper", "20220205", "20220107", "20220108", false),
- )
- DescribeTable("It's a exclusive lower and inclusive upper range",
- func(start, end, ts string, expected bool) {
- verifyFn(start, end, false, true, ts, expected)
- },
- Entry("is in the middle", "20220205", "20220107", "20220106", true),
- Entry("is at the lower", "20220205", "20220107", "20220105", false),
- Entry("is at the upper", "20220205", "20220107", "20220107", true),
- Entry("is before the lower", "20220205", "20220107", "20220104", false),
- Entry("is after the upper", "20220205", "20220107", "20220108", false),
- )
- })
- Context("Overlapping", func() {
- verifyFn := func(start1, end1, start2, end2 string, expected bool) {
- startTime1, _ := time.Parse("20060102", start1)
- endTime1, _ := time.Parse("20060102", end1)
- startTime2, _ := time.Parse("20060102", start2)
- endTime2, _ := time.Parse("20060102", end2)
- includes := []bool{true, false}
-
- for _, r1l := range includes {
- for _, r1u := range includes {
- for _, r2l := range includes {
- for _, r2u := range includes {
- By(fmt.Sprintf("r1 lower:%v upper:%v. r1 lower:%v upper:%v", r1l, r1u, r2l, r2u), func() {
- r1 := timestamp.NewTimeRange(startTime1, endTime1, r1l, r1u)
- r2 := timestamp.NewTimeRange(startTime2, endTime2, r2l, r2u)
- Expect(r1.Overlapping(r2)).To(Equal(expected))
- Expect(r2.Overlapping(r1)).To(Equal(expected))
- })
- }
- }
- }
- }
- }
- DescribeTable("Each range type's behavior is identical",
- verifyFn,
- Entry("is no overlapping", "20220205", "20220107", "20220108", "20220112", false),
- Entry("is the two range are identical", "20220105", "20220107", "20220105", "20220107", true),
- Entry("is the one includes the other", "20220102", "20220107", "20220103", "20220106", true),
- Entry("is the one includes the other, the upper bounds are identical", "20220102", "20220107", "20220103", "20220107", true),
- Entry("is the one includes the other, the lower bounds are identical", "20220102", "20220107", "20220102", "20220106", true),
- Entry("is they have an intersection", "20220102", "20220105", "20220103", "20220106", true),
- )
- adjacentVerifyFn := func(include1, include2, expected bool) {
- startTime1, _ := time.Parse("20060102", "20210105")
- endTime1, _ := time.Parse("20060102", "20210107")
- startTime2, _ := time.Parse("20060102", "20210107")
- endTime2, _ := time.Parse("20060102", "20210109")
- r1 := timestamp.NewTimeRange(startTime1, endTime1, false, include1)
- r2 := timestamp.NewTimeRange(startTime2, endTime2, include2, false)
- Expect(r1.Overlapping(r2)).To(Equal(expected))
- }
-
- DescribeTable("They are adjacent",
- adjacentVerifyFn,
- Entry("is they are inclusive", true, true, true),
- Entry("is that range1 includes upper, but range2 excludes lower", true, false, false),
- Entry("is that range1 excludes upper, but range2 includes lower", false, true, false),
- Entry("is they are exclusive", false, false, false),
- )
- })
- })
-})
diff --git a/banyand/tsdb/series_write.go b/banyand/tsdb/series_write.go
deleted file mode 100644
index 501689be..00000000
--- a/banyand/tsdb/series_write.go
+++ /dev/null
@@ -1,232 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package tsdb
-
-import (
- "bytes"
- "fmt"
- "time"
-
- "github.com/pkg/errors"
-
- "github.com/apache/skywalking-banyandb/api/common"
- "github.com/apache/skywalking-banyandb/pkg/convert"
- "github.com/apache/skywalking-banyandb/pkg/index"
-)
-
-// WriterBuilder is a helper to build a Writer.
-type WriterBuilder interface {
- Family(name []byte, val []byte) WriterBuilder
- Time(ts time.Time) WriterBuilder
- Val(val []byte) WriterBuilder
- Build() (Writer, error)
-}
-
-// Writer allow ingesting data into a tsdb.
-type Writer interface {
- IndexWriter
- Write() (GlobalItemID, error)
- ItemID() GlobalItemID
- String() string
-}
-
-var _ WriterBuilder = (*writerBuilder)(nil)
-
-type writerBuilder struct {
- series *seriesSpan
- block blockDelegate
- values []struct {
- family []byte
- val []byte
- }
- ts time.Time
- seriesIDBytes []byte
-}
-
-func (w *writerBuilder) Family(name []byte, val []byte) WriterBuilder {
- w.values = append(w.values, struct {
- family []byte
- val []byte
- }{family: name, val: val})
- return w
-}
-
-func (w *writerBuilder) Time(ts time.Time) WriterBuilder {
- w.ts = ts
- for _, b := range w.series.blocks {
- if b.contains(ts) {
- w.block = b
- break
- }
- }
- return w
-}
-
-func (w *writerBuilder) Val(val []byte) WriterBuilder {
- w.values = append(w.values, struct {
- family []byte
- val []byte
- }{val: val})
- return w
-}
-
-var (
- errNoTime = errors.New("no time specified")
- errNoVal = errors.New("no value specified")
- errDuplicatedFamily = errors.New("duplicated family")
-)
-
-func (w *writerBuilder) Build() (Writer, error) {
- if w.block == nil {
- return nil, errors.WithMessagef(errNoTime, "ts:%v", w.ts)
- }
- if len(w.values) < 1 {
- return nil, errors.WithStack(errNoVal)
- }
- for i, value := range w.values {
- for j := i + 1; j < len(w.values); j++ {
- if value.family == nil && w.values[j].family == nil {
- return nil, errors.Wrap(errDuplicatedFamily, "default family")
- }
- if bytes.Equal(value.family, w.values[j].family) {
- return nil, errors.Wrapf(errDuplicatedFamily, "family:%s", value.family)
- }
- }
- }
- segID, blockID := w.block.identity()
- return &writer{
- block: w.block,
- ts: w.ts,
- itemID: &GlobalItemID{
- ShardID: w.series.shardID,
- segID: segID,
- blockID: blockID,
- SeriesID: w.series.seriesID,
- ID: common.ItemID(uint64(w.ts.UnixNano())),
- },
- columns: w.values,
- }, nil
-}
-
-func newWriterBuilder(seriesSpan *seriesSpan) WriterBuilder {
- return &writerBuilder{
- series: seriesSpan,
- seriesIDBytes: seriesSpan.seriesID.Marshal(),
- }
-}
-
-var _ Writer = (*writer)(nil)
-
-type writer struct {
- ts time.Time
- block blockDelegate
- itemID *GlobalItemID
- columns []struct {
- family []byte
- val []byte
- }
-}
-
-func (w *writer) ItemID() GlobalItemID {
- return *w.itemID
-}
-
-func (w *writer) WriteLSMIndex(fields []index.Field) error {
- for i, f := range fields {
- f.Key.SeriesID = w.itemID.SeriesID
- fields[i] = f
- }
- return w.block.writeLSMIndex(fields, w.itemID.ID)
-}
-
-func (w *writer) WriteInvertedIndex(fields []index.Field) error {
- for i, f := range fields {
- f.Key.SeriesID = w.itemID.SeriesID
- fields[i] = f
- }
- return w.block.writeInvertedIndex(fields, w.itemID.ID)
-}
-
-func (w *writer) String() string {
- var buf []byte
- buf = append(buf, "block:"...)
- buf = append(buf, w.block.String()...)
- buf = append(buf, ",column:"...)
- buf = append(buf, fmt.Sprintf("%+v", w.columns)...)
- buf = append(buf, ",ts:"...)
- buf = append(buf, w.ts.String()...)
- return string(buf)
-}
-
-type dataBucket struct {
- family []byte
- seriesID common.SeriesID
-}
-
-func (d dataBucket) marshal() []byte {
- if d.family == nil {
- return d.seriesID.Marshal()
- }
- return bytes.Join([][]byte{
- d.seriesID.Marshal(),
- d.family,
- }, nil)
-}
-
-func (w *writer) Write() (GlobalItemID, error) {
- id := w.ItemID()
- for _, c := range w.columns {
- err := w.block.write(dataBucket{
- seriesID: w.itemID.SeriesID,
- family: c.family,
- }.marshal(),
- c.val, w.ts)
- if err != nil {
- return id, err
- }
- }
- return id, w.block.writePrimaryIndex(index.Field{
- Key: index.FieldKey{
- SeriesID: id.SeriesID,
- },
- Term: convert.Int64ToBytes(w.ts.UnixNano()),
- }, id.ID)
-}
-
-var _ IndexWriter = (*seriesIndexWriter)(nil)
-
-type seriesIndexWriter struct {
- seriesDB SeriesDatabase
- seriesID common.SeriesID
-}
-
-func (s *seriesIndexWriter) WriteInvertedIndex(fields []index.Field) error {
- return s.seriesDB.writeInvertedIndex(fields, s.seriesID)
-}
-
-func (s *seriesIndexWriter) WriteLSMIndex(fields []index.Field) error {
- return s.seriesDB.writeLSMIndex(fields, s.seriesID)
-}
-
-// NewSeriesIndexWriter returns a new series index writer.
-func NewSeriesIndexWriter(seriesID common.SeriesID, seriesDB SeriesDatabase) IndexWriter {
- return &seriesIndexWriter{
- seriesID: seriesID,
- seriesDB: seriesDB,
- }
-}
diff --git a/banyand/tsdb/seriesdb.go b/banyand/tsdb/seriesdb.go
deleted file mode 100644
index 94614959..00000000
--- a/banyand/tsdb/seriesdb.go
+++ /dev/null
@@ -1,779 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package tsdb
-
-import (
- "bytes"
- "context"
- "errors"
- "io"
- "math"
- "path"
- "sort"
- "strings"
- "sync"
- "time"
-
- "go.uber.org/multierr"
- "google.golang.org/protobuf/proto"
-
- "github.com/apache/skywalking-banyandb/api/common"
- databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
- modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
- "github.com/apache/skywalking-banyandb/banyand/kv"
- "github.com/apache/skywalking-banyandb/pkg/convert"
- "github.com/apache/skywalking-banyandb/pkg/index"
- "github.com/apache/skywalking-banyandb/pkg/index/inverted"
- "github.com/apache/skywalking-banyandb/pkg/index/lsm"
- "github.com/apache/skywalking-banyandb/pkg/index/posting"
- "github.com/apache/skywalking-banyandb/pkg/index/posting/roaring"
- "github.com/apache/skywalking-banyandb/pkg/logger"
- pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
- "github.com/apache/skywalking-banyandb/pkg/timestamp"
-)
-
-var (
- entityPrefix = []byte("entity_")
- entityPrefixLen = len(entityPrefix)
- seriesPrefix = []byte("series_")
- maxIntBytes = convert.Uint64ToBytes(math.MaxUint64)
- zeroIntBytes = convert.Uint64ToBytes(0)
-)
-
-// AnyEntry is the `*` for a regular expression. It could match "any" Entry in an Entity.
-var AnyEntry = Entry(nil)
-
-// Entry is an element in an Entity.
-type Entry []byte
-
-// Entity denotes an identity of a Series.
-// It defined by Stream or Measure schema.
-type Entity []Entry
-
-// Marshal encodes an Entity to bytes.
-func (e Entity) Marshal() []byte {
- data := make([][]byte, len(e))
- for i, entry := range e {
- data[i] = entry
- }
- return bytes.Join(data, nil)
-}
-
-// Prepend inserts an Entry before the first Entry as the prefix.
-func (e Entity) Prepend(entry Entry) Entity {
- d := e
- d = append(Entity{entry}, d...)
- return d
-}
-
-// Copy an Entity deeply.
-func (e Entity) Copy() Entity {
- a := make(Entity, len(e))
- copy(a, e)
- return a
-}
-
-// NewEntity return an Entity with an fixed length.
-func NewEntity(length int) Entity {
- e := make(Entity, length)
- for i := 0; i < length; i++ {
- e[i] = AnyEntry
- }
- return e
-}
-
-// EntityValue represents the value of a tag which is a part of an entity.
-type EntityValue = *modelv1.TagValue
-
-// EntityValueToEntry transforms EntityValue to Entry.
-func EntityValueToEntry(ev EntityValue) (Entry, error) {
- return pbv1.MarshalTagValue(ev)
-}
-
-// EntityValues is the encoded Entity.
-type EntityValues []EntityValue
-
-// Prepend inserts an EntityValue before the first EntityValue as the prefix.
-func (evs EntityValues) Prepend(scope EntityValue) EntityValues {
- return append(EntityValues{scope}, evs...)
-}
-
-// Encode EntityValues to tag values.
-func (evs EntityValues) Encode() (result []*modelv1.TagValue) {
- for _, v := range evs {
- result = append(result, v)
- }
- return
-}
-
-// ToEntity transforms EntityValues to Entity.
-func (evs EntityValues) ToEntity() (result Entity, err error) {
- for _, v := range evs {
- entry, errMarshal := EntityValueToEntry(v)
- if errMarshal != nil {
- return nil, err
- }
- result = append(result, entry)
- }
- return
-}
-
-// String outputs the string represent of an EntityValue.
-func (evs EntityValues) String() string {
- var strBuilder strings.Builder
- vv := evs.Encode()
- for i := 0; i < len(vv); i++ {
- strBuilder.WriteString(vv[i].String())
- if i < len(vv)-1 {
- strBuilder.WriteString(".")
- }
- }
- return strBuilder.String()
-}
-
-// DecodeEntityValues decodes tag values to EntityValues.
-func DecodeEntityValues(tvv []*modelv1.TagValue) (result EntityValues) {
- for _, tv := range tvv {
- result = append(result, tv)
- }
- return
-}
-
-// StrValue returns an EntityValue which wraps a string value.
-func StrValue(v string) EntityValue {
- return &modelv1.TagValue{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: v}}}
-}
-
-// Int64Value returns an EntityValue which wraps a int64 value.
-func Int64Value(v int64) EntityValue {
- return &modelv1.TagValue{Value: &modelv1.TagValue_Int{Int: &modelv1.Int{Value: v}}}
-}
-
-// MarshalEntityValues encodes EntityValues to bytes.
-func MarshalEntityValues(evs EntityValues) ([]byte, error) {
- data := &modelv1.TagFamilyForWrite{}
- for _, v := range evs {
- data.Tags = append(data.Tags, v)
- }
- return proto.Marshal(data)
-}
-
-// UnmarshalEntityValues decodes EntityValues from bytes.
-func UnmarshalEntityValues(evs []byte) (result EntityValues, err error) {
- data := &modelv1.TagFamilyForWrite{}
- result = make(EntityValues, len(data.Tags))
- if err = proto.Unmarshal(evs, data); err != nil {
- return nil, err
- }
- for _, tv := range data.Tags {
- result = append(result, tv)
- }
- return
-}
-
-// Path denotes a expression to match a Series.
-// It supports the fuzzy matching more than EQ by setting an entry to AnyEntry.
-type Path struct {
- prefix []byte
- seekKey []byte
- mask []byte
- template []byte
- isFull bool
- offset int
-}
-
-// NewPath return a Path with a matching expression.
-func NewPath(matchingExpression []Entry) Path {
- p := Path{
- seekKey: make([]byte, 0),
- mask: make([]byte, 0),
- template: make([]byte, 0),
- }
-
- var encounterAny bool
- for _, e := range matchingExpression {
- if e == nil {
- encounterAny = true
- p.mask = append(p.mask, zeroIntBytes...)
- p.template = append(p.template, zeroIntBytes...)
- continue
- }
- entry := Hash(e)
- if !encounterAny {
- p.offset += 8
- }
- p.mask = append(p.mask, maxIntBytes...)
- p.template = append(p.template, entry...)
- }
- if !encounterAny {
- p.isFull = true
- }
- p.extractPrefix()
- return p
-}
-
-func (p *Path) extractPrefix() {
- p.prefix = p.template[:p.offset]
- p.seekKey = p.seekKey[:0]
- p.seekKey = append(p.seekKey, p.prefix...)
- for i := 0; i < len(p.template)-p.offset; i++ {
- p.seekKey = append(p.seekKey, 0)
- }
-}
-
-func (p Path) prepend(entry Entry) Path {
- e := Hash(entry)
- p.template = prepend(p.template, e)
- p.offset += len(e)
- p.extractPrefix()
- p.mask = prepend(p.mask, maxIntBytes)
- return p
-}
-
-func prepend(src []byte, entry []byte) []byte {
- dst := make([]byte, len(src)+len(entry))
- copy(dst, entry)
- copy(dst[len(entry):], src)
- return dst
-}
-
-// OrderBy specifies the order of the result.
-type OrderBy struct {
- Index *databasev1.IndexRule
- Sort modelv1.Sort
-}
-
-// SeriesDatabase allows retrieving series.
-type SeriesDatabase interface {
- io.Closer
- GetByID(id common.SeriesID) (Series, error)
- Get(key []byte, entityValues EntityValues) (Series, error)
- List(ctx context.Context, path Path) (SeriesList, error)
- Search(ctx context.Context, path Path, filter index.Filter, order *OrderBy) (SeriesList, error)
- SizeOnDisk() int64
- writeInvertedIndex(fields []index.Field, seriesID common.SeriesID) error
- writeLSMIndex(fields []index.Field, seriesID common.SeriesID) error
-}
-
-type blockDatabase interface {
- shardID() common.ShardID
- span(ctx context.Context, timeRange timestamp.TimeRange) ([]blockDelegate, error)
- create(ctx context.Context, ts time.Time) (blockDelegate, error)
- block(ctx context.Context, id GlobalItemID) (blockDelegate, error)
-}
-
-var (
- _ SeriesDatabase = (*seriesDB)(nil)
- _ blockDatabase = (*seriesDB)(nil)
-
- dummySeriesList = make(SeriesList, 0)
-)
-
-type seriesDB struct {
- seriesMetadata kv.Store
- invertedIndex index.Store
- lsmIndex index.Store
- l *logger.Logger
- segCtrl *segmentController
- position common.Position
- sync.Mutex
- sID common.ShardID
-}
-
-func (s *seriesDB) writeInvertedIndex(fields []index.Field, seriesID common.SeriesID) error {
- if s.invertedIndex == nil {
- return errors.New("inverted index is not enabled")
- }
- return s.invertedIndex.Write(fields, uint64(seriesID))
-}
-
-func (s *seriesDB) writeLSMIndex(fields []index.Field, seriesID common.SeriesID) error {
- if s.lsmIndex == nil {
- return errors.New("lsm index is not enabled")
- }
- return s.lsmIndex.Write(fields, uint64(seriesID))
-}
-
-// nolint: contextcheck
-func (s *seriesDB) GetByID(id common.SeriesID) (Series, error) {
- var series string
- if e := s.l.Debug(); e.Enabled() {
- var buf bytes.Buffer
- buf.Write(seriesPrefix)
- buf.Write(id.Marshal())
- data, err := s.seriesMetadata.Get(buf.Bytes())
- if err != nil {
- e.Err(err).Msg("failed to get series id's literal")
- return newSeries(s.context(), id, "unknown", s), nil
- }
- entityValues, err := UnmarshalEntityValues(data)
- if err != nil {
- e.Err(err).Msg("malformed series id's literal")
- return newSeries(s.context(), id, "malformed", s), nil
- }
- series = entityValues.String()
- }
- return newSeries(s.context(), id, series, s), nil
-}
-
-func (s *seriesDB) block(ctx context.Context, id GlobalItemID) (blockDelegate, error) {
- seg := s.segCtrl.get(id.segID)
- if seg == nil {
- return nil, nil
- }
-
- return seg.blockController.get(ctx, id.blockID)
-}
-
-func (s *seriesDB) shardID() common.ShardID {
- return s.sID
-}
-
-func (s *seriesDB) Get(key []byte, entityValues EntityValues) (Series, error) {
- entityKey := prepend(key, entityPrefix)
- data, err := s.seriesMetadata.Get(entityKey)
- if errors.Is(err, kv.ErrKeyNotFound) {
- s.Lock()
- defer s.Unlock()
- seriesID := bytesToSeriesID(Hash(key))
- encodedData, entityValuesBytes, errDecode := encode(seriesID, entityValues)
- if errDecode != nil {
- return nil, errDecode
- }
- errDecode = s.seriesMetadata.Put(entityKey, encodedData)
- if errDecode != nil {
- receivedNumCounter.Inc(1, append(s.position.ShardLabelValues(), "series", "true")...)
- return nil, errDecode
- }
- receivedBytesCounter.Inc(float64(len(entityKey)+len(encodedData)), append(s.position.ShardLabelValues(), "series")...)
- receivedNumCounter.Inc(1, append(s.position.ShardLabelValues(), "series", "false")...)
-
- var series string
- if e := s.l.Debug(); e.Enabled() {
- errDecode = s.seriesMetadata.Put(prepend(seriesID.Marshal(), seriesPrefix), entityValuesBytes)
- if errDecode != nil {
- return nil, errDecode
- }
- series = entityValues.String()
- e.Str("series", series).
- Uint64("series_id", uint64(seriesID)).
- Msg("create a new series")
- }
- return newSeries(s.context(), seriesID, series, s), nil
- }
- if err != nil {
- return nil, err
- }
- seriesID, entityValues, err := decode(data)
- if err != nil {
- return nil, err
- }
-
- return newSeries(s.context(), seriesID, entityValues.String(), s), nil
-}
-
-func (s *seriesDB) SizeOnDisk() int64 {
- return s.seriesMetadata.SizeOnDisk()
-}
-
-func encode(seriesID common.SeriesID, evv EntityValues) ([]byte, []byte, error) {
- data, err := MarshalEntityValues(evv)
- if err != nil {
- return nil, nil, err
- }
- var buf bytes.Buffer
- buf.Write(convert.Uint64ToBytes(uint64(seriesID)))
- buf.Write(data)
- return buf.Bytes(), data, nil
-}
-
-func decode(value []byte) (common.SeriesID, EntityValues, error) {
- seriesID := convert.BytesToUint64(value[:8])
- entityValues, err := UnmarshalEntityValues(value[8:])
- if err != nil {
- return 0, nil, err
- }
- return common.SeriesID(seriesID), entityValues, nil
-}
-
-func (s *seriesDB) List(ctx context.Context, path Path) (SeriesList, error) {
- prefix := prepend(path.prefix, entityPrefix)
- l := logger.FetchOrDefault(ctx, "series_database", s.l)
- if path.isFull {
- data, err := s.seriesMetadata.Get(prefix)
- if err != nil && errors.Is(err, kv.ErrKeyNotFound) {
- if e := l.Debug(); e.Enabled() {
- e.Hex("path", path.prefix).Msg("doesn't get any series")
- }
- return dummySeriesList, nil
- }
- if err != nil {
- return nil, err
- }
- seriesID, entityValue, err := decode(data)
- if err != nil {
- return nil, err
- }
- var series string
- if e := l.Debug(); e.Enabled() {
- series = entityValue.String()
- e.Int("prefix_len", path.offset/8).
- Str("series", series).
- Uint64("series_id", uint64(seriesID)).
- Msg("got a series with a full path")
- }
- // nolint: contextcheck
- return []Series{newSeries(s.context(), seriesID, series, s)}, nil
- }
- result := make([]Series, 0)
- var err error
- errScan := s.seriesMetadata.Scan(prefix, prepend(path.seekKey, entityPrefix), kv.DefaultScanOpts, func(key []byte, getVal func() ([]byte, error)) error {
- key = key[entityPrefixLen:]
- comparableKey := make([]byte, len(key))
- // avoid slice out of bound
- if len(key) > len(path.mask) {
- return nil
- }
- for i, b := range key {
- comparableKey[i] = path.mask[i] & b
- }
- if bytes.Equal(path.template, comparableKey) {
- data, errGetVal := getVal()
- if errGetVal != nil {
- err = multierr.Append(err, errGetVal)
- return nil
- }
- seriesID, entityValue, errDecode := decode(data)
- if errDecode != nil {
- err = multierr.Append(err, errDecode)
- return nil
- }
- series := entityValue.String()
- if e := l.Debug(); e.Enabled() {
- e.Int("prefix_len", path.offset/8).
- Str("series", series).
- Uint64("series_id", uint64(seriesID)).
- Msg("match a series")
- }
- result = append(result, newSeries(s.context(), seriesID, series, s))
- }
- return nil
- })
- if errScan != nil {
- return nil, errScan
- }
- return result, err
-}
-
-func (s *seriesDB) Search(ctx context.Context, path Path, filter index.Filter, order *OrderBy) (SeriesList, error) {
- if s.invertedIndex == nil || s.lsmIndex == nil {
- return nil, errors.New("search is not supported")
- }
- if path.isFull {
- return s.List(ctx, path)
- }
- if order == nil {
- return s.filterSeries(ctx, path, filter)
- }
- fieldKey := index.FieldKey{
- IndexRuleID: order.Index.GetMetadata().Id,
- }
- var iter index.FieldIterator
- var err error
- switch order.Index.Type {
- case databasev1.IndexRule_TYPE_TREE:
- iter, err = s.lsmIndex.Iterator(fieldKey, rangeOpts, order.Sort, math.MaxInt)
- case databasev1.IndexRule_TYPE_INVERTED:
- iter, err = s.invertedIndex.Iterator(fieldKey, rangeOpts, order.Sort, math.MaxInt)
- default:
- return nil, errUnspecifiedIndexType
- }
- if err != nil {
- return nil, err
- }
- defer func() {
- err = multierr.Append(err, iter.Close())
- }()
- var pl posting.List
- if pl, err = s.seriesFilter(ctx, path, filter); err != nil {
- return nil, err
- }
- seriesList := make([]Series, 0)
- for iter.Next() {
- pv := iter.Val().Value
- if err = pv.Intersect(pl); err != nil {
- return nil, err
- }
- if pv.IsEmpty() {
- continue
- }
- pIter := pv.Iterator()
- for pIter.Next() {
- var series Series
- if series, err = s.GetByID(common.SeriesID(pIter.Current())); err != nil {
- return nil, multierr.Append(err, pIter.Close())
- }
- seriesList = append(seriesList, series)
- }
- if err = pIter.Close(); err != nil {
- return nil, err
- }
- }
- return seriesList, err
-}
-
-func (s *seriesDB) filterSeries(ctx context.Context, path Path, filter index.Filter) (SeriesList, error) {
- var seriesList SeriesList
- var err error
- if filter == nil {
- return s.List(ctx, path)
- }
- var pl posting.List
- if pl, err = filter.Execute(func(ruleType databasev1.IndexRule_Type) (index.Searcher, error) {
- switch ruleType {
- case databasev1.IndexRule_TYPE_TREE:
- return s.lsmIndex, nil
- case databasev1.IndexRule_TYPE_INVERTED:
- return s.invertedIndex, nil
- default:
- return nil, errUnspecifiedIndexType
- }
- }, 0); err != nil {
- return nil, err
- }
-
- if len(path.seekKey) == 0 {
- iter := pl.Iterator()
- defer func() {
- err = multierr.Append(err, iter.Close())
- }()
- for iter.Next() {
- var series Series
- if series, err = s.GetByID(common.SeriesID(iter.Current())); err != nil {
- return nil, err
- }
- seriesList = append(seriesList, series)
- }
- return seriesList, err
- }
- if seriesList, err = s.List(ctx, path); err != nil {
- return nil, err
- }
- // Remove a series from seriesList if its ID is not in the pl
- for i := 0; i < len(seriesList); i++ {
- if !pl.Contains(uint64(seriesList[i].ID())) {
- seriesList = append(seriesList[:i], seriesList[i+1:]...)
- i--
- }
- }
- return seriesList, nil
-}
-
-func (s *seriesDB) seriesFilter(ctx context.Context, path Path, filter index.Filter) (posting.List, error) {
- var sl SeriesList
- var err error
- if sl, err = s.filterSeries(ctx, path, filter); err != nil {
- return nil, err
- }
- pl := roaring.NewPostingList()
- for _, series := range sl {
- pl.Insert(uint64(series.ID()))
- }
- return pl, nil
-}
-
-func (s *seriesDB) span(ctx context.Context, timeRange timestamp.TimeRange) ([]blockDelegate, error) {
- result := make([]blockDelegate, 0)
- for _, s := range s.segCtrl.span(timeRange) {
- dd, err := s.blockController.span(ctx, timeRange)
- if err != nil {
- return nil, err
- }
- if dd == nil {
- continue
- }
- result = append(result, dd...)
- }
- return result, nil
-}
-
-func (s *seriesDB) create(ctx context.Context, ts time.Time) (blockDelegate, error) {
- s.Lock()
- defer s.Unlock()
- timeRange := timestamp.NewInclusiveTimeRange(ts, ts)
- ss := s.segCtrl.span(timeRange)
- if len(ss) > 0 {
- s := ss[0]
- dd, err := s.blockController.span(ctx, timeRange)
- if err != nil {
- return nil, err
- }
- if len(dd) > 0 {
- return dd[0], nil
- }
- block, err := s.blockController.create(timeRange.Start)
- if err != nil {
- return nil, err
- }
- return block.delegate(ctx)
- }
- seg, err := s.segCtrl.create(timeRange.Start)
- if err != nil {
- return nil, err
- }
- block, err := seg.blockController.create(timeRange.Start)
- if err != nil {
- return nil, err
- }
- return block.delegate(ctx)
-}
-
-func (s *seriesDB) context() context.Context {
- return context.WithValue(context.Background(), logger.ContextKey, s.l)
-}
-
-func (s *seriesDB) Close() error {
- err := s.seriesMetadata.Close()
- if s.invertedIndex != nil {
- err = multierr.Append(err, s.invertedIndex.Close())
- }
- if s.lsmIndex != nil {
- err = multierr.Append(err, s.lsmIndex.Close())
- }
- return err
-}
-
-func newSeriesDataBase(ctx context.Context, shardID common.ShardID, root string, segCtrl *segmentController) (SeriesDatabase, error) {
- sdb := &seriesDB{
- sID: shardID,
- segCtrl: segCtrl,
- l: logger.Fetch(ctx, "series_database"),
- position: common.GetPosition(ctx),
- }
- o := ctx.Value(OptionsKey)
- var options DatabaseOpts
- if o == nil {
- options = DatabaseOpts{}
- } else {
- options = o.(DatabaseOpts)
- }
- var memSize int64
- if options.SeriesMemSize > 1 {
- memSize = int64(options.SeriesMemSize)
- } else {
- memSize = defaultKVMemorySize
- }
- var err error
- if sdb.seriesMetadata, err = kv.OpenStore(root+"/md",
- kv.StoreWithNamedLogger("metadata", sdb.l),
- kv.StoreWithMemTableSize(memSize),
- ); err != nil {
- return nil, err
- }
- if options.IndexGranularity == IndexGranularitySeries {
- if sdb.invertedIndex, err = inverted.NewStore(inverted.StoreOpts{
- Path: path.Join(root, componentSecondInvertedIdx),
- Logger: sdb.l.Named(componentSecondInvertedIdx),
- BatchWaitSec: options.BlockInvertedIndex.BatchWaitSec,
- }); err != nil {
- return nil, err
- }
- if sdb.lsmIndex, err = lsm.NewStore(lsm.StoreOpts{
- Path: path.Join(root, componentSecondLSMIdx),
- Logger: sdb.l.Named(componentSecondLSMIdx),
- MemTableSize: defaultKVMemorySize,
- }); err != nil {
- return nil, err
- }
- }
- return sdb, nil
-}
-
-// HashEntity runs hash function (e.g. with xxhash algorithm) on each segment of the Entity,
-// and concatenates all uint64 in byte array. So the return length of the byte array will be
-// 8 (every uint64 has 8 bytes) * length of the input.
-func HashEntity(entity Entity) []byte {
- result := make([]byte, 0, len(entity)*8)
- for _, entry := range entity {
- result = append(result, Hash(entry)...)
- }
- return result
-}
-
-// SeriesID transforms Entity to common.SeriesID.
-func SeriesID(entity Entity) common.SeriesID {
- return common.SeriesID(convert.Hash(HashEntity(entity)))
-}
-
-// Hash encode Entry to 8 bytes.
-func Hash(entry []byte) []byte {
- return convert.Uint64ToBytes(convert.Hash(entry))
-}
-
-func bytesToSeriesID(data []byte) common.SeriesID {
- return common.SeriesID(convert.BytesToUint64(data))
-}
-
-// SeriesList is a collection of Series.
-type SeriesList []Series
-
-func (a SeriesList) Len() int {
- return len(a)
-}
-
-func (a SeriesList) Less(i, j int) bool {
- return a[i].ID() < a[j].ID()
-}
-
-func (a SeriesList) Swap(i, j int) {
- a[i], a[j] = a[j], a[i]
-}
-
-// Merge other SeriesList with this one to create a new SeriesList.
-func (a SeriesList) Merge(other SeriesList) SeriesList {
- if len(other) == 0 {
- return a
- }
- sort.Sort(other)
- if len(a) == 0 {
- return other
- }
- final := SeriesList{}
- i := 0
- j := 0
- for i < len(a) && j < len(other) {
- if a[i].ID() < other[j].ID() {
- final = append(final, a[i])
- i++
- } else {
- // deduplication
- if a[i].ID() == other[j].ID() {
- i++
- }
- final = append(final, other[j])
- j++
- }
- }
- for ; i < len(a); i++ {
- final = append(final, a[i])
- }
- for ; j < len(other); j++ {
- final = append(final, other[j])
- }
- return final
-}
diff --git a/banyand/tsdb/seriesdb_test.go b/banyand/tsdb/seriesdb_test.go
deleted file mode 100644
index 129f3208..00000000
--- a/banyand/tsdb/seriesdb_test.go
+++ /dev/null
@@ -1,579 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package tsdb
-
-import (
- "bytes"
- "context"
- "sort"
- "testing"
-
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-
- "github.com/apache/skywalking-banyandb/api/common"
- "github.com/apache/skywalking-banyandb/pkg/convert"
- "github.com/apache/skywalking-banyandb/pkg/logger"
- "github.com/apache/skywalking-banyandb/pkg/test"
- "github.com/apache/skywalking-banyandb/pkg/test/flags"
-)
-
-func TestEntity(t *testing.T) {
- tester := assert.New(t)
- entity := Entity{
- Entry("productpage"),
- Entry("10.0.0.1"),
- Entry(convert.Uint64ToBytes(0)),
- }
- entity = entity.Prepend(Entry("segment"))
- tester.Equal(Entity{
- Entry("segment"),
- Entry("productpage"),
- Entry("10.0.0.1"),
- Entry(convert.Uint64ToBytes(0)),
- }, entity)
-}
-
-func TestNewPath(t *testing.T) {
- tester := assert.New(t)
- tests := []struct {
- name string
- entity Entity
- scope Entry
- want Path
- }{
- {
- name: "general path",
- entity: Entity{
- Entry("productpage"),
- Entry("10.0.0.1"),
- Entry(convert.Uint64ToBytes(0)),
- },
- want: Path{
- isFull: true,
- prefix: bytes.Join([][]byte{
- Hash([]byte("productpage")),
- Hash([]byte("10.0.0.1")),
- Hash(convert.Uint64ToBytes(0)),
- }, nil),
- seekKey: bytes.Join([][]byte{
- Hash([]byte("productpage")),
- Hash([]byte("10.0.0.1")),
- Hash(convert.Uint64ToBytes(0)),
- }, nil),
- template: bytes.Join([][]byte{
- Hash([]byte("productpage")),
- Hash([]byte("10.0.0.1")),
- Hash(convert.Uint64ToBytes(0)),
- }, nil),
- mask: bytes.Join([][]byte{
- maxIntBytes,
- maxIntBytes,
- maxIntBytes,
- }, nil),
- offset: 24,
- },
- },
- {
- name: "the first is anyone",
- entity: Entity{
- AnyEntry,
- Entry("10.0.0.1"),
- Entry(convert.Uint64ToBytes(0)),
- },
- want: Path{
- prefix: []byte{},
- seekKey: bytes.Join([][]byte{
- zeroIntBytes,
- zeroIntBytes,
- zeroIntBytes,
- }, nil),
- template: bytes.Join([][]byte{
- zeroIntBytes,
- Hash([]byte("10.0.0.1")),
- Hash(convert.Uint64ToBytes(0)),
- }, nil),
- mask: bytes.Join([][]byte{
- zeroIntBytes,
- maxIntBytes,
- maxIntBytes,
- }, nil),
- offset: 0,
- },
- },
- {
- name: "the second is anyone",
- entity: Entity{
- Entry("productpage"),
- AnyEntry,
- Entry(convert.Uint64ToBytes(0)),
- },
- want: Path{
- prefix: bytes.Join([][]byte{
- Hash([]byte("productpage")),
- }, nil),
- seekKey: bytes.Join([][]byte{
- Hash([]byte("productpage")),
- zeroIntBytes,
- zeroIntBytes,
- }, nil),
- template: bytes.Join([][]byte{
- Hash([]byte("productpage")),
- zeroIntBytes,
- Hash(convert.Uint64ToBytes(0)),
- }, nil),
- mask: bytes.Join([][]byte{
- maxIntBytes,
- zeroIntBytes,
- maxIntBytes,
- }, nil),
- offset: 8,
- },
- },
- {
- name: "the last is anyone",
- entity: Entity{
- Entry("productpage"),
- Entry("10.0.0.1"),
- AnyEntry,
- },
- want: Path{
- prefix: bytes.Join([][]byte{
- Hash([]byte("productpage")),
- Hash([]byte("10.0.0.1")),
- }, nil),
- seekKey: bytes.Join([][]byte{
- Hash([]byte("productpage")),
- Hash([]byte("10.0.0.1")),
- zeroIntBytes,
- }, nil),
- template: bytes.Join([][]byte{
- Hash([]byte("productpage")),
- Hash([]byte("10.0.0.1")),
- zeroIntBytes,
- }, nil),
- mask: bytes.Join([][]byte{
- maxIntBytes,
- maxIntBytes,
- zeroIntBytes,
- }, nil),
- offset: 16,
- },
- },
- {
- name: "prepend a scope",
- entity: Entity{
- Entry("productpage"),
- Entry("10.0.0.1"),
- Entry(convert.Uint64ToBytes(0)),
- },
- scope: Entry("segment"),
- want: Path{
- isFull: true,
- prefix: bytes.Join([][]byte{
- Hash([]byte("segment")),
- Hash([]byte("productpage")),
- Hash([]byte("10.0.0.1")),
- Hash(convert.Uint64ToBytes(0)),
- }, nil),
- seekKey: bytes.Join([][]byte{
- Hash([]byte("segment")),
- Hash([]byte("productpage")),
- Hash([]byte("10.0.0.1")),
- Hash(convert.Uint64ToBytes(0)),
- }, nil),
- template: bytes.Join([][]byte{
- Hash([]byte("segment")),
- Hash([]byte("productpage")),
- Hash([]byte("10.0.0.1")),
- Hash(convert.Uint64ToBytes(0)),
- }, nil),
- mask: bytes.Join([][]byte{
- maxIntBytes,
- maxIntBytes,
- maxIntBytes,
- maxIntBytes,
- }, nil),
- offset: 32,
- },
- },
- {
- name: "prepend a scope to the entity whose first entry is anyone",
- entity: Entity{
- AnyEntry,
- Entry("10.0.0.1"),
- Entry(convert.Uint64ToBytes(0)),
- },
- scope: Entry("segment"),
- want: Path{
- prefix: Hash([]byte("segment")),
- seekKey: bytes.Join([][]byte{
- Hash([]byte("segment")),
- zeroIntBytes,
- zeroIntBytes,
- zeroIntBytes,
- }, nil),
- template: bytes.Join([][]byte{
- Hash([]byte("segment")),
- zeroIntBytes,
- Hash([]byte("10.0.0.1")),
- Hash(convert.Uint64ToBytes(0)),
- }, nil),
- mask: bytes.Join([][]byte{
- maxIntBytes,
- zeroIntBytes,
- maxIntBytes,
- maxIntBytes,
- }, nil),
- offset: 8,
- },
- },
- {
- name: "prepend a scope to the entity whose second entry is anyone",
- entity: Entity{
- Entry("productpage"),
- AnyEntry,
- Entry(convert.Uint64ToBytes(0)),
- },
- scope: Entry("segment"),
- want: Path{
- prefix: bytes.Join([][]byte{
- Hash([]byte("segment")),
- Hash([]byte("productpage")),
- }, nil),
- seekKey: bytes.Join([][]byte{
- Hash([]byte("segment")),
- Hash([]byte("productpage")),
- zeroIntBytes,
- zeroIntBytes,
- }, nil),
- template: bytes.Join([][]byte{
- Hash([]byte("segment")),
- Hash([]byte("productpage")),
- zeroIntBytes,
- Hash(convert.Uint64ToBytes(0)),
- }, nil),
- mask: bytes.Join([][]byte{
- maxIntBytes,
- maxIntBytes,
- zeroIntBytes,
- maxIntBytes,
- }, nil),
- offset: 16,
- },
- },
- {
- name: "prepend a scope to the entity whose last entry is anyone",
- entity: Entity{
- Entry("productpage"),
- Entry("10.0.0.1"),
- AnyEntry,
- },
- scope: Entry("segment"),
- want: Path{
- prefix: bytes.Join([][]byte{
- Hash([]byte("segment")),
- Hash([]byte("productpage")),
- Hash([]byte("10.0.0.1")),
- }, nil),
- seekKey: bytes.Join([][]byte{
- Hash([]byte("segment")),
- Hash([]byte("productpage")),
- Hash([]byte("10.0.0.1")),
- zeroIntBytes,
- }, nil),
- template: bytes.Join([][]byte{
- Hash([]byte("segment")),
- Hash([]byte("productpage")),
- Hash([]byte("10.0.0.1")),
- zeroIntBytes,
- }, nil),
- mask: bytes.Join([][]byte{
- maxIntBytes,
- maxIntBytes,
- maxIntBytes,
- zeroIntBytes,
- }, nil),
- offset: 24,
- },
- },
- {
- name: "prepend a scope to any",
- entity: Entity{
- AnyEntry,
- },
- scope: Entry("segment"),
- want: Path{
- prefix: bytes.Join([][]byte{
- Hash([]byte("segment")),
- }, nil),
- seekKey: bytes.Join([][]byte{
- Hash([]byte("segment")),
- zeroIntBytes,
- }, nil),
- template: bytes.Join([][]byte{
- Hash([]byte("segment")),
- zeroIntBytes,
- }, nil),
- mask: bytes.Join([][]byte{
- maxIntBytes,
- zeroIntBytes,
- }, nil),
- offset: 8,
- },
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got := NewPath(tt.entity)
- if tt.scope != nil {
- got = got.prepend(tt.scope)
- }
- tester.Equal(tt.want, got)
- })
- }
-}
-
-func Test_SeriesDatabase_Get_GetByID(t *testing.T) {
- tests := []struct {
- name string
- entities []Entity
- }{
- {
- name: "general entity",
- entities: []Entity{{
- Entry("productpage"),
- Entry("10.0.0.1"),
- convert.Uint64ToBytes(0),
- }},
- },
- {
- name: "duplicated entity",
- entities: []Entity{
- {
- Entry("productpage"),
- Entry("10.0.0.1"),
- convert.Uint64ToBytes(0),
- },
- {
- Entry("productpage"),
- Entry("10.0.0.1"),
- convert.Uint64ToBytes(0),
- },
- },
- },
- }
- tester := assert.New(t)
- tester.NoError(logger.Init(logger.Logging{
- Env: "dev",
- Level: flags.LogLevel,
- }))
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- dir, deferFunc := test.Space(require.New(t))
- defer deferFunc()
- s, err := newSeriesDataBase(context.WithValue(context.Background(), logger.ContextKey, logger.GetLogger("test")), 0, dir, nil)
- tester.NoError(err)
- for _, entity := range tt.entities {
- evv := toEntityValues(entity)
- series, err := s.Get(HashEntity(entity), evv)
- tester.NoError(err)
- tester.Greater(uint(series.ID()), uint(0))
- literal := series.String()
- if literal != "" {
- tester.Equal(evv.String(), literal)
- }
- series, err = s.GetByID(series.ID())
- tester.NoError(err)
- literal = series.String()
- if literal != "" {
- tester.Equal(evv.String(), literal)
- }
- }
- })
- }
-}
-
-func Test_SeriesDatabase_List(t *testing.T) {
- tester := assert.New(t)
- tester.NoError(logger.Init(logger.Logging{
- Env: "dev",
- Level: flags.LogLevel,
- }))
- dir, deferFunc := test.Space(require.New(t))
- defer deferFunc()
- s, err := newSeriesDataBase(context.WithValue(context.Background(), logger.ContextKey, logger.GetLogger("test")), 0, dir, nil)
- tester.NoError(err)
- data := setUpEntities(tester, s)
- tests := []struct {
- name string
- want SeriesList
- path Path
- wantErr bool
- }{
- {
- name: "equal",
- path: NewPath([]Entry{
- Entry("productpage"),
- Entry("10.0.0.1"),
- convert.Uint64ToBytes(0),
- }),
- want: SeriesList{
- newMockSeries(data[0].id, s.(*seriesDB)),
- },
- },
- {
- name: "all in an instance",
- path: NewPath([]Entry{
- Entry("productpage"),
- Entry("10.0.0.2"),
- AnyEntry,
- }),
- want: SeriesList{
- newMockSeries(data[1].id, s.(*seriesDB)),
- newMockSeries(data[2].id, s.(*seriesDB)),
- },
- },
- {
- name: "all in a service",
- path: NewPath([]Entry{
- Entry("productpage"),
- AnyEntry,
- AnyEntry,
- }),
- want: SeriesList{
- newMockSeries(data[0].id, s.(*seriesDB)),
- newMockSeries(data[1].id, s.(*seriesDB)),
- newMockSeries(data[2].id, s.(*seriesDB)),
- newMockSeries(data[3].id, s.(*seriesDB)),
- },
- },
- {
- name: "all successful",
- path: NewPath([]Entry{
- AnyEntry,
- AnyEntry,
- convert.Uint64ToBytes(0),
- }),
- want: SeriesList{
- newMockSeries(data[0].id, s.(*seriesDB)),
- newMockSeries(data[1].id, s.(*seriesDB)),
- newMockSeries(data[3].id, s.(*seriesDB)),
- },
- },
- {
- name: "all error",
- path: NewPath([]Entry{
- AnyEntry,
- AnyEntry,
- convert.Uint64ToBytes(1),
- }),
- want: SeriesList{
- newMockSeries(data[2].id, s.(*seriesDB)),
- newMockSeries(data[4].id, s.(*seriesDB)),
- },
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- series, err := s.List(context.Background(), tt.path)
- if tt.wantErr {
- tester.Error(err)
- return
- }
- tester.NoError(err)
- tester.Equal(transform(tt.want), transform(series))
- })
- }
-}
-
-type entityWithID struct {
- entity Entity
- id common.SeriesID
-}
-
-func setUpEntities(t *assert.Assertions, db SeriesDatabase) []*entityWithID {
- data := []*entityWithID{
- {
- entity: Entity{
- Entry("productpage"),
- Entry("10.0.0.1"),
- convert.Uint64ToBytes(0),
- },
- },
- {
- entity: Entity{
- Entry("productpage"),
- Entry("10.0.0.2"),
- convert.Uint64ToBytes(0),
- },
- },
- {
- entity: Entity{
- Entry("productpage"),
- Entry("10.0.0.2"),
- convert.Uint64ToBytes(1),
- },
- },
- {
- entity: Entity{
- Entry("productpage"),
- Entry("10.0.0.3"),
- convert.Uint64ToBytes(0),
- },
- },
- {
- entity: Entity{
- Entry("payment"),
- Entry("10.0.0.2"),
- convert.Uint64ToBytes(1),
- },
- },
- }
- for _, d := range data {
- d.id = common.SeriesID(convert.BytesToUint64(Hash(HashEntity(d.entity))))
- series, err := db.Get(HashEntity(d.entity), toEntityValues(d.entity))
- t.NoError(err)
- t.Greater(uint(series.ID()), uint(0))
- }
- return data
-}
-
-func toEntityValues(entity Entity) (result EntityValues) {
- for i, e := range entity {
- if len(e) == 8 && i == len(entity)-1 {
- result = append(result, Int64Value(int64(convert.BytesToUint64(e))))
- } else {
- result = append(result, StrValue(string(e)))
- }
- }
- return
-}
-
-func newMockSeries(id common.SeriesID, blockDB *seriesDB) *series {
- return newSeries(context.TODO(), id, "", blockDB)
-}
-
-func transform(list SeriesList) (seriesIDs []common.SeriesID) {
- sort.Sort(list)
- for _, s := range list {
- seriesIDs = append(seriesIDs, s.ID())
- }
- return seriesIDs
-}
diff --git a/banyand/tsdb/shard.go b/banyand/tsdb/shard.go
deleted file mode 100644
index 5ab48ca0..00000000
--- a/banyand/tsdb/shard.go
+++ /dev/null
@@ -1,333 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package tsdb
-
-import (
- "context"
- "sort"
- "strconv"
- "strings"
- "sync"
- "time"
-
- "github.com/pkg/errors"
- "github.com/shirou/gopsutil/v3/disk"
- "go.uber.org/multierr"
-
- "github.com/apache/skywalking-banyandb/api/common"
- "github.com/apache/skywalking-banyandb/banyand/observability"
- "github.com/apache/skywalking-banyandb/banyand/tsdb/bucket"
- "github.com/apache/skywalking-banyandb/pkg/logger"
- "github.com/apache/skywalking-banyandb/pkg/meter"
- "github.com/apache/skywalking-banyandb/pkg/timestamp"
-)
-
-const (
- defaultBlockQueueSize = 2
- defaultMaxBlockQueueSize = 64
- defaultKVMemorySize = 4 << 20
- defaultNumBufferShards = 2
- defaultWriteConcurrency = 1000
-)
-
-var (
- _ Shard = (*shard)(nil)
-
- shardProvider = observability.NewMeterProvider(meterTSDB.SubScope("shard"))
- diskStateGetter = disk.UsageWithContext
-
- diskStateGauge meter.Gauge
- flushBytes meter.Counter
- flushNum meter.Counter
- flushLatency meter.Histogram
- receivedBytesCounter meter.Counter
- receivedNumCounter meter.Counter
- onDiskBytesGauge meter.Gauge
-)
-
-type contextBufferSupplierKey struct{}
-
-var bufferSupplierKey = contextBufferSupplierKey{}
-
-func init() {
- labelNames := common.ShardLabelNames()
- diskStateGauge = shardProvider.Gauge("disk_state", append(labelNames, "kind")...)
- flushBytes = shardProvider.Counter("flush_bytes", labelNames...)
- flushNum = shardProvider.Counter("flush_num", append(labelNames, "is_error")...)
- flushLatency = shardProvider.Histogram("flush_latency", meter.DefBuckets, labelNames...)
- receivedBytesCounter = shardProvider.Counter("received_bytes", append(labelNames, "kind")...)
- receivedNumCounter = shardProvider.Counter("received_num", append(labelNames, "kind", "is_error")...)
- onDiskBytesGauge = shardProvider.Gauge("on_disk_bytes", append(labelNames, "kind")...)
-}
-
-type shard struct {
- seriesDatabase SeriesDatabase
- indexDatabase IndexDatabase
- l *logger.Logger
- segmentController *segmentController
- segmentManageStrategy *bucket.Strategy
- scheduler *timestamp.Scheduler
- bufferSupplier *BufferSupplier
- position common.Position
- closeOnce sync.Once
- id common.ShardID
-}
-
-// OpenShard returns an existed Shard or create a new one if not existed.
-func OpenShard(ctx context.Context, id common.ShardID,
- root string, segmentSize, blockSize, ttl IntervalRule, openedBlockSize, maxOpenedBlockSize int, enableWAL bool,
-) (Shard, error) {
- path, err := mkdir(shardTemplate, root, int(id))
- if err != nil {
- return nil, errors.Wrapf(err, "make the directory of the shard %d ", int(id))
- }
- l := logger.Fetch(ctx, "shard"+strconv.Itoa(int(id)))
- l.Info().Int("shard_id", int(id)).Str("path", path).Msg("creating a shard")
- if openedBlockSize < 1 {
- openedBlockSize = defaultBlockQueueSize
- }
- shardCtx := context.WithValue(ctx, logger.ContextKey, l)
- shardCtx = common.SetPosition(shardCtx, func(p common.Position) common.Position {
- p.Shard = strconv.Itoa(int(id))
- return p
- })
- clock, _ := timestamp.GetClock(shardCtx)
-
- scheduler := timestamp.NewScheduler(l, clock)
- s := &shard{
- id: id,
- l: l,
- scheduler: scheduler,
- position: common.GetPosition(shardCtx),
- }
- s.bufferSupplier = NewBufferSupplier(l, s.position, defaultWriteConcurrency, defaultNumBufferShards, enableWAL, path)
- shardCtx = context.WithValue(shardCtx, bufferSupplierKey, s.bufferSupplier)
- s.segmentController, err = newSegmentController(shardCtx, path, segmentSize, blockSize, openedBlockSize, maxOpenedBlockSize, l, scheduler)
- if err != nil {
- return nil, errors.Wrapf(err, "create the segment controller of the shard %d", int(id))
- }
- err = s.segmentController.open()
- if err != nil {
- return nil, err
- }
- seriesPath, err := mkdir(seriesTemplate, path)
- if err != nil {
- return nil, err
- }
- sdb, err := newSeriesDataBase(shardCtx, s.id, seriesPath, s.segmentController)
- if err != nil {
- return nil, err
- }
- s.seriesDatabase = sdb
- idb := newIndexDatabase(shardCtx, s.id, s.segmentController)
- s.indexDatabase = idb
- s.segmentManageStrategy, err = bucket.NewStrategy(s.segmentController, bucket.WithLogger(s.l))
- if err != nil {
- return nil, err
- }
- s.segmentManageStrategy.Run()
- retentionTask := newRetentionTask(s.segmentController, ttl)
- if err := scheduler.Register("retention", retentionTask.option, retentionTask.expr, retentionTask.run); err != nil {
- return nil, err
- }
- plv := s.position.ShardLabelValues()
- observability.MetricsCollector.Register(strings.Join(plv, "-"), func() {
- if stat, err := diskStateGetter(ctx, path); err != nil {
- s.l.Error().Err(err).Msg("get disk usage stat")
- } else {
- diskStateGauge.Set(stat.UsedPercent, append(plv, "used_percent")...)
- diskStateGauge.Set(float64(stat.Free), append(plv, "free")...)
- diskStateGauge.Set(float64(stat.Total), append(plv, "total")...)
- diskStateGauge.Set(float64(stat.Used), append(plv, "used")...)
- diskStateGauge.Set(float64(stat.InodesUsed), append(plv, "inodes_used")...)
- diskStateGauge.Set(float64(stat.InodesFree), append(plv, "inodes_free")...)
- diskStateGauge.Set(float64(stat.InodesTotal), append(plv, "inodes_total")...)
- diskStateGauge.Set(stat.InodesUsedPercent, append(plv, "inodes_used_percent")...)
- }
- s.collectSizeOnDisk(plv)
- })
- return s, nil
-}
-
-func (s *shard) ID() common.ShardID {
- return s.id
-}
-
-func (s *shard) Series() SeriesDatabase {
- return s.seriesDatabase
-}
-
-func (s *shard) Index() IndexDatabase {
- return s.indexDatabase
-}
-
-func (s *shard) collectSizeOnDisk(labelsValues []string) {
- var globalIndex, localLSM, localInverted, tst int64
- for _, seg := range s.segmentController.segments() {
- if seg.globalIndex != nil {
- globalIndex += seg.globalIndex.SizeOnDisk()
- }
- for _, b := range seg.blockController.blocks() {
- if b.Closed() {
- continue
- }
- if b.tsTable != nil {
- tst += b.tsTable.SizeOnDisk()
- }
- if b.lsmIndex != nil {
- localLSM += b.lsmIndex.SizeOnDisk()
- }
- if b.invertedIndex != nil {
- localInverted += b.invertedIndex.SizeOnDisk()
- }
- }
- }
- onDiskBytesGauge.Set(float64(s.seriesDatabase.SizeOnDisk()), append(labelsValues, "series")...)
- onDiskBytesGauge.Set(float64(globalIndex), append(labelsValues, "global_index")...)
- onDiskBytesGauge.Set(float64(localLSM), append(labelsValues, "local_lsm")...)
- onDiskBytesGauge.Set(float64(localInverted), append(labelsValues, "local_inverted")...)
- onDiskBytesGauge.Set(float64(tst), append(labelsValues, "tst")...)
-}
-
-func (s *shard) State() (shardState ShardState) {
- shardState.StrategyManagers = append(shardState.StrategyManagers, s.segmentManageStrategy.String())
- for _, seg := range s.segmentController.segments() {
- if seg.blockManageStrategy != nil {
- shardState.StrategyManagers = append(shardState.StrategyManagers, seg.blockManageStrategy.String())
- }
- for _, b := range seg.blockController.blocks() {
- shardState.Blocks = append(shardState.Blocks, BlockState{
- ID: BlockID{
- SegID: b.segID,
- BlockID: b.blockID,
- },
- TimeRange: b.TimeRange,
- Closed: b.Closed(),
- })
- }
- }
- all := s.segmentController.blockQueue.All()
- shardState.OpenBlocks = make([]BlockID, len(all))
- for i, v := range all {
- shardState.OpenBlocks[i] = v.(BlockID)
- }
- sort.Slice(shardState.OpenBlocks, func(i, j int) bool {
- x := shardState.OpenBlocks[i]
- y := shardState.OpenBlocks[j]
- if x.SegID == y.SegID {
- return x.BlockID < y.BlockID
- }
- return x.SegID < y.SegID
- })
- s.l.Trace().Interface("", shardState).Msg("shard state")
- return shardState
-}
-
-func (s *shard) TriggerSchedule(task string) bool {
- return s.scheduler.Trigger(task)
-}
-
-func (s *shard) Close() (err error) {
- s.closeOnce.Do(func() {
- s.scheduler.Close()
- s.segmentManageStrategy.Close()
- ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
- defer cancel()
- err = multierr.Combine(s.bufferSupplier.Close(), s.segmentController.close(ctx), s.seriesDatabase.Close())
- })
- return err
-}
-
-// IntervalUnit denotes the unit of a time point.
-type IntervalUnit int
-
-// Available IntervalUnits. HOUR and DAY are adequate for the APM scenario.
-const (
- HOUR IntervalUnit = iota
- DAY
-)
-
-func (iu IntervalUnit) String() string {
- switch iu {
- case HOUR:
- return "hour"
- case DAY:
- return "day"
- }
- panic("invalid interval unit")
-}
-
-// IntervalRule defines a length of two points in time.
-type IntervalRule struct {
- Unit IntervalUnit
- Num int
-}
-
-func (ir IntervalRule) nextTime(current time.Time) time.Time {
- switch ir.Unit {
- case HOUR:
- return current.Add(time.Hour * time.Duration(ir.Num))
- case DAY:
- return current.AddDate(0, 0, ir.Num)
- }
- panic("invalid interval unit")
-}
-
-func (ir IntervalRule) estimatedDuration() time.Duration {
- switch ir.Unit {
- case HOUR:
- return time.Hour * time.Duration(ir.Num)
- case DAY:
- return 24 * time.Hour * time.Duration(ir.Num)
- }
- panic("invalid interval unit")
-}
-
-type parser interface {
- Parse(value string) (time.Time, error)
-}
-
-func loadSections(root, prefix string, parser parser, intervalRule IntervalRule, loadFn func(start, end time.Time) error) error {
- var startTimeLst []time.Time
- if err := walkDir(
- root,
- prefix,
- func(suffix string) error {
- startTime, err := parser.Parse(suffix)
- if err != nil {
- return err
- }
- startTimeLst = append(startTimeLst, startTime)
- return nil
- }); err != nil {
- return err
- }
- sort.Slice(startTimeLst, func(i, j int) bool { return i < j })
- for i, start := range startTimeLst {
- var end time.Time
- if i < len(startTimeLst)-1 {
- end = startTimeLst[i+1]
- } else {
- end = intervalRule.nextTime(start)
- }
- if err := loadFn(start, end); err != nil {
- return err
- }
- }
- return nil
-}
diff --git a/banyand/tsdb/shard_test.go b/banyand/tsdb/shard_test.go
deleted file mode 100644
index ae9abfd8..00000000
--- a/banyand/tsdb/shard_test.go
+++ /dev/null
@@ -1,605 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package tsdb_test
-
-import (
- "context"
- "time"
-
- . "github.com/onsi/ginkgo/v2"
- . "github.com/onsi/gomega"
- "github.com/onsi/gomega/gleak"
-
- "github.com/apache/skywalking-banyandb/api/common"
- "github.com/apache/skywalking-banyandb/banyand/tsdb"
- "github.com/apache/skywalking-banyandb/pkg/test"
- "github.com/apache/skywalking-banyandb/pkg/test/flags"
- "github.com/apache/skywalking-banyandb/pkg/timestamp"
-)
-
-var _ = Describe("Shard", func() {
- Describe("Generate segments and blocks", func() {
- var tmp string
- var deferFn func()
- var shard tsdb.Shard
- var clock timestamp.MockClock
- var goods []gleak.Goroutine
-
- BeforeEach(func() {
- goods = gleak.Goroutines()
- var err error
- tmp, deferFn, err = test.NewSpace()
- Expect(err).NotTo(HaveOccurred())
- clock = timestamp.NewMockClock()
- clock.Set(time.Date(1970, 0o1, 0o1, 0, 0, 0, 0, time.Local))
- Expect(err).NotTo(HaveOccurred())
- })
- AfterEach(func() {
- GinkgoWriter.Printf("shard state:%+v \n", shard.State())
- shard.Close()
- deferFn()
- Eventually(gleak.Goroutines, flags.EventuallyTimeout).ShouldNot(gleak.HaveLeaked(goods))
- })
- started := func(tasks ...string) {
- for _, task := range tasks {
- Eventually(func() bool {
- return shard.TriggerSchedule(task)
- }, flags.EventuallyTimeout).Should(BeTrue())
- }
- }
- forward := func(hours int, tasks ...string) {
- for i := 0; i < hours; i++ {
- clock.Add(1 * time.Hour)
- for _, task := range tasks {
- shard.TriggerSchedule(task)
- }
- }
- }
- It("generates several segments and blocks", func() {
- By("open 4 blocks")
- var err error
- shard, err = tsdb.OpenShard(context.WithValue(timestamp.SetClock(context.Background(), clock),
- tsdb.OptionsKey, tsdb.DatabaseOpts{
- TSTableFactory: tsdb.NewByPassTSTableFactory(),
- }), common.ShardID(0), tmp,
- tsdb.IntervalRule{
- Unit: tsdb.DAY,
- Num: 1,
- },
- tsdb.IntervalRule{
- Unit: tsdb.HOUR,
- Num: 12,
- },
- tsdb.IntervalRule{
- Unit: tsdb.DAY,
- Num: 7,
- },
- 2,
- 3,
- false,
- )
- Expect(err).NotTo(HaveOccurred())
- started("BlockID-19700101-1970010100-1", "SegID-19700101-1")
- By("01/01 00:00 1st block is opened")
- t1 := clock.Now()
- Eventually(func() []tsdb.BlockState {
- return shard.State().Blocks
- }, flags.EventuallyTimeout).Should(Equal([]tsdb.BlockState{
- {
- ID: tsdb.BlockID{
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700101),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010100),
- },
- TimeRange: timestamp.NewTimeRangeDuration(t1, 12*time.Hour, true, false),
- },
- }))
- By("01/01 11:00 2nd block is opened")
- forward(11, "BlockID-19700101-1970010100-1", "SegID-19700101-1")
- t2 := clock.Now().Add(1 * time.Hour)
- Eventually(func() []tsdb.BlockState {
- return shard.State().Blocks
- }, flags.EventuallyTimeout).Should(Equal([]tsdb.BlockState{
- {
- ID: tsdb.BlockID{
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700101),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010100),
- },
- TimeRange: timestamp.NewTimeRangeDuration(t1, 12*time.Hour, true, false),
- },
- {
- ID: tsdb.BlockID{
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700101),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010112),
- },
- TimeRange: timestamp.NewTimeRangeDuration(t2, 12*time.Hour, true, false),
- },
- }))
- Eventually(func() []tsdb.BlockID {
- return shard.State().OpenBlocks
- }, flags.EventuallyTimeout).Should(Equal([]tsdb.BlockID{}))
- By("01/01 13:00 moves to the 2nd block")
- forward(2, "BlockID-19700101-1970010100-1", "SegID-19700101-1")
- started("BlockID-19700101-1970010112-1")
- Eventually(func() []tsdb.BlockID {
- return shard.State().OpenBlocks
- }, flags.EventuallyTimeout).Should(Equal([]tsdb.BlockID{
- {
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700101),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010100),
- },
- }))
- By("01/01 23:00 3rd block is opened")
- forward(10, "BlockID-19700101-1970010112-1", "SegID-19700101-1")
- t3 := clock.Now().Add(1 * time.Hour)
- Eventually(func() []tsdb.BlockState {
- return shard.State().Blocks
- }, flags.EventuallyTimeout).Should(Equal([]tsdb.BlockState{
- {
- ID: tsdb.BlockID{
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700101),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010100),
- },
- TimeRange: timestamp.NewTimeRangeDuration(t1, 12*time.Hour, true, false),
- },
- {
- ID: tsdb.BlockID{
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700101),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010112),
- },
- TimeRange: timestamp.NewTimeRangeDuration(t2, 12*time.Hour, true, false),
- },
- {
- ID: tsdb.BlockID{
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700102),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010200),
- },
- TimeRange: timestamp.NewTimeRangeDuration(t3, 12*time.Hour, true, false),
- },
- }))
- By("01/02 01:00 moves to 3rd block")
- forward(2, "BlockID-19700101-1970010112-1", "SegID-19700101-1")
- started("BlockID-19700102-1970010200-1", "SegID-19700102-1")
- Eventually(func() []tsdb.BlockID {
- if clock.TriggerTimer() {
- GinkgoWriter.Println("01/02 01:00 has been triggered")
- }
- return shard.State().OpenBlocks
- }, flags.EventuallyTimeout).Should(Equal([]tsdb.BlockID{
- {
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700101),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010100),
- },
- {
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700101),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010112),
- },
- }))
- By("01/02 11:00 4th block is opened")
- forward(10, "BlockID-19700102-1970010200-1", "SegID-19700102-1")
- t4 := clock.Now().Add(1 * time.Hour)
- Eventually(func() []tsdb.BlockState {
- if clock.TriggerTimer() {
- GinkgoWriter.Println("01/02 10:00 has been triggered")
- }
- return shard.State().Blocks
- }, flags.EventuallyTimeout).Should(Equal([]tsdb.BlockState{
- {
- ID: tsdb.BlockID{
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700101),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010100),
- },
- TimeRange: timestamp.NewTimeRangeDuration(t1, 12*time.Hour, true, false),
- },
- {
- ID: tsdb.BlockID{
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700101),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010112),
- },
- TimeRange: timestamp.NewTimeRangeDuration(t2, 12*time.Hour, true, false),
- },
- {
- ID: tsdb.BlockID{
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700102),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010200),
- },
- TimeRange: timestamp.NewTimeRangeDuration(t3, 12*time.Hour, true, false),
- },
- {
- ID: tsdb.BlockID{
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700102),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010212),
- },
- TimeRange: timestamp.NewTimeRangeDuration(t4, 12*time.Hour, true, false),
- },
- }))
- By("01/02 13:00 moves to 4th block")
- forward(2, "BlockID-19700102-1970010200-1", "SegID-19700102-1")
- started("BlockID-19700102-1970010212-1")
- Eventually(func() []tsdb.BlockID {
- return shard.State().OpenBlocks
- }, flags.EventuallyTimeout).Should(Equal([]tsdb.BlockID{
- {
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700101),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010100),
- },
- {
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700101),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010112),
- },
- {
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700102),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010200),
- },
- }))
- By("01/02 23:00 5th block is opened")
- forward(10, "BlockID-19700102-1970010212-1", "SegID-19700102-1")
- t5 := clock.Now().Add(1 * time.Hour)
- Eventually(func() []tsdb.BlockState {
- return shard.State().Blocks
- }, flags.EventuallyTimeout).Should(Equal([]tsdb.BlockState{
- {
- ID: tsdb.BlockID{
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700101),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010100),
- },
- TimeRange: timestamp.NewTimeRangeDuration(t1, 12*time.Hour, true, false),
- },
- {
- ID: tsdb.BlockID{
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700101),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010112),
- },
- TimeRange: timestamp.NewTimeRangeDuration(t2, 12*time.Hour, true, false),
- },
- {
- ID: tsdb.BlockID{
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700102),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010200),
- },
- TimeRange: timestamp.NewTimeRangeDuration(t3, 12*time.Hour, true, false),
- },
- {
- ID: tsdb.BlockID{
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700102),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010212),
- },
- TimeRange: timestamp.NewTimeRangeDuration(t4, 12*time.Hour, true, false),
- },
- {
- ID: tsdb.BlockID{
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700103),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010300),
- },
- TimeRange: timestamp.NewTimeRangeDuration(t5, 12*time.Hour, true, false),
- },
- }))
- By("01/03 01:00 close 1st block by adding 5th block")
- forward(2, "BlockID-19700102-1970010212-1", "SegID-19700102-1")
- started("BlockID-19700103-1970010300-1", "SegID-19700103-1")
- Eventually(func() []tsdb.BlockID {
- return shard.State().OpenBlocks
- }, flags.EventuallyTimeout).Should(Equal([]tsdb.BlockID{
- {
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700101),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010112),
- },
- {
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700102),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010200),
- },
- {
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700102),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010212),
- },
- }))
- Eventually(func() []tsdb.BlockState {
- return shard.State().Blocks
- }, flags.EventuallyTimeout).Should(Equal([]tsdb.BlockState{
- {
- ID: tsdb.BlockID{
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700101),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010100),
- },
- TimeRange: timestamp.NewTimeRangeDuration(t1, 12*time.Hour, true, false),
- Closed: true,
- },
- {
- ID: tsdb.BlockID{
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700101),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010112),
- },
- TimeRange: timestamp.NewTimeRangeDuration(t2, 12*time.Hour, true, false),
- },
- {
- ID: tsdb.BlockID{
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700102),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010200),
- },
- TimeRange: timestamp.NewTimeRangeDuration(t3, 12*time.Hour, true, false),
- },
- {
- ID: tsdb.BlockID{
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700102),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010212),
- },
- TimeRange: timestamp.NewTimeRangeDuration(t4, 12*time.Hour, true, false),
- },
- {
- ID: tsdb.BlockID{
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700103),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010300),
- },
- TimeRange: timestamp.NewTimeRangeDuration(t5, 12*time.Hour, true, false),
- },
- }))
- By("reopen 1st block")
- series, err := shard.Series().GetByID(common.SeriesID(11))
- Expect(err).NotTo(HaveOccurred())
- t1Range := timestamp.NewInclusiveTimeRangeDuration(t1, 1*time.Hour)
- span, err := series.Span(context.Background(), t1Range)
- Expect(err).NotTo(HaveOccurred())
- defer span.Close()
- writer, err := span.WriterBuilder().Family([]byte("test"), []byte("test")).Time(t1Range.End).Build()
- Expect(err).NotTo(HaveOccurred())
- _, err = writer.Write()
- Expect(err).NotTo(HaveOccurred())
- Eventually(func() []tsdb.BlockID {
- return shard.State().OpenBlocks
- }, flags.EventuallyTimeout).Should(Equal([]tsdb.BlockID{
- {
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700101),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010100),
- },
- {
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700102),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010200),
- },
- {
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700102),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010212),
- },
- }))
- Eventually(func() []tsdb.BlockState {
- return shard.State().Blocks
- }, flags.EventuallyTimeout).Should(Equal([]tsdb.BlockState{
- {
- ID: tsdb.BlockID{
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700101),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010100),
- },
- TimeRange: timestamp.NewTimeRangeDuration(t1, 12*time.Hour, true, false),
- },
- {
- ID: tsdb.BlockID{
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700101),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010112),
- },
- TimeRange: timestamp.NewTimeRangeDuration(t2, 12*time.Hour, true, false),
- Closed: true,
- },
- {
- ID: tsdb.BlockID{
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700102),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010200),
- },
- TimeRange: timestamp.NewTimeRangeDuration(t3, 12*time.Hour, true, false),
- },
- {
- ID: tsdb.BlockID{
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700102),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010212),
- },
- TimeRange: timestamp.NewTimeRangeDuration(t4, 12*time.Hour, true, false),
- },
- {
- ID: tsdb.BlockID{
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700103),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010300),
- },
- TimeRange: timestamp.NewTimeRangeDuration(t5, 12*time.Hour, true, false),
- },
- }))
- })
- It("retention", func() {
- var err error
- shard, err = tsdb.OpenShard(context.WithValue(timestamp.SetClock(context.Background(), clock),
- tsdb.OptionsKey, tsdb.DatabaseOpts{
- TSTableFactory: tsdb.NewByPassTSTableFactory(),
- }), common.ShardID(0), tmp,
- tsdb.IntervalRule{
- Unit: tsdb.DAY,
- Num: 1,
- },
- tsdb.IntervalRule{
- Unit: tsdb.HOUR,
- Num: 12,
- },
- tsdb.IntervalRule{
- Unit: tsdb.DAY,
- Num: 1,
- },
- 10,
- 15,
- false,
- )
- Expect(err).NotTo(HaveOccurred())
- started("BlockID-19700101-1970010100-1", "SegID-19700101-1", "retention")
- By("01/01 00:00 1st block is opened")
- t1 := clock.Now()
- By("01/01 11:00 2nd block is opened")
- forward(11, "BlockID-19700101-1970010100-1", "SegID-19700101-1", "retention")
- t2 := clock.Now().Add(1 * time.Hour)
- By("01/01 13:00 moves to the 2nd block")
- forward(2, "BlockID-19700101-1970010100-1", "SegID-19700101-1", "retention")
- started("BlockID-19700101-1970010112-1", "retention")
- By("01/01 23:00 3rd block is opened")
- forward(10, "BlockID-19700101-1970010112-1", "SegID-19700101-1", "retention")
- t3 := clock.Now().Add(1 * time.Hour)
- By("01/02 01:00 moves to 3rd block")
- forward(2, "BlockID-19700101-1970010112-1", "SegID-19700101-1", "retention")
- started("BlockID-19700102-1970010200-1", "SegID-19700102-1", "retention")
- By("01/02 11:00 4th block is opened")
- forward(10, "BlockID-19700102-1970010200-1", "SegID-19700102-1", "retention")
- t4 := clock.Now().Add(1 * time.Hour)
-
- Eventually(func() []tsdb.BlockState {
- return shard.State().Blocks
- }, flags.EventuallyTimeout).Should(Equal([]tsdb.BlockState{
- {
- ID: tsdb.BlockID{
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700101),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010100),
- },
- TimeRange: timestamp.NewTimeRangeDuration(t1, 12*time.Hour, true, false),
- },
- {
- ID: tsdb.BlockID{
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700101),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010112),
- },
- TimeRange: timestamp.NewTimeRangeDuration(t2, 12*time.Hour, true, false),
- },
- {
- ID: tsdb.BlockID{
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700102),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010200),
- },
- TimeRange: timestamp.NewTimeRangeDuration(t3, 12*time.Hour, true, false),
- },
- {
- ID: tsdb.BlockID{
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700102),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010212),
- },
- TimeRange: timestamp.NewTimeRangeDuration(t4, 12*time.Hour, true, false),
- },
- }))
- By("01/02 13:00 moves to 4th block")
- forward(2, "BlockID-19700102-1970010200-1", "SegID-19700102-1", "retention")
- started("BlockID-19700102-1970010212-1", "retention")
- By("01/02 23:00 5th block is opened")
- forward(10, "BlockID-19700102-1970010212-1", "SegID-19700102-1", "retention")
- t5 := clock.Now().Add(1 * time.Hour)
- By("01/03 01:00 close 1st block by adding 5th block")
- forward(2, "BlockID-19700102-1970010212-1", "SegID-19700102-1", "retention")
- started("BlockID-19700103-1970010300-1", "SegID-19700103-1", "retention")
- Eventually(func() []tsdb.BlockState {
- started("retention")
- return shard.State().Blocks
- }, flags.EventuallyTimeout).Should(Equal([]tsdb.BlockState{
- {
- ID: tsdb.BlockID{
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700102),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010200),
- },
- TimeRange: timestamp.NewTimeRangeDuration(t3, 12*time.Hour, true, false),
- },
- {
- ID: tsdb.BlockID{
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700102),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010212),
- },
- TimeRange: timestamp.NewTimeRangeDuration(t4, 12*time.Hour, true, false),
- },
- {
- ID: tsdb.BlockID{
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700103),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010300),
- },
- TimeRange: timestamp.NewTimeRangeDuration(t5, 12*time.Hour, true, false),
- },
- }))
- Eventually(func() []tsdb.BlockID {
- return shard.State().OpenBlocks
- }, flags.EventuallyTimeout).Should(Equal([]tsdb.BlockID{
- {
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700102),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010200),
- },
- {
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700102),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010212),
- },
- }))
- })
- It("creates arbitrary blocks", func() {
- clock.Set(time.Date(1970, 0o1, 0o1, 1, 0, 0, 0, time.Local))
- By("open 1 block")
- var err error
- shard, err = tsdb.OpenShard(context.WithValue(timestamp.SetClock(context.Background(), clock),
- tsdb.OptionsKey, tsdb.DatabaseOpts{
- TSTableFactory: tsdb.NewByPassTSTableFactory(),
- }), common.ShardID(0), tmp,
- tsdb.IntervalRule{
- Unit: tsdb.DAY,
- Num: 1,
- },
- tsdb.IntervalRule{
- Unit: tsdb.HOUR,
- Num: 12,
- },
- tsdb.IntervalRule{
- Unit: tsdb.DAY,
- Num: 7,
- },
- 2,
- 3,
- false,
- )
- Expect(err).NotTo(HaveOccurred())
- started("BlockID-19700101-1970010101-1", "SegID-19700101-1", "retention")
- By("01/01 00:01 1st block is opened")
- t1 := clock.Now()
- Eventually(func() []tsdb.BlockState {
- return shard.State().Blocks
- }, flags.EventuallyTimeout).Should(Equal([]tsdb.BlockState{
- {
- ID: tsdb.BlockID{
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700101),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010101),
- },
- TimeRange: timestamp.NewTimeRangeDuration(t1, 12*time.Hour, true, false),
- },
- }))
- By("01/01 12:00 2nd block is opened")
- forward(11, "BlockID-19700101-1970010101-1", "SegID-19700101-1")
- t2 := clock.Now().Add(1 * time.Hour)
- By("01/01 14:00 moves to the 2nd block")
- forward(2, "BlockID-19700101-1970010101-1", "SegID-19700101-1")
- started("BlockID-19700101-1970010113-1")
- Eventually(func() []tsdb.BlockState {
- return shard.State().Blocks
- }, flags.EventuallyTimeout).Should(Equal([]tsdb.BlockState{
- {
- ID: tsdb.BlockID{
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700101),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010101),
- },
- TimeRange: timestamp.NewTimeRangeDuration(t1, 12*time.Hour, true, false),
- },
- {
- ID: tsdb.BlockID{
- SegID: tsdb.GenerateInternalID(tsdb.DAY, 19700101),
- BlockID: tsdb.GenerateInternalID(tsdb.HOUR, 1970010113),
- },
- // The last block only takes 11 hours to align the segment's size
- TimeRange: timestamp.NewTimeRangeDuration(t2, 11*time.Hour, true, false),
- },
- }))
- })
- })
-})
diff --git a/banyand/tsdb/tsdb.go b/banyand/tsdb/tsdb.go
deleted file mode 100644
index e8b8a311..00000000
--- a/banyand/tsdb/tsdb.go
+++ /dev/null
@@ -1,404 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-// Package tsdb implements a time-series-based storage engine.
-// It provides:
-// - Partition data based on a time axis.
-// - Sharding data based on a series id which represents a unique entity of stream/measure
-// - Retrieving data based on index.Filter.
-// - Cleaning expired data, or the data retention.
-package tsdb
-
-import (
- "context"
- "fmt"
- "io"
- "os"
- "strconv"
- "strings"
- "sync"
- "sync/atomic"
-
- "github.com/pkg/errors"
- "go.uber.org/multierr"
-
- "github.com/apache/skywalking-banyandb/api/common"
- "github.com/apache/skywalking-banyandb/banyand/observability"
- "github.com/apache/skywalking-banyandb/pkg/convert"
- "github.com/apache/skywalking-banyandb/pkg/encoding"
- "github.com/apache/skywalking-banyandb/pkg/logger"
- "github.com/apache/skywalking-banyandb/pkg/run"
- "github.com/apache/skywalking-banyandb/pkg/timestamp"
-)
-
-const (
- shardPathPrefix = "shard"
- pathSeparator = string(os.PathSeparator)
- rootPrefix = "%s" + pathSeparator
- shardTemplate = rootPrefix + shardPathPrefix + "-%d"
- seriesTemplate = rootPrefix + "series"
- segPathPrefix = "seg"
- segTemplate = rootPrefix + segPathPrefix + "-%s"
- blockPathPrefix = "block"
- blockTemplate = rootPrefix + blockPathPrefix + "-%s"
- globalIndexTemplate = rootPrefix + "index"
-
- hourFormat = "2006010215"
- dayFormat = "20060102"
-
- dirPerm = 0o700
-)
-
-var (
- // ErrUnknownShard indicates that the shard is not found.
- ErrUnknownShard = errors.New("unknown shard")
- errOpenDatabase = errors.New("fails to open the database")
-
- // OptionsKey is the key of options in context.
- OptionsKey = contextOptionsKey{}
- meterStorage = observability.RootScope.SubScope("storage")
- meterTSDB = meterStorage.SubScope("tsdb")
-)
-
-type contextOptionsKey struct{}
-
-// Supplier allows getting a tsdb's runtime.
-type Supplier interface {
- SupplyTSDB() Database
-}
-
-// Database allows listing and getting shard details.
-type Database interface {
- io.Closer
- CreateShardsAndGetByID(id common.ShardID) (Shard, error)
- Shards() []Shard
- Shard(id common.ShardID) (Shard, error)
-}
-
-// Shard allows accessing data of tsdb.
-type Shard interface {
- io.Closer
- ID() common.ShardID
- Series() SeriesDatabase
- Index() IndexDatabase
- State() ShardState
- // Only works with MockClock
- TriggerSchedule(task string) bool
-}
-
-// IndexGranularity denotes the granularity of the local index.
-type IndexGranularity int
-
-// The options of the local index granularity.
-const (
- IndexGranularityBlock IndexGranularity = iota
- IndexGranularitySeries
-)
-
-var _ Database = (*database)(nil)
-
-// DatabaseOpts wraps options to create a tsdb.
-type DatabaseOpts struct {
- TSTableFactory TSTableFactory
- Location string
- SegmentInterval IntervalRule
- BlockInterval IntervalRule
- TTL IntervalRule
- BlockInvertedIndex InvertedIndexOpts
- SeriesMemSize run.Bytes
- GlobalIndexMemSize run.Bytes
- IndexGranularity IndexGranularity
- ShardNum uint32
- EnableGlobalIndex bool
- EnableWAL bool
-}
-
-// InvertedIndexOpts wraps options to create the block inverted index.
-type InvertedIndexOpts struct {
- BatchWaitSec int64
-}
-
-// EncodingMethod wraps encoder/decoder pools to flush/compact data on disk.
-type EncodingMethod struct {
- EncoderPool encoding.SeriesEncoderPool
- DecoderPool encoding.SeriesDecoderPool
- ChunkSizeInBytes int
-}
-
-// CompressionMethod denotes how to compress a single chunk.
-type CompressionMethod struct {
- Type CompressionType
- ChunkSizeInBytes int
-}
-
-// CompressionType specifies how a chunk should be compressed.
-type CompressionType int
-
-const (
- // CompressionTypeNone mode indicates that a chunk is not compressed.
- CompressionTypeNone CompressionType = iota
- // CompressionTypeZSTD mode indicates that a chunk is compressed using CompressionTypeZSTD algorithm.
- CompressionTypeZSTD
-)
-
-type (
- // SectionID is the kind of a block/segment.
- SectionID uint32
-
- // BlockID is the identity of a block in a shard.
- BlockID struct {
- SegID SectionID
- BlockID SectionID
- }
-)
-
-func (b BlockID) String() string {
- return fmt.Sprintf("BlockID-%d-%d", parseSuffix(b.SegID), parseSuffix(b.BlockID))
-}
-
-// GenerateInternalID returns a identity of a section(segment or block) based on IntervalRule.
-func GenerateInternalID(unit IntervalUnit, suffix int) SectionID {
- return SectionID(unit)<<31 | ((SectionID(suffix) << 1) >> 1)
-}
-
-func parseSuffix(id SectionID) int {
- return int((id << 1) >> 1)
-}
-
-func sectionIDToBytes(id SectionID) []byte {
- return convert.Uint32ToBytes(uint32(id))
-}
-
-func readSectionID(data []byte, offset int) (SectionID, int) {
- end := offset + 4
- return SectionID(convert.BytesToUint32(data[offset:end])), end
-}
-
-// BlockState is a sample of a block's runtime state.
-type BlockState struct {
- TimeRange timestamp.TimeRange
- ID BlockID
- Closed bool
-}
-
-// ShardState is a sample of a shard's runtime state.
-type ShardState struct {
- Blocks []BlockState
- OpenBlocks []BlockID
- StrategyManagers []string
-}
-
-type database struct {
- shardCreationCtx context.Context
- logger *logger.Logger
- location string
- sLst []Shard
- segmentSize IntervalRule
- blockSize IntervalRule
- ttl IntervalRule
- sync.RWMutex
- shardNum uint32
- shardCreationState uint32
- enableWAL bool
-}
-
-func (d *database) CreateShardsAndGetByID(id common.ShardID) (Shard, error) {
- if atomic.LoadUint32(&d.shardCreationState) != 0 {
- return d.shard(id)
- }
- d.Lock()
- defer d.Unlock()
- if atomic.LoadUint32(&d.shardCreationState) != 0 {
- return d.shard(id)
- }
- loadedShardsNum := len(d.sLst)
- if loadedShardsNum < int(d.shardNum) {
- _, err := createDatabase(d, loadedShardsNum)
- if err != nil {
- return nil, errors.WithMessage(err, "create the database failed")
- }
- }
- atomic.StoreUint32(&d.shardCreationState, 1)
- return d.shard(id)
-}
-
-func (d *database) Shards() []Shard {
- d.RLock()
- defer d.RUnlock()
- return d.sLst
-}
-
-func (d *database) Shard(id common.ShardID) (Shard, error) {
- d.RLock()
- defer d.RUnlock()
- return d.shard(id)
-}
-
-func (d *database) shard(id common.ShardID) (Shard, error) {
- if uint(id) >= uint(len(d.sLst)) {
- return nil, ErrUnknownShard
- }
- return d.sLst[id], nil
-}
-
-func (d *database) Close() error {
- d.Lock()
- defer d.Unlock()
- var err error
- for _, s := range d.sLst {
- innerErr := s.Close()
- if innerErr != nil {
- err = multierr.Append(err, innerErr)
- }
- }
- return err
-}
-
-// OpenDatabase returns a new tsdb runtime. This constructor will create a new database if it's absent,
-// or load an existing one.
-func OpenDatabase(ctx context.Context, opts DatabaseOpts) (Database, error) {
- if _, err := mkdir(opts.Location); err != nil {
- return nil, err
- }
- if opts.SegmentInterval.Num == 0 {
- return nil, errors.Wrap(errOpenDatabase, "segment interval is absent")
- }
- if opts.BlockInterval.Num == 0 {
- return nil, errors.Wrap(errOpenDatabase, "block interval is absent")
- }
- if opts.BlockInterval.estimatedDuration() > opts.SegmentInterval.estimatedDuration() {
- return nil, errors.Wrapf(errOpenDatabase, "the block size is bigger than the segment size")
- }
- if opts.TTL.Num == 0 {
- return nil, errors.Wrap(errOpenDatabase, "ttl is absent")
- }
- p := common.GetPosition(ctx)
- db := &database{
- location: opts.Location,
- shardNum: opts.ShardNum,
- logger: logger.Fetch(ctx, p.Database),
- segmentSize: opts.SegmentInterval,
- blockSize: opts.BlockInterval,
- ttl: opts.TTL,
- enableWAL: opts.EnableWAL,
- }
- db.logger.Info().Str("path", opts.Location).Msg("initialized")
- var entries []os.DirEntry
- var err error
- if entries, err = os.ReadDir(opts.Location); err != nil {
- return nil, errors.Wrap(err, "failed to read directory contents failed")
- }
- thisContext := context.WithValue(ctx, logger.ContextKey, db.logger)
- thisContext = context.WithValue(thisContext, OptionsKey, opts)
- db.shardCreationCtx = thisContext
- if len(entries) > 0 {
- return loadDatabase(thisContext, db)
- }
- return db, nil
-}
-
-func createDatabase(db *database, startID int) (Database, error) {
- var err error
- for i := startID; i < int(db.shardNum); i++ {
- db.logger.Info().Int("shard_id", i).Msg("creating a shard")
- so, errNewShard := OpenShard(db.shardCreationCtx, common.ShardID(i),
- db.location, db.segmentSize, db.blockSize, db.ttl, defaultBlockQueueSize, defaultMaxBlockQueueSize, db.enableWAL)
- if errNewShard != nil {
- err = multierr.Append(err, errNewShard)
- continue
- }
- db.sLst = append(db.sLst, so)
- }
- return db, err
-}
-
-func loadDatabase(ctx context.Context, db *database) (Database, error) {
- // TODO: open the lock file
- // TODO: open the manifest file
- db.Lock()
- defer db.Unlock()
- err := walkDir(db.location, shardPathPrefix, func(suffix string) error {
- shardID, err := strconv.Atoi(suffix)
- if err != nil {
- return err
- }
- if shardID >= int(db.shardNum) {
- return nil
- }
- db.logger.Info().Int("shard_id", shardID).Msg("opening a existing shard")
- so, errOpenShard := OpenShard(
- context.WithValue(ctx, logger.ContextKey, db.logger),
- common.ShardID(shardID),
- db.location,
- db.segmentSize,
- db.blockSize,
- db.ttl,
- defaultBlockQueueSize,
- defaultMaxBlockQueueSize,
- db.enableWAL,
- )
- if errOpenShard != nil {
- return errOpenShard
- }
- db.sLst = append(db.sLst, so)
- return nil
- })
- if err != nil {
- return nil, errors.WithMessage(err, "load the database failed")
- }
- return db, nil
-}
-
-type walkFn func(suffix string) error
-
-func walkDir(root, prefix string, wf walkFn) error {
- files, err := os.ReadDir(root)
- if err != nil {
- return errors.Wrapf(err, "failed to walk the database path: %s", root)
- }
- for _, f := range files {
- if !f.IsDir() || !strings.HasPrefix(f.Name(), prefix) {
- continue
- }
- segs := strings.Split(f.Name(), "-")
- errWalk := wf(segs[len(segs)-1])
- if errWalk != nil {
- return errors.WithMessagef(errWalk, "failed to load: %s", f.Name())
- }
- }
- return nil
-}
-
-func mkdir(format string, a ...interface{}) (path string, err error) {
- path = fmt.Sprintf(format, a...)
- if err = os.MkdirAll(path, dirPerm); err != nil {
- return "", errors.Wrapf(err, "failed to create %s", path)
- }
- return path, err
-}
-
-func mkdirIfNotExist(format string, a ...interface{}) (err error) {
- path := fmt.Sprintf(format, a...)
- if _, err = os.Stat(path); errors.Is(err, os.ErrNotExist) {
- if err = os.MkdirAll(path, dirPerm); err != nil {
- return errors.Wrapf(err, "failed to create %s", path)
- }
- } else {
- return os.ErrExist
- }
- return err
-}
diff --git a/banyand/tsdb/tsdb_suite_test.go b/banyand/tsdb/tsdb_suite_test.go
deleted file mode 100644
index 5456e59f..00000000
--- a/banyand/tsdb/tsdb_suite_test.go
+++ /dev/null
@@ -1,39 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package tsdb_test
-
-import (
- "testing"
-
- . "github.com/onsi/ginkgo/v2"
- . "github.com/onsi/gomega"
-
- "github.com/apache/skywalking-banyandb/pkg/logger"
- "github.com/apache/skywalking-banyandb/pkg/test/flags"
-)
-
-func TestTsdb(t *testing.T) {
- RegisterFailHandler(Fail)
- RunSpecs(t, "Tsdb Suite", Label("banyand"))
-}
-
-var _ = BeforeSuite(func() {
- Expect(logger.Init(logger.Logging{
- Env: "dev",
- Level: flags.LogLevel,
- })).Should(Succeed())
-})
diff --git a/banyand/tsdb/tsdb_test.go b/banyand/tsdb/tsdb_test.go
deleted file mode 100644
index 8c2dbd71..00000000
--- a/banyand/tsdb/tsdb_test.go
+++ /dev/null
@@ -1,177 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package tsdb
-
-import (
- "context"
- "fmt"
- "os"
- "sync"
- "testing"
- "time"
-
- "github.com/dgraph-io/badger/v3"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-
- "github.com/apache/skywalking-banyandb/api/common"
- "github.com/apache/skywalking-banyandb/pkg/logger"
- "github.com/apache/skywalking-banyandb/pkg/test"
- "github.com/apache/skywalking-banyandb/pkg/test/flags"
- "github.com/apache/skywalking-banyandb/pkg/timestamp"
-)
-
-func TestOpenDatabase(t *testing.T) {
- tester := assert.New(t)
- req := require.New(t)
- tempDir, deferFunc := test.Space(req)
- db := openDatabase(context.Background(), req, tempDir)
- defer func() {
- req.NoError(db.Close())
- deferFunc()
- }()
- verifyDatabaseStructure(tester, tempDir, time.Now())
-}
-
-func TestReOpenDatabase(t *testing.T) {
- tester := assert.New(t)
- req := require.New(t)
- tempDir, deferFunc := test.Space(req)
- defer deferFunc()
- db := openDatabase(context.Background(), req, tempDir)
- req.NoError(db.Close())
- verifyDatabaseStructure(tester, tempDir, time.Now())
- db = openDatabase(context.Background(), req, tempDir)
- req.NoError(db.Close())
- verifyDatabaseStructure(tester, tempDir, time.Now())
-}
-
-func TestReOpenDatabaseNextBlock(t *testing.T) {
- tester := assert.New(t)
- req := require.New(t)
- tempDir, deferFunc := test.Space(req)
- defer deferFunc()
- clock := timestamp.NewMockClock()
- clock.Set(time.Date(1970, 0o1, 0o1, 0, 0, 0, 0, time.Local))
- db := openDatabase(timestamp.SetClock(context.Background(), clock), req, tempDir)
- req.NoError(db.Close())
- verifyDatabaseStructure(tester, tempDir, clock.Now())
- clock.Add(5 * time.Hour)
- db = openDatabase(timestamp.SetClock(context.Background(), clock), req, tempDir)
- req.NoError(db.Close())
- verifyDatabaseStructure(tester, tempDir, clock.Now())
-}
-
-func TestReOpenDatabaseNextDay(t *testing.T) {
- tester := assert.New(t)
- req := require.New(t)
- tempDir, deferFunc := test.Space(req)
- defer deferFunc()
- clock := timestamp.NewMockClock()
- clock.Set(time.Date(1970, 0o1, 0o1, 0, 0, 0, 0, time.Local))
- db := openDatabase(timestamp.SetClock(context.Background(), clock), req, tempDir)
- req.NoError(db.Close())
- verifyDatabaseStructure(tester, tempDir, clock.Now())
- clock.Add(26 * time.Hour)
- db = openDatabase(timestamp.SetClock(context.Background(), clock), req, tempDir)
- req.NoError(db.Close())
- verifyDatabaseStructure(tester, tempDir, clock.Now())
-}
-
-func verifyDatabaseStructure(tester *assert.Assertions, tempDir string, now time.Time) {
- shardPath := fmt.Sprintf(shardTemplate, tempDir, 0)
- validateDirectory(tester, shardPath)
- seriesPath := fmt.Sprintf(seriesTemplate, shardPath)
- validateDirectory(tester, seriesPath)
- segPath := fmt.Sprintf(segTemplate, shardPath, now.Format(dayFormat))
- validateDirectory(tester, segPath)
- validateDirectory(tester, fmt.Sprintf(blockTemplate, segPath, now.Format(hourFormat)))
-}
-
-func openDatabase(ctx context.Context, t *require.Assertions, path string) (db Database) {
- t.NoError(logger.Init(logger.Logging{
- Env: "dev",
- Level: flags.LogLevel,
- }))
- db, err := OpenDatabase(
- context.WithValue(ctx, logger.ContextKey, logger.GetLogger("test")),
- DatabaseOpts{
- Location: path,
- ShardNum: 1,
- BlockInterval: IntervalRule{Num: 2},
- SegmentInterval: IntervalRule{Num: 1, Unit: DAY},
- TTL: IntervalRule{Num: 7, Unit: DAY},
- TSTableFactory: bypassTSTableFactory{},
- })
- t.NoError(err)
- t.NotNil(db)
- shard, err := db.CreateShardsAndGetByID(0)
- t.NoError(err)
- t.Equal(common.ShardID(0), shard.ID())
- return db
-}
-
-func validateDirectory(t *assert.Assertions, dir string) {
- info, err := os.Stat(dir)
- t.False(os.IsNotExist(err), "Directory does not exist: %v", dir)
- t.NoError(err, "Directory error: %v", dir)
- t.True(info.IsDir(), "Directory is a file, not a directory: %#v\n", dir)
-}
-
-// bypassTSTableFactory is a bypass implementation of TSTableFactory.
-type bypassTSTableFactory struct{}
-
-// NewByPassTSTableFactory creates a bypass implementation of TSTableFactory.
-func NewByPassTSTableFactory() TSTableFactory {
- return bypassTSTableFactory{}
-}
-
-func (bypassTSTableFactory) NewTSTable(_ *BufferSupplier, _ string, _ common.Position, _ *logger.Logger, _ timestamp.TimeRange) (TSTable, error) {
- return newBypassTSTable()
-}
-
-// bypassTSTable is a bypass implementation of TSTable.
-type bypassTSTable struct{}
-
-func newBypassTSTable() (TSTable, error) {
- return &bypassTSTable{}, nil
-}
-
-func (b *bypassTSTable) OpenBuffer() error {
- return nil
-}
-
-func (b *bypassTSTable) Put(_, _ []byte, _ time.Time) error {
- return nil
-}
-
-func (b *bypassTSTable) Get(_ []byte, _ time.Time) ([]byte, error) {
- return nil, nil
-}
-
-func (b *bypassTSTable) CollectStats() *badger.Statistics {
- return &badger.Statistics{TableBuilderSize: &sync.Map{}}
-}
-
-func (b *bypassTSTable) SizeOnDisk() int64 {
- return 0
-}
-
-func (b *bypassTSTable) Close() error {
- return nil
-}
diff --git a/banyand/tsdb/tstable.go b/banyand/tsdb/tstable.go
deleted file mode 100644
index d23a5b8c..00000000
--- a/banyand/tsdb/tstable.go
+++ /dev/null
@@ -1,49 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package tsdb
-
-import (
- "io"
- "time"
-
- "github.com/dgraph-io/badger/v3"
-
- "github.com/apache/skywalking-banyandb/api/common"
- "github.com/apache/skywalking-banyandb/pkg/logger"
- "github.com/apache/skywalking-banyandb/pkg/timestamp"
-)
-
-// TSTable is time series table.
-type TSTable interface {
- // Put a value with a timestamp/version
- Put(key, val []byte, ts time.Time) error
- // Get a value by its key and timestamp/version
- Get(key []byte, ts time.Time) ([]byte, error)
- // CollectStats collects statistics of the underlying storage.
- CollectStats() *badger.Statistics
- // SizeOnDisk returns the size of the underlying storage.
- SizeOnDisk() int64
- io.Closer
-}
-
-// TSTableFactory is the factory of TSTable.
-type TSTableFactory interface {
- // NewTSTable creates a new TSTable.
- NewTSTable(bufferSupplier *BufferSupplier, root string, position common.Position,
- l *logger.Logger, timeRange timestamp.TimeRange) (TSTable, error)
-}
diff --git a/docs/api-reference.md b/docs/api-reference.md
index f61a6f80..a4e2df3f 100644
--- a/docs/api-reference.md
+++ b/docs/api-reference.md
@@ -67,7 +67,6 @@
- [EncodingMethod](#banyandb-database-v1-EncodingMethod)
- [FieldType](#banyandb-database-v1-FieldType)
- [IndexRule.Analyzer](#banyandb-database-v1-IndexRule-Analyzer)
- - [IndexRule.Location](#banyandb-database-v1-IndexRule-Location)
- [IndexRule.Type](#banyandb-database-v1-IndexRule-Type)
- [TagType](#banyandb-database-v1-TagType)
@@ -330,7 +329,7 @@ Metadata is for multi-tenant, multi-model use
| ----- | ---- | ----- | ----------- |
| group | [string](#string) | | group contains a set of options, like retention policy, max |
| name | [string](#string) | | name of the entity |
-| id | [uint32](#uint32) | | |
+| id | [uint32](#uint32) | | id is the unique identifier of the entity if id is not set, the system will generate a unique id |
| create_revision | [int64](#int64) | | readonly. create_revision is the revision of last creation on this key. |
| mod_revision | [int64](#int64) | | readonly. mod_revision is the revision of last modification on this key. |
@@ -348,7 +347,6 @@ Metadata is for multi-tenant, multi-model use
| Field | Type | Label | Description |
| ----- | ---- | ----- | ----------- |
| shard_num | [uint32](#uint32) | | shard_num is the number of shards |
-| block_interval | [IntervalRule](#banyandb-common-v1-IntervalRule) | | block_interval indicates the length of a block block_interval should be less than or equal to segment_interval |
| segment_interval | [IntervalRule](#banyandb-common-v1-IntervalRule) | | segment_interval indicates the length of a segment |
| ttl | [IntervalRule](#banyandb-common-v1-IntervalRule) | | ttl indicates time to live, how long the data will be cached |
@@ -897,7 +895,6 @@ IndexRule should bind to a subject through an IndexRuleBinding to generate prope
| metadata | [banyandb.common.v1.Metadata](#banyandb-common-v1-Metadata) | | metadata define the rule's identity |
| tags | [string](#string) | repeated | tags are the combination that refers to an indexed object If the elements in tags are more than 1, the object will generate a multi-tag index Caveat: All tags in a multi-tag MUST have an identical IndexType |
| type | [IndexRule.Type](#banyandb-database-v1-IndexRule-Type) | | type is the IndexType of this IndexObject. |
-| location | [IndexRule.Location](#banyandb-database-v1-IndexRule-Location) | | location indicates where to store index. |
| updated_at | [google.protobuf.Timestamp](#google-protobuf-Timestamp) | | updated_at indicates when the IndexRule is updated |
| analyzer | [IndexRule.Analyzer](#banyandb-database-v1-IndexRule-Analyzer) | | analyzer analyzes tag value to support the full-text searching for TYPE_INVERTED indices. |
@@ -1093,19 +1090,6 @@ TopNAggregation generates offline TopN statistics for a measure's TopN appro
-<a name="banyandb-database-v1-IndexRule-Location"></a>
-
-### IndexRule.Location
-
-
-| Name | Number | Description |
-| ---- | ------ | ----------- |
-| LOCATION_UNSPECIFIED | 0 | |
-| LOCATION_SERIES | 1 | |
-| LOCATION_GLOBAL | 2 | |
-
-
-
<a name="banyandb-database-v1-IndexRule-Type"></a>
### IndexRule.Type
@@ -1114,8 +1098,7 @@ Type determine the index structure under the hood
| Name | Number | Description |
| ---- | ------ | ----------- |
| TYPE_UNSPECIFIED | 0 | |
-| TYPE_TREE | 1 | |
-| TYPE_INVERTED | 2 | |
+| TYPE_INVERTED | 1 | |
diff --git a/go.mod b/go.mod
index 607bef95..a9d0243b 100644
--- a/go.mod
+++ b/go.mod
@@ -8,7 +8,6 @@ require (
github.com/benbjohnson/clock v1.3.0
github.com/blugelabs/bluge v0.2.2
github.com/cespare/xxhash v1.1.0
- github.com/dgraph-io/badger/v3 v3.2011.1
github.com/emirpasic/gods v1.18.1
github.com/envoyproxy/protoc-gen-validate v1.0.2
github.com/go-chi/chi/v5 v5.0.11
@@ -53,6 +52,7 @@ require (
github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect
github.com/machinebox/graphql v0.2.2 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
+ github.com/spaolacci/murmur3 v1.1.0 // indirect
)
require (
@@ -73,7 +73,6 @@ require (
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/dchest/siphash v1.2.3 // indirect
- github.com/dgraph-io/ristretto v0.1.1 // indirect
github.com/dgryski/go-metro v0.0.0-20211217172704-adc40b04c140 // indirect
github.com/dustin/go-humanize v1.0.1
github.com/fsnotify/fsnotify v1.7.0 // indirect
@@ -83,12 +82,9 @@ require (
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
- github.com/golang/glog v1.2.0 // indirect
- github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/golang/snappy v0.0.4
github.com/google/btree v1.1.2 // indirect
- github.com/google/flatbuffers v23.5.26+incompatible // indirect
github.com/google/pprof v0.0.0-20231212022811-ec68065c825e // indirect
github.com/gorilla/websocket v1.5.1 // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
@@ -136,7 +132,6 @@ require (
go.etcd.io/etcd/client/v2 v2.305.11 // indirect
go.etcd.io/etcd/pkg/v3 v3.5.11 // indirect
go.etcd.io/etcd/raft/v3 v3.5.11 // indirect
- go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 // indirect
go.opentelemetry.io/otel v1.21.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 // indirect
@@ -164,5 +159,4 @@ replace (
github.com/blugelabs/bluge => github.com/zinclabs/bluge v1.1.5
github.com/blugelabs/bluge_segment_api => github.com/zinclabs/bluge_segment_api v1.0.0
github.com/blugelabs/ice => github.com/zinclabs/ice v1.1.3
- github.com/dgraph-io/badger/v3 v3.2011.1 => github.com/SkyAPM/badger/v3 v3.0.0-20230511223516-583c6a825854
)
diff --git a/go.sum b/go.sum
index 7847d402..9546f716 100644
--- a/go.sum
+++ b/go.sum
@@ -11,8 +11,6 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAE
github.com/RoaringBitmap/roaring v0.9.4/go.mod h1:icnadbWcNyfEHlYdr+tDlOTih1Bf/h+rzPpv4sbomAA=
github.com/RoaringBitmap/roaring v1.7.0 h1:OZF303tJCER1Tj3x+aArx/S5X7hrT186ri6JjrGvG68=
github.com/RoaringBitmap/roaring v1.7.0/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90=
-github.com/SkyAPM/badger/v3 v3.0.0-20230511223516-583c6a825854 h1:GlNxZHaMREV6V6+T838i4xr8lBS8ekjOfenlJF26Z3U=
-github.com/SkyAPM/badger/v3 v3.0.0-20230511223516-583c6a825854/go.mod h1:4WETftF8A4mEeFgIsYB/VvGo5kfTVl/neYgEqlVW9Ag=
github.com/SkyAPM/clock v1.3.1-0.20220809233656-dc7607c94a97 h1:FKuhJ+6n/DHspGeLleeNbziWnKr9gHKYN4q7NcoCp4s=
github.com/SkyAPM/clock v1.3.1-0.20220809233656-dc7607c94a97/go.mod h1:2xGRl9H1pllhxTbEGO1W3gDkip8P9GQaHPni/wpdR44=
github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
@@ -49,8 +47,6 @@ github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyY
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
-github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
@@ -79,14 +75,9 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8Yc
github.com/dchest/siphash v1.2.2/go.mod h1:q+IRvb2gOSrUnYoPqHiyHXS0FOBBOdl6tONBlVnOnt4=
github.com/dchest/siphash v1.2.3 h1:QXwFc8cFOR2dSa/gE6o/HokBMWtLUaNDVd+22aKHeEA=
github.com/dchest/siphash v1.2.3/go.mod h1:0NvQU092bT0ipiFN++/rXm69QG9tVxLAlQHIXMPAkHc=
-github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8=
-github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA=
-github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
-github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc/go.mod h1:c9O8+fpSOX1DM8cPNSkX/qsBWdkD4yd2dpciOWQjpBw=
github.com/dgryski/go-metro v0.0.0-20211217172704-adc40b04c140 h1:y7y0Oa6UawqTFPCDw9JG6pdKt4F9pAhHv0B7FMGaGD0=
github.com/dgryski/go-metro v0.0.0-20211217172704-adc40b04c140/go.mod h1:c9O8+fpSOX1DM8cPNSkX/qsBWdkD4yd2dpciOWQjpBw=
-github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
@@ -131,15 +122,8 @@ github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOW
github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68=
-github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w=
-github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
-github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
@@ -154,21 +138,15 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU=
github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=
-github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
-github.com/google/flatbuffers v23.5.26+incompatible h1:M9dgRyhJemaM4Sw8+66GHBu8ioaQmyPLg1b8VwK5WJg=
-github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
@@ -214,7 +192,6 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/kkdai/maglev v0.2.0 h1:w6DCW0kAA6fstZqXkrBrlgIC3jeIRXkjOYea/m6EK/Y=
github.com/kkdai/maglev v0.2.0/go.mod h1:d+mt8Lmt3uqi9aRb/BnPjzD0fy+ETs1vVXiGRnqHVZ4=
-github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
github.com/klauspost/compress v1.15.2/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4=
github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
@@ -318,8 +295,9 @@ github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js=
github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0=
github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
-github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
+github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8=
github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY=
@@ -348,7 +326,6 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
@@ -399,9 +376,6 @@ go.etcd.io/etcd/raft/v3 v3.5.11 h1:eeimaNIT9DjV4bdLSy4FjLQ/KGSAiG1L5T1nTf5VoZg=
go.etcd.io/etcd/raft/v3 v3.5.11/go.mod h1:Tp7kZJVtWJWLiMCPrgkimiOB5ZYi8YM93onQihpG724=
go.etcd.io/etcd/server/v3 v3.5.11 h1:FEa0ImvoXdIPa81/vZUKpnJ74fpQ5ZivseoIKMPzfpg=
go.etcd.io/etcd/server/v3 v3.5.11/go.mod h1:CS0+TwcuRlhg1I5CpA3YlisOcoqJB1h1GMRgje75uDs=
-go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
-go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
-go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 h1:SpGay3w+nEwMpfVnbqOLH5gY52/foP8RE8UzTZ1pdSE=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1/go.mod h1:4UoMYEZOC0yN/sPGH76KPkkU7zgiEWYWL9vwmbnTJPE=
go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc=
@@ -469,7 +443,6 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
@@ -487,7 +460,6 @@ golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCA
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -499,7 +471,6 @@ golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -516,7 +487,6 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -575,7 +545,6 @@ google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7
google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM=
google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
@@ -588,13 +557,11 @@ google.golang.org/genproto/googleapis/api v0.0.0-20231212172506-995d672761c0/go.
google.golang.org/genproto/googleapis/rpc v0.0.0-20231212172506-995d672761c0 h1:/jFB8jK5R3Sq3i/lmeZO0cATSzFfZaJq1J2Euan3XKU=
google.golang.org/genproto/googleapis/rpc v0.0.0-20231212172506-995d672761c0/go.mod h1:FUoWkonphQm3RhTS+kOEhF8h0iDpm4tdXolVCeZ9KKA=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
-google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
@@ -615,7 +582,6 @@ google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs
google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
diff --git a/pkg/bytes/buffer.go b/pkg/bytes/buffer.go
index c55c2a33..595ce54c 100644
--- a/pkg/bytes/buffer.go
+++ b/pkg/bytes/buffer.go
@@ -126,3 +126,10 @@ func (bp *BufferPool) Release(b *Buffer) {
b.Reset()
bp.p.Put(b)
}
+
+// Copy copies a to a new slice.
+func Copy(a []byte) []byte {
+ b := make([]byte, len(a))
+ copy(b, a)
+ return b
+}
diff --git a/pkg/index/inverted/inverted.go b/pkg/index/inverted/inverted.go
index 28962031..8707befd 100644
--- a/pkg/index/inverted/inverted.go
+++ b/pkg/index/inverted/inverted.go
@@ -33,11 +33,11 @@ import (
"github.com/blugelabs/bluge/analysis/analyzer"
blugeIndex "github.com/blugelabs/bluge/index"
"github.com/blugelabs/bluge/search"
- "github.com/dgraph-io/badger/v3/y"
"go.uber.org/multierr"
databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+ pbytes "github.com/apache/skywalking-banyandb/pkg/bytes"
"github.com/apache/skywalking-banyandb/pkg/convert"
"github.com/apache/skywalking-banyandb/pkg/index"
"github.com/apache/skywalking-banyandb/pkg/index/posting"
@@ -488,7 +488,7 @@ func (bmi *blugeMatchIterator) nextTerm() bool {
i++
}
if field == bmi.fieldKey {
- v := y.Copy(value)
+ v := pbytes.Copy(value)
if bmi.shouldDecodeTerm {
termRaw = v
term = index.UnmarshalTerm(v)
diff --git a/pkg/index/lsm/iterator.go b/pkg/index/lsm/iterator.go
deleted file mode 100644
index 1fbf082a..00000000
--- a/pkg/index/lsm/iterator.go
+++ /dev/null
@@ -1,219 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package lsm
-
-import (
- "bytes"
- "math"
- "sync"
-
- "go.uber.org/multierr"
-
- modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
- "github.com/apache/skywalking-banyandb/banyand/kv"
- "github.com/apache/skywalking-banyandb/pkg/convert"
- "github.com/apache/skywalking-banyandb/pkg/index"
- "github.com/apache/skywalking-banyandb/pkg/logger"
- "github.com/apache/skywalking-banyandb/pkg/run"
-)
-
-type compositePostingValueFn = func(term, value []byte, delegated kv.Iterator) (*index.PostingValue, error)
-
-var (
- _ index.FieldIterator = (*fieldIteratorTemplate)(nil)
- defaultUpper = convert.Uint64ToBytes(math.MaxUint64)
- defaultLower = convert.Uint64ToBytes(0)
-)
-
-type fieldIteratorTemplate struct {
- err error
- delegated *delegateIterator
- cur *index.PostingValue
- fn compositePostingValueFn
- closer *run.Closer
- seekKey []byte
- termRange index.RangeOpts
- closeOnce sync.Once
- init bool
- reverse bool
-}
-
-func (f *fieldIteratorTemplate) Next() bool {
- if !f.init {
- f.init = true
- f.delegated.Seek(f.seekKey)
- }
- if !f.delegated.Valid() {
- return false
- }
- pv, err := f.fn(f.delegated.Field().Term, f.delegated.Val(), f.delegated)
- if err != nil {
- f.err = err
- return false
- }
- in := f.termRange.Between(pv.Term)
- switch {
- case in > 0:
- if f.reverse {
- return f.Next()
- }
- return false
- case in < 0:
- if f.reverse {
- return false
- }
- return f.Next()
- }
- f.cur = pv
- return true
-}
-
-func (f *fieldIteratorTemplate) Val() *index.PostingValue {
- return f.cur
-}
-
-func (f *fieldIteratorTemplate) Close() (err error) {
- f.closeOnce.Do(func() {
- defer f.closer.Done()
- err = multierr.Combine(f.err, f.delegated.Close())
- })
- return err
-}
-
-func newFieldIteratorTemplate(l *logger.Logger, fieldKey index.FieldKey, termRange index.RangeOpts, order modelv1.Sort, iterable kv.Iterable,
- closer *run.Closer, fn compositePostingValueFn,
-) *fieldIteratorTemplate {
- if termRange.Upper == nil {
- termRange.Upper = defaultUpper
- }
- if termRange.Lower == nil {
- termRange.Lower = defaultLower
- }
- var reverse bool
- var term []byte
- switch order {
- case modelv1.Sort_SORT_ASC, modelv1.Sort_SORT_UNSPECIFIED:
- term = termRange.Lower
- reverse = false
- case modelv1.Sort_SORT_DESC:
- term = termRange.Upper
- reverse = true
- }
- iter := iterable.NewIterator(kv.ScanOpts{
- Prefix: fieldKey.Marshal(),
- Reverse: reverse,
- })
- field := index.Field{
- Key: fieldKey,
- Term: term,
- }
- return &fieldIteratorTemplate{
- delegated: newDelegateIterator(iter, fieldKey, l),
- termRange: termRange,
- fn: fn,
- reverse: reverse,
- seekKey: field.Marshal(),
- closer: closer,
- }
-}
-
-func parseKey(fieldKey index.FieldKey, key []byte) (index.Field, error) {
- f := &index.Field{
- Key: fieldKey,
- }
- err := f.Unmarshal(key)
- if err != nil {
- return *f, err
- }
- return *f, nil
-}
-
-var _ kv.Iterator = (*delegateIterator)(nil)
-
-type delegateIterator struct {
- delegated kv.Iterator
- l *logger.Logger
- fieldKeyBytes []byte
- curField index.Field
- fieldKey index.FieldKey
- closed bool
-}
-
-func newDelegateIterator(delegated kv.Iterator, fieldKey index.FieldKey, l *logger.Logger) *delegateIterator {
- fieldKeyBytes := fieldKey.Marshal()
- return &delegateIterator{
- delegated: delegated,
- fieldKey: fieldKey,
- fieldKeyBytes: fieldKeyBytes,
- l: l,
- }
-}
-
-func (di *delegateIterator) Next() {
- di.delegated.Next()
-}
-
-func (di *delegateIterator) Rewind() {
- di.delegated.Rewind()
-}
-
-func (di *delegateIterator) Seek(key []byte) {
- di.delegated.Seek(key)
-}
-
-func (di *delegateIterator) Key() []byte {
- return di.delegated.Key()
-}
-
-func (di *delegateIterator) RawKey() []byte {
- return di.delegated.RawKey()
-}
-
-func (di *delegateIterator) Field() index.Field {
- return di.curField
-}
-
-func (di *delegateIterator) Val() []byte {
- return di.delegated.Val()
-}
-
-func (di *delegateIterator) Valid() bool {
- if di.closed || !di.delegated.Valid() {
- return false
- }
- var err error
- di.curField, err = parseKey(di.fieldKey, di.Key())
- if err != nil {
- di.l.Error().Err(err).Msg("fail to parse field from key")
- return false
- }
- if !bytes.Equal(di.curField.Key.Marshal(), di.fieldKeyBytes) {
- if e := di.l.Debug(); e.Enabled() {
- e.Uint64("series_id", uint64(di.fieldKey.SeriesID)).
- Uint32("index_rule_id", di.fieldKey.IndexRuleID).
- Msg("reached the limitation of the field(series_id+index_rule_id)")
- }
- return false
- }
- return true
-}
-
-func (di *delegateIterator) Close() error {
- di.closed = true
- return di.delegated.Close()
-}
diff --git a/pkg/index/lsm/lsm.go b/pkg/index/lsm/lsm.go
deleted file mode 100644
index d7cb0a21..00000000
--- a/pkg/index/lsm/lsm.go
+++ /dev/null
@@ -1,89 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-// Package lsm implements a tree-based index repository.
-package lsm
-
-import (
- "sync"
-
- "go.uber.org/multierr"
-
- "github.com/apache/skywalking-banyandb/banyand/kv"
- "github.com/apache/skywalking-banyandb/pkg/convert"
- "github.com/apache/skywalking-banyandb/pkg/index"
- "github.com/apache/skywalking-banyandb/pkg/logger"
- "github.com/apache/skywalking-banyandb/pkg/run"
-)
-
-var _ index.Store = (*store)(nil)
-
-type store struct {
- lsm kv.Store
- l *logger.Logger
- closer *run.Closer
- closeOnce sync.Once
-}
-
-// Batch implements index.Store.
-func (*store) Batch(_ index.Batch) error {
- panic("unimplemented")
-}
-
-func (s *store) Close() (err error) {
- s.closeOnce.Do(func() {
- s.closer.Done()
- s.closer.CloseThenWait()
- err = s.lsm.Close()
- })
- return err
-}
-
-func (s *store) Write(fields []index.Field, docID uint64) (err error) {
- for _, field := range fields {
- err = multierr.Append(err, s.lsm.PutWithVersion(field.Marshal(), convert.Uint64ToBytes(docID), docID))
- }
- return err
-}
-
-func (s *store) SizeOnDisk() int64 {
- return s.lsm.SizeOnDisk()
-}
-
-// StoreOpts wraps options to create the lsm repository.
-type StoreOpts struct {
- Logger *logger.Logger
- Path string
- MemTableSize int64
-}
-
-// NewStore creates a new lsm index repository.
-func NewStore(opts StoreOpts) (index.Store, error) {
- var err error
- var lsm kv.Store
- if lsm, err = kv.OpenStore(
- opts.Path,
- kv.StoreWithLogger(opts.Logger),
- kv.StoreWithMemTableSize(opts.MemTableSize)); err != nil {
- return nil, err
- }
- return &store{
- lsm: lsm,
- l: opts.Logger,
- closer: run.NewCloser(1),
- }, nil
-}
diff --git a/pkg/index/lsm/lsm_test.go b/pkg/index/lsm/lsm_test.go
deleted file mode 100644
index 3a7f32b7..00000000
--- a/pkg/index/lsm/lsm_test.go
+++ /dev/null
@@ -1,72 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package lsm
-
-import (
- "testing"
-
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-
- "github.com/apache/skywalking-banyandb/pkg/index/testcases"
- "github.com/apache/skywalking-banyandb/pkg/logger"
- "github.com/apache/skywalking-banyandb/pkg/test"
- "github.com/apache/skywalking-banyandb/pkg/test/flags"
-)
-
-func TestStore_MatchTerm(t *testing.T) {
- tester := assert.New(t)
- path, fn := setUp(require.New(t))
- s, err := NewStore(StoreOpts{
- Path: path,
- Logger: logger.GetLogger("test"),
- MemTableSize: 1, // invalid size
- })
- defer func() {
- tester.NoError(s.Close())
- fn()
- }()
- tester.NoError(err)
- testcases.SetUp(tester, s)
- testcases.RunServiceName(t, s)
-}
-
-func TestStore_Iterator(t *testing.T) {
- tester := assert.New(t)
- path, fn := setUp(require.New(t))
- s, err := NewStore(StoreOpts{
- Path: path,
- Logger: logger.GetLogger("test"),
- })
- defer func() {
- tester.NoError(s.Close())
- fn()
- }()
- tester.NoError(err)
- data := testcases.SetUpDuration(tester, s)
- testcases.RunDuration(t, data, s)
-}
-
-func setUp(t *require.Assertions) (tempDir string, deferFunc func()) {
- t.NoError(logger.Init(logger.Logging{
- Env: "dev",
- Level: flags.LogLevel,
- }))
- tempDir, deferFunc = test.Space(t)
- return tempDir, deferFunc
-}
diff --git a/pkg/index/lsm/search.go b/pkg/index/lsm/search.go
deleted file mode 100644
index eba435cb..00000000
--- a/pkg/index/lsm/search.go
+++ /dev/null
@@ -1,101 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package lsm
-
-import (
- "bytes"
- "math"
-
- "github.com/pkg/errors"
- "go.uber.org/multierr"
-
- modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
- "github.com/apache/skywalking-banyandb/banyand/kv"
- "github.com/apache/skywalking-banyandb/pkg/convert"
- "github.com/apache/skywalking-banyandb/pkg/index"
- "github.com/apache/skywalking-banyandb/pkg/index/posting"
- "github.com/apache/skywalking-banyandb/pkg/index/posting/roaring"
-)
-
-var errUnsupportedOperation = errors.New("unsupported operation")
-
-func (s *store) MatchField(fieldKey index.FieldKey) (list posting.List, err error) {
- return s.Range(fieldKey, index.RangeOpts{})
-}
-
-func (s *store) MatchTerms(field index.Field) (list posting.List, err error) {
- list = roaring.NewPostingList()
- err = s.lsm.GetAll(field.Marshal(), func(itemID []byte) error {
- list.Insert(convert.BytesToUint64(itemID))
- return nil
- })
- if errors.Is(err, kv.ErrKeyNotFound) {
- return roaring.DummyPostingList, nil
- }
- return
-}
-
-func (s *store) Range(fieldKey index.FieldKey, opts index.RangeOpts) (list posting.List, err error) {
- iter, err := s.Iterator(fieldKey, opts, modelv1.Sort_SORT_ASC, math.MaxInt)
- if err != nil {
- return roaring.DummyPostingList, err
- }
- list = roaring.NewPostingList()
- for iter.Next() {
- err = multierr.Append(err, list.Union(iter.Val().Value))
- }
- err = multierr.Append(err, iter.Close())
- return
-}
-
-func (s *store) Iterator(fieldKey index.FieldKey, termRange index.RangeOpts, order modelv1.Sort, _ int) (index.FieldIterator, error) {
- if !s.closer.AddRunning() {
- return nil, errors.New("lsm index store is closed")
- }
- return newFieldIteratorTemplate(s.l, fieldKey, termRange, order, s.lsm, s.closer,
- func(term, value []byte, delegated kv.Iterator) (*index.PostingValue, error) {
- pv := &index.PostingValue{
- Term: term,
- Value: roaring.NewPostingListWithInitialData(convert.BytesToUint64(value)),
- }
-
- for ; delegated.Valid(); delegated.Next() {
- f := index.Field{}
- err := f.Unmarshal(delegated.Key())
- if err != nil {
- return nil, err
- }
- if !bytes.Equal(f.Term, term) {
- break
- }
- itemID := convert.BytesToUint64(delegated.Val())
- if e := s.l.Debug(); e.Enabled() {
- e.Uint64("series_id", uint64(fieldKey.SeriesID)).
- Uint64("index_rule_id", uint64(fieldKey.IndexRuleID)).
- Uint64("item_id", itemID).
- Msg("fetched item from the index")
- }
- pv.Value.Insert(itemID)
- }
- return pv, nil
- }), nil
-}
-
-func (s *store) Match(_ index.FieldKey, _ []string) (posting.List, error) {
- return nil, errors.WithMessage(errUnsupportedOperation, "LSM-Tree index doesn't support full-text searching")
-}
diff --git a/pkg/partition/entity.go b/pkg/partition/entity.go
index 22433fc6..45dc8bf3 100644
--- a/pkg/partition/entity.go
+++ b/pkg/partition/entity.go
@@ -25,7 +25,6 @@ import (
"github.com/apache/skywalking-banyandb/api/common"
databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
- "github.com/apache/skywalking-banyandb/banyand/tsdb"
pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
)
@@ -57,9 +56,9 @@ func NewEntityLocator(families []*databasev1.TagFamilySpec, entity *databasev1.E
}
// Find the entity from a tag family, prepend a subject to the entity.
-func (e EntityLocator) Find(subject string, value []*modelv1.TagFamilyForWrite) (tsdb.Entity, tsdb.EntityValues, error) {
- entityValues := make(tsdb.EntityValues, len(e.TagLocators)+1)
- entityValues[0] = tsdb.StrValue(subject)
+func (e EntityLocator) Find(subject string, value []*modelv1.TagFamilyForWrite) (pbv1.Entity, pbv1.EntityValues, error) {
+ entityValues := make(pbv1.EntityValues, len(e.TagLocators)+1)
+ entityValues[0] = pbv1.StrValue(subject)
for i, index := range e.TagLocators {
tag, err := GetTagByOffset(value, index.FamilyOffset, index.TagOffset)
if err != nil {
@@ -75,7 +74,7 @@ func (e EntityLocator) Find(subject string, value []*modelv1.TagFamilyForWrite)
}
// Locate a shard and find the entity from a tag family, prepend a subject to the entity.
-func (e EntityLocator) Locate(subject string, value []*modelv1.TagFamilyForWrite, shardNum uint32) (tsdb.Entity, tsdb.EntityValues, common.ShardID, error) {
+func (e EntityLocator) Locate(subject string, value []*modelv1.TagFamilyForWrite, shardNum uint32) (pbv1.Entity, pbv1.EntityValues, common.ShardID, error) {
entity, tagValues, err := e.Find(subject, value)
if err != nil {
return nil, nil, 0, err
diff --git a/pkg/pb/v1/entity.go b/pkg/pb/v1/entity.go
new file mode 100644
index 00000000..d95f10a2
--- /dev/null
+++ b/pkg/pb/v1/entity.go
@@ -0,0 +1,101 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package v1
+
+import (
+ "bytes"
+ "strings"
+
+ modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+ "github.com/apache/skywalking-banyandb/pkg/convert"
+)
+
+// Entry is an element in an Entity.
+type Entry []byte
+
+// Entity denotes an identity of a Series.
+// It defined by Stream or Measure schema.
+type Entity []Entry
+
+// Marshal encodes an Entity to bytes.
+func (e Entity) Marshal() []byte {
+ data := make([][]byte, len(e))
+ for i, entry := range e {
+ data[i] = entry
+ }
+ return bytes.Join(data, nil)
+}
+
+// EntityValue represents the value of a tag which is a part of an entity.
+type EntityValue = *modelv1.TagValue
+
+// EntityValues is the encoded Entity.
+type EntityValues []EntityValue
+
+// Encode EntityValues to tag values.
+func (evs EntityValues) Encode() (result []*modelv1.TagValue) {
+ for _, v := range evs {
+ result = append(result, v)
+ }
+ return
+}
+
+// ToEntity transforms EntityValues to Entity.
+func (evs EntityValues) ToEntity() (result Entity, err error) {
+ for _, v := range evs {
+ entry, errMarshal := MarshalTagValue(v)
+ if errMarshal != nil {
+ return nil, err
+ }
+ result = append(result, entry)
+ }
+ return
+}
+
+// String outputs the string represent of an EntityValue.
+func (evs EntityValues) String() string {
+ var strBuilder strings.Builder
+ vv := evs.Encode()
+ for i := 0; i < len(vv); i++ {
+ strBuilder.WriteString(vv[i].String())
+ if i < len(vv)-1 {
+ strBuilder.WriteString(".")
+ }
+ }
+ return strBuilder.String()
+}
+
+// StrValue returns an EntityValue which wraps a string value.
+func StrValue(v string) EntityValue {
+ return &modelv1.TagValue{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: v}}}
+}
+
+// HashEntity runs hash function (e.g. with xxhash algorithm) on each segment of the Entity,
+// and concatenates all uint64 in byte array. So the return length of the byte array will be
+// 8 (every uint64 has 8 bytes) * length of the input.
+func HashEntity(entity Entity) []byte {
+ result := make([]byte, 0, len(entity)*8)
+ for _, entry := range entity {
+ result = append(result, hash(entry)...)
+ }
+ return result
+}
+
+func hash(entry []byte) []byte {
+ return convert.Uint64ToBytes(convert.Hash(entry))
+}
diff --git a/pkg/pb/v1/tsdb/options.go b/pkg/pb/v1/tsdb/options.go
deleted file mode 100644
index 6cd3f9c1..00000000
--- a/pkg/pb/v1/tsdb/options.go
+++ /dev/null
@@ -1,42 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-// Package tsdb implements helpers around tsdb.IntervalRule.
-package tsdb
-
-import (
- "github.com/pkg/errors"
-
- commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
- "github.com/apache/skywalking-banyandb/banyand/tsdb"
-)
-
-var errUnknown = errors.New("unknown")
-
-// ToIntervalRule decodes commonv1.IntervalRule to tsdb.IntervalRule.
-func ToIntervalRule(ir *commonv1.IntervalRule) (result tsdb.IntervalRule, err error) {
- switch ir.Unit {
- case commonv1.IntervalRule_UNIT_DAY:
- result.Unit = tsdb.DAY
- case commonv1.IntervalRule_UNIT_HOUR:
- result.Unit = tsdb.HOUR
- default:
- return result, errors.WithMessagef(errUnknown, "interval rule:%v", ir)
- }
- result.Num = int(ir.Num)
- return result, err
-}
diff --git a/pkg/query/logical/common.go b/pkg/query/logical/common.go
index 25e78885..69bf90aa 100644
--- a/pkg/query/logical/common.go
+++ b/pkg/query/logical/common.go
@@ -21,7 +21,6 @@ import (
"github.com/pkg/errors"
modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
- "github.com/apache/skywalking-banyandb/banyand/tsdb"
)
var (
@@ -32,12 +31,6 @@ var (
errIndexNotDefined = errors.New("index is not define for the tag")
)
-type (
- // SeekerBuilder wraps the execution of tsdb.SeekerBuilder.
- // TODO:// we could have a chance to remove this wrapper.
- SeekerBuilder func(builder tsdb.SeekerBuilder)
-)
-
// Tag represents the combination of tag family and tag name.
// It's a tag's identity.
type Tag struct {
diff --git a/pkg/query/logical/index_filter.go b/pkg/query/logical/index_filter.go
index 98d0b98c..e4e88455 100644
--- a/pkg/query/logical/index_filter.go
+++ b/pkg/query/logical/index_filter.go
@@ -28,8 +28,6 @@ import (
"github.com/apache/skywalking-banyandb/api/common"
databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
- "github.com/apache/skywalking-banyandb/banyand/tsdb"
- "github.com/apache/skywalking-banyandb/pkg/convert"
"github.com/apache/skywalking-banyandb/pkg/index"
"github.com/apache/skywalking-banyandb/pkg/index/posting"
pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
@@ -46,265 +44,6 @@ type GlobalIndexError struct {
func (g GlobalIndexError) Error() string { return g.IndexRule.String() }
-// BuildLocalFilterDeprecated returns a new index.Filter for local indices.
-// It could parse series Path at the same time.
-func BuildLocalFilterDeprecated(criteria *modelv1.Criteria, schema Schema, entityDict map[string]int,
- entity tsdb.Entity, mandatoryIndexRule bool,
-) (index.Filter, []tsdb.Entity, error) {
- if criteria == nil {
- return nil, []tsdb.Entity{entity}, nil
- }
- switch criteria.GetExp().(type) {
- case *modelv1.Criteria_Condition:
- cond := criteria.GetCondition()
- expr, parsedEntity, err := parseExprOrEntityDeprecated(entityDict, entity, cond)
- if err != nil {
- return nil, nil, err
- }
- if parsedEntity != nil {
- return nil, parsedEntity, nil
- }
- if ok, indexRule := schema.IndexDefined(cond.Name); ok {
- if indexRule.Location == databasev1.IndexRule_LOCATION_GLOBAL {
- switch cond.Op {
- case modelv1.Condition_BINARY_OP_EQ, modelv1.Condition_BINARY_OP_IN:
- return nil, nil, GlobalIndexError{
- IndexRule: indexRule,
- Expr: expr,
- }
- default:
- return nil, nil, errors.Wrapf(errUnsupportedConditionOp, "gobal index conf:%s", cond)
- }
- }
- return parseConditionDeprecated(cond, indexRule, expr, entity)
- } else if mandatoryIndexRule {
- return nil, nil, errors.Wrapf(errUnsupportedConditionOp, "mandatory index rule conf:%s", cond)
- }
- return Enode, []tsdb.Entity{entity}, nil
- case *modelv1.Criteria_Le:
- le := criteria.GetLe()
- if le.GetLeft() == nil && le.GetRight() == nil {
- return nil, nil, errors.WithMessagef(errInvalidLogicalExpression, "both sides(left and right) of [%v] are empty", criteria)
- }
- if le.GetLeft() == nil {
- return BuildLocalFilterDeprecated(le.Right, schema, entityDict, entity, mandatoryIndexRule)
- }
- if le.GetRight() == nil {
- return BuildLocalFilterDeprecated(le.Left, schema, entityDict, entity, mandatoryIndexRule)
- }
- left, leftEntities, err := BuildLocalFilterDeprecated(le.Left, schema, entityDict, entity, mandatoryIndexRule)
- if err != nil {
- return nil, nil, err
- }
- right, rightEntities, err := BuildLocalFilterDeprecated(le.Right, schema, entityDict, entity, mandatoryIndexRule)
- if err != nil {
- return nil, nil, err
- }
- entities := parseEntitiesDeprecated(le.Op, entity, leftEntities, rightEntities)
- if entities == nil {
- return nil, nil, nil
- }
- if left == nil && right == nil {
- return nil, entities, nil
- }
- switch le.Op {
- case modelv1.LogicalExpression_LOGICAL_OP_AND:
- and := newAnd(2)
- and.append(left).append(right)
- return and, entities, nil
- case modelv1.LogicalExpression_LOGICAL_OP_OR:
- or := newOr(2)
- or.append(left).append(right)
- return or, entities, nil
- }
- }
- return nil, nil, errInvalidCriteriaType
-}
-
-func parseConditionDeprecated(cond *modelv1.Condition, indexRule *databasev1.IndexRule, expr LiteralExpr, entity tsdb.Entity) (index.Filter, []tsdb.Entity, error) {
- switch cond.Op {
- case modelv1.Condition_BINARY_OP_GT:
- return newRange(indexRule, index.RangeOpts{
- Lower: bytes.Join(expr.Bytes(), nil),
- }), []tsdb.Entity{entity}, nil
- case modelv1.Condition_BINARY_OP_GE:
- return newRange(indexRule, index.RangeOpts{
- IncludesLower: true,
- Lower: bytes.Join(expr.Bytes(), nil),
- }), []tsdb.Entity{entity}, nil
- case modelv1.Condition_BINARY_OP_LT:
- return newRange(indexRule, index.RangeOpts{
- Upper: bytes.Join(expr.Bytes(), nil),
- }), []tsdb.Entity{entity}, nil
- case modelv1.Condition_BINARY_OP_LE:
- return newRange(indexRule, index.RangeOpts{
- IncludesUpper: true,
- Upper: bytes.Join(expr.Bytes(), nil),
- }), []tsdb.Entity{entity}, nil
- case modelv1.Condition_BINARY_OP_EQ:
- return newEq(indexRule, expr), []tsdb.Entity{entity}, nil
- case modelv1.Condition_BINARY_OP_MATCH:
- return newMatch(indexRule, expr), []tsdb.Entity{entity}, nil
- case modelv1.Condition_BINARY_OP_NE:
- return newNot(indexRule, newEq(indexRule, expr)), []tsdb.Entity{entity}, nil
- case modelv1.Condition_BINARY_OP_HAVING:
- bb := expr.Bytes()
- and := newAnd(len(bb))
- for _, b := range bb {
- and.append(newEq(indexRule, newBytesLiteral(b)))
- }
- return and, []tsdb.Entity{entity}, nil
- case modelv1.Condition_BINARY_OP_NOT_HAVING:
- bb := expr.Bytes()
- and := newAnd(len(bb))
- for _, b := range bb {
- and.append(newEq(indexRule, newBytesLiteral(b)))
- }
- return newNot(indexRule, and), []tsdb.Entity{entity}, nil
- case modelv1.Condition_BINARY_OP_IN:
- bb := expr.Bytes()
- or := newOr(len(bb))
- for _, b := range bb {
- or.append(newEq(indexRule, newBytesLiteral(b)))
- }
- return or, []tsdb.Entity{entity}, nil
- case modelv1.Condition_BINARY_OP_NOT_IN:
- bb := expr.Bytes()
- or := newOr(len(bb))
- for _, b := range bb {
- or.append(newEq(indexRule, newBytesLiteral(b)))
- }
- return newNot(indexRule, or), []tsdb.Entity{entity}, nil
- }
- return nil, nil, errors.WithMessagef(errUnsupportedConditionOp, "index filter parses %v", cond)
-}
-
-func parseExprOrEntityDeprecated(entityDict map[string]int, entity tsdb.Entity, cond *modelv1.Condition) (LiteralExpr, []tsdb.Entity, error) {
- entityIdx, ok := entityDict[cond.Name]
- if ok && cond.Op != modelv1.Condition_BINARY_OP_EQ && cond.Op != modelv1.Condition_BINARY_OP_IN {
- return nil, nil, errors.WithMessagef(errUnsupportedConditionOp, "tag belongs to the entity only supports EQ or IN operation in condition(%v)", cond)
- }
- switch v := cond.Value.Value.(type) {
- case *modelv1.TagValue_Str:
- if ok {
- parsedEntity := make(tsdb.Entity, len(entity))
- copy(parsedEntity, entity)
- parsedEntity[entityIdx] = []byte(v.Str.GetValue())
- return nil, []tsdb.Entity{parsedEntity}, nil
- }
- return str(v.Str.GetValue()), nil, nil
- case *modelv1.TagValue_StrArray:
- if ok && cond.Op == modelv1.Condition_BINARY_OP_IN {
- entities := make([]tsdb.Entity, len(v.StrArray.Value))
- for i, va := range v.StrArray.Value {
- parsedEntity := make(tsdb.Entity, len(entity))
- copy(parsedEntity, entity)
- parsedEntity[entityIdx] = []byte(va)
- entities[i] = parsedEntity
- }
- return nil, entities, nil
- }
- return &strArrLiteral{
- arr: v.StrArray.GetValue(),
- }, nil, nil
- case *modelv1.TagValue_Int:
- if ok {
- parsedEntity := make(tsdb.Entity, len(entity))
- copy(parsedEntity, entity)
- parsedEntity[entityIdx] = convert.Int64ToBytes(v.Int.GetValue())
- return nil, []tsdb.Entity{parsedEntity}, nil
- }
- return &int64Literal{
- int64: v.Int.GetValue(),
- }, nil, nil
- case *modelv1.TagValue_IntArray:
- if ok && cond.Op == modelv1.Condition_BINARY_OP_IN {
- entities := make([]tsdb.Entity, len(v.IntArray.Value))
- for i, va := range v.IntArray.Value {
- parsedEntity := make(tsdb.Entity, len(entity))
- copy(parsedEntity, entity)
- parsedEntity[entityIdx] = convert.Int64ToBytes(va)
- entities[i] = parsedEntity
- }
- return nil, entities, nil
- }
- return &int64ArrLiteral{
- arr: v.IntArray.GetValue(),
- }, nil, nil
- case *modelv1.TagValue_Null:
- return nullLiteralExpr, nil, nil
- }
- return nil, nil, errors.WithMessagef(errUnsupportedConditionValue, "index filter parses %v", cond)
-}
-
-func parseEntitiesDeprecated(op modelv1.LogicalExpression_LogicalOp, input tsdb.Entity, left, right []tsdb.Entity) []tsdb.Entity {
- count := len(input)
- result := make(tsdb.Entity, count)
- anyEntity := func(entities []tsdb.Entity) bool {
- for _, entity := range entities {
- for _, entry := range entity {
- if !bytes.Equal(entry, tsdb.AnyEntry) {
- return false
- }
- }
- }
- return true
- }
- leftAny := anyEntity(left)
- rightAny := anyEntity(right)
-
- mergedEntities := make([]tsdb.Entity, 0, len(left)+len(right))
-
- switch op {
- case modelv1.LogicalExpression_LOGICAL_OP_AND:
- if leftAny && !rightAny {
- return right
- }
- if !leftAny && rightAny {
- return left
- }
- mergedEntities = append(mergedEntities, left...)
- mergedEntities = append(mergedEntities, right...)
- for i := 0; i < count; i++ {
- entry := tsdb.AnyEntry
- for j := 0; j < len(mergedEntities); j++ {
- e := mergedEntities[j][i]
- if bytes.Equal(e, tsdb.AnyEntry) {
- continue
- }
- if bytes.Equal(entry, tsdb.AnyEntry) {
- entry = e
- } else if !bytes.Equal(entry, e) {
- return nil
- }
- }
- result[i] = entry
- }
- case modelv1.LogicalExpression_LOGICAL_OP_OR:
- if leftAny {
- return left
- }
- if rightAny {
- return right
- }
- mergedEntities = append(mergedEntities, left...)
- mergedEntities = append(mergedEntities, right...)
- for i := 0; i < count; i++ {
- entry := tsdb.AnyEntry
- for j := 0; j < len(mergedEntities); j++ {
- e := mergedEntities[j][i]
- if bytes.Equal(entry, tsdb.AnyEntry) {
- entry = e
- } else if !bytes.Equal(entry, e) {
- return mergedEntities
- }
- }
- result[i] = entry
- }
- }
- return []tsdb.Entity{result}
-}
-
// BuildLocalFilter returns a new index.Filter for local indices.
func BuildLocalFilter(criteria *modelv1.Criteria, schema Schema, entityDict map[string]int,
entity []*modelv1.TagValue, mandatoryIndexRule bool,
diff --git a/pkg/query/logical/measure/schema.go b/pkg/query/logical/measure/schema.go
index b389053f..fe8e8947 100644
--- a/pkg/query/logical/measure/schema.go
+++ b/pkg/query/logical/measure/schema.go
@@ -23,7 +23,6 @@ import (
"github.com/pkg/errors"
databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
- "github.com/apache/skywalking-banyandb/banyand/tsdb"
"github.com/apache/skywalking-banyandb/pkg/query/logical"
)
@@ -36,10 +35,6 @@ type schema struct {
common *logical.CommonSchema
}
-func (m *schema) Scope() tsdb.Entry {
- return tsdb.Entry(m.measure.Metadata.Name)
-}
-
func (m *schema) EntityList() []string {
return m.common.EntityList
}
diff --git a/pkg/query/logical/schema.go b/pkg/query/logical/schema.go
index cc588e93..0f9b0ffb 100644
--- a/pkg/query/logical/schema.go
+++ b/pkg/query/logical/schema.go
@@ -21,7 +21,6 @@ import (
"github.com/pkg/errors"
databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
- "github.com/apache/skywalking-banyandb/banyand/tsdb"
)
// IndexChecker allows checking the existence of a specific index rule.
@@ -49,7 +48,6 @@ type TagSpecRegistry interface {
type Schema interface {
TagSpecRegistry
IndexChecker
- Scope() tsdb.Entry
EntityList() []string
CreateTagRef(tags ...[]*Tag) ([][]*TagRef, error)
CreateFieldRef(fields ...*Field) ([]*FieldRef, error)
diff --git a/pkg/query/logical/stream/schema.go b/pkg/query/logical/stream/schema.go
index 47f468d5..a48b789e 100644
--- a/pkg/query/logical/stream/schema.go
+++ b/pkg/query/logical/stream/schema.go
@@ -22,7 +22,6 @@ import (
"github.com/google/go-cmp/cmp"
databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
- "github.com/apache/skywalking-banyandb/banyand/tsdb"
"github.com/apache/skywalking-banyandb/pkg/query/logical"
)
@@ -84,7 +83,3 @@ func (s *schema) ProjTags(refs ...[]*logical.TagRef) logical.Schema {
func (s *schema) ProjFields(...*logical.FieldRef) logical.Schema {
panic("stream does not support field")
}
-
-func (s *schema) Scope() tsdb.Entry {
- return tsdb.Entry(s.stream.Metadata.Name)
-}