You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ha...@apache.org on 2024/01/19 05:44:26 UTC
(skywalking-banyandb) branch stream-column updated: Implement filtering for stream (#370)
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch stream-column
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
The following commit(s) were added to refs/heads/stream-column by this push:
new b9d395b5 Implement filtering for stream (#370)
b9d395b5 is described below
commit b9d395b56f790adb33f0da4992a8f0d1e2c650ca
Author: Huang Youliang <52...@users.noreply.github.com>
AuthorDate: Fri Jan 19 13:44:21 2024 +0800
Implement filtering for stream (#370)
* Flush measure memory data to disk. (#360)
* Implement measure merger (#364)
* Add the measure merger loop
---------
Signed-off-by: Gao Hongtao <ha...@gmail.com>
Co-authored-by: Gao Hongtao <ha...@gmail.com>
Co-authored-by: Caican Cai <77...@users.noreply.github.com>
---
CHANGES.md | 5 +
README.md | 2 +-
banyand/internal/storage/index.go | 4 +-
banyand/internal/storage/retention.go | 10 +-
banyand/internal/storage/segment.go | 56 +--
banyand/internal/storage/shard.go | 21 +-
banyand/internal/storage/storage.go | 6 +-
banyand/internal/storage/tsdb.go | 32 +-
banyand/liaison/grpc/stream.go | 5 +-
banyand/measure/block.go | 220 ++++++++++-
banyand/measure/block_metadata.go | 25 +-
banyand/measure/block_reader.go | 212 ++++++++++-
banyand/measure/block_reader_test.go | 165 ++++++++
banyand/measure/block_test.go | 393 ++++++++++++++++++-
banyand/measure/block_writer.go | 46 ++-
banyand/measure/column.go | 23 ++
banyand/measure/datapoints.go | 2 +-
banyand/measure/flusher.go | 97 ++++-
banyand/measure/gc.go | 97 ++---
banyand/measure/gc_test.go | 94 ++---
banyand/measure/introducer.go | 59 ++-
banyand/measure/measure.go | 6 +
banyand/measure/merger.go | 378 +++++++++++++++++++
banyand/measure/merger_test.go | 356 ++++++++++++++++++
banyand/measure/metadata.go | 9 +-
banyand/measure/part.go | 24 +-
banyand/measure/part_iter.go | 146 ++++++++
banyand/measure/part_iter_test.go | 69 +++-
banyand/measure/part_metadata.go | 14 +-
banyand/measure/part_test.go | 2 +-
banyand/measure/primary_metadata.go | 5 +-
banyand/measure/query.go | 6 +-
banyand/measure/query_test.go | 44 ++-
banyand/measure/service.go | 2 +
banyand/measure/snapshot.go | 31 +-
banyand/measure/snapshot_test.go | 138 ++++++-
banyand/measure/tstable.go | 56 ++-
banyand/measure/tstable_test.go | 338 +++++++++++------
banyand/stream/block.go | 310 ++++++++++++---
banyand/stream/block_metadata.go | 27 +-
banyand/stream/block_reader.go | 212 ++++++++++-
banyand/stream/block_reader_test.go | 165 ++++++++
banyand/stream/block_test.go | 415 ++++++++++++++++++++-
banyand/stream/block_writer.go | 46 ++-
banyand/stream/elements.go | 3 +-
banyand/stream/flusher.go | 97 ++++-
banyand/stream/gc.go | 97 ++---
banyand/stream/gc_test.go | 94 ++---
banyand/stream/index.go | 81 +++-
banyand/stream/introducer.go | 59 ++-
banyand/stream/iter.go | 7 +-
banyand/stream/iter_builder.go | 16 +-
banyand/stream/merger.go | 378 +++++++++++++++++++
banyand/stream/merger_test.go | 328 ++++++++++++++++
banyand/stream/metadata.go | 9 +-
banyand/stream/part.go | 76 +++-
banyand/stream/part_iter.go | 146 ++++++++
banyand/stream/part_iter_test.go | 68 +++-
banyand/stream/part_metadata.go | 14 +-
banyand/stream/part_test.go | 2 +-
banyand/stream/primary_metadata.go | 9 +-
banyand/stream/query.go | 9 +-
banyand/stream/service.go | 2 +
banyand/stream/snapshot.go | 31 +-
banyand/stream/snapshot_test.go | 138 ++++++-
banyand/stream/stream.go | 136 +++++--
banyand/stream/tag.go | 23 ++
banyand/stream/tstable.go | 58 ++-
banyand/stream/tstable_test.go | 375 ++++++++++++-------
banyand/stream/write.go | 19 +-
pkg/bytes/buffer.go | 30 ++
pkg/fs/file_system.go | 24 +-
pkg/fs/local_file_system.go | 27 +-
pkg/fs/local_file_system_test.go | 5 +-
pkg/index/inverted/inverted.go | 8 +-
pkg/index/inverted/inverted_test.go | 8 +-
pkg/pb/v1/metadata.go | 41 +-
pkg/query/executor/interface.go | 3 +-
pkg/query/logical/index_filter.go | 18 +-
.../logical/stream/stream_plan_indexscan_local.go | 82 +++-
pkg/watcher/watcher.go | 3 +
scripts/build/version.mk | 2 +-
test/cases/stream/stream.go | 31 +-
83 files changed, 5998 insertions(+), 932 deletions(-)
diff --git a/CHANGES.md b/CHANGES.md
index b5e03537..e63ee3a3 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -15,6 +15,11 @@ Release Notes.
- Measure column-based storage:
- Data ingestion and retrieval.
- Flush memory data to disk.
+ - Merge memory data and disk data.
+- Stream column-based storage:
+ - Data ingestion and retrieval.
+ - Flush memory data to disk.
+ - Merge memory data and disk data.
### Bugs
diff --git a/README.md b/README.md
index 4cf9d42d..80b07eb9 100644
--- a/README.md
+++ b/README.md
@@ -15,7 +15,7 @@ It's designed to handle observability data generated by observability platform a
BanyanDB, as an observability database, aims to ingest, analyze and store Metrics, Tracing, and Logging data. It's designed to handle observability data generated by Apache SkyWalking. Before BanyanDB emerges, the Databases that SkyWalking adopted are not ideal for the APM data model, especially for saving tracing and logging data. Consequently, There’s room to improve the performance and resource usage based on the nature of SkyWalking data patterns.
-The database research community usually uses [RUM conjecture](http://daslab.seas.harvard.edu/rum-conjecture/) to describe how a database access data. BanyanDB combines several access methods to build a comprehensive APM database to balance read cost, update cost, and memory overhead.
+The database research community usually uses [RUM Conjecture](http://daslab.seas.harvard.edu/rum-conjecture/) to describe how a database access data. BanyanDB combines several access methods to build a comprehensive APM database to balance read cost, update cost, and memory overhead.
## Contact us
diff --git a/banyand/internal/storage/index.go b/banyand/internal/storage/index.go
index eaef4967..11d35f9e 100644
--- a/banyand/internal/storage/index.go
+++ b/banyand/internal/storage/index.go
@@ -33,11 +33,11 @@ import (
pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
)
-func (d *database[T]) IndexDB() IndexDB {
+func (d *database[T, O]) IndexDB() IndexDB {
return d.index
}
-func (d *database[T]) Lookup(ctx context.Context, series *pbv1.Series) (pbv1.SeriesList, error) {
+func (d *database[T, O]) Lookup(ctx context.Context, series *pbv1.Series) (pbv1.SeriesList, error) {
return d.index.searchPrimary(ctx, series)
}
diff --git a/banyand/internal/storage/retention.go b/banyand/internal/storage/retention.go
index ec76f1c1..ee5f66ea 100644
--- a/banyand/internal/storage/retention.go
+++ b/banyand/internal/storage/retention.go
@@ -25,14 +25,14 @@ import (
"github.com/apache/skywalking-banyandb/pkg/logger"
)
-type retentionTask[T TSTable] struct {
- segment *segmentController[T]
+type retentionTask[T TSTable, O any] struct {
+ segment *segmentController[T, O]
expr string
option cron.ParseOption
duration time.Duration
}
-func newRetentionTask[T TSTable](segment *segmentController[T], ttl IntervalRule) *retentionTask[T] {
+func newRetentionTask[T TSTable, O any](segment *segmentController[T, O], ttl IntervalRule) *retentionTask[T, O] {
var expr string
switch ttl.Unit {
case HOUR:
@@ -42,7 +42,7 @@ func newRetentionTask[T TSTable](segment *segmentController[T], ttl IntervalRule
// Every day on 00:05
expr = "5 0"
}
- return &retentionTask[T]{
+ return &retentionTask[T, O]{
segment: segment,
option: cron.Minute | cron.Hour,
expr: expr,
@@ -50,7 +50,7 @@ func newRetentionTask[T TSTable](segment *segmentController[T], ttl IntervalRule
}
}
-func (rc *retentionTask[T]) run(now time.Time, l *logger.Logger) bool {
+func (rc *retentionTask[T, O]) run(now time.Time, l *logger.Logger) bool {
if err := rc.segment.remove(now.Add(-rc.duration)); err != nil {
l.Error().Err(err)
}
diff --git a/banyand/internal/storage/segment.go b/banyand/internal/storage/segment.go
index d4b0d7ff..2ec4d16b 100644
--- a/banyand/internal/storage/segment.go
+++ b/banyand/internal/storage/segment.go
@@ -115,11 +115,12 @@ func (s *segment[T]) String() string {
return "SegID-" + s.suffix
}
-type segmentController[T TSTable] struct {
+type segmentController[T TSTable, O any] struct {
clock timestamp.Clock
+ option O
scheduler *timestamp.Scheduler
l *logger.Logger
- tsTableCreator TSTableCreator[T]
+ tsTableCreator TSTableCreator[T, O]
position common.Position
location string
lst []*segment[T]
@@ -127,12 +128,12 @@ type segmentController[T TSTable] struct {
sync.RWMutex
}
-func newSegmentController[T TSTable](ctx context.Context, location string,
+func newSegmentController[T TSTable, O any](ctx context.Context, location string,
segmentSize IntervalRule, l *logger.Logger, scheduler *timestamp.Scheduler,
- tsTableCreator TSTableCreator[T],
-) *segmentController[T] {
+ tsTableCreator TSTableCreator[T, O], option O,
+) *segmentController[T, O] {
clock, _ := timestamp.GetClock(ctx)
- return &segmentController[T]{
+ return &segmentController[T, O]{
location: location,
segmentSize: segmentSize,
l: l,
@@ -140,10 +141,11 @@ func newSegmentController[T TSTable](ctx context.Context, location string,
scheduler: scheduler,
position: common.GetPosition(ctx),
tsTableCreator: tsTableCreator,
+ option: option,
}
}
-func (sc *segmentController[T]) selectTSTables(timeRange timestamp.TimeRange) (tt []TSTableWrapper[T]) {
+func (sc *segmentController[T, O]) selectTSTables(timeRange timestamp.TimeRange) (tt []TSTableWrapper[T]) {
sc.RLock()
defer sc.RUnlock()
last := len(sc.lst) - 1
@@ -157,7 +159,7 @@ func (sc *segmentController[T]) selectTSTables(timeRange timestamp.TimeRange) (t
return tt
}
-func (sc *segmentController[T]) createTSTable(ts time.Time) (TSTableWrapper[T], error) {
+func (sc *segmentController[T, O]) createTSTable(ts time.Time) (TSTableWrapper[T], error) {
s, err := sc.create(ts)
if err != nil {
return nil, err
@@ -166,7 +168,7 @@ func (sc *segmentController[T]) createTSTable(ts time.Time) (TSTableWrapper[T],
return s, nil
}
-func (sc *segmentController[T]) segments() (ss []*segment[T]) {
+func (sc *segmentController[T, O]) segments() (ss []*segment[T]) {
sc.RLock()
defer sc.RUnlock()
r := make([]*segment[T], len(sc.lst))
@@ -177,7 +179,7 @@ func (sc *segmentController[T]) segments() (ss []*segment[T]) {
return r
}
-func (sc *segmentController[T]) Current() (bucket.Reporter, error) {
+func (sc *segmentController[T, O]) Current() (bucket.Reporter, error) {
now := sc.Standard(sc.clock.Now())
ns := uint64(now.UnixNano())
if b := func() bucket.Reporter {
@@ -195,7 +197,7 @@ func (sc *segmentController[T]) Current() (bucket.Reporter, error) {
return sc.create(now)
}
-func (sc *segmentController[T]) Next() (bucket.Reporter, error) {
+func (sc *segmentController[T, O]) Next() (bucket.Reporter, error) {
c, err := sc.Current()
if err != nil {
return nil, err
@@ -208,7 +210,7 @@ func (sc *segmentController[T]) Next() (bucket.Reporter, error) {
return reporter, err
}
-func (sc *segmentController[T]) OnMove(prev bucket.Reporter, next bucket.Reporter) {
+func (sc *segmentController[T, O]) OnMove(prev bucket.Reporter, next bucket.Reporter) {
event := sc.l.Info()
if prev != nil {
event.Stringer("prev", prev)
@@ -219,7 +221,7 @@ func (sc *segmentController[T]) OnMove(prev bucket.Reporter, next bucket.Reporte
event.Msg("move to the next segment")
}
-func (sc *segmentController[T]) Standard(t time.Time) time.Time {
+func (sc *segmentController[T, O]) Standard(t time.Time) time.Time {
switch sc.segmentSize.Unit {
case HOUR:
return time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), 0, 0, 0, t.Location())
@@ -229,7 +231,7 @@ func (sc *segmentController[T]) Standard(t time.Time) time.Time {
panic("invalid interval unit")
}
-func (sc *segmentController[T]) Format(tm time.Time) string {
+func (sc *segmentController[T, O]) Format(tm time.Time) string {
switch sc.segmentSize.Unit {
case HOUR:
return tm.Format(hourFormat)
@@ -239,7 +241,7 @@ func (sc *segmentController[T]) Format(tm time.Time) string {
panic("invalid interval unit")
}
-func (sc *segmentController[T]) Parse(value string) (time.Time, error) {
+func (sc *segmentController[T, O]) Parse(value string) (time.Time, error) {
switch sc.segmentSize.Unit {
case HOUR:
return time.ParseInLocation(hourFormat, value, time.Local)
@@ -249,7 +251,7 @@ func (sc *segmentController[T]) Parse(value string) (time.Time, error) {
panic("invalid interval unit")
}
-func (sc *segmentController[T]) open() error {
+func (sc *segmentController[T, O]) open() error {
sc.Lock()
defer sc.Unlock()
return loadSegments(sc.location, segPathPrefix, sc, sc.segmentSize, func(start, end time.Time) error {
@@ -261,7 +263,7 @@ func (sc *segmentController[T]) open() error {
})
}
-func (sc *segmentController[T]) create(start time.Time) (*segment[T], error) {
+func (sc *segmentController[T, O]) create(start time.Time) (*segment[T], error) {
sc.Lock()
defer sc.Unlock()
start = sc.Standard(start)
@@ -285,24 +287,22 @@ func (sc *segmentController[T]) create(start time.Time) (*segment[T], error) {
return sc.load(start, end, sc.location)
}
-func (sc *segmentController[T]) sortLst() {
+func (sc *segmentController[T, O]) sortLst() {
sort.Slice(sc.lst, func(i, j int) bool {
return sc.lst[i].id < sc.lst[j].id
})
}
-func (sc *segmentController[T]) load(start, end time.Time, root string) (seg *segment[T], err error) {
+func (sc *segmentController[T, O]) load(start, end time.Time, root string) (seg *segment[T], err error) {
suffix := sc.Format(start)
segPath := path.Join(root, fmt.Sprintf(segTemplate, suffix))
var tsTable T
- if tsTable, err = sc.tsTableCreator(lfs, segPath, sc.position, sc.l, timestamp.NewSectionTimeRange(start, end)); err != nil {
+ p := sc.position
+ p.Segment = suffix
+ if tsTable, err = sc.tsTableCreator(lfs, segPath, p, sc.l, timestamp.NewSectionTimeRange(start, end), sc.option); err != nil {
return nil, err
}
- ctx := context.WithValue(context.Background(), logger.ContextKey, sc.l)
- seg, err = openSegment[T](common.SetPosition(ctx, func(p common.Position) common.Position {
- p.Segment = suffix
- return p
- }), start, end, segPath, suffix, sc.segmentSize, sc.scheduler, tsTable)
+ seg, err = openSegment[T](context.WithValue(context.Background(), logger.ContextKey, sc.l), start, end, segPath, suffix, sc.segmentSize, sc.scheduler, tsTable)
if err != nil {
return nil, err
}
@@ -311,7 +311,7 @@ func (sc *segmentController[T]) load(start, end time.Time, root string) (seg *se
return seg, nil
}
-func (sc *segmentController[T]) remove(deadline time.Time) (err error) {
+func (sc *segmentController[T, O]) remove(deadline time.Time) (err error) {
sc.l.Info().Time("deadline", deadline).Msg("start to remove before deadline")
for _, s := range sc.segments() {
if s.End.Before(deadline) || s.Contains(uint64(deadline.UnixNano())) {
@@ -330,7 +330,7 @@ func (sc *segmentController[T]) remove(deadline time.Time) (err error) {
return err
}
-func (sc *segmentController[T]) removeSeg(segID segmentID) {
+func (sc *segmentController[T, O]) removeSeg(segID segmentID) {
for i, b := range sc.lst {
if b.id == segID {
sc.lst = append(sc.lst[:i], sc.lst[i+1:]...)
@@ -339,7 +339,7 @@ func (sc *segmentController[T]) removeSeg(segID segmentID) {
}
}
-func (sc *segmentController[T]) close() {
+func (sc *segmentController[T, O]) close() {
sc.Lock()
defer sc.Unlock()
for _, s := range sc.lst {
diff --git a/banyand/internal/storage/shard.go b/banyand/internal/storage/shard.go
index 19150f9c..3ea2f050 100644
--- a/banyand/internal/storage/shard.go
+++ b/banyand/internal/storage/shard.go
@@ -30,9 +30,9 @@ import (
"github.com/apache/skywalking-banyandb/pkg/timestamp"
)
-type shard[T TSTable] struct {
+type shard[T TSTable, O any] struct {
l *logger.Logger
- segmentController *segmentController[T]
+ segmentController *segmentController[T, O]
segmentManageStrategy *bucket.Strategy
scheduler *timestamp.Scheduler
position common.Position
@@ -40,7 +40,7 @@ type shard[T TSTable] struct {
id common.ShardID
}
-func (d *database[T]) openShard(ctx context.Context, id common.ShardID) (*shard[T], error) {
+func (d *database[T, O]) openShard(ctx context.Context, id common.ShardID) (*shard[T, O], error) {
location := path.Join(d.location, fmt.Sprintf(shardTemplate, int(id)))
lfs.MkdirIfNotExist(location, dirPerm)
l := logger.Fetch(ctx, "shard"+strconv.Itoa(int(id)))
@@ -53,12 +53,13 @@ func (d *database[T]) openShard(ctx context.Context, id common.ShardID) (*shard[
clock, _ := timestamp.GetClock(shardCtx)
scheduler := timestamp.NewScheduler(l, clock)
- s := &shard[T]{
- id: id,
- l: l,
- scheduler: scheduler,
- position: common.GetPosition(shardCtx),
- segmentController: newSegmentController[T](shardCtx, location, d.opts.SegmentInterval, l, scheduler, d.opts.TSTableCreator),
+ s := &shard[T, O]{
+ id: id,
+ l: l,
+ scheduler: scheduler,
+ position: common.GetPosition(shardCtx),
+ segmentController: newSegmentController[T](shardCtx, location,
+ d.opts.SegmentInterval, l, scheduler, d.opts.TSTableCreator, d.opts.Option),
}
var err error
if err = s.segmentController.open(); err != nil {
@@ -75,7 +76,7 @@ func (d *database[T]) openShard(ctx context.Context, id common.ShardID) (*shard[
return s, nil
}
-func (s *shard[T]) close() {
+func (s *shard[T, O]) close() {
s.closeOnce.Do(func() {
s.scheduler.Close()
s.segmentManageStrategy.Close()
diff --git a/banyand/internal/storage/storage.go b/banyand/internal/storage/storage.go
index 911c5410..a2feb960 100644
--- a/banyand/internal/storage/storage.go
+++ b/banyand/internal/storage/storage.go
@@ -70,7 +70,7 @@ type IndexDB interface {
}
// TSDB allows listing and getting shard details.
-type TSDB[T TSTable] interface {
+type TSDB[T TSTable, O any] interface {
io.Closer
Lookup(ctx context.Context, series *pbv1.Series) (pbv1.SeriesList, error)
CreateTSTableIfNotExist(shardID common.ShardID, ts time.Time) (TSTableWrapper[T], error)
@@ -92,8 +92,8 @@ type TSTableWrapper[T TSTable] interface {
}
// TSTableCreator creates a TSTable.
-type TSTableCreator[T TSTable] func(fileSystem fs.FileSystem, root string, position common.Position,
- l *logger.Logger, timeRange timestamp.TimeRange) (T, error)
+type TSTableCreator[T TSTable, O any] func(fileSystem fs.FileSystem, root string, position common.Position,
+ l *logger.Logger, timeRange timestamp.TimeRange, option O) (T, error)
// IntervalUnit denotes the unit of a time point.
type IntervalUnit int
diff --git a/banyand/internal/storage/tsdb.go b/banyand/internal/storage/tsdb.go
index fd5b648a..16b5fb09 100644
--- a/banyand/internal/storage/tsdb.go
+++ b/banyand/internal/storage/tsdb.go
@@ -43,8 +43,9 @@ const (
)
// TSDBOpts wraps options to create a tsdb.
-type TSDBOpts[T TSTable] struct {
- TSTableCreator TSTableCreator[T]
+type TSDBOpts[T TSTable, O any] struct {
+ Option O
+ TSTableCreator TSTableCreator[T, O]
Location string
SegmentInterval IntervalRule
TTL IntervalRule
@@ -59,17 +60,18 @@ func generateSegID(unit IntervalUnit, suffix int) segmentID {
return segmentID(unit)<<31 | ((segmentID(suffix) << 1) >> 1)
}
-type database[T TSTable] struct {
+type database[T TSTable, O any] struct {
logger *logger.Logger
index *seriesIndex
+ p common.Position
location string
- sLst []*shard[T]
- opts TSDBOpts[T]
+ sLst []*shard[T, O]
+ opts TSDBOpts[T, O]
sync.RWMutex
sLen uint32
}
-func (d *database[T]) Close() error {
+func (d *database[T, O]) Close() error {
d.Lock()
defer d.Unlock()
for _, s := range d.sLst {
@@ -80,7 +82,7 @@ func (d *database[T]) Close() error {
// OpenTSDB returns a new tsdb runtime. This constructor will create a new database if it's absent,
// or load an existing one.
-func OpenTSDB[T TSTable](ctx context.Context, opts TSDBOpts[T]) (TSDB[T], error) {
+func OpenTSDB[T TSTable, O any](ctx context.Context, opts TSDBOpts[T, O]) (TSDB[T, O], error) {
if opts.SegmentInterval.Num == 0 {
return nil, errors.Wrap(errOpenDatabase, "segment interval is absent")
}
@@ -94,11 +96,12 @@ func OpenTSDB[T TSTable](ctx context.Context, opts TSDBOpts[T]) (TSDB[T], error)
if err != nil {
return nil, errors.Wrap(errOpenDatabase, errors.WithMessage(err, "create series index failed").Error())
}
- db := &database[T]{
+ db := &database[T, O]{
location: location,
logger: logger.Fetch(ctx, p.Database),
index: si,
opts: opts,
+ p: p,
}
db.logger.Info().Str("path", opts.Location).Msg("initialized")
if err = db.loadDatabase(); err != nil {
@@ -107,7 +110,7 @@ func OpenTSDB[T TSTable](ctx context.Context, opts TSDBOpts[T]) (TSDB[T], error)
return db, nil
}
-func (d *database[T]) CreateTSTableIfNotExist(shardID common.ShardID, ts time.Time) (TSTableWrapper[T], error) {
+func (d *database[T, O]) CreateTSTableIfNotExist(shardID common.ShardID, ts time.Time) (TSTableWrapper[T], error) {
id := uint32(shardID)
if id >= atomic.LoadUint32(&d.sLen) {
return func() (TSTableWrapper[T], error) {
@@ -129,7 +132,7 @@ func (d *database[T]) CreateTSTableIfNotExist(shardID common.ShardID, ts time.Ti
return d.createTSTTable(shardID, ts)
}
-func (d *database[T]) createTSTTable(shardID common.ShardID, ts time.Time) (TSTableWrapper[T], error) {
+func (d *database[T, O]) createTSTTable(shardID common.ShardID, ts time.Time) (TSTableWrapper[T], error) {
timeRange := timestamp.NewInclusiveTimeRange(ts, ts)
ss := d.sLst[shardID].segmentController.selectTSTables(timeRange)
if len(ss) > 0 {
@@ -138,7 +141,7 @@ func (d *database[T]) createTSTTable(shardID common.ShardID, ts time.Time) (TSTa
return d.sLst[shardID].segmentController.createTSTable(ts)
}
-func (d *database[T]) SelectTSTables(timeRange timestamp.TimeRange) []TSTableWrapper[T] {
+func (d *database[T, O]) SelectTSTables(timeRange timestamp.TimeRange) []TSTableWrapper[T] {
var result []TSTableWrapper[T]
d.RLock()
for i := range d.sLst {
@@ -148,8 +151,11 @@ func (d *database[T]) SelectTSTables(timeRange timestamp.TimeRange) []TSTableWra
return result
}
-func (d *database[T]) registerShard(id int) error {
+func (d *database[T, O]) registerShard(id int) error {
ctx := context.WithValue(context.Background(), logger.ContextKey, d.logger)
+ ctx = common.SetPosition(ctx, func(p common.Position) common.Position {
+ return d.p
+ })
so, err := d.openShard(ctx, common.ShardID(id))
if err != nil {
return err
@@ -159,7 +165,7 @@ func (d *database[T]) registerShard(id int) error {
return nil
}
-func (d *database[T]) loadDatabase() error {
+func (d *database[T, O]) loadDatabase() error {
d.Lock()
defer d.Unlock()
return walkDir(d.location, shardPathPrefix, func(suffix string) error {
diff --git a/banyand/liaison/grpc/stream.go b/banyand/liaison/grpc/stream.go
index 37dc44ea..9ad52102 100644
--- a/banyand/liaison/grpc/stream.go
+++ b/banyand/liaison/grpc/stream.go
@@ -116,9 +116,8 @@ func (s *streamService) Write(stream streamv1.StreamService_WriteServer) error {
Request: writeEntity,
ShardId: uint32(shardID),
SeriesHash: tsdb.HashEntity(entity),
- }
- if s.log.Debug().Enabled() {
- iwr.EntityValues = tagValues.Encode()
+ // TODO: remove the first value (stream name) of tagValues
+ EntityValues: tagValues[1:].Encode(),
}
nodeID, errPickNode := s.nodeRegistry.Locate(writeEntity.GetMetadata().GetGroup(), writeEntity.GetMetadata().GetName(), uint32(shardID))
if errPickNode != nil {
diff --git a/banyand/measure/block.go b/banyand/measure/block.go
index d3f10459..7da65663 100644
--- a/banyand/measure/block.go
+++ b/banyand/measure/block.go
@@ -134,7 +134,7 @@ func (b *block) mustWriteTo(sid common.SeriesID, bm *blockMetadata, ww *writers)
bm.uncompressedSizeBytes = b.uncompressedSizeBytes()
bm.count = uint64(b.Len())
- mustWriteTimestampsTo(&bm.timestamps, b.timestamps, ww.timestampsWriter)
+ mustWriteTimestampsTo(&bm.timestamps, b.timestamps, &ww.timestampsWriter)
for ti := range b.tagFamilies {
b.marshalTagFamily(b.tagFamilies[ti], bm, ww)
@@ -212,8 +212,11 @@ func (b *block) unmarshalTagFamily(decoder *encoding.BytesBlockDecoder, tfIndex
}
bigValuePool.Release(bb)
b.tagFamilies[tfIndex].name = name
- cc := b.tagFamilies[tfIndex].resizeColumns(len(tagProjection))
+ if len(tagProjection) < 1 {
+ return
+ }
+ cc := b.tagFamilies[tfIndex].resizeColumns(len(tagProjection))
for j := range tagProjection {
for i := range cfm.columnMetadata {
if tagProjection[j] == cfm.columnMetadata[i].name {
@@ -224,6 +227,30 @@ func (b *block) unmarshalTagFamily(decoder *encoding.BytesBlockDecoder, tfIndex
}
}
+func (b *block) unmarshalTagFamilyFromSeqReaders(decoder *encoding.BytesBlockDecoder, tfIndex int, name string,
+ columnFamilyMetadataBlock *dataBlock, metaReader, valueReader *seqReader,
+) {
+ if columnFamilyMetadataBlock.offset != metaReader.bytesRead {
+ logger.Panicf("offset %d must be equal to bytesRead %d", columnFamilyMetadataBlock.offset, metaReader.bytesRead)
+ }
+ bb := bigValuePool.Generate()
+ bb.Buf = bytes.ResizeExact(bb.Buf, int(columnFamilyMetadataBlock.size))
+ metaReader.mustReadFull(bb.Buf)
+ cfm := generateColumnFamilyMetadata()
+ defer releaseColumnFamilyMetadata(cfm)
+ _, err := cfm.unmarshal(bb.Buf)
+ if err != nil {
+ logger.Panicf("%s: cannot unmarshal columnFamilyMetadata: %v", metaReader.Path(), err)
+ }
+ bigValuePool.Release(bb)
+ b.tagFamilies[tfIndex].name = name
+
+ cc := b.tagFamilies[tfIndex].resizeColumns(len(cfm.columnMetadata))
+ for i := range cfm.columnMetadata {
+ cc[i].mustSeqReadValues(decoder, valueReader, cfm.columnMetadata[i], uint64(b.Len()))
+ }
+}
+
func (b *block) uncompressedSizeBytes() uint64 {
dataPointsCount := uint64(b.Len())
@@ -261,6 +288,11 @@ func (b *block) mustReadFrom(decoder *encoding.BytesBlockDecoder, p *part, bm bl
b.timestamps = mustReadTimestampsFrom(b.timestamps, &bm.timestamps, int(bm.count), p.timestamps)
+ cc := b.field.resizeColumns(len(bm.field.columnMetadata))
+ for i := range cc {
+ cc[i].mustReadValues(decoder, p.fieldValues, bm.field.columnMetadata[i], bm.count)
+ }
+
_ = b.resizeTagFamilies(len(bm.tagProjection))
for i := range bm.tagProjection {
name := bm.tagProjection[i].Family
@@ -268,11 +300,31 @@ func (b *block) mustReadFrom(decoder *encoding.BytesBlockDecoder, p *part, bm bl
if !ok {
continue
}
- b.unmarshalTagFamily(decoder, i, name, block, bm.tagProjection[i].Names, p.tagFamilyMetadata[name], p.tagFamilies[name])
+ b.unmarshalTagFamily(decoder, i, name, block,
+ bm.tagProjection[i].Names, p.tagFamilyMetadata[name],
+ p.tagFamilies[name])
}
+}
+
+func (b *block) mustSeqReadFrom(decoder *encoding.BytesBlockDecoder, seqReaders *seqReaders, bm blockMetadata) {
+ b.reset()
+
+ b.timestamps = mustSeqReadTimestampsFrom(b.timestamps, &bm.timestamps, int(bm.count), &seqReaders.timestamps)
+
cc := b.field.resizeColumns(len(bm.field.columnMetadata))
for i := range cc {
- cc[i].mustReadValues(decoder, p.fieldValues, bm.field.columnMetadata[i], bm.count)
+ cc[i].mustSeqReadValues(decoder, &seqReaders.fieldValues, bm.field.columnMetadata[i], bm.count)
+ }
+ _ = b.resizeTagFamilies(len(bm.tagFamilies))
+ keys := make([]string, 0, len(bm.tagFamilies))
+ for k := range bm.tagFamilies {
+ keys = append(keys, k)
+ }
+ sort.Strings(keys)
+ for i, name := range keys {
+ block := bm.tagFamilies[name]
+ b.unmarshalTagFamilyFromSeqReaders(decoder, i, name, block,
+ seqReaders.tagFamilyMetadata[name], seqReaders.tagFamilies[name])
}
}
@@ -283,7 +335,7 @@ func (b *block) sortTagFamilies() {
})
}
-func mustWriteTimestampsTo(tm *timestampsMetadata, timestamps []int64, timestampsWriter writer) {
+func mustWriteTimestampsTo(tm *timestampsMetadata, timestamps []int64, timestampsWriter *writer) {
tm.reset()
bb := bigValuePool.Generate()
@@ -311,6 +363,22 @@ func mustReadTimestampsFrom(dst []int64, tm *timestampsMetadata, count int, read
return dst
}
+func mustSeqReadTimestampsFrom(dst []int64, tm *timestampsMetadata, count int, reader *seqReader) []int64 {
+ if tm.offset != reader.bytesRead {
+ logger.Panicf("offset %d must be equal to bytesRead %d", tm.offset, reader.bytesRead)
+ }
+ bb := bigValuePool.Generate()
+ defer bigValuePool.Release(bb)
+ bb.Buf = bytes.ResizeExact(bb.Buf, int(tm.size))
+ reader.mustReadFull(bb.Buf)
+ var err error
+ dst, err = encoding.BytesToInt64List(dst, bb.Buf, tm.encodeType, tm.min, count)
+ if err != nil {
+ logger.Panicf("%s: cannot unmarshal timestamps: %v", reader.Path(), err)
+ }
+ return dst
+}
+
func generateBlock() *block {
v := blockPool.Get()
if v == nil {
@@ -369,9 +437,20 @@ func (bc *blockCursor) init(p *part, bm blockMetadata, queryOpts queryOptions) {
bc.fieldProjection = queryOpts.FieldProjection
}
-func (bc *blockCursor) copyAllTo(r *pbv1.Result) {
+func (bc *blockCursor) copyAllTo(r *pbv1.Result, desc bool) {
+ var idx, offset int
+ if desc {
+ idx = 0
+ offset = bc.idx + 1
+ } else {
+ idx = bc.idx
+ offset = len(bc.timestamps)
+ }
+ if offset <= idx {
+ return
+ }
r.SID = bc.bm.seriesID
- r.Timestamps = append(r.Timestamps, bc.timestamps...)
+ r.Timestamps = append(r.Timestamps, bc.timestamps[idx:offset]...)
for _, cf := range bc.tagFamilies {
tf := pbv1.TagFamily{
Name: cf.name,
@@ -380,7 +459,7 @@ func (bc *blockCursor) copyAllTo(r *pbv1.Result) {
t := pbv1.Tag{
Name: c.name,
}
- for _, v := range c.values {
+ for _, v := range c.values[idx:offset] {
t.Values = append(t.Values, mustDecodeTagValue(c.valueType, v))
}
tf.Tags = append(tf.Tags, t)
@@ -391,7 +470,7 @@ func (bc *blockCursor) copyAllTo(r *pbv1.Result) {
f := pbv1.Field{
Name: c.name,
}
- for _, v := range c.values {
+ for _, v := range c.values[idx:offset] {
f.Values = append(f.Values, mustDecodeFieldValue(c.valueType, v))
}
r.Fields = append(r.Fields, f)
@@ -502,6 +581,7 @@ func (bc *blockCursor) loadData(tmpBlock *block) bool {
name: tmpBlock.field.columns[i].name,
valueType: tmpBlock.field.columns[i].valueType,
}
+
c.values = append(c.values, tmpBlock.field.columns[i].values[start:end]...)
bc.fields.columns = append(bc.fields.columns, c)
}
@@ -527,10 +607,21 @@ func findRange(timestamps []int64, min int64, max int64) (int, int, bool) {
}
}
- if start == -1 || end == -1 || start >= end {
+ if start == -1 && end == -1 {
return 0, 0, false
}
+ if start == -1 {
+ start = 0
+ }
+
+ if end == -1 {
+ end = l
+ }
+
+ if start >= end {
+ return 0, 0, false
+ }
return start, end, true
}
@@ -548,3 +639,112 @@ func releaseBlockCursor(bc *blockCursor) {
bc.reset()
blockCursorPool.Put(bc)
}
+
+type blockPointer struct {
+ block
+ bm blockMetadata
+ idx int
+ lastPartID uint64
+}
+
+func (bi *blockPointer) updateMetadata() {
+ if len(bi.block.timestamps) == 0 {
+ return
+ }
+ // only update timestamps since they are used for merging
+ // blockWriter will recompute all fields
+ bi.bm.timestamps.min = bi.block.timestamps[0]
+ bi.bm.timestamps.max = bi.block.timestamps[len(bi.timestamps)-1]
+}
+
+func (bi *blockPointer) copyFrom(src *blockPointer) {
+ bi.idx = 0
+ bi.bm.copyFrom(&src.bm)
+ bi.appendAll(src)
+}
+
+func (bi *blockPointer) appendAll(b *blockPointer) {
+ if len(b.timestamps) == 0 {
+ return
+ }
+ bi.append(b, len(b.timestamps))
+}
+
+func (bi *blockPointer) append(b *blockPointer, offset int) {
+ if offset <= b.idx {
+ return
+ }
+ if len(bi.tagFamilies) == 0 && len(b.tagFamilies) > 0 {
+ for _, tf := range b.tagFamilies {
+ tagFamily := columnFamily{name: tf.name}
+ for _, c := range tf.columns {
+ col := column{name: c.name, valueType: c.valueType}
+ assertIdxAndOffset(col.name, len(c.values), b.idx, offset)
+ col.values = append(col.values, c.values[b.idx:offset]...)
+ tagFamily.columns = append(tagFamily.columns, col)
+ }
+ bi.tagFamilies = append(bi.tagFamilies, tagFamily)
+ }
+ } else {
+ if len(bi.tagFamilies) != len(b.tagFamilies) {
+ logger.Panicf("unexpected number of tag families: got %d; want %d", len(bi.tagFamilies), len(b.tagFamilies))
+ }
+ for i := range bi.tagFamilies {
+ if bi.tagFamilies[i].name != b.tagFamilies[i].name {
+ logger.Panicf("unexpected tag family name: got %q; want %q", bi.tagFamilies[i].name, b.tagFamilies[i].name)
+ }
+ if len(bi.tagFamilies[i].columns) != len(b.tagFamilies[i].columns) {
+ logger.Panicf("unexpected number of tags for tag family %q: got %d; want %d", bi.tagFamilies[i].name, len(bi.tagFamilies[i].columns), len(b.tagFamilies[i].columns))
+ }
+ for j := range bi.tagFamilies[i].columns {
+ if bi.tagFamilies[i].columns[j].name != b.tagFamilies[i].columns[j].name {
+ logger.Panicf("unexpected tag name for tag family %q: got %q; want %q", bi.tagFamilies[i].name, bi.tagFamilies[i].columns[j].name, b.tagFamilies[i].columns[j].name)
+ }
+ assertIdxAndOffset(b.tagFamilies[i].columns[j].name, len(b.tagFamilies[i].columns[j].values), b.idx, offset)
+ bi.tagFamilies[i].columns[j].values = append(bi.tagFamilies[i].columns[j].values, b.tagFamilies[i].columns[j].values[b.idx:offset]...)
+ }
+ }
+ }
+
+ if len(bi.field.columns) == 0 && len(b.field.columns) > 0 {
+ for _, c := range b.field.columns {
+ col := column{name: c.name, valueType: c.valueType}
+ assertIdxAndOffset(col.name, len(c.values), b.idx, offset)
+ col.values = append(col.values, c.values[b.idx:offset]...)
+ bi.field.columns = append(bi.field.columns, col)
+ }
+ } else {
+ if len(bi.field.columns) != len(b.field.columns) {
+ logger.Panicf("unexpected number of fields: got %d; want %d", len(bi.field.columns), len(b.field.columns))
+ }
+ for i := range bi.field.columns {
+ assertIdxAndOffset(b.field.columns[i].name, len(b.field.columns[i].values), b.idx, offset)
+ bi.field.columns[i].values = append(bi.field.columns[i].values, b.field.columns[i].values[b.idx:offset]...)
+ }
+ }
+
+ assertIdxAndOffset("timestamps", len(b.timestamps), bi.idx, offset)
+ bi.timestamps = append(bi.timestamps, b.timestamps[b.idx:offset]...)
+
+ bi.lastPartID = b.lastPartID
+}
+
+func assertIdxAndOffset(name string, length int, idx int, offset int) {
+ if idx >= offset {
+ logger.Panicf("%q idx %d must be less than offset %d", name, idx, offset)
+ }
+ if offset > length {
+ logger.Panicf("%q offset %d must be less than or equal to length %d", name, offset, length)
+ }
+}
+
+func (bi *blockPointer) isFull() bool {
+ return bi.bm.count >= maxBlockLength || bi.bm.uncompressedSizeBytes >= maxUncompressedBlockSize
+}
+
+func (bi *blockPointer) reset() {
+ bi.idx = 0
+ bi.lastPartID = 0
+ bi.block.reset()
+ bi.bm = blockMetadata{}
+}
diff --git a/banyand/measure/block_metadata.go b/banyand/measure/block_metadata.go
index e239162c..6202c9eb 100644
--- a/banyand/measure/block_metadata.go
+++ b/banyand/measure/block_metadata.go
@@ -20,6 +20,7 @@ package measure
import (
"errors"
"fmt"
+ "sort"
"sync"
"github.com/apache/skywalking-banyandb/api/common"
@@ -74,6 +75,21 @@ type blockMetadata struct {
count uint64
}
+func (bh *blockMetadata) copyFrom(src *blockMetadata) {
+ bh.seriesID = src.seriesID
+ bh.uncompressedSizeBytes = src.uncompressedSizeBytes
+ bh.count = src.count
+ bh.timestamps.copyFrom(&src.timestamps)
+ for k, db := range src.tagFamilies {
+ if bh.tagFamilies == nil {
+ bh.tagFamilies = make(map[string]*dataBlock)
+ }
+ bh.tagFamilies[k] = &dataBlock{}
+ bh.tagFamilies[k].copyFrom(db)
+ }
+ bh.field.copyFrom(&src.field)
+}
+
func (bh *blockMetadata) getTagFamilyMetadata(name string) *dataBlock {
if bh.tagFamilies == nil {
bh.tagFamilies = make(map[string]*dataBlock)
@@ -105,7 +121,14 @@ func (bh *blockMetadata) marshal(dst []byte) []byte {
dst = encoding.VarUint64ToBytes(dst, bh.count)
dst = bh.timestamps.marshal(dst)
dst = encoding.VarUint64ToBytes(dst, uint64(len(bh.tagFamilies)))
- for name, cf := range bh.tagFamilies {
+ // make sure the order of tagFamilies is stable
+ keys := make([]string, 0, len(bh.tagFamilies))
+ for k := range bh.tagFamilies {
+ keys = append(keys, k)
+ }
+ sort.Strings(keys)
+ for _, name := range keys {
+ cf := bh.tagFamilies[name]
dst = encoding.EncodeBytes(dst, convert.StringToBytes(name))
dst = cf.marshal(dst)
}
diff --git a/banyand/measure/block_reader.go b/banyand/measure/block_reader.go
index de5c5c2f..219d0a94 100644
--- a/banyand/measure/block_reader.go
+++ b/banyand/measure/block_reader.go
@@ -17,23 +17,215 @@
package measure
-import "github.com/apache/skywalking-banyandb/pkg/fs"
+import (
+ "container/heap"
+ "errors"
+ "fmt"
+ "io"
+ "sync"
-type reader struct {
+ "github.com/apache/skywalking-banyandb/pkg/encoding"
+ "github.com/apache/skywalking-banyandb/pkg/fs"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+)
+
+type seqReader struct {
+ sr io.Reader
r fs.Reader
bytesRead uint64
}
-func newReader(r fs.Reader) *reader {
- return &reader{r: r}
+func (sr *seqReader) reset() {
+ sr.r = nil
+ sr.sr = nil
+ sr.bytesRead = 0
+}
+
+func (sr *seqReader) Path() string {
+ return sr.r.Path()
+}
+
+func (sr *seqReader) init(r fs.Reader) {
+ sr.reset()
+ sr.sr = r.StreamRead()
+ sr.r = r
+}
+
+func (sr *seqReader) mustReadFull(data []byte) {
+ n, err := io.ReadFull(sr.sr, data)
+ if err != nil {
+ if errors.Is(err, io.EOF) {
+ return
+ }
+ logger.Panicf("cannot read data: %v", err)
+ }
+ if n != len(data) {
+ logger.Panicf("cannot read full data: %d/%d", n, len(data))
+ }
+ sr.bytesRead += uint64(n)
+}
+
+func generateSeqReader() *seqReader {
+ if v := seqReaderPool.Get(); v != nil {
+ return v.(*seqReader)
+ }
+ return &seqReader{}
+}
+
+func releaseSeqReader(sr *seqReader) {
+ sr.reset()
+ seqReaderPool.Put(sr)
+}
+
+var seqReaderPool sync.Pool
+
+type seqReaders struct {
+ tagFamilyMetadata map[string]*seqReader
+ tagFamilies map[string]*seqReader
+ primary seqReader
+ timestamps seqReader
+ fieldValues seqReader
+}
+
+func (sr *seqReaders) reset() {
+ sr.primary.reset()
+ sr.timestamps.reset()
+ sr.fieldValues.reset()
+ if sr.tagFamilyMetadata != nil {
+ for k, r := range sr.tagFamilyMetadata {
+ releaseSeqReader(r)
+ delete(sr.tagFamilyMetadata, k)
+ }
+ }
+ if sr.tagFamilies != nil {
+ for k, r := range sr.tagFamilies {
+ releaseSeqReader(r)
+ delete(sr.tagFamilies, k)
+ }
+ }
}
-func (r *reader) Path() string {
- return r.r.Path()
+func (sr *seqReaders) init(p *part) {
+ sr.reset()
+ sr.primary.init(p.primary)
+ sr.timestamps.init(p.timestamps)
+ sr.fieldValues.init(p.fieldValues)
+ if sr.tagFamilies == nil {
+ sr.tagFamilies = make(map[string]*seqReader)
+ sr.tagFamilyMetadata = make(map[string]*seqReader)
+ }
+
+ for k, r := range p.tagFamilies {
+ sr.tagFamilies[k] = generateSeqReader()
+ sr.tagFamilies[k].init(r)
+ sr.tagFamilyMetadata[k] = generateSeqReader()
+ sr.tagFamilyMetadata[k].init(p.tagFamilyMetadata[k])
+ }
+}
+
+type blockReader struct {
+ err error
+ block *blockPointer
+ pih partMergeIterHeap
+ nextBlockNoop bool
+}
+
+func (br *blockReader) reset() {
+ br.block = nil
+ for i := range br.pih {
+ br.pih[i] = nil
+ }
+ br.pih = br.pih[:0]
+ br.nextBlockNoop = false
+ br.err = nil
+}
+
+func (br *blockReader) init(pii []*partMergeIter) {
+ br.reset()
+ for _, pi := range pii {
+ if pi.nextBlockMetadata() {
+ br.pih = append(br.pih, pi)
+ continue
+ }
+ if err := pi.error(); err != nil {
+ br.err = fmt.Errorf("can't get the block to merge: %w", err)
+ return
+ }
+ }
+ if len(br.pih) == 0 {
+ br.err = io.EOF
+ return
+ }
+ heap.Init(&br.pih)
+ br.block = &br.pih[0].block
+ br.nextBlockNoop = true
+}
+
+func (br *blockReader) nextBlockMetadata() bool {
+ if br.err != nil {
+ return false
+ }
+ if br.nextBlockNoop {
+ br.nextBlockNoop = false
+ return true
+ }
+
+ br.err = br.nextMetadata()
+ if br.err != nil {
+ if errors.Is(br.err, io.EOF) {
+ return false
+ }
+ br.err = fmt.Errorf("can't get the block to merge: %w", br.err)
+ return false
+ }
+ return true
+}
+
+func (br *blockReader) nextMetadata() error {
+ head := br.pih[0]
+ if head.nextBlockMetadata() {
+ heap.Fix(&br.pih, 0)
+ br.block = &br.pih[0].block
+ return nil
+ }
+
+ if err := head.error(); err != nil {
+ br.block = nil
+ return err
+ }
+
+ heap.Pop(&br.pih)
+
+ if len(br.pih) == 0 {
+ br.block = nil
+ return io.EOF
+ }
+
+ br.block = &br.pih[0].block
+ return nil
+}
+
+func (br *blockReader) loadBlockData(decoder *encoding.BytesBlockDecoder) {
+ br.pih[0].mustLoadBlockData(decoder, br.block)
+}
+
+func (br *blockReader) error() error {
+ if errors.Is(br.err, io.EOF) {
+ return nil
+ }
+ return br.err
+}
+
+var blockReaderPool sync.Pool
+
+func generateBlockReader() *blockReader {
+ if v := blockReaderPool.Get(); v != nil {
+ return v.(*blockReader)
+ }
+ return &blockReader{}
}
-func (r *reader) Read(p []byte) (int, error) {
- n, err := r.r.Read(0, p)
- r.bytesRead += uint64(n)
- return n, err
+func releaseBlockReader(br *blockReader) {
+ br.reset()
+ blockReaderPool.Put(br)
}
diff --git a/banyand/measure/block_reader_test.go b/banyand/measure/block_reader_test.go
new file mode 100644
index 00000000..e7b44287
--- /dev/null
+++ b/banyand/measure/block_reader_test.go
@@ -0,0 +1,165 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package measure
+
+import (
+ "errors"
+ "sort"
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+ "github.com/stretchr/testify/require"
+
+ "github.com/apache/skywalking-banyandb/pkg/fs"
+ "github.com/apache/skywalking-banyandb/pkg/test"
+)
+
+func Test_blockReader_nextBlock(t *testing.T) {
+ tests := []struct {
+ wantErr error
+ name string
+ dpsList []*dataPoints
+ want []blockMetadata
+ }{
+ {
+ name: "Test with no data points",
+ dpsList: []*dataPoints{},
+ },
+ {
+ name: "Test with single part",
+ dpsList: []*dataPoints{dpsTS1},
+ want: []blockMetadata{
+ {seriesID: 1, count: 1, uncompressedSizeBytes: 1676},
+ {seriesID: 2, count: 1, uncompressedSizeBytes: 55},
+ {seriesID: 3, count: 1, uncompressedSizeBytes: 24},
+ },
+ },
+ {
+ name: "Test with multiple parts with different ts",
+ dpsList: []*dataPoints{dpsTS1, dpsTS2},
+ want: []blockMetadata{
+ {seriesID: 1, count: 1, uncompressedSizeBytes: 1676},
+ {seriesID: 1, count: 1, uncompressedSizeBytes: 1676},
+ {seriesID: 2, count: 1, uncompressedSizeBytes: 55},
+ {seriesID: 2, count: 1, uncompressedSizeBytes: 55},
+ {seriesID: 3, count: 1, uncompressedSizeBytes: 24},
+ {seriesID: 3, count: 1, uncompressedSizeBytes: 24},
+ },
+ },
+ {
+ name: "Test with multiple parts with same ts",
+ dpsList: []*dataPoints{dpsTS1, dpsTS1},
+ want: []blockMetadata{
+ {seriesID: 1, count: 1, uncompressedSizeBytes: 1676},
+ {seriesID: 1, count: 1, uncompressedSizeBytes: 1676},
+ {seriesID: 2, count: 1, uncompressedSizeBytes: 55},
+ {seriesID: 2, count: 1, uncompressedSizeBytes: 55},
+ {seriesID: 3, count: 1, uncompressedSizeBytes: 24},
+ {seriesID: 3, count: 1, uncompressedSizeBytes: 24},
+ },
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ verify := func(pp []*part) {
+ var pii []*partMergeIter
+ for _, p := range pp {
+ pmi := &partMergeIter{}
+ pmi.mustInitFromPart(p)
+ pii = append(pii, pmi)
+ }
+ reader := generateBlockReader()
+ defer releaseBlockReader(reader)
+ reader.init(pii)
+ var got []blockMetadata
+ for reader.nextBlockMetadata() {
+ if reader.block.bm.seriesID == 0 {
+ t.Errorf("Expected curBlock to be initialized, but it was nil")
+ }
+ var tagFamilyNames []string
+ for _, cf := range reader.block.block.tagFamilies {
+ tagFamilyNames = append(tagFamilyNames, cf.name)
+ }
+ if !sort.IsSorted(sort.StringSlice(tagFamilyNames)) {
+ t.Errorf("Expected tagFamilyNames %q to be sorted, but it was not", tagFamilyNames)
+ }
+ got = append(got, reader.block.bm)
+ }
+
+ if !errors.Is(reader.error(), tt.wantErr) {
+ t.Errorf("Unexpected error: got %v, want %v", reader.err, tt.wantErr)
+ }
+
+ if diff := cmp.Diff(got, tt.want,
+ cmpopts.IgnoreFields(blockMetadata{}, "timestamps"),
+ cmpopts.IgnoreFields(blockMetadata{}, "field"),
+ cmpopts.IgnoreFields(blockMetadata{}, "tagFamilies"),
+ cmp.AllowUnexported(blockMetadata{}),
+ ); diff != "" {
+ t.Errorf("Unexpected blockMetadata (-got +want):\n%s", diff)
+ }
+ }
+
+ t.Run("memory parts", func(t *testing.T) {
+ var mpp []*memPart
+ defer func() {
+ for _, mp := range mpp {
+ releaseMemPart(mp)
+ }
+ }()
+ var pp []*part
+ for _, dps := range tt.dpsList {
+ mp := generateMemPart()
+ mpp = append(mpp, mp)
+ mp.mustInitFromDataPoints(dps)
+ pp = append(pp, openMemPart(mp))
+ }
+ verify(pp)
+ })
+
+ t.Run("file parts", func(t *testing.T) {
+ var mpp []*memPart
+ var fpp []*partWrapper
+ tmpPath, defFn := test.Space(require.New(t))
+ defer func() {
+ for _, mp := range mpp {
+ releaseMemPart(mp)
+ }
+ for _, pw := range fpp {
+ pw.decRef()
+ }
+ defFn()
+ }()
+ var pp []*part
+ fileSystem := fs.NewLocalFileSystem()
+ for i, dps := range tt.dpsList {
+ mp := generateMemPart()
+ mpp = append(mpp, mp)
+ mp.mustInitFromDataPoints(dps)
+ mp.mustFlush(fileSystem, partPath(tmpPath, uint64(i)))
+ filePW := newPartWrapper(nil, mustOpenFilePart(uint64(i), tmpPath, fileSystem))
+ filePW.p.partMetadata.ID = uint64(i)
+ fpp = append(fpp, filePW)
+ pp = append(pp, filePW.p)
+ }
+ verify(pp)
+ })
+ })
+ }
+}
diff --git a/banyand/measure/block_test.go b/banyand/measure/block_test.go
index e7b776c1..b51e33c2 100644
--- a/banyand/measure/block_test.go
+++ b/banyand/measure/block_test.go
@@ -264,7 +264,7 @@ func Test_mustWriteAndReadTimestamps(t *testing.T) {
}()
tm := ×tampsMetadata{}
b := &bytes.Buffer{}
- mustWriteTimestampsTo(tm, tt.args, writer{w: b})
+ mustWriteTimestampsTo(tm, tt.args, &writer{w: b})
timestamps := mustReadTimestampsFrom(nil, tm, len(tt.args), b)
if !reflect.DeepEqual(timestamps, tt.args) {
t.Errorf("mustReadTimestampsFrom() = %v, want %v", timestamps, tt.args)
@@ -325,6 +325,26 @@ func Test_marshalAndUnmarshalTagFamily(t *testing.T) {
); diff != "" {
t.Errorf("block.unmarshalTagFamily() (-got +want):\n%s", diff)
}
+
+ unmarshaled2 := generateBlock()
+ defer releaseBlock(unmarshaled2)
+ unmarshaled2.timestamps = make([]int64, len(b.timestamps))
+ unmarshaled2.resizeTagFamilies(1)
+
+ metaReader := generateSeqReader()
+ defer releaseSeqReader(metaReader)
+ metaReader.init(metaBuffer)
+ valueReader := generateSeqReader()
+ defer releaseSeqReader(valueReader)
+ valueReader.init(dataBuffer)
+
+ unmarshaled2.unmarshalTagFamilyFromSeqReaders(decoder, tfIndex, name, bm.getTagFamilyMetadata(name), metaReader, valueReader)
+
+ if diff := cmp.Diff(unmarshaled2.tagFamilies[0], b.tagFamilies[0],
+ cmp.AllowUnexported(columnFamily{}, column{}),
+ ); diff != "" {
+ t.Errorf("block.unmarshalTagFamilyFromSeqReaders() (-got +want):\n%s", diff)
+ }
}
func Test_marshalAndUnmarshalBlock(t *testing.T) {
@@ -339,6 +359,7 @@ func Test_marshalAndUnmarshalBlock(t *testing.T) {
fieldValuesWriter: writer{w: fieldBuffer},
}
p := &part{
+ primary: &bytes.Buffer{},
timestamps: timestampBuffer,
fieldValues: fieldBuffer,
}
@@ -378,6 +399,17 @@ func Test_marshalAndUnmarshalBlock(t *testing.T) {
if !reflect.DeepEqual(b, unmarshaled) {
t.Errorf("block.mustReadFrom() = %+v, want %+v", unmarshaled, b)
}
+
+ unmarshaled2 := generateBlock()
+ defer releaseBlock(unmarshaled2)
+ var sr seqReaders
+ sr.init(p)
+ defer sr.reset()
+
+ unmarshaled2.mustSeqReadFrom(decoder, &sr, bm)
+ if !reflect.DeepEqual(b, unmarshaled2) {
+ t.Errorf("block.mustSeqReadFrom() = %+v, want %+v", unmarshaled, b)
+ }
}
func Test_findRange(t *testing.T) {
@@ -464,3 +496,362 @@ func Test_findRange(t *testing.T) {
})
}
}
+
+func Test_blockPointer_append(t *testing.T) {
+ type fields struct {
+ timestamps []int64
+ tagFamilies []columnFamily
+ field columnFamily
+ partID uint64
+ }
+ type args struct {
+ b *blockPointer
+ offset int
+ }
+ tests := []struct {
+ want *blockPointer
+ name string
+ args args
+ fields fields
+ wantPanic bool
+ }{
+ {
+ name: "Test append with empty block",
+ fields: fields{
+ timestamps: []int64{1, 2},
+ tagFamilies: []columnFamily{
+ {
+ name: "arrTag",
+ columns: []column{
+ {
+ name: "strArrTag", valueType: pbv1.ValueTypeStrArr,
+ values: [][]byte{marshalStrArr([][]byte{[]byte("value5"), []byte("value6")}), marshalStrArr([][]byte{[]byte("value7"), []byte("value8")})},
+ },
+ },
+ },
+ },
+ field: columnFamily{
+ columns: []column{
+ {name: "strField", valueType: pbv1.ValueTypeStr, values: [][]byte{[]byte("field3"), []byte("field4")}},
+ },
+ },
+ },
+ args: args{
+ b: &blockPointer{
+ block: block{
+ timestamps: []int64{},
+ tagFamilies: []columnFamily{},
+ field: columnFamily{},
+ },
+ idx: 0,
+ },
+ offset: 0,
+ },
+ want: &blockPointer{
+ block: block{
+ timestamps: []int64{1, 2},
+ tagFamilies: []columnFamily{
+ {
+ name: "arrTag",
+ columns: []column{
+ {
+ name: "strArrTag", valueType: pbv1.ValueTypeStrArr,
+ values: [][]byte{marshalStrArr([][]byte{[]byte("value5"), []byte("value6")}), marshalStrArr([][]byte{[]byte("value7"), []byte("value8")})},
+ },
+ },
+ },
+ },
+ field: columnFamily{
+ columns: []column{
+ {name: "strField", valueType: pbv1.ValueTypeStr, values: [][]byte{[]byte("field3"), []byte("field4")}},
+ },
+ },
+ },
+ idx: 0,
+ },
+ },
+ {
+ name: "Test append to a empty block",
+ fields: fields{
+ timestamps: nil,
+ tagFamilies: nil,
+ field: columnFamily{
+ columns: nil,
+ },
+ partID: 0,
+ },
+ args: args{
+ b: &blockPointer{
+ lastPartID: 2,
+ block: block{
+ timestamps: []int64{4, 5},
+ tagFamilies: []columnFamily{
+ {
+ name: "arrTag",
+ columns: []column{
+ {
+ name: "strArrTag", valueType: pbv1.ValueTypeStrArr,
+ values: [][]byte{marshalStrArr([][]byte{[]byte("value9"), []byte("value10")}), marshalStrArr([][]byte{[]byte("value11"), []byte("value12")})},
+ },
+ },
+ },
+ },
+ field: columnFamily{
+ columns: []column{
+ {name: "strField", valueType: pbv1.ValueTypeStr, values: [][]byte{[]byte("field5"), []byte("field6")}},
+ },
+ },
+ },
+ idx: 0,
+ },
+ offset: 2,
+ },
+ want: &blockPointer{
+ lastPartID: 2,
+ block: block{
+ timestamps: []int64{4, 5},
+ tagFamilies: []columnFamily{
+ {
+ name: "arrTag",
+ columns: []column{
+ {
+ name: "strArrTag", valueType: pbv1.ValueTypeStrArr,
+ values: [][]byte{marshalStrArr([][]byte{[]byte("value9"), []byte("value10")}), marshalStrArr([][]byte{[]byte("value11"), []byte("value12")})},
+ },
+ },
+ },
+ },
+ field: columnFamily{
+ columns: []column{
+ {name: "strField", valueType: pbv1.ValueTypeStr, values: [][]byte{[]byte("field5"), []byte("field6")}},
+ },
+ },
+ },
+ idx: 0,
+ },
+ },
+ {
+ name: "Test append with offset equals to the data size. All data",
+ fields: fields{
+ timestamps: []int64{1, 2},
+ tagFamilies: []columnFamily{
+ {
+ name: "arrTag",
+ columns: []column{
+ {
+ name: "strArrTag", valueType: pbv1.ValueTypeStrArr,
+ values: [][]byte{marshalStrArr([][]byte{[]byte("value5"), []byte("value6")}), marshalStrArr([][]byte{[]byte("value7"), []byte("value8")})},
+ },
+ },
+ },
+ },
+ field: columnFamily{
+ columns: []column{
+ {name: "strField", valueType: pbv1.ValueTypeStr, values: [][]byte{[]byte("field3"), []byte("field4")}},
+ },
+ },
+ partID: 1,
+ },
+ args: args{
+ b: &blockPointer{
+ lastPartID: 2,
+ block: block{
+ timestamps: []int64{4, 5},
+ tagFamilies: []columnFamily{
+ {
+ name: "arrTag",
+ columns: []column{
+ {
+ name: "strArrTag", valueType: pbv1.ValueTypeStrArr,
+ values: [][]byte{marshalStrArr([][]byte{[]byte("value9"), []byte("value10")}), marshalStrArr([][]byte{[]byte("value11"), []byte("value12")})},
+ },
+ },
+ },
+ },
+ field: columnFamily{
+ columns: []column{
+ {name: "strField", valueType: pbv1.ValueTypeStr, values: [][]byte{[]byte("field5"), []byte("field6")}},
+ },
+ },
+ },
+ idx: 0,
+ },
+ offset: 2,
+ },
+ want: &blockPointer{
+ lastPartID: 2,
+ block: block{
+ timestamps: []int64{1, 2, 4, 5},
+ tagFamilies: []columnFamily{
+ {
+ name: "arrTag",
+ columns: []column{
+ {
+ name: "strArrTag", valueType: pbv1.ValueTypeStrArr,
+ values: [][]byte{
+ marshalStrArr([][]byte{[]byte("value5"), []byte("value6")}),
+ marshalStrArr([][]byte{[]byte("value7"), []byte("value8")}), marshalStrArr([][]byte{[]byte("value9"), []byte("value10")}),
+ marshalStrArr([][]byte{[]byte("value11"), []byte("value12")}),
+ },
+ },
+ },
+ },
+ },
+ field: columnFamily{
+ columns: []column{
+ {name: "strField", valueType: pbv1.ValueTypeStr, values: [][]byte{[]byte("field3"), []byte("field4"), []byte("field5"), []byte("field6")}},
+ },
+ },
+ },
+ idx: 0,
+ },
+ },
+ {
+ name: "Test append with non-empty block and offset less than timestamps",
+ fields: fields{
+ timestamps: []int64{1, 2},
+ tagFamilies: []columnFamily{
+ {
+ name: "arrTag",
+ columns: []column{
+ {
+ name: "strArrTag", valueType: pbv1.ValueTypeStrArr,
+ values: [][]byte{marshalStrArr([][]byte{[]byte("value5"), []byte("value6")}), marshalStrArr([][]byte{[]byte("value7"), []byte("value8")})},
+ },
+ },
+ },
+ },
+ field: columnFamily{
+ columns: []column{
+ {name: "strField", valueType: pbv1.ValueTypeStr, values: [][]byte{[]byte("field3"), []byte("field4")}},
+ },
+ },
+ partID: 3,
+ },
+ args: args{
+ b: &blockPointer{
+ lastPartID: 2,
+ block: block{
+ timestamps: []int64{4, 5},
+ tagFamilies: []columnFamily{
+ {
+ name: "arrTag",
+ columns: []column{
+ {
+ name: "strArrTag", valueType: pbv1.ValueTypeStrArr,
+ values: [][]byte{marshalStrArr([][]byte{[]byte("value9"), []byte("value10")}), marshalStrArr([][]byte{[]byte("value11"), []byte("value12")})},
+ },
+ },
+ },
+ },
+ field: columnFamily{
+ columns: []column{
+ {name: "strField", valueType: pbv1.ValueTypeStr, values: [][]byte{[]byte("field5"), []byte("field6")}},
+ },
+ },
+ },
+ idx: 0,
+ },
+ offset: 1,
+ },
+ want: &blockPointer{
+ lastPartID: 2,
+ block: block{
+ timestamps: []int64{1, 2, 4},
+ tagFamilies: []columnFamily{
+ {
+ name: "arrTag",
+ columns: []column{
+ {
+ name: "strArrTag", valueType: pbv1.ValueTypeStrArr,
+ values: [][]byte{marshalStrArr([][]byte{[]byte("value5"), []byte("value6")}), marshalStrArr([][]byte{
+ []byte("value7"),
+ []byte("value8"),
+ }), marshalStrArr([][]byte{[]byte("value9"), []byte("value10")})},
+ },
+ },
+ },
+ },
+ field: columnFamily{
+ columns: []column{
+ {name: "strField", valueType: pbv1.ValueTypeStr, values: [][]byte{[]byte("field3"), []byte("field4"), []byte("field5")}},
+ },
+ },
+ },
+ idx: 0,
+ },
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ defer func() {
+ r := recover()
+ if (r != nil) != tt.wantPanic {
+ t.Errorf("blockPointer.append() recover = %v, wantPanic = %v", r, tt.wantPanic)
+ }
+ }()
+ bi := &blockPointer{
+ block: block{
+ timestamps: tt.fields.timestamps,
+ tagFamilies: tt.fields.tagFamilies,
+ field: tt.fields.field,
+ },
+ lastPartID: tt.fields.partID,
+ }
+ bi.append(tt.args.b, tt.args.offset)
+ if !reflect.DeepEqual(bi, tt.want) {
+ t.Errorf("blockPointer.append() = %+v, want %+v", bi, tt.want)
+ }
+ })
+ }
+}
+
+func Test_blockPointer_copyFrom(t *testing.T) {
+ type fields struct {
+ bm blockMetadata
+ idx int
+ }
+ type args struct {
+ src *blockPointer
+ }
+ tests := []struct {
+ args args
+ want *blockPointer
+ name string
+ fields fields
+ }{
+ {
+ name: "Test copyFrom",
+ fields: fields{
+ bm: blockMetadata{},
+ idx: 0,
+ },
+ args: args{
+ src: &blockPointer{
+ bm: blockMetadata{count: 1},
+ idx: 0,
+ block: conventionalBlock,
+ lastPartID: 2,
+ },
+ },
+ want: &blockPointer{
+ bm: blockMetadata{count: 1},
+ idx: 0,
+ block: conventionalBlock,
+ lastPartID: 2,
+ },
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ bi := &blockPointer{
+ bm: tt.fields.bm,
+ idx: tt.fields.idx,
+ }
+ bi.copyFrom(tt.args.src)
+ if !reflect.DeepEqual(bi, tt.want) {
+ t.Errorf("blockPointer.copyFrom() = %+v, want %+v", bi, tt.want)
+ }
+ })
+ }
+}
diff --git a/banyand/measure/block_writer.go b/banyand/measure/block_writer.go
index fc840084..35d8d3ca 100644
--- a/banyand/measure/block_writer.go
+++ b/banyand/measure/block_writer.go
@@ -18,6 +18,7 @@
package measure
import (
+ "path/filepath"
"sync"
"github.com/apache/skywalking-banyandb/api/common"
@@ -170,6 +171,19 @@ func (bw *blockWriter) MustInitForMemPart(mp *memPart) {
bw.writers.fieldValuesWriter.init(&mp.fieldValues)
}
+func (bw *blockWriter) mustInitForFilePart(fileSystem fs.FileSystem, path string) {
+ bw.reset()
+ fileSystem.MkdirPanicIfExist(path, dirPermission)
+ bw.writers.mustCreateTagFamilyWriters = func(name string) (fs.Writer, fs.Writer) {
+ return fs.MustCreateFile(fileSystem, filepath.Join(path, name+tagFamiliesMetadataFilenameExt), filePermission),
+ fs.MustCreateFile(fileSystem, filepath.Join(path, name+tagFamiliesFilenameExt), filePermission)
+ }
+ bw.writers.metaWriter.init(fs.MustCreateFile(fileSystem, filepath.Join(path, metaFilename), filePermission))
+ bw.writers.primaryWriter.init(fs.MustCreateFile(fileSystem, filepath.Join(path, primaryFilename), filePermission))
+ bw.writers.timestampsWriter.init(fs.MustCreateFile(fileSystem, filepath.Join(path, timestampsFilename), filePermission))
+ bw.writers.fieldValuesWriter.init(fs.MustCreateFile(fileSystem, filepath.Join(path, fieldValuesFilename), filePermission))
+}
+
func (bw *blockWriter) MustWriteDataPoints(sid common.SeriesID, timestamps []int64, tagFamilies [][]nameValues, fields []nameValues) {
if len(timestamps) == 0 {
return
@@ -178,6 +192,10 @@ func (bw *blockWriter) MustWriteDataPoints(sid common.SeriesID, timestamps []int
b := generateBlock()
defer releaseBlock(b)
b.mustInitFromDataPoints(timestamps, tagFamilies, fields)
+ bw.mustWriteBlock(sid, b)
+}
+
+func (bw *blockWriter) mustWriteBlock(sid common.SeriesID, b *block) {
if b.Len() == 0 {
return
}
@@ -192,9 +210,9 @@ func (bw *blockWriter) MustWriteDataPoints(sid common.SeriesID, timestamps []int
isSeenSid := sid == bw.sidLast
bw.sidLast = sid
- bh := generateBlockMetadata()
- b.mustWriteTo(sid, bh, &bw.writers)
- th := &bh.timestamps
+ bm := generateBlockMetadata()
+ b.mustWriteTo(sid, bm, &bw.writers)
+ th := &bm.timestamps
if bw.totalCount == 0 || th.min < bw.totalMinTimestamp {
bw.totalMinTimestamp = th.min
}
@@ -212,12 +230,12 @@ func (bw *blockWriter) MustWriteDataPoints(sid common.SeriesID, timestamps []int
}
bw.minTimestampLast = th.min
- bw.totalUncompressedSizeBytes += bh.uncompressedSizeBytes
- bw.totalCount += bh.count
+ bw.totalUncompressedSizeBytes += bm.uncompressedSizeBytes
+ bw.totalCount += bm.count
bw.totalBlocksCount++
- bw.primaryBlockData = bh.marshal(bw.primaryBlockData)
- releaseBlockMetadata(bh)
+ bw.primaryBlockData = bm.marshal(bw.primaryBlockData)
+ releaseBlockMetadata(bm)
if len(bw.primaryBlockData) > maxUncompressedPrimaryBlockSize {
bw.mustFlushPrimaryBlock(bw.primaryBlockData)
bw.primaryBlockData = bw.primaryBlockData[:0]
@@ -235,12 +253,12 @@ func (bw *blockWriter) mustFlushPrimaryBlock(data []byte) {
bw.sidFirst = 0
}
-func (bw *blockWriter) Flush(ph *partMetadata) {
- ph.UncompressedSizeBytes = bw.totalUncompressedSizeBytes
- ph.TotalCount = bw.totalCount
- ph.BlocksCount = bw.totalBlocksCount
- ph.MinTimestamp = bw.totalMinTimestamp
- ph.MaxTimestamp = bw.totalMaxTimestamp
+func (bw *blockWriter) Flush(pm *partMetadata) {
+ pm.UncompressedSizeBytes = bw.totalUncompressedSizeBytes
+ pm.TotalCount = bw.totalCount
+ pm.BlocksCount = bw.totalBlocksCount
+ pm.MinTimestamp = bw.totalMinTimestamp
+ pm.MaxTimestamp = bw.totalMaxTimestamp
bw.mustFlushPrimaryBlock(bw.primaryBlockData)
@@ -249,7 +267,7 @@ func (bw *blockWriter) Flush(ph *partMetadata) {
bw.writers.metaWriter.MustWrite(bb.Buf)
bigValuePool.Release(bb)
- ph.CompressedSizeBytes = bw.writers.totalBytesWritten()
+ pm.CompressedSizeBytes = bw.writers.totalBytesWritten()
bw.writers.MustClose()
bw.reset()
diff --git a/banyand/measure/column.go b/banyand/measure/column.go
index e84062fc..2f17d186 100644
--- a/banyand/measure/column.go
+++ b/banyand/measure/column.go
@@ -91,6 +91,29 @@ func (c *column) mustReadValues(decoder *encoding.BytesBlockDecoder, reader fs.R
}
}
+func (c *column) mustSeqReadValues(decoder *encoding.BytesBlockDecoder, reader *seqReader, cm columnMetadata, count uint64) {
+ c.name = cm.name
+ c.valueType = cm.valueType
+ if cm.offset != reader.bytesRead {
+ logger.Panicf("%s: offset mismatch: %d vs %d", reader.Path(), cm.offset, reader.bytesRead)
+ }
+ valuesSize := cm.size
+ if valuesSize > maxValuesBlockSize {
+ logger.Panicf("%s: block size cannot exceed %d bytes; got %d bytes", reader.Path(), maxValuesBlockSize, valuesSize)
+ }
+
+ bb := bigValuePool.Generate()
+ defer bigValuePool.Release(bb)
+
+ bb.Buf = bytes.ResizeOver(bb.Buf, int(valuesSize))
+ reader.mustReadFull(bb.Buf)
+ var err error
+ c.values, err = decoder.Decode(c.values[:0], bb.Buf, count)
+ if err != nil {
+ logger.Panicf("%s: cannot decode values: %v", reader.Path(), err)
+ }
+}
+
var bigValuePool bytes.BufferPool
type columnFamily struct {
diff --git a/banyand/measure/datapoints.go b/banyand/measure/datapoints.go
index f8136705..a06dfe8c 100644
--- a/banyand/measure/datapoints.go
+++ b/banyand/measure/datapoints.go
@@ -147,7 +147,7 @@ type dataPointsInTable struct {
}
type dataPointsInGroup struct {
- tsdb storage.TSDB[*tsTable]
+ tsdb storage.TSDB[*tsTable, option]
docs index.Documents
tables []*dataPointsInTable
diff --git a/banyand/measure/flusher.go b/banyand/measure/flusher.go
index 4fe26408..16ae914d 100644
--- a/banyand/measure/flusher.go
+++ b/banyand/measure/flusher.go
@@ -18,22 +18,34 @@
package measure
import (
+ "errors"
+ "math"
+ "time"
+
"github.com/apache/skywalking-banyandb/pkg/watcher"
)
-func (tst *tsTable) flusherLoop(flushCh chan *flusherIntroduction, introducerWatcher watcher.Channel, epoch uint64) {
+func (tst *tsTable) flusherLoop(flushCh chan *flusherIntroduction, mergeCh chan *mergerIntroduction, introducerWatcher, flusherWatcher watcher.Channel, epoch uint64) {
defer tst.loopCloser.Done()
- epochWatcher := introducerWatcher.Add(0, tst.loopCloser.CloseNotify())
+ epochWatcher := introducerWatcher.Add(epoch, tst.loopCloser.CloseNotify())
if epochWatcher == nil {
return
}
+ var flusherWatchers watcher.Epochs
for {
select {
case <-tst.loopCloser.CloseNotify():
return
+ case e := <-flusherWatcher:
+ flusherWatchers.Add(e)
case <-epochWatcher.Watch():
- var curSnapshot *snapshot
+ curSnapshot := tst.currentSnapshot()
+ if curSnapshot != nil {
+ flusherWatchers = tst.pauseFlusherToPileupMemParts(epoch, flusherWatcher, flusherWatchers)
+ curSnapshot.decRef()
+ curSnapshot = nil
+ }
tst.RLock()
if tst.snapshot != nil && tst.snapshot.epoch > epoch {
curSnapshot = tst.snapshot
@@ -41,11 +53,20 @@ func (tst *tsTable) flusherLoop(flushCh chan *flusherIntroduction, introducerWat
}
tst.RUnlock()
if curSnapshot != nil {
- epoch = tst.flush(curSnapshot, flushCh)
- if epoch == 0 {
- return
+ merged, err := tst.mergeMemParts(curSnapshot, mergeCh)
+ if err != nil {
+ tst.l.Logger.Warn().Err(err).Msgf("cannot merge snapshot: %d", curSnapshot.epoch)
+ curSnapshot.decRef()
+ continue
+ }
+ if !merged {
+ tst.flush(curSnapshot, flushCh)
}
- tst.persistSnapshot(curSnapshot)
+ epoch = curSnapshot.epoch
+ // Notify merger to start a new round of merge.
+ // This round might have be triggered in pauseFlusherToPileupMemParts.
+ flusherWatchers.Notify(math.MaxUint64)
+ flusherWatchers = nil
curSnapshot.decRef()
if tst.currentEpoch() != epoch {
continue
@@ -55,12 +76,60 @@ func (tst *tsTable) flusherLoop(flushCh chan *flusherIntroduction, introducerWat
if epochWatcher == nil {
return
}
- tst.gc.clean()
}
}
}
-func (tst *tsTable) flush(snapshot *snapshot, flushCh chan *flusherIntroduction) uint64 {
+// pauseFlusherToPileupMemParts takes a pause to wait for in-memory parts to pile up.
+// If there is no in-memory part, we can skip the pause.
+// When a merging is finished, we can skip the pause.
+func (tst *tsTable) pauseFlusherToPileupMemParts(epoch uint64, flushWatcher watcher.Channel, flusherWatchers watcher.Epochs) watcher.Epochs {
+ curSnapshot := tst.currentSnapshot()
+ if curSnapshot == nil {
+ return flusherWatchers
+ }
+ curSnapshot.decRef()
+ flusherWatchers.Notify(epoch)
+ select {
+ case <-tst.loopCloser.CloseNotify():
+ case <-time.After(tst.option.flushTimeout):
+ case e := <-flushWatcher:
+ flusherWatchers.Add(e)
+ flusherWatchers.Notify(epoch)
+ }
+ return flusherWatchers
+}
+
+func (tst *tsTable) mergeMemParts(snp *snapshot, mergeCh chan *mergerIntroduction) (bool, error) {
+ var memParts []*partWrapper
+ mergedIDs := make(map[uint64]struct{})
+ for i := range snp.parts {
+ if snp.parts[i].mp != nil {
+ memParts = append(memParts, snp.parts[i])
+ mergedIDs[snp.parts[i].ID()] = struct{}{}
+ continue
+ }
+ }
+ if len(memParts) < 2 {
+ return false, nil
+ }
+ // merge memory must not be closed by the tsTable.close
+ closeCh := make(chan struct{})
+ newPart, err := tst.mergePartsThenSendIntroduction(snapshotCreatorMergedFlusher, memParts, mergedIDs, mergeCh, closeCh)
+ close(closeCh)
+ if err != nil {
+ if errors.Is(err, errClosed) {
+ return true, nil
+ }
+ return false, err
+ }
+ if newPart == nil {
+ return false, nil
+ }
+ return true, nil
+}
+
+func (tst *tsTable) flush(snapshot *snapshot, flushCh chan *flusherIntroduction) {
ind := generateFlusherIntroduction()
defer releaseFlusherIntroduction(ind)
for _, pw := range snapshot.parts {
@@ -69,24 +138,22 @@ func (tst *tsTable) flush(snapshot *snapshot, flushCh chan *flusherIntroduction)
}
partPath := partPath(tst.root, pw.ID())
pw.mp.mustFlush(tst.fileSystem, partPath)
- newPW := newPartWrapper(nil, mustOpenFilePart(partPath, tst.fileSystem), tst.fileSystem)
+ newPW := newPartWrapper(nil, mustOpenFilePart(pw.ID(), tst.root, tst.fileSystem))
newPW.p.partMetadata.ID = pw.ID()
ind.flushed[newPW.ID()] = newPW
}
if len(ind.flushed) < 1 {
- return snapshot.epoch
+ return
}
ind.applied = make(chan struct{})
select {
case flushCh <- ind:
case <-tst.loopCloser.CloseNotify():
- return 0
+ return
}
select {
case <-ind.applied:
- return snapshot.epoch
case <-tst.loopCloser.CloseNotify():
- return 0
}
}
@@ -96,5 +163,5 @@ func (tst *tsTable) persistSnapshot(snapshot *snapshot) {
partNames = append(partNames, partName(snapshot.parts[i].ID()))
}
tst.mustWriteSnapshot(snapshot.epoch, partNames)
- tst.gc.registerSnapshot(snapshot.epoch)
+ tst.gc.registerSnapshot(snapshot)
}
diff --git a/banyand/measure/gc.go b/banyand/measure/gc.go
index 8fb6f608..039d5295 100644
--- a/banyand/measure/gc.go
+++ b/banyand/measure/gc.go
@@ -18,69 +18,70 @@
package measure
import (
- "errors"
- "path"
- "sort"
-
- "github.com/apache/skywalking-banyandb/pkg/fs"
+ "path/filepath"
)
type garbageCleaner struct {
- parent *tsTable
- snapshots []uint64
- parts []uint64
-}
-
-func (g *garbageCleaner) registerSnapshot(snapshot uint64) {
- g.snapshots = append(g.snapshots, snapshot)
+ parent *tsTable
+ liveParts map[uint64]map[uint64]struct{}
+ knownPartFiles map[uint64]struct{}
+ deletableEpochs []uint64
+ liveEpoch uint64
}
-func (g *garbageCleaner) submitParts(parts ...uint64) {
- g.parts = append(g.parts, parts...)
+func (g *garbageCleaner) init(parent *tsTable) {
+ g.parent = parent
+ g.liveParts = make(map[uint64]map[uint64]struct{})
+ g.knownPartFiles = make(map[uint64]struct{})
}
-func (g garbageCleaner) clean() {
- if len(g.snapshots) > 1 {
- g.cleanSnapshots()
+func (g *garbageCleaner) registerSnapshot(snapshot *snapshot) {
+ parts := make(map[uint64]struct{})
+ for _, part := range snapshot.parts {
+ parts[part.ID()] = struct{}{}
+ g.knownPartFiles[part.ID()] = struct{}{}
}
- if len(g.parts) > 0 {
- g.cleanParts()
+ g.liveParts[snapshot.epoch] = parts
+
+ if g.liveEpoch > 0 {
+ g.deletableEpochs = append(g.deletableEpochs, g.liveEpoch)
}
+ g.liveEpoch = snapshot.epoch
+}
+
+func (g *garbageCleaner) submitParts(partID uint64) {
+ g.knownPartFiles[partID] = struct{}{}
+}
+
+func (g *garbageCleaner) clean() {
+ g.cleanSnapshots()
+ g.cleanParts()
}
func (g *garbageCleaner) cleanSnapshots() {
- if len(g.snapshots) < 2 {
- return
- }
- if !sort.SliceIsSorted(g.snapshots, func(i, j int) bool {
- return g.snapshots[i] < g.snapshots[j]
- }) {
- sort.Slice(g.snapshots, func(i, j int) bool {
- return g.snapshots[i] < g.snapshots[j]
- })
- }
- // keep the latest snapshot
- var remainingSnapshots []uint64
- for i := 0; i < len(g.snapshots)-1; i++ {
- filePath := path.Join(g.parent.root, snapshotName(g.snapshots[i]))
- if err := g.parent.fileSystem.DeleteFile(filePath); err != nil {
- var notExistErr *fs.FileSystemError
- if errors.As(err, ¬ExistErr) && notExistErr.Code != fs.IsNotExistError {
- g.parent.l.Warn().Err(err).Str("path", filePath).Msg("failed to delete snapshot, will retry in next round. Please check manually")
- remainingSnapshots = append(remainingSnapshots, g.snapshots[i])
- }
+ var remainingEpochs []uint64
+ for _, deletableEpoch := range g.deletableEpochs {
+ path := filepath.Join(g.parent.root, snapshotName(deletableEpoch))
+ err := g.parent.fileSystem.DeleteFile(path)
+ if err == nil {
+ delete(g.liveParts, deletableEpoch)
+ continue
}
+ g.parent.l.Warn().Err(err).Msgf("cannot delete snapshot file: %s", path)
+ remainingEpochs = append(remainingEpochs, deletableEpoch)
}
- if remainingSnapshots == nil {
- g.snapshots = g.snapshots[len(g.snapshots)-1:]
- return
- }
- remained := g.snapshots[len(g.snapshots)-1]
- g.snapshots = g.snapshots[:0]
- g.snapshots = append(g.snapshots, remainingSnapshots...)
- g.snapshots = append(g.snapshots, remained)
+ g.deletableEpochs = remainingEpochs
}
func (g garbageCleaner) cleanParts() {
- panic("implement me")
+OUTER:
+ for partID := range g.knownPartFiles {
+ for _, partInSnapshot := range g.liveParts {
+ if _, ok := partInSnapshot[partID]; ok {
+ continue OUTER
+ }
+ }
+ g.parent.fileSystem.MustRMAll(partPath(g.parent.root, partID))
+ delete(g.knownPartFiles, partID)
+ }
}
diff --git a/banyand/measure/gc_test.go b/banyand/measure/gc_test.go
index 2d6d8901..e31dc46e 100644
--- a/banyand/measure/gc_test.go
+++ b/banyand/measure/gc_test.go
@@ -18,88 +18,54 @@
package measure
import (
- "path/filepath"
- "sort"
+ "fmt"
+ "reflect"
"testing"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-
"github.com/apache/skywalking-banyandb/pkg/fs"
- "github.com/apache/skywalking-banyandb/pkg/logger"
- "github.com/apache/skywalking-banyandb/pkg/test"
)
-func TestGarbageCleanerCleanSnapshot(t *testing.T) {
+func TestDeletableEpochs(t *testing.T) {
tests := []struct {
name string
- snapshotsOnDisk []uint64
- snapshotsInGC []uint64
- wantOnDisk []uint64
- wantInGC []uint64
+ knownEpochs []uint64
+ deletableEpochs []uint64
+ n int
}{
{
- name: "Test with no snapshots on disk and no snapshots in GC",
- },
- {
- name: "Test with some snapshots on disk and no snapshots in GC",
- snapshotsInGC: nil,
- snapshotsOnDisk: []uint64{1, 2, 3},
- wantInGC: nil,
- wantOnDisk: []uint64{1, 2, 3},
- },
- {
- name: "Test with no snapshots on disk and some snapshots in GC",
- snapshotsOnDisk: nil,
- snapshotsInGC: []uint64{1, 2, 3},
- wantOnDisk: nil,
- // gc won't fix the miss match between inmemory and disk
- wantInGC: []uint64{3},
+ name: "empty",
+ n: 1,
+ knownEpochs: nil,
+ deletableEpochs: nil,
},
{
- name: "Test with some snapshots on disk and some snapshots in GC",
- snapshotsOnDisk: []uint64{1, 2, 3, 4, 5},
- snapshotsInGC: []uint64{1, 3, 5},
- wantOnDisk: []uint64{2, 4, 5},
- wantInGC: []uint64{5},
+ name: "one",
+ n: 1,
+ knownEpochs: []uint64{1},
+ deletableEpochs: nil,
},
{
- name: "Test with unsorted",
- snapshotsOnDisk: []uint64{1, 3, 2, 5, 4},
- snapshotsInGC: []uint64{5, 1, 3},
- wantOnDisk: []uint64{2, 4, 5},
- wantInGC: []uint64{5},
+ name: "many",
+ n: 1,
+ knownEpochs: []uint64{1, 2, 3, 4},
+ deletableEpochs: []uint64{1, 2, 3},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- root, defFn := test.Space(require.New(t))
- defer defFn()
- parent := &tsTable{
- root: root,
- l: logger.GetLogger("test"),
+
+ for _, test := range tests {
+ test := test
+ t.Run(fmt.Sprintf("%s-%d", test.name, test.n), func(t *testing.T) {
+ var gc garbageCleaner
+ gc.init(&tsTable{
fileSystem: fs.NewLocalFileSystem(),
+ })
+
+ for _, epoch := range test.knownEpochs {
+ gc.registerSnapshot(&snapshot{epoch: epoch})
}
- for i := range tt.snapshotsOnDisk {
- filePath := filepath.Join(parent.root, snapshotName(tt.snapshotsOnDisk[i]))
- file, err := parent.fileSystem.CreateFile(filePath, filePermission)
- require.NoError(t, err)
- require.NoError(t, file.Close())
- }
- g := &garbageCleaner{
- parent: parent,
- snapshots: tt.snapshotsInGC,
- }
- g.cleanSnapshots()
- var got []uint64
- for _, de := range parent.fileSystem.ReadDir(parent.root) {
- s, err := parseSnapshot(de.Name())
- require.NoError(t, err, "failed to parse snapshot name:%s", de.Name())
- got = append(got, s)
+ if !reflect.DeepEqual(gc.deletableEpochs, test.deletableEpochs) {
+ t.Errorf("expected deletable: %#v, got %#v", test.deletableEpochs, gc.deletableEpochs)
}
- sort.Slice(got, func(i, j int) bool { return got[i] < got[j] })
- assert.Equal(t, tt.wantOnDisk, got)
- assert.Equal(t, tt.wantInGC, g.snapshots)
})
}
}
diff --git a/banyand/measure/introducer.go b/banyand/measure/introducer.go
index 17a4edef..e3bb2e01 100644
--- a/banyand/measure/introducer.go
+++ b/banyand/measure/introducer.go
@@ -77,7 +77,38 @@ func releaseFlusherIntroduction(i *flusherIntroduction) {
flusherIntroductionPool.Put(i)
}
-func (tst *tsTable) introducerLoop(flushCh chan *flusherIntroduction, watcherCh watcher.Channel, epoch uint64) {
+type mergerIntroduction struct {
+ merged map[uint64]struct{}
+ newPart *partWrapper
+ applied chan struct{}
+ creator snapshotCreator
+}
+
+func (i *mergerIntroduction) reset() {
+ for k := range i.merged {
+ delete(i.merged, k)
+ }
+ i.newPart = nil
+ i.applied = nil
+ i.creator = 0
+}
+
+var mergerIntroductionPool = sync.Pool{}
+
+func generateMergerIntroduction() *mergerIntroduction {
+ v := mergerIntroductionPool.Get()
+ if v == nil {
+ return &mergerIntroduction{}
+ }
+ return v.(*mergerIntroduction)
+}
+
+func releaseMergerIntroduction(i *mergerIntroduction) {
+ i.reset()
+ mergerIntroductionPool.Put(i)
+}
+
+func (tst *tsTable) introducerLoop(flushCh chan *flusherIntroduction, mergeCh chan *mergerIntroduction, watcherCh watcher.Channel, epoch uint64) {
var introducerWatchers watcher.Epochs
defer tst.loopCloser.Done()
for {
@@ -89,6 +120,11 @@ func (tst *tsTable) introducerLoop(flushCh chan *flusherIntroduction, watcherCh
epoch++
case next := <-flushCh:
tst.introduceFlushed(next, epoch)
+ tst.gc.cleanSnapshots()
+ epoch++
+ case next := <-mergeCh:
+ tst.introduceMerged(next, epoch)
+ tst.gc.clean()
epoch++
case epochWatcher := <-watcherCh:
introducerWatchers.Add(epochWatcher)
@@ -108,8 +144,8 @@ func (tst *tsTable) introduceMemPart(nextIntroduction *introduction, epoch uint6
next := nextIntroduction.memPart
nextSnp := cur.copyAllTo(epoch)
- next.p.partMetadata.ID = epoch
nextSnp.parts = append(nextSnp.parts, next)
+ nextSnp.creator = snapshotCreatorMemPart
tst.replaceSnapshot(&nextSnp)
if nextIntroduction.applied != nil {
close(nextIntroduction.applied)
@@ -123,7 +159,26 @@ func (tst *tsTable) introduceFlushed(nextIntroduction *flusherIntroduction, epoc
}
defer cur.decRef()
nextSnp := cur.merge(epoch, nextIntroduction.flushed)
+ nextSnp.creator = snapshotCreatorFlusher
+ tst.replaceSnapshot(&nextSnp)
+ tst.persistSnapshot(&nextSnp)
+ if nextIntroduction.applied != nil {
+ close(nextIntroduction.applied)
+ }
+}
+
+func (tst *tsTable) introduceMerged(nextIntroduction *mergerIntroduction, epoch uint64) {
+ cur := tst.currentSnapshot()
+ if cur == nil {
+ tst.l.Panic().Msg("current snapshot is nil")
+ return
+ }
+ defer cur.decRef()
+ nextSnp := cur.remove(epoch, nextIntroduction.merged)
+ nextSnp.parts = append(nextSnp.parts, nextIntroduction.newPart)
+ nextSnp.creator = nextIntroduction.creator
tst.replaceSnapshot(&nextSnp)
+ tst.persistSnapshot(&nextSnp)
if nextIntroduction.applied != nil {
close(nextIntroduction.applied)
}
diff --git a/banyand/measure/measure.go b/banyand/measure/measure.go
index 3f258275..cec744ed 100644
--- a/banyand/measure/measure.go
+++ b/banyand/measure/measure.go
@@ -41,8 +41,14 @@ const (
maxUncompressedPrimaryBlockSize = 128 * 1024
maxBlockLength = 8 * 1024
+
+ defaultFlushTimeout = 5 * time.Second
)
+type option struct {
+ flushTimeout time.Duration
+}
+
type measure struct {
databaseSupplier schema.Supplier
l *logger.Logger
diff --git a/banyand/measure/merger.go b/banyand/measure/merger.go
new file mode 100644
index 00000000..1f03ca1e
--- /dev/null
+++ b/banyand/measure/merger.go
@@ -0,0 +1,378 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package measure
+
+import (
+ "errors"
+ "fmt"
+ "sync/atomic"
+ "time"
+
+ "github.com/dustin/go-humanize"
+
+ "github.com/apache/skywalking-banyandb/pkg/encoding"
+ "github.com/apache/skywalking-banyandb/pkg/fs"
+ "github.com/apache/skywalking-banyandb/pkg/watcher"
+)
+
+func (tst *tsTable) mergeLoop(merges chan *mergerIntroduction, flusherNotifier watcher.Channel) {
+ defer tst.loopCloser.Done()
+
+ var epoch uint64
+
+ ew := flusherNotifier.Add(0, tst.loopCloser.CloseNotify())
+ if ew == nil {
+ return
+ }
+
+ for {
+ select {
+ case <-tst.loopCloser.CloseNotify():
+ return
+ case <-ew.Watch():
+ curSnapshot := tst.currentSnapshot()
+ if curSnapshot == nil {
+ continue
+ }
+ if curSnapshot.epoch != epoch {
+ if err := tst.mergeSnapshot(curSnapshot, merges); err != nil {
+ if errors.Is(err, errClosed) {
+ curSnapshot.decRef()
+ return
+ }
+ tst.l.Logger.Warn().Err(err).Msgf("cannot merge snapshot: %d", curSnapshot.epoch)
+ curSnapshot.decRef()
+ continue
+ }
+ epoch = curSnapshot.epoch
+ }
+ curSnapshot.decRef()
+ ew = flusherNotifier.Add(epoch, tst.loopCloser.CloseNotify())
+ if ew == nil {
+ return
+ }
+ }
+ }
+}
+
+func (tst *tsTable) mergeSnapshot(curSnapshot *snapshot, merges chan *mergerIntroduction) error {
+ maxFanOut := tst.freeDiskSpace(tst.root) / 2
+ partsToMerge, toBeMerged := tst.getPartsToMerge(curSnapshot, maxFanOut)
+ if len(partsToMerge) < 2 {
+ return nil
+ }
+ if _, err := tst.mergePartsThenSendIntroduction(snapshotCreatorMerger, partsToMerge,
+ toBeMerged, merges, tst.loopCloser.CloseNotify()); err != nil {
+ return err
+ }
+ return nil
+}
+
+func (tst *tsTable) mergePartsThenSendIntroduction(creator snapshotCreator, parts []*partWrapper, merged map[uint64]struct{}, merges chan *mergerIntroduction,
+ closeCh <-chan struct{},
+) (*partWrapper, error) {
+ reservedSpace := tst.reserveSpace(parts)
+ defer releaseDiskSpace(reservedSpace)
+ start := time.Now()
+ newPart, err := mergeParts(tst.fileSystem, closeCh, parts, atomic.AddUint64(&tst.curPartID, 1), tst.root)
+ if err != nil {
+ return nil, err
+ }
+ elapsed := time.Since(start)
+ if elapsed > 30*time.Second {
+ var totalCount uint64
+ for _, pw := range parts {
+ totalCount += pw.p.partMetadata.TotalCount
+ }
+ tst.l.Warn().
+ Uint64("beforeTotalCount", totalCount).
+ Uint64("afterTotalCount", newPart.p.partMetadata.TotalCount).
+ Int("beforePartCount", len(parts)).
+ Dur("elapsed", elapsed).
+ Msg("background merger takes too long")
+ } else if snapshotCreatorMerger == creator && tst.l.Info().Enabled() && len(parts) > 2 {
+ var minSize, maxSize, totalSize, totalCount uint64
+ for _, pw := range parts {
+ totalCount += pw.p.partMetadata.TotalCount
+ totalSize += pw.p.partMetadata.CompressedSizeBytes
+ if minSize == 0 || minSize > pw.p.partMetadata.CompressedSizeBytes {
+ minSize = pw.p.partMetadata.CompressedSizeBytes
+ }
+ if maxSize < pw.p.partMetadata.CompressedSizeBytes {
+ maxSize = pw.p.partMetadata.CompressedSizeBytes
+ }
+ }
+ if totalSize > 10<<20 && minSize*uint64(len(parts)) < maxSize {
+ // it's a unbalanced merge. but it's ok when the size is small.
+ tst.l.Info().
+ Str("beforeTotalCount", humanize.Comma(int64(totalCount))).
+ Str("afterTotalCount", humanize.Comma(int64(newPart.p.partMetadata.TotalCount))).
+ Int("beforePartCount", len(parts)).
+ Str("minSize", humanize.IBytes(minSize)).
+ Str("maxSize", humanize.IBytes(maxSize)).
+ Dur("elapsedMS", elapsed).
+ Msg("background merger merges unbalanced parts")
+ }
+ }
+ mi := generateMergerIntroduction()
+ defer releaseMergerIntroduction(mi)
+ mi.creator = creator
+ mi.newPart = newPart
+ mi.merged = merged
+ mi.applied = make(chan struct{})
+ select {
+ case merges <- mi:
+ case <-tst.loopCloser.CloseNotify():
+ return newPart, errClosed
+ }
+ select {
+ case <-mi.applied:
+ case <-tst.loopCloser.CloseNotify():
+ return newPart, errClosed
+ }
+ return newPart, nil
+}
+
+func (tst *tsTable) freeDiskSpace(path string) uint64 {
+ free := tst.fileSystem.MustGetFreeSpace(path)
+ reserved := atomic.LoadUint64(&reservedDiskSpace)
+ if free < reserved {
+ return 0
+ }
+ return free - reserved
+}
+
+func (tst *tsTable) tryReserveDiskSpace(n uint64) bool {
+ available := tst.fileSystem.MustGetFreeSpace(tst.root)
+ reserved := reserveDiskSpace(n)
+ if available > reserved {
+ return true
+ }
+ releaseDiskSpace(n)
+ return false
+}
+
+func reserveDiskSpace(n uint64) uint64 {
+ return atomic.AddUint64(&reservedDiskSpace, n)
+}
+
+func releaseDiskSpace(n uint64) {
+ atomic.AddUint64(&reservedDiskSpace, ^(n - 1))
+}
+
+var reservedDiskSpace uint64
+
+func (tst *tsTable) getPartsToMerge(snapshot *snapshot, maxFanOut uint64) ([]*partWrapper, map[uint64]struct{}) {
+ var parts []*partWrapper
+ toBeMerged := make(map[uint64]struct{})
+ for _, pw := range snapshot.parts {
+ if pw.mp != nil || pw.p.partMetadata.TotalCount < 1 || pw.p.partMetadata.CompressedSizeBytes > maxFanOut {
+ continue
+ }
+ parts = append(parts, pw)
+ toBeMerged[pw.ID()] = struct{}{}
+ }
+ // TODO: select proper parts to generate the lowest write amplification
+ return parts, toBeMerged
+}
+
+func (tst *tsTable) reserveSpace(parts []*partWrapper) uint64 {
+ var needSize uint64
+ for i := range parts {
+ needSize = +parts[i].p.partMetadata.CompressedSizeBytes
+ }
+ if tst.tryReserveDiskSpace(needSize) {
+ return needSize
+ }
+ return 0
+}
+
+var errNoPartToMerge = fmt.Errorf("no part to merge")
+
+func mergeParts(fileSystem fs.FileSystem, closeCh <-chan struct{}, parts []*partWrapper, partID uint64, root string) (*partWrapper, error) {
+ if len(parts) == 0 {
+ return nil, errNoPartToMerge
+ }
+ dstPath := partPath(root, partID)
+ pii := make([]*partMergeIter, 0, len(parts))
+ for i := range parts {
+ pmi := generatePartMergeIter()
+ pmi.mustInitFromPart(parts[i].p)
+ pii = append(pii, pmi)
+ }
+ br := generateBlockReader()
+ br.init(pii)
+ bw := generateBlockWriter()
+ bw.mustInitForFilePart(fileSystem, dstPath)
+
+ pm, err := mergeBlocks(closeCh, bw, br)
+ releaseBlockWriter(bw)
+ releaseBlockReader(br)
+ for i := range pii {
+ releasePartMergeIter(pii[i])
+ }
+ if err != nil {
+ return nil, err
+ }
+ pm.mustWriteMetadata(fileSystem, dstPath)
+ fileSystem.SyncPath(dstPath)
+ p := mustOpenFilePart(partID, root, fileSystem)
+ return newPartWrapper(nil, p), nil
+}
+
+var errClosed = fmt.Errorf("the merger is closed")
+
+func mergeBlocks(closeCh <-chan struct{}, bw *blockWriter, br *blockReader) (*partMetadata, error) {
+ pendingBlockIsEmpty := true
+ pendingBlock := &blockPointer{}
+ var tmpBlock, tmpBlock2 *blockPointer
+ var decoder *encoding.BytesBlockDecoder
+ getDecoder := func() *encoding.BytesBlockDecoder {
+ if decoder == nil {
+ decoder = generateColumnValuesDecoder()
+ }
+ return decoder
+ }
+ releaseDecoder := func() {
+ if decoder != nil {
+ releaseColumnValuesDecoder(decoder)
+ decoder = nil
+ }
+ }
+ for br.nextBlockMetadata() {
+ select {
+ case <-closeCh:
+ return nil, errClosed
+ default:
+ }
+ b := br.block
+
+ if pendingBlockIsEmpty {
+ br.loadBlockData(getDecoder())
+ pendingBlock.copyFrom(b)
+ pendingBlockIsEmpty = false
+ continue
+ }
+
+ if pendingBlock.bm.seriesID != b.bm.seriesID ||
+ (pendingBlock.isFull() && pendingBlock.bm.timestamps.max <= b.bm.timestamps.min) {
+ bw.mustWriteBlock(pendingBlock.bm.seriesID, &pendingBlock.block)
+ releaseDecoder()
+ pendingBlock.reset()
+ br.loadBlockData(getDecoder())
+ pendingBlock.copyFrom(b)
+ continue
+ }
+
+ if tmpBlock == nil {
+ tmpBlock = &blockPointer{}
+ }
+ tmpBlock.reset()
+ tmpBlock.bm.seriesID = b.bm.seriesID
+ br.loadBlockData(getDecoder())
+ mergeTwoBlocks(tmpBlock, pendingBlock, b)
+ if len(tmpBlock.timestamps) <= maxBlockLength && tmpBlock.uncompressedSizeBytes() <= maxUncompressedBlockSize {
+ if len(tmpBlock.timestamps) == 0 {
+ pendingBlockIsEmpty = true
+ }
+ pendingBlock, tmpBlock = tmpBlock, pendingBlock
+ continue
+ }
+
+ if len(tmpBlock.timestamps) <= maxBlockLength {
+ bw.mustWriteBlock(tmpBlock.bm.seriesID, &tmpBlock.block)
+ releaseDecoder()
+ continue
+ }
+ tmpBlock.idx = maxBlockLength
+ pendingBlock.copyFrom(tmpBlock)
+ l := tmpBlock.idx
+ tmpBlock.idx = 0
+ if tmpBlock2 == nil {
+ tmpBlock2 = &blockPointer{}
+ }
+ tmpBlock2.reset()
+ tmpBlock2.append(tmpBlock, l)
+ bw.mustWriteBlock(tmpBlock.bm.seriesID, &tmpBlock2.block)
+ releaseDecoder()
+ }
+ if err := br.error(); err != nil {
+ return nil, fmt.Errorf("cannot read block to merge: %w", err)
+ }
+ if !pendingBlockIsEmpty {
+ bw.mustWriteBlock(pendingBlock.bm.seriesID, &pendingBlock.block)
+ }
+ releaseDecoder()
+ var result partMetadata
+ bw.Flush(&result)
+ return &result, nil
+}
+
+func mergeTwoBlocks(target, left, right *blockPointer) {
+ appendIfEmpty := func(ib1, ib2 *blockPointer) bool {
+ if ib1.idx >= len(ib1.timestamps) {
+ target.appendAll(ib2)
+ return true
+ }
+ return false
+ }
+
+ defer target.updateMetadata()
+
+ if left.bm.timestamps.max < right.bm.timestamps.min {
+ target.appendAll(left)
+ target.appendAll(right)
+ return
+ }
+ if right.bm.timestamps.max < left.bm.timestamps.min {
+ target.appendAll(right)
+ target.appendAll(left)
+ return
+ }
+ if appendIfEmpty(left, right) || appendIfEmpty(right, left) {
+ return
+ }
+
+ for {
+ i := left.idx
+ ts2 := right.timestamps[right.idx]
+ for i < len(left.timestamps) && left.timestamps[i] <= ts2 {
+ i++
+ }
+ if left.timestamps[i-1] == ts2 {
+ if left.lastPartID >= right.lastPartID {
+ target.append(left, i)
+ } else {
+ target.append(left, i-1) // skip left
+ target.append(right, right.idx+1)
+ }
+ left.idx = i
+ right.idx++ // skip right
+ if appendIfEmpty(right, left) {
+ return
+ }
+ } else {
+ target.append(left, i)
+ left.idx = i
+ }
+ if appendIfEmpty(left, right) {
+ return
+ }
+ left, right = right, left
+ }
+}
diff --git a/banyand/measure/merger_test.go b/banyand/measure/merger_test.go
new file mode 100644
index 00000000..1e55df34
--- /dev/null
+++ b/banyand/measure/merger_test.go
@@ -0,0 +1,356 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package measure
+
+import (
+ "errors"
+ "reflect"
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+ "github.com/stretchr/testify/require"
+
+ "github.com/apache/skywalking-banyandb/pkg/fs"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
+ "github.com/apache/skywalking-banyandb/pkg/test"
+)
+
+func Test_mergeTwoBlocks(t *testing.T) {
+ tests := []struct {
+ left *blockPointer
+ right *blockPointer
+ want *blockPointer
+ name string
+ }{
+ {
+ name: "Merge two empty blocks",
+ left: &blockPointer{},
+ right: &blockPointer{},
+ want: &blockPointer{},
+ },
+ {
+ name: "Merge left is non-empty right is empty",
+ left: &blockPointer{block: conventionalBlock},
+ right: &blockPointer{},
+ want: &blockPointer{block: conventionalBlock, bm: blockMetadata{timestamps: timestampsMetadata{min: 1, max: 2}}},
+ },
+ {
+ name: "Merge left is empty right is non-empty",
+ left: &blockPointer{},
+ right: &blockPointer{block: conventionalBlock},
+ want: &blockPointer{block: conventionalBlock, bm: blockMetadata{timestamps: timestampsMetadata{min: 1, max: 2}}},
+ },
+ {
+ name: "Merge two non-empty blocks without overlap",
+ left: &blockPointer{
+ block: block{
+ timestamps: []int64{1, 2},
+ tagFamilies: []columnFamily{
+ {
+ name: "arrTag",
+ columns: []column{
+ {
+ name: "strArrTag", valueType: pbv1.ValueTypeStrArr,
+ values: [][]byte{marshalStrArr([][]byte{[]byte("value1"), []byte("value2")}), marshalStrArr([][]byte{[]byte("value3"), []byte("value4")})},
+ },
+ },
+ },
+ },
+ field: columnFamily{
+ columns: []column{
+ {name: "strField", valueType: pbv1.ValueTypeStr, values: [][]byte{[]byte("field1"), []byte("field2")}},
+ },
+ },
+ },
+ },
+ right: &blockPointer{
+ block: block{
+ timestamps: []int64{3, 4},
+ tagFamilies: []columnFamily{
+ {
+ name: "arrTag",
+ columns: []column{
+ {
+ name: "strArrTag", valueType: pbv1.ValueTypeStrArr,
+ values: [][]byte{marshalStrArr([][]byte{[]byte("value5"), []byte("value6")}), marshalStrArr([][]byte{[]byte("value7"), []byte("value8")})},
+ },
+ },
+ },
+ },
+ field: columnFamily{
+ columns: []column{
+ {name: "strField", valueType: pbv1.ValueTypeStr, values: [][]byte{[]byte("field3"), []byte("field4")}},
+ },
+ },
+ },
+ },
+ want: &blockPointer{block: mergedBlock, bm: blockMetadata{timestamps: timestampsMetadata{min: 1, max: 4}}},
+ },
+ {
+ name: "Merge two non-empty blocks without duplicated timestamps",
+ left: &blockPointer{
+ block: block{
+ timestamps: []int64{1, 3},
+ tagFamilies: []columnFamily{
+ {
+ name: "arrTag",
+ columns: []column{
+ {
+ name: "strArrTag", valueType: pbv1.ValueTypeStrArr,
+ values: [][]byte{marshalStrArr([][]byte{[]byte("value1"), []byte("value2")}), marshalStrArr([][]byte{[]byte("value5"), []byte("value6")})},
+ },
+ },
+ },
+ },
+ field: columnFamily{
+ columns: []column{
+ {name: "strField", valueType: pbv1.ValueTypeStr, values: [][]byte{[]byte("field1"), []byte("field3")}},
+ },
+ },
+ },
+ },
+ right: &blockPointer{
+ block: block{
+ timestamps: []int64{2, 4},
+ tagFamilies: []columnFamily{
+ {
+ name: "arrTag",
+ columns: []column{
+ {
+ name: "strArrTag", valueType: pbv1.ValueTypeStrArr,
+ values: [][]byte{marshalStrArr([][]byte{[]byte("value3"), []byte("value4")}), marshalStrArr([][]byte{[]byte("value7"), []byte("value8")})},
+ },
+ },
+ },
+ },
+ field: columnFamily{
+ columns: []column{
+ {name: "strField", valueType: pbv1.ValueTypeStr, values: [][]byte{[]byte("field2"), []byte("field4")}},
+ },
+ },
+ },
+ },
+ want: &blockPointer{block: mergedBlock, bm: blockMetadata{timestamps: timestampsMetadata{min: 1, max: 4}}},
+ },
+ {
+ name: "Merge two non-empty blocks with duplicated timestamps",
+ left: &blockPointer{
+ lastPartID: 1, // the less partID will be skipped
+ block: block{
+ timestamps: []int64{1, 2, 3},
+ tagFamilies: []columnFamily{
+ {
+ name: "arrTag",
+ columns: []column{
+ {
+ name: "strArrTag", valueType: pbv1.ValueTypeStrArr,
+ values: [][]byte{
+ marshalStrArr([][]byte{[]byte("value1"), []byte("value2")}),
+ marshalStrArr([][]byte{[]byte("duplicated1")}),
+ marshalStrArr([][]byte{[]byte("duplicated2")}),
+ },
+ },
+ },
+ },
+ },
+ field: columnFamily{
+ columns: []column{
+ {name: "strField", valueType: pbv1.ValueTypeStr, values: [][]byte{[]byte("field1"), []byte("duplicated1"), []byte("duplicated2")}},
+ },
+ },
+ },
+ },
+ right: &blockPointer{
+ lastPartID: 2, // the greater partID will be appended
+ block: block{
+ timestamps: []int64{2, 3, 4},
+ tagFamilies: []columnFamily{
+ {
+ name: "arrTag",
+ columns: []column{
+ {
+ name: "strArrTag", valueType: pbv1.ValueTypeStrArr,
+ values: [][]byte{
+ marshalStrArr([][]byte{[]byte("value3"), []byte("value4")}), marshalStrArr([][]byte{[]byte("value5"), []byte("value6")}),
+ marshalStrArr([][]byte{[]byte("value7"), []byte("value8")}),
+ },
+ },
+ },
+ },
+ },
+ field: columnFamily{
+ columns: []column{
+ {name: "strField", valueType: pbv1.ValueTypeStr, values: [][]byte{[]byte("field2"), []byte("field3"), []byte("field4")}},
+ },
+ },
+ },
+ },
+ want: &blockPointer{block: mergedBlock, lastPartID: 2, bm: blockMetadata{timestamps: timestampsMetadata{min: 1, max: 4}}},
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ target := &blockPointer{}
+ mergeTwoBlocks(target, tt.left, tt.right)
+ if !reflect.DeepEqual(target, tt.want) {
+ t.Errorf("mergeTwoBlocks() = %v, want %v", target, tt.want)
+ }
+ })
+ }
+}
+
+var mergedBlock = block{
+ timestamps: []int64{1, 2, 3, 4},
+ tagFamilies: []columnFamily{
+ {
+ name: "arrTag",
+ columns: []column{
+ {
+ name: "strArrTag", valueType: pbv1.ValueTypeStrArr,
+ values: [][]byte{
+ marshalStrArr([][]byte{[]byte("value1"), []byte("value2")}), marshalStrArr([][]byte{[]byte("value3"), []byte("value4")}),
+ marshalStrArr([][]byte{[]byte("value5"), []byte("value6")}), marshalStrArr([][]byte{[]byte("value7"), []byte("value8")}),
+ },
+ },
+ },
+ },
+ },
+ field: columnFamily{
+ columns: []column{
+ {name: "strField", valueType: pbv1.ValueTypeStr, values: [][]byte{[]byte("field1"), []byte("field2"), []byte("field3"), []byte("field4")}},
+ },
+ },
+}
+
+func Test_mergeParts(t *testing.T) {
+ tests := []struct {
+ wantErr error
+ name string
+ dpsList []*dataPoints
+ want []blockMetadata
+ }{
+ {
+ name: "Test with no data points",
+ dpsList: []*dataPoints{},
+ wantErr: errNoPartToMerge,
+ },
+ {
+ name: "Test with single part",
+ dpsList: []*dataPoints{dpsTS1},
+ want: []blockMetadata{
+ {seriesID: 1, count: 1, uncompressedSizeBytes: 1676},
+ {seriesID: 2, count: 1, uncompressedSizeBytes: 55},
+ {seriesID: 3, count: 1, uncompressedSizeBytes: 24},
+ },
+ },
+ {
+ name: "Test with multiple parts with different ts",
+ dpsList: []*dataPoints{dpsTS1, dpsTS2, dpsTS2},
+ want: []blockMetadata{
+ {seriesID: 1, count: 2, uncompressedSizeBytes: 3352},
+ {seriesID: 2, count: 2, uncompressedSizeBytes: 110},
+ {seriesID: 3, count: 2, uncompressedSizeBytes: 48},
+ },
+ },
+ {
+ name: "Test with multiple parts with same ts",
+ dpsList: []*dataPoints{dpsTS1, dpsTS1, dpsTS1},
+ want: []blockMetadata{
+ {seriesID: 1, count: 1, uncompressedSizeBytes: 1676},
+ {seriesID: 2, count: 1, uncompressedSizeBytes: 55},
+ {seriesID: 3, count: 1, uncompressedSizeBytes: 24},
+ },
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ verify := func(t *testing.T, pp []*partWrapper, fileSystem fs.FileSystem, root string, partID uint64) {
+ closeCh := make(chan struct{})
+ defer close(closeCh)
+ p, err := mergeParts(fileSystem, closeCh, pp, partID, root)
+ if tt.wantErr != nil {
+ if !errors.Is(err, tt.wantErr) {
+ t.Fatalf("Unexpected error: got %v, want %v", err, tt.wantErr)
+ }
+ return
+ }
+ defer p.decRef()
+ pmi := &partMergeIter{}
+ pmi.mustInitFromPart(p.p)
+ reader := &blockReader{}
+ reader.init([]*partMergeIter{pmi})
+ var got []blockMetadata
+ for reader.nextBlockMetadata() {
+ got = append(got, reader.block.bm)
+ }
+ require.NoError(t, reader.error())
+
+ if diff := cmp.Diff(got, tt.want,
+ cmpopts.IgnoreFields(blockMetadata{}, "timestamps"),
+ cmpopts.IgnoreFields(blockMetadata{}, "field"),
+ cmpopts.IgnoreFields(blockMetadata{}, "tagFamilies"),
+ cmp.AllowUnexported(blockMetadata{}),
+ ); diff != "" {
+ t.Errorf("Unexpected blockMetadata (-got +want):\n%s", diff)
+ }
+ }
+
+ t.Run("memory parts", func(t *testing.T) {
+ var pp []*partWrapper
+ tmpPath, defFn := test.Space(require.New(t))
+ defer func() {
+ for _, pw := range pp {
+ pw.decRef()
+ }
+ defFn()
+ }()
+ for _, dps := range tt.dpsList {
+ mp := generateMemPart()
+ mp.mustInitFromDataPoints(dps)
+ pp = append(pp, newPartWrapper(mp, openMemPart(mp)))
+ }
+ verify(t, pp, fs.NewLocalFileSystem(), tmpPath, 1)
+ })
+
+ t.Run("file parts", func(t *testing.T) {
+ var fpp []*partWrapper
+ tmpPath, defFn := test.Space(require.New(t))
+ defer func() {
+ for _, pw := range fpp {
+ pw.decRef()
+ }
+ defFn()
+ }()
+ fileSystem := fs.NewLocalFileSystem()
+ for i, dps := range tt.dpsList {
+ mp := generateMemPart()
+ mp.mustInitFromDataPoints(dps)
+ mp.mustFlush(fileSystem, partPath(tmpPath, uint64(i)))
+ filePW := newPartWrapper(nil, mustOpenFilePart(uint64(i), tmpPath, fileSystem))
+ filePW.p.partMetadata.ID = uint64(i)
+ fpp = append(fpp, filePW)
+ releaseMemPart(mp)
+ }
+ verify(t, fpp, fileSystem, tmpPath, uint64(len(tt.dpsList)))
+ })
+ })
+ }
+}
diff --git a/banyand/measure/metadata.go b/banyand/measure/metadata.go
index ba3cafef..14c940d6 100644
--- a/banyand/measure/metadata.go
+++ b/banyand/measure/metadata.go
@@ -301,12 +301,12 @@ func (sr *schemaRepo) loadMeasure(metadata *commonv1.Metadata) (*measure, bool)
return s, ok
}
-func (sr *schemaRepo) loadTSDB(groupName string) (storage.TSDB[*tsTable], error) {
+func (sr *schemaRepo) loadTSDB(groupName string) (storage.TSDB[*tsTable, option], error) {
g, ok := sr.LoadGroup(groupName)
if !ok {
return nil, fmt.Errorf("group %s not found", groupName)
}
- return g.SupplyTSDB().(storage.TSDB[*tsTable]), nil
+ return g.SupplyTSDB().(storage.TSDB[*tsTable, option]), nil
}
var _ resourceSchema.ResourceSupplier = (*supplier)(nil)
@@ -316,6 +316,7 @@ type supplier struct {
pipeline queue.Queue
l *logger.Logger
path string
+ option option
}
func newSupplier(path string, svc *service) *supplier {
@@ -324,6 +325,7 @@ func newSupplier(path string, svc *service) *supplier {
metadata: svc.metadata,
l: svc.l,
pipeline: svc.localPipeline,
+ option: svc.option,
}
}
@@ -343,12 +345,13 @@ func (s *supplier) ResourceSchema(md *commonv1.Metadata) (resourceSchema.Resourc
}
func (s *supplier) OpenDB(groupSchema *commonv1.Group) (io.Closer, error) {
- opts := storage.TSDBOpts[*tsTable]{
+ opts := storage.TSDBOpts[*tsTable, option]{
ShardNum: groupSchema.ResourceOpts.ShardNum,
Location: path.Join(s.path, groupSchema.Metadata.Name),
TSTableCreator: newTSTable,
SegmentInterval: storage.MustToIntervalRule(groupSchema.ResourceOpts.SegmentInterval),
TTL: storage.MustToIntervalRule(groupSchema.ResourceOpts.Ttl),
+ Option: s.option,
}
name := groupSchema.Metadata.Name
return storage.OpenTSDB(
diff --git a/banyand/measure/part.go b/banyand/measure/part.go
index 81fcce0e..16b8d156 100644
--- a/banyand/measure/part.go
+++ b/banyand/measure/part.go
@@ -74,7 +74,7 @@ func openMemPart(mp *memPart) *part {
var p part
p.partMetadata = mp.partMetadata
- p.primaryBlockMetadata = mustReadPrimaryBlockMetadata(p.primaryBlockMetadata[:0], newReader(&mp.meta))
+ p.primaryBlockMetadata = mustReadPrimaryBlockMetadata(p.primaryBlockMetadata[:0], &mp.meta)
// Open data files
p.meta = &mp.meta
@@ -221,15 +221,13 @@ func releaseMemPart(mp *memPart) {
var memPartPool sync.Pool
type partWrapper struct {
- fileSystem fs.FileSystem
- mp *memPart
- p *part
- ref int32
- mustBeDeleted uint32
+ mp *memPart
+ p *part
+ ref int32
}
-func newPartWrapper(mp *memPart, p *part, fileSystem fs.FileSystem) *partWrapper {
- return &partWrapper{mp: mp, p: p, fileSystem: fileSystem, ref: 1}
+func newPartWrapper(mp *memPart, p *part) *partWrapper {
+ return &partWrapper{mp: mp, p: p, ref: 1}
}
func (pw *partWrapper) incRef() {
@@ -248,24 +246,22 @@ func (pw *partWrapper) decRef() {
return
}
pw.p.close()
- if atomic.LoadUint32(&pw.mustBeDeleted) == 0 {
- return
- }
- pw.fileSystem.MustRMAll(pw.p.path)
}
func (pw *partWrapper) ID() uint64 {
return pw.p.partMetadata.ID
}
-func mustOpenFilePart(partPath string, fileSystem fs.FileSystem) *part {
+func mustOpenFilePart(id uint64, root string, fileSystem fs.FileSystem) *part {
var p part
+ partPath := partPath(root, id)
p.path = partPath
p.partMetadata.mustReadMetadata(fileSystem, partPath)
+ p.partMetadata.ID = id
metaPath := path.Join(partPath, metaFilename)
pr := mustOpenReader(metaPath, fileSystem)
- p.primaryBlockMetadata = mustReadPrimaryBlockMetadata(p.primaryBlockMetadata[:0], newReader(pr))
+ p.primaryBlockMetadata = mustReadPrimaryBlockMetadata(p.primaryBlockMetadata[:0], pr)
fs.MustClose(pr)
p.primary = mustOpenReader(path.Join(partPath, primaryFilename), fileSystem)
diff --git a/banyand/measure/part_iter.go b/banyand/measure/part_iter.go
index 14b4c8c3..c44d114a 100644
--- a/banyand/measure/part_iter.go
+++ b/banyand/measure/part_iter.go
@@ -22,10 +22,12 @@ import (
"fmt"
"io"
"sort"
+ "sync"
"github.com/apache/skywalking-banyandb/api/common"
"github.com/apache/skywalking-banyandb/pkg/bytes"
"github.com/apache/skywalking-banyandb/pkg/compress/zstd"
+ "github.com/apache/skywalking-banyandb/pkg/encoding"
"github.com/apache/skywalking-banyandb/pkg/fs"
"github.com/apache/skywalking-banyandb/pkg/logger"
)
@@ -233,3 +235,147 @@ func (pi *partIter) findBlock() bool {
pi.bms = nil
return false
}
+
+type partMergeIter struct {
+ seqReaders seqReaders
+ err error
+ primaryBlockMetadata []primaryBlockMetadata
+ compressedPrimaryBuf []byte
+ primaryBuf []byte
+ block blockPointer
+ partID uint64
+ primaryMetadataIdx int
+}
+
+func (pmi *partMergeIter) reset() {
+ pmi.err = nil
+ pmi.seqReaders.reset()
+ pmi.primaryBlockMetadata = nil
+ pmi.primaryMetadataIdx = 0
+ pmi.partID = 0
+ pmi.primaryBuf = pmi.primaryBuf[:0]
+ pmi.compressedPrimaryBuf = pmi.compressedPrimaryBuf[:0]
+ pmi.block.reset()
+}
+
+func (pmi *partMergeIter) mustInitFromPart(p *part) {
+ pmi.reset()
+ pmi.seqReaders.init(p)
+ pmi.primaryBlockMetadata = p.primaryBlockMetadata
+ pmi.partID = p.partMetadata.ID
+}
+
+func (pmi *partMergeIter) error() error {
+ if pmi.err == nil || errors.Is(pmi.err, io.EOF) {
+ return nil
+ }
+ return pmi.err
+}
+
+func (pmi *partMergeIter) nextBlockMetadata() bool {
+ if pmi.err != nil {
+ return false
+ }
+ pmi.block.reset()
+ if len(pmi.primaryBuf) == 0 {
+ if err := pmi.loadPrimaryBuf(); err != nil {
+ pmi.err = err
+ return false
+ }
+ }
+ if err := pmi.loadBlockMetadata(); err != nil {
+ pmi.err = err
+ return false
+ }
+ return true
+}
+
+func (pmi *partMergeIter) loadPrimaryBuf() error {
+ if pmi.primaryMetadataIdx >= len(pmi.primaryBlockMetadata) {
+ return io.EOF
+ }
+ pm := pmi.primaryBlockMetadata[pmi.primaryMetadataIdx]
+ pmi.compressedPrimaryBuf = bytes.ResizeOver(pmi.compressedPrimaryBuf, int(pm.size))
+ pmi.seqReaders.primary.mustReadFull(pmi.compressedPrimaryBuf)
+ var err error
+ pmi.primaryBuf, err = zstd.Decompress(pmi.primaryBuf[:0], pmi.compressedPrimaryBuf)
+ if err != nil {
+ return fmt.Errorf("cannot decompress primary block: %w", err)
+ }
+ pmi.primaryMetadataIdx++
+ return nil
+}
+
+func (pmi *partMergeIter) loadBlockMetadata() error {
+ pmi.block.reset()
+ var err error
+ pmi.primaryBuf, err = pmi.block.bm.unmarshal(pmi.primaryBuf)
+ if err != nil {
+ pm := pmi.primaryBlockMetadata[pmi.primaryMetadataIdx-1]
+ return fmt.Errorf("can't read block metadata from primary at %d: %w", pm.offset, err)
+ }
+
+ pmi.block.lastPartID = pmi.partID
+ return nil
+}
+
+func (pmi *partMergeIter) mustLoadBlockData(decoder *encoding.BytesBlockDecoder, block *blockPointer) {
+ block.block.mustSeqReadFrom(decoder, &pmi.seqReaders, pmi.block.bm)
+}
+
+func generatePartMergeIter() *partMergeIter {
+ v := pmiPool.Get()
+ if v == nil {
+ return &partMergeIter{}
+ }
+ return v.(*partMergeIter)
+}
+
+func releasePartMergeIter(pmi *partMergeIter) {
+ pmi.reset()
+ pmiPool.Put(pmi)
+}
+
+var pmiPool sync.Pool
+
+type partMergeIterHeap []*partMergeIter
+
+func (pih *partMergeIterHeap) Len() int {
+ return len(*pih)
+}
+
+func (pih *partMergeIterHeap) Less(i, j int) bool {
+ x := *pih
+ return x[i].block.bm.less(x[j].block.bm)
+}
+
+func (pih *partMergeIterHeap) Swap(i, j int) {
+ x := *pih
+ x[i], x[j] = x[j], x[i]
+}
+
+func (pih *partMergeIterHeap) Push(x interface{}) {
+ *pih = append(*pih, x.(*partMergeIter))
+}
+
+func (pih *partMergeIterHeap) Pop() interface{} {
+ a := *pih
+ v := a[len(a)-1]
+ *pih = a[:len(a)-1]
+ return v
+}
+
+func generateColumnValuesDecoder() *encoding.BytesBlockDecoder {
+ v := columnValuesDecoderPool.Get()
+ if v == nil {
+ return &encoding.BytesBlockDecoder{}
+ }
+ return v.(*encoding.BytesBlockDecoder)
+}
+
+func releaseColumnValuesDecoder(d *encoding.BytesBlockDecoder) {
+ d.Reset()
+ columnValuesDecoderPool.Put(d)
+}
+
+var columnValuesDecoderPool sync.Pool
diff --git a/banyand/measure/part_iter_test.go b/banyand/measure/part_iter_test.go
index 4aa6d7a9..bc2a3301 100644
--- a/banyand/measure/part_iter_test.go
+++ b/banyand/measure/part_iter_test.go
@@ -26,6 +26,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/apache/skywalking-banyandb/api/common"
+ "github.com/apache/skywalking-banyandb/pkg/encoding"
"github.com/apache/skywalking-banyandb/pkg/fs"
"github.com/apache/skywalking-banyandb/pkg/test"
)
@@ -150,8 +151,74 @@ func Test_partIter_nextBlock(t *testing.T) {
partPath := partPath(tmpDir, epoch)
fileSystem := fs.NewLocalFileSystem()
mp.mustFlush(fileSystem, partPath)
- p = mustOpenFilePart(partPath, fileSystem)
+ p = mustOpenFilePart(epoch, tmpDir, fileSystem)
verifyPart(p)
})
}
}
+
+func Test_partMergeIter_nextBlock(t *testing.T) {
+ tests := []struct {
+ wantErr error
+ name string
+ dps *dataPoints
+ want []blockMetadata
+ }{
+ {
+ name: "Test with all data",
+ dps: dps,
+ want: []blockMetadata{
+ {seriesID: 1, count: 2}, {seriesID: 2, count: 2}, {seriesID: 3, count: 2},
+ },
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ verifyPart := func(p *part, decoder *encoding.BytesBlockDecoder) {
+ defer p.close()
+ pi := generatePartMergeIter()
+ defer releasePartMergeIter(pi)
+ pi.mustInitFromPart(p)
+ var got []blockMetadata
+ for pi.nextBlockMetadata() {
+ got = append(got, pi.block.bm)
+ require.Nil(t, pi.block.bm.tagProjection)
+ pi.mustLoadBlockData(decoder, &pi.block)
+ require.Equal(t, len(pi.block.bm.tagFamilies), len(pi.block.tagFamilies))
+ require.Equal(t, len(pi.block.bm.field.columnMetadata), len(pi.block.field.columns))
+ }
+
+ if !errors.Is(pi.error(), tt.wantErr) {
+ t.Errorf("Unexpected error: got %v, want %v", pi.err, tt.wantErr)
+ }
+
+ if diff := cmp.Diff(got, tt.want,
+ cmpopts.IgnoreFields(blockMetadata{}, "uncompressedSizeBytes"),
+ cmpopts.IgnoreFields(blockMetadata{}, "timestamps"),
+ cmpopts.IgnoreFields(blockMetadata{}, "field"),
+ cmpopts.IgnoreFields(blockMetadata{}, "tagFamilies"),
+ cmp.AllowUnexported(blockMetadata{}),
+ ); diff != "" {
+ t.Errorf("Unexpected blockMetadata (-got +want):\n%s", diff)
+ }
+ }
+ mp := generateMemPart()
+ releaseMemPart(mp)
+ mp.mustInitFromDataPoints(tt.dps)
+
+ decoder := generateColumnValuesDecoder()
+ defer releaseColumnValuesDecoder(decoder)
+ p := openMemPart(mp)
+ verifyPart(p, decoder)
+ tmpDir, defFn := test.Space(require.New(t))
+ defer defFn()
+ epoch := uint64(1)
+ partPath := partPath(tmpDir, epoch)
+ fileSystem := fs.NewLocalFileSystem()
+ mp.mustFlush(fileSystem, partPath)
+ p = mustOpenFilePart(epoch, tmpDir, fileSystem)
+ verifyPart(p, decoder)
+ })
+ }
+}
diff --git a/banyand/measure/part_metadata.go b/banyand/measure/part_metadata.go
index 0323e660..9b64c612 100644
--- a/banyand/measure/part_metadata.go
+++ b/banyand/measure/part_metadata.go
@@ -26,13 +26,13 @@ import (
)
type partMetadata struct {
- CompressedSizeBytes uint64
- UncompressedSizeBytes uint64
- TotalCount uint64
- BlocksCount uint64
- MinTimestamp int64
- MaxTimestamp int64
- ID uint64
+ CompressedSizeBytes uint64 `json:"compressedSizeBytes"`
+ UncompressedSizeBytes uint64 `json:"uncompressedSizeBytes"`
+ TotalCount uint64 `json:"totalCount"`
+ BlocksCount uint64 `json:"blocksCount"`
+ MinTimestamp int64 `json:"minTimestamp"`
+ MaxTimestamp int64 `json:"maxTimestamp"`
+ ID uint64 `json:"-"`
}
func (pm *partMetadata) reset() {
diff --git a/banyand/measure/part_test.go b/banyand/measure/part_test.go
index 5473fd76..1044a46c 100644
--- a/banyand/measure/part_test.go
+++ b/banyand/measure/part_test.go
@@ -103,7 +103,7 @@ func TestMustInitFromDataPoints(t *testing.T) {
fileSystem := fs.NewLocalFileSystem()
mp.mustFlush(fileSystem, path)
- p := mustOpenFilePart(path, fileSystem)
+ p := mustOpenFilePart(epoch, tmpPath, fileSystem)
defer p.close()
assert.Equal(t, tt.want.BlocksCount, p.partMetadata.BlocksCount)
assert.Equal(t, tt.want.MinTimestamp, p.partMetadata.MinTimestamp)
diff --git a/banyand/measure/primary_metadata.go b/banyand/measure/primary_metadata.go
index 7b09503e..ed3e4a82 100644
--- a/banyand/measure/primary_metadata.go
+++ b/banyand/measure/primary_metadata.go
@@ -24,6 +24,7 @@ import (
"github.com/apache/skywalking-banyandb/api/common"
"github.com/apache/skywalking-banyandb/pkg/compress/zstd"
"github.com/apache/skywalking-banyandb/pkg/encoding"
+ "github.com/apache/skywalking-banyandb/pkg/fs"
"github.com/apache/skywalking-banyandb/pkg/logger"
)
@@ -81,8 +82,8 @@ func (ph *primaryBlockMetadata) unmarshal(src []byte) ([]byte, error) {
return src[8:], nil
}
-func mustReadPrimaryBlockMetadata(dst []primaryBlockMetadata, r *reader) []primaryBlockMetadata {
- data, err := io.ReadAll(r)
+func mustReadPrimaryBlockMetadata(dst []primaryBlockMetadata, r fs.Reader) []primaryBlockMetadata {
+ data, err := io.ReadAll(r.StreamRead())
if err != nil {
logger.Panicf("cannot read primaryBlockMetadata entries from %s: %s", r.Path(), err)
}
diff --git a/banyand/measure/query.go b/banyand/measure/query.go
index d0b17d7f..c327e9fb 100644
--- a/banyand/measure/query.go
+++ b/banyand/measure/query.go
@@ -71,7 +71,7 @@ func (s *measure) Query(ctx context.Context, mqo pbv1.MeasureQueryOptions) (pbv1
if len(mqo.TagProjection) == 0 && len(mqo.FieldProjection) == 0 {
return nil, errors.New("invalid query options: tagProjection or fieldProjection is required")
}
- tsdb := s.databaseSupplier.SupplyTSDB().(storage.TSDB[*tsTable])
+ tsdb := s.databaseSupplier.SupplyTSDB().(storage.TSDB[*tsTable, option])
tabWrappers := tsdb.SelectTSTables(*mqo.TimeRange)
defer func() {
for i := range tabWrappers {
@@ -103,7 +103,7 @@ func (s *measure) Query(ctx context.Context, mqo pbv1.MeasureQueryOptions) (pbv1
if s == nil {
continue
}
- parts, n = s.getParts(parts, qo)
+ parts, n = s.getParts(parts, qo.minTimestamp, qo.maxTimestamp)
if n < 1 {
s.decRef()
continue
@@ -349,7 +349,7 @@ func (qr *queryResult) Pull() *pbv1.Result {
if len(qr.data) == 1 {
r := &pbv1.Result{}
bc := qr.data[0]
- bc.copyAllTo(r)
+ bc.copyAllTo(r, qr.orderByTimestampDesc())
qr.data = qr.data[:0]
return r
}
diff --git a/banyand/measure/query_test.go b/banyand/measure/query_test.go
index 4226dfc7..a5d1ad55 100644
--- a/banyand/measure/query_test.go
+++ b/banyand/measure/query_test.go
@@ -24,7 +24,6 @@ import (
"time"
"github.com/google/go-cmp/cmp"
- "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"google.golang.org/protobuf/testing/protocmp"
@@ -371,7 +370,7 @@ func TestQueryResult(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- verify := func(tst *tsTable) uint64 {
+ verify := func(t *testing.T, tst *tsTable) {
defer tst.Close()
queryOpts := queryOptions{
minTimestamp: tt.minTimestamp,
@@ -380,8 +379,7 @@ func TestQueryResult(t *testing.T) {
s := tst.currentSnapshot()
require.NotNil(t, s)
defer s.decRef()
- pp, n := s.getParts(nil, queryOpts)
- require.Equal(t, len(tt.dpsList), n, "parts: %+v", pp)
+ pp, _ := s.getParts(nil, queryOpts.minTimestamp, queryOpts.maxTimestamp)
sids := make([]common.SeriesID, len(tt.sids))
copy(sids, tt.sids)
sort.Slice(sids, func(i, j int) bool {
@@ -430,22 +428,27 @@ func TestQueryResult(t *testing.T) {
protocmp.IgnoreUnknown(), protocmp.Transform()); diff != "" {
t.Errorf("Unexpected []pbv1.Result (-got +want):\n%s", diff)
}
- return s.epoch
}
t.Run("memory snapshot", func(t *testing.T) {
+ tmpPath, defFn := test.Space(require.New(t))
+ defer defFn()
tst := &tsTable{
loopCloser: run.NewCloser(2),
introductions: make(chan *introduction),
+ fileSystem: fs.NewLocalFileSystem(),
+ root: tmpPath,
}
+ tst.gc.init(tst)
flushCh := make(chan *flusherIntroduction)
+ mergeCh := make(chan *mergerIntroduction)
introducerWatcher := make(watcher.Channel, 1)
- go tst.introducerLoop(flushCh, introducerWatcher, 1)
+ go tst.introducerLoop(flushCh, mergeCh, introducerWatcher, 1)
for _, dps := range tt.dpsList {
tst.mustAddDataPoints(dps)
time.Sleep(100 * time.Millisecond)
}
- _ = verify(tst)
+ verify(t, tst)
})
t.Run("file snapshot", func(t *testing.T) {
@@ -453,18 +456,37 @@ func TestQueryResult(t *testing.T) {
tmpPath, defFn := test.Space(require.New(t))
fileSystem := fs.NewLocalFileSystem()
defer defFn()
- tst, err := newTSTable(fileSystem, tmpPath, common.Position{}, logger.GetLogger("test"), timestamp.TimeRange{})
+ tst, err := newTSTable(fileSystem, tmpPath, common.Position{},
+ logger.GetLogger("test"), timestamp.TimeRange{}, option{flushTimeout: defaultFlushTimeout})
require.NoError(t, err)
for _, dps := range tt.dpsList {
tst.mustAddDataPoints(dps)
time.Sleep(100 * time.Millisecond)
}
- epoch := verify(tst)
+ // wait until the introducer is done
+ if len(tt.dpsList) > 0 {
+ for {
+ snp := tst.currentSnapshot()
+ if snp == nil {
+ time.Sleep(100 * time.Millisecond)
+ continue
+ }
+ if len(snp.parts) == len(tt.dpsList) {
+ snp.decRef()
+ tst.Close()
+ break
+ }
+ snp.decRef()
+ time.Sleep(100 * time.Millisecond)
+ }
+ }
// reopen the table
- tst, err = newTSTable(fileSystem, tmpPath, common.Position{}, logger.GetLogger("test"), timestamp.TimeRange{})
+ tst, err = newTSTable(fileSystem, tmpPath, common.Position{},
+ logger.GetLogger("test"), timestamp.TimeRange{}, option{flushTimeout: defaultFlushTimeout})
require.NoError(t, err)
- assert.Equal(t, epoch, verify(tst))
+
+ verify(t, tst)
})
})
}
diff --git a/banyand/measure/service.go b/banyand/measure/service.go
index 9a84f308..f937b670 100644
--- a/banyand/measure/service.go
+++ b/banyand/measure/service.go
@@ -59,6 +59,7 @@ type service struct {
localPipeline queue.Queue
l *logger.Logger
root string
+ option option
}
func (s *service) Measure(metadata *commonv1.Metadata) (Measure, error) {
@@ -76,6 +77,7 @@ func (s *service) LoadGroup(name string) (resourceSchema.Group, bool) {
func (s *service) FlagSet() *run.FlagSet {
flagS := run.NewFlagSet("storage")
flagS.StringVar(&s.root, "measure-root-path", "/tmp", "the root path of database")
+ flagS.DurationVar(&s.option.flushTimeout, "measure-flush-timeout", defaultFlushTimeout, "the memory data timeout of measure")
return flagS
}
diff --git a/banyand/measure/snapshot.go b/banyand/measure/snapshot.go
index ddc995dc..93a1fdeb 100644
--- a/banyand/measure/snapshot.go
+++ b/banyand/measure/snapshot.go
@@ -35,18 +35,28 @@ func (tst *tsTable) currentSnapshot() *snapshot {
return s
}
+type snapshotCreator rune
+
+const (
+ snapshotCreatorMemPart = iota
+ snapshotCreatorFlusher
+ snapshotCreatorMerger
+ snapshotCreatorMergedFlusher
+)
+
type snapshot struct {
- parts []*partWrapper
- epoch uint64
+ parts []*partWrapper
+ epoch uint64
+ creator snapshotCreator
ref int32
}
-func (s *snapshot) getParts(dst []*part, opts queryOptions) ([]*part, int) {
+func (s *snapshot) getParts(dst []*part, minTimestamp, maxTimestamp int64) ([]*part, int) {
var count int
for _, p := range s.parts {
pm := p.p.partMetadata
- if opts.maxTimestamp < pm.MinTimestamp || opts.minTimestamp > pm.MaxTimestamp {
+ if maxTimestamp < pm.MinTimestamp || minTimestamp > pm.MaxTimestamp {
continue
}
dst = append(dst, p.p)
@@ -96,6 +106,19 @@ func (s *snapshot) merge(nextEpoch uint64, nextParts map[uint64]*partWrapper) sn
return result
}
+func (s *snapshot) remove(nextEpoch uint64, merged map[uint64]struct{}) snapshot {
+ var result snapshot
+ result.epoch = nextEpoch
+ result.ref = 1
+ for i := 0; i < len(s.parts); i++ {
+ if _, ok := merged[s.parts[i].ID()]; !ok {
+ s.parts[i].incRef()
+ result.parts = append(result.parts, s.parts[i])
+ }
+ }
+ return result
+}
+
func snapshotName(snapshot uint64) string {
return fmt.Sprintf("%016x%s", snapshot, snapshotSuffix)
}
diff --git a/banyand/measure/snapshot_test.go b/banyand/measure/snapshot_test.go
index f571d4c9..80cf41b1 100644
--- a/banyand/measure/snapshot_test.go
+++ b/banyand/measure/snapshot_test.go
@@ -21,6 +21,8 @@ import (
"testing"
"github.com/stretchr/testify/assert"
+
+ "github.com/apache/skywalking-banyandb/pkg/bytes"
)
func TestSnapshotGetParts(t *testing.T) {
@@ -110,7 +112,7 @@ func TestSnapshotGetParts(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- result, count := tt.snapshot.getParts(tt.dst, tt.opts)
+ result, count := tt.snapshot.getParts(tt.dst, tt.opts.minTimestamp, tt.opts.maxTimestamp)
assert.Equal(t, tt.expected, result)
assert.Equal(t, tt.count, count)
})
@@ -285,3 +287,137 @@ func TestSnapshotMerge(t *testing.T) {
})
}
}
+
+func TestSnapshotRemove(t *testing.T) {
+ tests := []struct {
+ snapshot *snapshot
+ mergedParts map[uint64]struct{}
+ name string
+ expected snapshot
+ nextEpoch uint64
+ closePrev bool
+ }{
+ {
+ name: "Test with empty snapshot and no parts to remove",
+ snapshot: &snapshot{
+ parts: []*partWrapper{},
+ },
+ nextEpoch: 1,
+ mergedParts: map[uint64]struct{}{},
+ expected: snapshot{
+ epoch: 1,
+ ref: 1,
+ parts: nil,
+ },
+ },
+ {
+ name: "Test with non-empty snapshot and no parts to remove",
+ snapshot: &snapshot{
+ parts: []*partWrapper{
+ {p: &part{partMetadata: partMetadata{ID: 1}}, ref: 1},
+ {p: &part{partMetadata: partMetadata{ID: 2}}, ref: 2},
+ },
+ },
+ nextEpoch: 1,
+ mergedParts: map[uint64]struct{}{},
+ expected: snapshot{
+ epoch: 1,
+ ref: 1,
+ parts: []*partWrapper{
+ {p: &part{partMetadata: partMetadata{ID: 1}}, ref: 2},
+ {p: &part{partMetadata: partMetadata{ID: 2}}, ref: 3},
+ },
+ },
+ },
+ {
+ name: "Test with non-empty snapshot and some parts to remove",
+ snapshot: &snapshot{
+ parts: []*partWrapper{
+ {p: &part{partMetadata: partMetadata{ID: 1}}, ref: 1},
+ {p: &part{partMetadata: partMetadata{ID: 2}}, ref: 2},
+ },
+ },
+ nextEpoch: 1,
+ mergedParts: map[uint64]struct{}{
+ 1: {},
+ },
+ expected: snapshot{
+ epoch: 1,
+ ref: 1,
+ parts: []*partWrapper{
+ {p: &part{partMetadata: partMetadata{ID: 2}}, ref: 3},
+ },
+ },
+ },
+ {
+ name: "Test with empty snapshot, no parts to remove, and closePrev=true",
+ snapshot: &snapshot{
+ parts: []*partWrapper{},
+ },
+ nextEpoch: 1,
+ mergedParts: map[uint64]struct{}{},
+ expected: snapshot{
+ epoch: 1,
+ ref: 1,
+ parts: nil,
+ },
+ closePrev: true,
+ },
+ {
+ name: "Test with non-empty snapshot, no parts to remove, and closePrev=true",
+ snapshot: &snapshot{
+ parts: []*partWrapper{
+ {p: &part{partMetadata: partMetadata{ID: 1}}, ref: 1},
+ {p: &part{partMetadata: partMetadata{ID: 2}}, ref: 2},
+ },
+ },
+ nextEpoch: 1,
+ mergedParts: map[uint64]struct{}{},
+ expected: snapshot{
+ epoch: 1,
+ ref: 1,
+ parts: []*partWrapper{
+ {p: &part{partMetadata: partMetadata{ID: 1}}, ref: 1},
+ {p: &part{partMetadata: partMetadata{ID: 2}}, ref: 2},
+ },
+ },
+ closePrev: true,
+ },
+ {
+ name: "Test with non-empty snapshot, some parts to remove, and closePrev=true",
+ snapshot: &snapshot{
+ parts: []*partWrapper{
+ {p: &part{
+ partMetadata: partMetadata{ID: 1},
+ timestamps: &bytes.Buffer{},
+ primary: &bytes.Buffer{},
+ fieldValues: &bytes.Buffer{},
+ }, ref: 1},
+ {p: &part{partMetadata: partMetadata{ID: 2}}, ref: 2},
+ },
+ },
+ nextEpoch: 1,
+ mergedParts: map[uint64]struct{}{
+ 1: {},
+ },
+ expected: snapshot{
+ epoch: 1,
+ ref: 1,
+ parts: []*partWrapper{
+ {p: &part{partMetadata: partMetadata{ID: 2}}, ref: 2},
+ },
+ },
+ closePrev: true,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ result := tt.snapshot.remove(tt.nextEpoch, tt.mergedParts)
+ if tt.closePrev {
+ tt.snapshot.decRef()
+ }
+ assert.Equal(t, tt.expected, result)
+ })
+ }
+}
diff --git a/banyand/measure/tstable.go b/banyand/measure/tstable.go
index f3c445dc..14650e08 100644
--- a/banyand/measure/tstable.go
+++ b/banyand/measure/tstable.go
@@ -27,6 +27,7 @@ import (
"sort"
"strconv"
"sync"
+ "sync/atomic"
"time"
"github.com/apache/skywalking-banyandb/api/common"
@@ -43,12 +44,17 @@ const (
dirPermission = 0o700
)
-func newTSTable(fileSystem fs.FileSystem, rootPath string, _ common.Position, l *logger.Logger, _ timestamp.TimeRange) (*tsTable, error) {
- var tst tsTable
- tst.fileSystem = fileSystem
- tst.root = rootPath
- tst.l = l
- tst.gc.parent = &tst
+func newTSTable(fileSystem fs.FileSystem, rootPath string, p common.Position,
+ l *logger.Logger, _ timestamp.TimeRange, option option,
+) (*tsTable, error) {
+ tst := tsTable{
+ fileSystem: fileSystem,
+ root: rootPath,
+ option: option,
+ l: l,
+ p: p,
+ }
+ tst.gc.init(&tst)
ee := fileSystem.ReadDir(rootPath)
if len(ee) == 0 {
t := &tst
@@ -79,7 +85,6 @@ func newTSTable(fileSystem fs.FileSystem, rootPath string, _ common.Position, l
continue
}
loadedSnapshots = append(loadedSnapshots, snapshot)
- tst.gc.registerSnapshot(snapshot)
}
for i := range needToDelete {
if err := fileSystem.DeleteFile(filepath.Join(rootPath, needToDelete[i])); err != nil {
@@ -107,8 +112,11 @@ type tsTable struct {
snapshot *snapshot
introductions chan *introduction
loopCloser *run.Closer
+ p common.Position
root string
gc garbageCleaner
+ curPartID uint64
+ option option
sync.RWMutex
}
@@ -117,21 +125,25 @@ func (tst *tsTable) loadSnapshot(epoch uint64, loadedParts []uint64) {
snp := snapshot{
epoch: epoch,
}
- for _, partName := range loadedParts {
+ for _, id := range loadedParts {
var find bool
for j := range parts {
- if partName == parts[j] {
+ if id == parts[j] {
find = true
break
}
}
if !find {
- tst.gc.submitParts(partName)
+ tst.gc.submitParts(id)
+ }
+ p := mustOpenFilePart(id, tst.root, tst.fileSystem)
+ p.partMetadata.ID = id
+ snp.parts = append(snp.parts, newPartWrapper(nil, p))
+ if tst.curPartID < id {
+ tst.curPartID = id
}
- p := mustOpenFilePart(partPath(tst.root, partName), tst.fileSystem)
- p.partMetadata.ID = partName
- snp.parts = append(snp.parts, newPartWrapper(nil, p, tst.fileSystem))
}
+ tst.gc.registerSnapshot(&snp)
tst.gc.clean()
if len(snp.parts) < 1 {
return
@@ -141,12 +153,15 @@ func (tst *tsTable) loadSnapshot(epoch uint64, loadedParts []uint64) {
}
func (tst *tsTable) startLoop(cur uint64) {
- tst.loopCloser = run.NewCloser(3)
+ tst.loopCloser = run.NewCloser(1 + 3)
tst.introductions = make(chan *introduction)
flushCh := make(chan *flusherIntroduction)
+ mergeCh := make(chan *mergerIntroduction)
introducerWatcher := make(watcher.Channel, 1)
- go tst.introducerLoop(flushCh, introducerWatcher, cur+1)
- go tst.flusherLoop(flushCh, introducerWatcher, cur)
+ flusherWatcher := make(watcher.Channel, 1)
+ go tst.introducerLoop(flushCh, mergeCh, introducerWatcher, cur+1)
+ go tst.flusherLoop(flushCh, mergeCh, introducerWatcher, flusherWatcher, cur)
+ go tst.mergeLoop(mergeCh, flusherWatcher)
}
func parseEpoch(epochStr string) (uint64, error) {
@@ -204,9 +219,11 @@ func (tst *tsTable) Close() error {
}
tst.RLock()
defer tst.RUnlock()
- if tst.snapshot != nil {
- tst.snapshot.decRef()
+ if tst.snapshot == nil {
+ return nil
}
+ tst.snapshot.decRef()
+ tst.snapshot = nil
return nil
}
@@ -222,7 +239,8 @@ func (tst *tsTable) mustAddDataPoints(dps *dataPoints) {
ind := generateIntroduction()
defer releaseIntroduction(ind)
ind.applied = make(chan struct{})
- ind.memPart = newPartWrapper(mp, p, tst.fileSystem)
+ ind.memPart = newPartWrapper(mp, p)
+ ind.memPart.p.partMetadata.ID = atomic.AddUint64(&tst.curPartID, 1)
select {
case tst.introductions <- ind:
diff --git a/banyand/measure/tstable_test.go b/banyand/measure/tstable_test.go
index ca231e4f..8b3c9505 100644
--- a/banyand/measure/tstable_test.go
+++ b/banyand/measure/tstable_test.go
@@ -99,8 +99,9 @@ func Test_tsTable_mustAddDataPoints(t *testing.T) {
introductions: make(chan *introduction),
}
flushCh := make(chan *flusherIntroduction)
+ mergeCh := make(chan *mergerIntroduction)
introducerWatcher := make(watcher.Channel, 1)
- go tst.introducerLoop(flushCh, introducerWatcher, 1)
+ go tst.introducerLoop(flushCh, mergeCh, introducerWatcher, 1)
defer tst.Close()
for _, dps := range tt.dpsList {
tst.mustAddDataPoints(dps)
@@ -126,7 +127,7 @@ func Test_tsTable_mustAddDataPoints(t *testing.T) {
}
func Test_tstIter(t *testing.T) {
- tests := []struct {
+ type testCtx struct {
wantErr error
name string
dpsList []*dataPoints
@@ -134,131 +135,252 @@ func Test_tstIter(t *testing.T) {
want []blockMetadata
minTimestamp int64
maxTimestamp int64
- }{
- {
- name: "Test with no data points",
- dpsList: []*dataPoints{},
- sids: []common.SeriesID{1, 2, 3},
- minTimestamp: 1,
- maxTimestamp: 1,
- },
- {
- name: "Test with single part",
- dpsList: []*dataPoints{dpsTS1},
- sids: []common.SeriesID{1, 2, 3},
- minTimestamp: 1,
- maxTimestamp: 1,
- want: []blockMetadata{
- {seriesID: 1, count: 1, uncompressedSizeBytes: 1676},
- {seriesID: 2, count: 1, uncompressedSizeBytes: 55},
- {seriesID: 3, count: 1, uncompressedSizeBytes: 24},
- },
- },
- {
- name: "Test with multiple parts with different ts",
- dpsList: []*dataPoints{dpsTS1, dpsTS2},
- sids: []common.SeriesID{1, 2, 3},
- minTimestamp: 1,
- maxTimestamp: 2,
- want: []blockMetadata{
- {seriesID: 1, count: 1, uncompressedSizeBytes: 1676},
- {seriesID: 1, count: 1, uncompressedSizeBytes: 1676},
- {seriesID: 2, count: 1, uncompressedSizeBytes: 55},
- {seriesID: 2, count: 1, uncompressedSizeBytes: 55},
- {seriesID: 3, count: 1, uncompressedSizeBytes: 24},
- {seriesID: 3, count: 1, uncompressedSizeBytes: 24},
- },
- },
- {
- name: "Test with multiple parts with same ts",
- dpsList: []*dataPoints{dpsTS1, dpsTS1},
- sids: []common.SeriesID{1, 2, 3},
- minTimestamp: 1,
- maxTimestamp: 2,
- want: []blockMetadata{
- {seriesID: 1, count: 1, uncompressedSizeBytes: 1676},
- {seriesID: 1, count: 1, uncompressedSizeBytes: 1676},
- {seriesID: 2, count: 1, uncompressedSizeBytes: 55},
- {seriesID: 2, count: 1, uncompressedSizeBytes: 55},
- {seriesID: 3, count: 1, uncompressedSizeBytes: 24},
- {seriesID: 3, count: 1, uncompressedSizeBytes: 24},
- },
- },
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- verify := func(tst *tsTable) uint64 {
- defer tst.Close()
- s := tst.currentSnapshot()
- if s == nil {
- s = new(snapshot)
- }
- defer s.decRef()
- pp, n := s.getParts(nil, queryOptions{
- minTimestamp: tt.minTimestamp,
- maxTimestamp: tt.maxTimestamp,
- })
- require.Equal(t, len(s.parts), n)
- ti := &tstIter{}
- ti.init(pp, tt.sids, tt.minTimestamp, tt.maxTimestamp)
- var got []blockMetadata
- for ti.nextBlock() {
- if ti.piHeap[0].curBlock.seriesID == 0 {
- t.Errorf("Expected curBlock to be initialized, but it was nil")
- }
- got = append(got, ti.piHeap[0].curBlock)
- }
-
- if !errors.Is(ti.Error(), tt.wantErr) {
- t.Errorf("Unexpected error: got %v, want %v", ti.err, tt.wantErr)
- }
- if diff := cmp.Diff(got, tt.want,
- cmpopts.IgnoreFields(blockMetadata{}, "timestamps"),
- cmpopts.IgnoreFields(blockMetadata{}, "field"),
- cmpopts.IgnoreFields(blockMetadata{}, "tagFamilies"),
- cmp.AllowUnexported(blockMetadata{}),
- ); diff != "" {
- t.Errorf("Unexpected blockMetadata (-got +want):\n%s", diff)
- }
- return s.epoch
+ verify := func(t *testing.T, tt testCtx, tst *tsTable) uint64 {
+ defer tst.Close()
+ s := tst.currentSnapshot()
+ if s == nil {
+ s = new(snapshot)
+ }
+ defer s.decRef()
+ pp, n := s.getParts(nil, tt.minTimestamp, tt.maxTimestamp)
+ require.Equal(t, len(s.parts), n)
+ ti := &tstIter{}
+ ti.init(pp, tt.sids, tt.minTimestamp, tt.maxTimestamp)
+ var got []blockMetadata
+ for ti.nextBlock() {
+ if ti.piHeap[0].curBlock.seriesID == 0 {
+ t.Errorf("Expected curBlock to be initialized, but it was nil")
}
+ got = append(got, ti.piHeap[0].curBlock)
+ }
+
+ if !errors.Is(ti.Error(), tt.wantErr) {
+ t.Errorf("Unexpected error: got %v, want %v", ti.err, tt.wantErr)
+ }
+
+ if diff := cmp.Diff(got, tt.want,
+ cmpopts.IgnoreFields(blockMetadata{}, "timestamps"),
+ cmpopts.IgnoreFields(blockMetadata{}, "field"),
+ cmpopts.IgnoreFields(blockMetadata{}, "tagFamilies"),
+ cmp.AllowUnexported(blockMetadata{}),
+ ); diff != "" {
+ t.Errorf("Unexpected blockMetadata (-got +want):\n%s", diff)
+ }
+ return s.epoch
+ }
- t.Run("memory snapshot", func(t *testing.T) {
+ t.Run("memory snapshot", func(t *testing.T) {
+ tests := []testCtx{
+ {
+ name: "Test with no data points",
+ dpsList: []*dataPoints{},
+ sids: []common.SeriesID{1, 2, 3},
+ minTimestamp: 1,
+ maxTimestamp: 1,
+ },
+ {
+ name: "Test with single part",
+ dpsList: []*dataPoints{dpsTS1},
+ sids: []common.SeriesID{1, 2, 3},
+ minTimestamp: 1,
+ maxTimestamp: 1,
+ want: []blockMetadata{
+ {seriesID: 1, count: 1, uncompressedSizeBytes: 1676},
+ {seriesID: 2, count: 1, uncompressedSizeBytes: 55},
+ {seriesID: 3, count: 1, uncompressedSizeBytes: 24},
+ },
+ },
+ {
+ name: "Test with multiple parts with different ts",
+ dpsList: []*dataPoints{dpsTS1, dpsTS2},
+ sids: []common.SeriesID{1, 2, 3},
+ minTimestamp: 1,
+ maxTimestamp: 2,
+ want: []blockMetadata{
+ {seriesID: 1, count: 1, uncompressedSizeBytes: 1676},
+ {seriesID: 1, count: 1, uncompressedSizeBytes: 1676},
+ {seriesID: 2, count: 1, uncompressedSizeBytes: 55},
+ {seriesID: 2, count: 1, uncompressedSizeBytes: 55},
+ {seriesID: 3, count: 1, uncompressedSizeBytes: 24},
+ {seriesID: 3, count: 1, uncompressedSizeBytes: 24},
+ },
+ },
+ {
+ name: "Test with multiple parts with same ts",
+ dpsList: []*dataPoints{dpsTS1, dpsTS1},
+ sids: []common.SeriesID{1, 2, 3},
+ minTimestamp: 1,
+ maxTimestamp: 2,
+ want: []blockMetadata{
+ {seriesID: 1, count: 1, uncompressedSizeBytes: 1676},
+ {seriesID: 1, count: 1, uncompressedSizeBytes: 1676},
+ {seriesID: 2, count: 1, uncompressedSizeBytes: 55},
+ {seriesID: 2, count: 1, uncompressedSizeBytes: 55},
+ {seriesID: 3, count: 1, uncompressedSizeBytes: 24},
+ {seriesID: 3, count: 1, uncompressedSizeBytes: 24},
+ },
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ tmpPath, defFn := test.Space(require.New(t))
+ defer defFn()
tst := &tsTable{
loopCloser: run.NewCloser(2),
introductions: make(chan *introduction),
+ fileSystem: fs.NewLocalFileSystem(),
+ root: tmpPath,
}
+ tst.gc.init(tst)
flushCh := make(chan *flusherIntroduction)
+ mergeCh := make(chan *mergerIntroduction)
introducerWatcher := make(watcher.Channel, 1)
- go tst.introducerLoop(flushCh, introducerWatcher, 1)
+ go tst.introducerLoop(flushCh, mergeCh, introducerWatcher, 1)
for _, dps := range tt.dpsList {
tst.mustAddDataPoints(dps)
time.Sleep(100 * time.Millisecond)
}
- verify(tst)
+ verify(t, tt, tst)
})
+ }
+ })
- t.Run("file snapshot", func(t *testing.T) {
- tmpPath, defFn := test.Space(require.New(t))
- fileSystem := fs.NewLocalFileSystem()
- defer defFn()
+ t.Run("file snapshot", func(t *testing.T) {
+ tests := []testCtx{
+ {
+ name: "Test with no data points",
+ dpsList: []*dataPoints{},
+ sids: []common.SeriesID{1, 2, 3},
+ minTimestamp: 1,
+ maxTimestamp: 1,
+ },
+ {
+ name: "Test with single part",
+ dpsList: []*dataPoints{dpsTS1},
+ sids: []common.SeriesID{1, 2, 3},
+ minTimestamp: 1,
+ maxTimestamp: 1,
+ want: []blockMetadata{
+ {seriesID: 1, count: 1, uncompressedSizeBytes: 1676},
+ {seriesID: 2, count: 1, uncompressedSizeBytes: 55},
+ {seriesID: 3, count: 1, uncompressedSizeBytes: 24},
+ },
+ },
+ {
+ name: "Test with multiple parts with different ts, the block will be merged",
+ dpsList: []*dataPoints{dpsTS1, dpsTS2, dpsTS2},
+ sids: []common.SeriesID{1, 2, 3},
+ minTimestamp: 1,
+ maxTimestamp: 2,
+ want: []blockMetadata{
+ {seriesID: 1, count: 2, uncompressedSizeBytes: 3352},
+ {seriesID: 2, count: 2, uncompressedSizeBytes: 110},
+ {seriesID: 3, count: 2, uncompressedSizeBytes: 48},
+ },
+ },
+ {
+ name: "Test with multiple parts with same ts, duplicated blocks will be merged",
+ dpsList: []*dataPoints{dpsTS1, dpsTS1},
+ sids: []common.SeriesID{1, 2, 3},
+ minTimestamp: 1,
+ maxTimestamp: 2,
+ want: []blockMetadata{
+ {seriesID: 1, count: 1, uncompressedSizeBytes: 1676},
+ {seriesID: 2, count: 1, uncompressedSizeBytes: 55},
+ {seriesID: 3, count: 1, uncompressedSizeBytes: 24},
+ },
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ t.Run("merging on the fly", func(t *testing.T) {
+ tmpPath, defFn := test.Space(require.New(t))
+ fileSystem := fs.NewLocalFileSystem()
+ defer defFn()
- tst, err := newTSTable(fileSystem, tmpPath, common.Position{}, logger.GetLogger("test"), timestamp.TimeRange{})
- require.NoError(t, err)
- for _, dps := range tt.dpsList {
- tst.mustAddDataPoints(dps)
- time.Sleep(100 * time.Millisecond)
- }
- epoch := verify(tst)
+ tst, err := newTSTable(fileSystem, tmpPath, common.Position{},
+ logger.GetLogger("test"), timestamp.TimeRange{}, option{flushTimeout: 0})
+ require.NoError(t, err)
+ for i, dps := range tt.dpsList {
+ tst.mustAddDataPoints(dps)
+ for {
+ snp := tst.currentSnapshot()
+ if snp == nil {
+ t.Logf("waiting for snapshot %d to be introduced", i)
+ time.Sleep(100 * time.Millisecond)
+ continue
+ }
+ if snp.creator != snapshotCreatorMemPart {
+ snp.decRef()
+ break
+ }
+ t.Logf("waiting for snapshot %d to be flushed or merged: current creator:%d, parts: %+v",
+ i, snp.creator, snp.parts)
+ snp.decRef()
+ time.Sleep(100 * time.Millisecond)
+ }
+ }
+ // wait until some parts are merged
+ if len(tt.dpsList) > 0 {
+ for {
+ snp := tst.currentSnapshot()
+ if snp == nil {
+ time.Sleep(100 * time.Millisecond)
+ continue
+ }
+ if len(snp.parts) == 1 || len(snp.parts) < len(tt.dpsList) {
+ snp.decRef()
+ break
+ }
+ t.Logf("waiting for snapshot to be merged: current creator:%d, parts: %+v", snp.creator, snp.parts)
+ snp.decRef()
+ time.Sleep(100 * time.Millisecond)
+ }
+ }
+ verify(t, tt, tst)
+ })
- // reopen the table
- tst, err = newTSTable(fileSystem, tmpPath, common.Position{}, logger.GetLogger("test"), timestamp.TimeRange{})
- require.NoError(t, err)
- assert.Equal(t, epoch, verify(tst))
+ t.Run("merging on close", func(t *testing.T) {
+ t.Skip("the test is flaky due to unpredictable merge loop schedule.")
+ tmpPath, defFn := test.Space(require.New(t))
+ fileSystem := fs.NewLocalFileSystem()
+ defer defFn()
+
+ tst, err := newTSTable(fileSystem, tmpPath, common.Position{},
+ logger.GetLogger("test"), timestamp.TimeRange{}, option{flushTimeout: defaultFlushTimeout})
+ require.NoError(t, err)
+ for _, dps := range tt.dpsList {
+ tst.mustAddDataPoints(dps)
+ time.Sleep(100 * time.Millisecond)
+ }
+ // wait until the introducer is done
+ if len(tt.dpsList) > 0 {
+ for {
+ snp := tst.currentSnapshot()
+ if snp == nil {
+ time.Sleep(100 * time.Millisecond)
+ continue
+ }
+ if len(snp.parts) == len(tt.dpsList) {
+ snp.decRef()
+ tst.Close()
+ break
+ }
+ snp.decRef()
+ time.Sleep(100 * time.Millisecond)
+ }
+ }
+ // reopen the table
+ tst, err = newTSTable(fileSystem, tmpPath, common.Position{},
+ logger.GetLogger("test"), timestamp.TimeRange{}, option{flushTimeout: defaultFlushTimeout})
+ require.NoError(t, err)
+ verify(t, tt, tst)
+ })
})
- })
- }
+ }
+ })
}
var tagProjections = map[int][]pbv1.TagProjection{
diff --git a/banyand/stream/block.go b/banyand/stream/block.go
index e8d1e950..1b6c99fd 100644
--- a/banyand/stream/block.go
+++ b/banyand/stream/block.go
@@ -121,8 +121,8 @@ func (b *block) mustWriteTo(sid common.SeriesID, bm *blockMetadata, ww *writers)
bm.uncompressedSizeBytes = b.uncompressedSizeBytes()
bm.count = uint64(b.Len())
- mustWriteTimestampsTo(&bm.timestamps, b.timestamps, ww.timestampsWriter)
- mustWriteElementIDsTo(&bm.elementIDs, b.elementIDs, ww.elementIDsWriter)
+ mustWriteTimestampsTo(&bm.timestamps, b.timestamps, &ww.timestampsWriter)
+ mustWriteElementIDsTo(&bm.elementIDs, b.elementIDs, &ww.elementIDsWriter)
for ti := range b.tagFamilies {
b.marshalTagFamily(b.tagFamilies[ti], bm, ww)
@@ -190,8 +190,10 @@ func (b *block) unmarshalTagFamily(decoder *encoding.BytesBlockDecoder, tfIndex
}
bigValuePool.Release(bb)
b.tagFamilies[tfIndex].name = name
+ if len(tagProjection) < 1 {
+ return
+ }
cc := b.tagFamilies[tfIndex].resizeTags(len(tagProjection))
-
for j := range tagProjection {
for i := range cfm.tagMetadata {
if tagProjection[j] == cfm.tagMetadata[i].name {
@@ -202,6 +204,30 @@ func (b *block) unmarshalTagFamily(decoder *encoding.BytesBlockDecoder, tfIndex
}
}
+func (b *block) unmarshalTagFamilyFromSeqReaders(decoder *encoding.BytesBlockDecoder, tfIndex int, name string,
+ columnFamilyMetadataBlock *dataBlock, metaReader, valueReader *seqReader,
+) {
+ if columnFamilyMetadataBlock.offset != metaReader.bytesRead {
+ logger.Panicf("offset %d must be equal to bytesRead %d", columnFamilyMetadataBlock.offset, metaReader.bytesRead)
+ }
+ bb := bigValuePool.Generate()
+ bb.Buf = bytes.ResizeExact(bb.Buf, int(columnFamilyMetadataBlock.size))
+ metaReader.mustReadFull(bb.Buf)
+ cfm := generateTagFamilyMetadata()
+ defer releaseTagFamilyMetadata(cfm)
+ _, err := cfm.unmarshal(bb.Buf)
+ if err != nil {
+ logger.Panicf("%s: cannot unmarshal columnFamilyMetadata: %v", metaReader.Path(), err)
+ }
+ bigValuePool.Release(bb)
+ b.tagFamilies[tfIndex].name = name
+
+ cc := b.tagFamilies[tfIndex].resizeTags(len(cfm.tagMetadata))
+ for i := range cfm.tagMetadata {
+ cc[i].mustSeqReadValues(decoder, valueReader, cfm.tagMetadata[i], uint64(b.Len()))
+ }
+}
+
func (b *block) uncompressedSizeBytes() uint64 {
elementsCount := uint64(b.Len())
@@ -236,7 +262,28 @@ func (b *block) mustReadFrom(decoder *encoding.BytesBlockDecoder, p *part, bm bl
if !ok {
continue
}
- b.unmarshalTagFamily(decoder, i, name, block, bm.tagProjection[i].Names, p.tagFamilyMetadata[name], p.tagFamilies[name])
+ b.unmarshalTagFamily(decoder, i, name, block,
+ bm.tagProjection[i].Names, p.tagFamilyMetadata[name],
+ p.tagFamilies[name])
+ }
+}
+
+func (b *block) mustSeqReadFrom(decoder *encoding.BytesBlockDecoder, seqReaders *seqReaders, bm blockMetadata) {
+ b.reset()
+
+ b.timestamps = mustSeqReadTimestampsFrom(b.timestamps, &bm.timestamps, int(bm.count), &seqReaders.timestamps)
+ b.elementIDs = mustSeqReadElementIDsFrom(b.elementIDs, &bm.elementIDs, int(bm.count), &seqReaders.elementIDs)
+
+ _ = b.resizeTagFamilies(len(bm.tagFamilies))
+ keys := make([]string, 0, len(bm.tagFamilies))
+ for k := range bm.tagFamilies {
+ keys = append(keys, k)
+ }
+ sort.Strings(keys)
+ for i, name := range keys {
+ block := bm.tagFamilies[name]
+ b.unmarshalTagFamilyFromSeqReaders(decoder, i, name, block,
+ seqReaders.tagFamilyMetadata[name], seqReaders.tagFamilies[name])
}
}
@@ -247,7 +294,7 @@ func (b *block) sortTagFamilies() {
})
}
-func mustWriteTimestampsTo(tm *timestampsMetadata, timestamps []int64, timestampsWriter writer) {
+func mustWriteTimestampsTo(tm *timestampsMetadata, timestamps []int64, timestampsWriter *writer) {
tm.reset()
bb := bigValuePool.Generate()
@@ -275,7 +322,7 @@ func mustReadTimestampsFrom(dst []int64, tm *timestampsMetadata, count int, read
return dst
}
-func mustWriteElementIDsTo(em *elementIDsMetadata, elementIDs []string, elementIDsWriter writer) {
+func mustWriteElementIDsTo(em *elementIDsMetadata, elementIDs []string, elementIDsWriter *writer) {
em.reset()
bb := bigValuePool.Generate()
@@ -311,6 +358,42 @@ func mustReadElementIDsFrom(dst []string, em *elementIDsMetadata, count int, rea
return dst
}
+func mustSeqReadTimestampsFrom(dst []int64, tm *timestampsMetadata, count int, reader *seqReader) []int64 {
+ if tm.offset != reader.bytesRead {
+ logger.Panicf("offset %d must be equal to bytesRead %d", tm.offset, reader.bytesRead)
+ }
+ bb := bigValuePool.Generate()
+ defer bigValuePool.Release(bb)
+ bb.Buf = bytes.ResizeExact(bb.Buf, int(tm.size))
+ reader.mustReadFull(bb.Buf)
+ var err error
+ dst, err = encoding.BytesToInt64List(dst, bb.Buf, tm.encodeType, tm.min, count)
+ if err != nil {
+ logger.Panicf("%s: cannot unmarshal timestamps: %v", reader.Path(), err)
+ }
+ return dst
+}
+
+func mustSeqReadElementIDsFrom(dst []string, em *elementIDsMetadata, count int, reader *seqReader) []string {
+ if em.offset != reader.bytesRead {
+ logger.Panicf("offset %d must be equal to bytesRead %d", em.offset, reader.bytesRead)
+ }
+ bb := bigValuePool.Generate()
+ defer bigValuePool.Release(bb)
+ bb.Buf = bytes.ResizeExact(bb.Buf, int(em.size))
+ reader.mustReadFull(bb.Buf)
+ decoder := encoding.BytesBlockDecoder{}
+ var elementIDsByteSlice [][]byte
+ elementIDsByteSlice, err := decoder.Decode(elementIDsByteSlice, bb.Buf, uint64(count))
+ if err != nil {
+ logger.Panicf("%s: cannot unmarshal elementIDs: %v", reader.Path(), err)
+ }
+ for _, elementID := range elementIDsByteSlice {
+ dst = append(dst, string(elementID))
+ }
+ return dst
+}
+
func generateBlock() *block {
v := blockPool.Get()
if v == nil {
@@ -327,16 +410,18 @@ func releaseBlock(b *block) {
var blockPool sync.Pool
type blockCursor struct {
- p *part
- timestamps []int64
- elementIDs []string
- tagFamilies []tagFamily
- tagValuesDecoder encoding.BytesBlockDecoder
- tagProjection []pbv1.TagProjection
- bm blockMetadata
- idx int
- minTimestamp int64
- maxTimestamp int64
+ p *part
+ timestamps []int64
+ elementIDs []string
+ tagFamilies []tagFamily
+ tagValuesDecoder encoding.BytesBlockDecoder
+ tagProjection []pbv1.TagProjection
+ bm blockMetadata
+ idx int
+ minTimestamp int64
+ maxTimestamp int64
+ includeMinTimestamp bool
+ includeMaxTimestamp bool
}
func (bc *blockCursor) reset() {
@@ -345,6 +430,8 @@ func (bc *blockCursor) reset() {
bc.bm = blockMetadata{}
bc.minTimestamp = 0
bc.maxTimestamp = 0
+ bc.includeMinTimestamp = false
+ bc.includeMaxTimestamp = false
bc.tagProjection = bc.tagProjection[:0]
bc.timestamps = bc.timestamps[:0]
@@ -363,13 +450,26 @@ func (bc *blockCursor) init(p *part, bm blockMetadata, queryOpts queryOptions) {
bc.bm = bm
bc.minTimestamp = queryOpts.minTimestamp
bc.maxTimestamp = queryOpts.maxTimestamp
+ bc.includeMinTimestamp = queryOpts.includeMin
+ bc.includeMaxTimestamp = queryOpts.includeMax
bc.tagProjection = queryOpts.TagProjection
}
-func (bc *blockCursor) copyAllTo(r *pbv1.Result) {
+func (bc *blockCursor) copyAllTo(r *pbv1.Result, desc bool) {
+ var idx, offset int
+ if desc {
+ idx = 0
+ offset = bc.idx + 1
+ } else {
+ idx = bc.idx
+ offset = len(bc.timestamps)
+ }
+ if offset <= idx {
+ return
+ }
r.SID = bc.bm.seriesID
- r.Timestamps = append(r.Timestamps, bc.timestamps...)
- r.ElementIDs = append(r.ElementIDs, bc.elementIDs...)
+ r.Timestamps = append(r.Timestamps, bc.timestamps[idx:offset]...)
+ r.ElementIDs = append(r.ElementIDs, bc.elementIDs[idx:offset]...)
for _, cf := range bc.tagFamilies {
tf := pbv1.TagFamily{
Name: cf.name,
@@ -378,7 +478,7 @@ func (bc *blockCursor) copyAllTo(r *pbv1.Result) {
t := pbv1.Tag{
Name: c.name,
}
- for _, v := range c.values {
+ for _, v := range c.values[idx:offset] {
t.Values = append(t.Values, mustDecodeTagValue(c.valueType, v))
}
tf.Tags = append(tf.Tags, t)
@@ -413,7 +513,11 @@ func (bc *blockCursor) copyTo(r *pbv1.Result) {
logger.Panicf("unexpected number of tags: got %d; want %d", len(r.TagFamilies[i].Tags), len(bc.tagProjection[i].Names))
}
for i2, c := range cf.tags {
- r.TagFamilies[i].Tags[i2].Values = append(r.TagFamilies[i].Tags[i2].Values, mustDecodeTagValue(c.valueType, c.values[bc.idx]))
+ if c.values != nil {
+ r.TagFamilies[i].Tags[i2].Values = append(r.TagFamilies[i].Tags[i2].Values, mustDecodeTagValue(c.valueType, c.values[bc.idx]))
+ } else {
+ r.TagFamilies[i].Tags[i2].Values = append(r.TagFamilies[i].Tags[i2].Values, pbv1.NullTagValue)
+ }
}
}
}
@@ -432,60 +536,63 @@ func (bc *blockCursor) loadData(tmpBlock *block) bool {
bc.bm.tagFamilies = tf
tmpBlock.mustReadFrom(&bc.tagValuesDecoder, bc.p, bc.bm)
- start, end, ok := findRange(tmpBlock.timestamps, bc.minTimestamp, bc.maxTimestamp)
+ start, end, ok := findRange(tmpBlock.timestamps, bc.minTimestamp, bc.maxTimestamp, bc.includeMinTimestamp, bc.includeMaxTimestamp)
if !ok {
return false
}
- bc.timestamps = append(bc.timestamps, tmpBlock.timestamps[start:end]...)
- bc.elementIDs = append(bc.elementIDs, tmpBlock.elementIDs[start:end]...)
+ bc.timestamps = append(bc.timestamps, tmpBlock.timestamps[start:end+1]...)
+ bc.elementIDs = append(bc.elementIDs, tmpBlock.elementIDs[start:end+1]...)
- for _, cf := range tmpBlock.tagFamilies {
+ for i, projection := range bc.bm.tagProjection {
tf := tagFamily{
- name: cf.name,
+ name: projection.Family,
}
- for i := range cf.tags {
- tag := tag{
- name: cf.tags[i].name,
- valueType: cf.tags[i].valueType,
- }
- if len(cf.tags[i].values) == 0 {
- continue
+ blockIndex := 0
+ for _, name := range projection.Names {
+ t := tag{
+ name: name,
}
- if len(cf.tags[i].values) != len(tmpBlock.timestamps) {
- logger.Panicf("unexpected number of values for tags %q: got %d; want %d", cf.tags[i].name, len(cf.tags[i].values), len(tmpBlock.timestamps))
+ if tmpBlock.tagFamilies[i].tags[blockIndex].name == name {
+ t.valueType = tmpBlock.tagFamilies[i].tags[blockIndex].valueType
+ if len(tmpBlock.tagFamilies[i].tags[blockIndex].values) != len(tmpBlock.timestamps) {
+ logger.Panicf("unexpected number of values for tags %q: got %d; want %d",
+ tmpBlock.tagFamilies[i].tags[blockIndex].name, len(tmpBlock.tagFamilies[i].tags[blockIndex].values), len(tmpBlock.timestamps))
+ }
+ t.values = append(t.values, tmpBlock.tagFamilies[i].tags[blockIndex].values[start:end+1]...)
+ blockIndex++
}
- tag.values = append(tag.values, cf.tags[i].values[start:end]...)
- tf.tags = append(tf.tags, tag)
+ tf.tags = append(tf.tags, t)
}
bc.tagFamilies = append(bc.tagFamilies, tf)
}
return true
}
-func findRange(timestamps []int64, min int64, max int64) (int, int, bool) {
- l := len(timestamps)
- start, end := -1, -1
+func findRange(timestamps []int64, min, max int64, includeMin, includeMax bool) (int, int, bool) {
+ if len(timestamps) == 0 {
+ return -1, -1, false
+ }
+ if timestamps[0] > max || !includeMin && timestamps[0] == max {
+ return -1, -1, false
+ }
+ if timestamps[len(timestamps)-1] < min || !includeMax && timestamps[len(timestamps)-1] == min {
+ return -1, -1, false
+ }
- for i := 0; i < l; i++ {
- if timestamps[i] > max || timestamps[l-i-1] < min {
+ start, end := -1, len(timestamps)
+ for start < len(timestamps)-1 {
+ start++
+ if timestamps[start] > min || (includeMin && timestamps[start] == min) {
break
}
- if timestamps[i] >= min && start == -1 {
- start = i
- }
- if timestamps[l-i-1] <= max && end == -1 {
- end = l - i
- }
- if start != -1 && end != -1 {
+ }
+ for end > 0 {
+ end--
+ if timestamps[end] < max || (includeMax && timestamps[end] == max) {
break
}
}
-
- if start == -1 || end == -1 || start >= end {
- return 0, 0, false
- }
-
- return start, end, true
+ return start, end, start <= end
}
var blockCursorPool sync.Pool
@@ -502,3 +609,96 @@ func releaseBlockCursor(bc *blockCursor) {
bc.reset()
blockCursorPool.Put(bc)
}
+
+type blockPointer struct {
+ block
+ bm blockMetadata
+ idx int
+ lastPartID uint64
+}
+
+func (bi *blockPointer) updateMetadata() {
+ if len(bi.block.timestamps) == 0 {
+ return
+ }
+ // only update timestamps since they are used for merging
+ // blockWriter will recompute all fields
+ bi.bm.timestamps.min = bi.block.timestamps[0]
+ bi.bm.timestamps.max = bi.block.timestamps[len(bi.timestamps)-1]
+}
+
+func (bi *blockPointer) copyFrom(src *blockPointer) {
+ bi.idx = 0
+ bi.bm.copyFrom(&src.bm)
+ bi.appendAll(src)
+}
+
+func (bi *blockPointer) appendAll(b *blockPointer) {
+ if len(b.timestamps) == 0 {
+ return
+ }
+ bi.append(b, len(b.timestamps))
+}
+
+func (bi *blockPointer) append(b *blockPointer, offset int) {
+ if offset <= b.idx {
+ return
+ }
+ if len(bi.tagFamilies) == 0 && len(b.tagFamilies) > 0 {
+ for _, tf := range b.tagFamilies {
+ tFamily := tagFamily{name: tf.name}
+ for _, c := range tf.tags {
+ col := tag{name: c.name, valueType: c.valueType}
+ assertIdxAndOffset(col.name, len(c.values), b.idx, offset)
+ col.values = append(col.values, c.values[b.idx:offset]...)
+ tFamily.tags = append(tFamily.tags, col)
+ }
+ bi.tagFamilies = append(bi.tagFamilies, tFamily)
+ }
+ } else {
+ if len(bi.tagFamilies) != len(b.tagFamilies) {
+ logger.Panicf("unexpected number of tag families: got %d; want %d", len(bi.tagFamilies), len(b.tagFamilies))
+ }
+ for i := range bi.tagFamilies {
+ if bi.tagFamilies[i].name != b.tagFamilies[i].name {
+ logger.Panicf("unexpected tag family name: got %q; want %q", bi.tagFamilies[i].name, b.tagFamilies[i].name)
+ }
+ if len(bi.tagFamilies[i].tags) != len(b.tagFamilies[i].tags) {
+ logger.Panicf("unexpected number of tags for tag family %q: got %d; want %d", bi.tagFamilies[i].name, len(bi.tagFamilies[i].tags), len(b.tagFamilies[i].tags))
+ }
+ for j := range bi.tagFamilies[i].tags {
+ if bi.tagFamilies[i].tags[j].name != b.tagFamilies[i].tags[j].name {
+ logger.Panicf("unexpected tag name for tag family %q: got %q; want %q", bi.tagFamilies[i].name, bi.tagFamilies[i].tags[j].name, b.tagFamilies[i].tags[j].name)
+ }
+ assertIdxAndOffset(b.tagFamilies[i].tags[j].name, len(b.tagFamilies[i].tags[j].values), b.idx, offset)
+ bi.tagFamilies[i].tags[j].values = append(bi.tagFamilies[i].tags[j].values, b.tagFamilies[i].tags[j].values[b.idx:offset]...)
+ }
+ }
+ }
+
+ assertIdxAndOffset("timestamps", len(b.timestamps), bi.idx, offset)
+ bi.timestamps = append(bi.timestamps, b.timestamps[b.idx:offset]...)
+ bi.elementIDs = append(bi.elementIDs, b.elementIDs[b.idx:offset]...)
+
+ bi.lastPartID = b.lastPartID
+}
+
+func assertIdxAndOffset(name string, length int, idx int, offset int) {
+ if idx >= offset {
+ logger.Panicf("%q idx %d must be less than offset %d", name, idx, offset)
+ }
+ if offset > length {
+ logger.Panicf("%q offset %d must be less than or equal to length %d", name, offset, length)
+ }
+}
+
+func (bi *blockPointer) isFull() bool {
+ return bi.bm.count >= maxBlockLength || bi.bm.uncompressedSizeBytes >= maxUncompressedBlockSize
+}
+
+func (bi *blockPointer) reset() {
+ bi.idx = 0
+ bi.lastPartID = 0
+ bi.block.reset()
+ bi.bm = blockMetadata{}
+}
diff --git a/banyand/stream/block_metadata.go b/banyand/stream/block_metadata.go
index ddc0a6ec..3366ac1e 100644
--- a/banyand/stream/block_metadata.go
+++ b/banyand/stream/block_metadata.go
@@ -20,6 +20,7 @@ package stream
import (
"errors"
"fmt"
+ "sort"
"sync"
"github.com/apache/skywalking-banyandb/api/common"
@@ -74,6 +75,21 @@ type blockMetadata struct {
count uint64
}
+func (bh *blockMetadata) copyFrom(src *blockMetadata) {
+ bh.seriesID = src.seriesID
+ bh.uncompressedSizeBytes = src.uncompressedSizeBytes
+ bh.count = src.count
+ bh.timestamps.copyFrom(&src.timestamps)
+ bh.elementIDs.copyFrom(&src.elementIDs)
+ for k, db := range src.tagFamilies {
+ if bh.tagFamilies == nil {
+ bh.tagFamilies = make(map[string]*dataBlock)
+ }
+ bh.tagFamilies[k] = &dataBlock{}
+ bh.tagFamilies[k].copyFrom(db)
+ }
+}
+
func (bh *blockMetadata) getTagFamilyMetadata(name string) *dataBlock {
if bh.tagFamilies == nil {
bh.tagFamilies = make(map[string]*dataBlock)
@@ -106,7 +122,14 @@ func (bh *blockMetadata) marshal(dst []byte) []byte {
dst = bh.timestamps.marshal(dst)
dst = bh.elementIDs.marshal(dst)
dst = encoding.VarUint64ToBytes(dst, uint64(len(bh.tagFamilies)))
- for name, cf := range bh.tagFamilies {
+ // make sure the order of tagFamilies is stable
+ keys := make([]string, 0, len(bh.tagFamilies))
+ for k := range bh.tagFamilies {
+ keys = append(keys, k)
+ }
+ sort.Strings(keys)
+ for _, name := range keys {
+ cf := bh.tagFamilies[name]
dst = encoding.EncodeBytes(dst, convert.StringToBytes(name))
dst = cf.marshal(dst)
}
@@ -241,7 +264,6 @@ func (th *elementIDsMetadata) reset() {
th.encodeType = 0
}
-// nolint: unused
func (th *elementIDsMetadata) copyFrom(src *elementIDsMetadata) {
th.dataBlock.copyFrom(&src.dataBlock)
th.encodeType = src.encodeType
@@ -258,7 +280,6 @@ func (th *elementIDsMetadata) unmarshal(src []byte) ([]byte, error) {
if err != nil {
return nil, fmt.Errorf("cannot unmarshal dataBlock: %w", err)
}
- // src = src[8:]
th.encodeType = encoding.EncodeType(src[0])
return src[1:], nil
}
diff --git a/banyand/stream/block_reader.go b/banyand/stream/block_reader.go
index 86820d77..4fc18f2a 100644
--- a/banyand/stream/block_reader.go
+++ b/banyand/stream/block_reader.go
@@ -17,23 +17,215 @@
package stream
-import "github.com/apache/skywalking-banyandb/pkg/fs"
+import (
+ "container/heap"
+ "errors"
+ "fmt"
+ "io"
+ "sync"
-type reader struct {
+ "github.com/apache/skywalking-banyandb/pkg/encoding"
+ "github.com/apache/skywalking-banyandb/pkg/fs"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+)
+
+type seqReader struct {
+ sr io.Reader
r fs.Reader
bytesRead uint64
}
-func newReader(r fs.Reader) *reader {
- return &reader{r: r}
+func (sr *seqReader) reset() {
+ sr.r = nil
+ sr.sr = nil
+ sr.bytesRead = 0
+}
+
+func (sr *seqReader) Path() string {
+ return sr.r.Path()
+}
+
+func (sr *seqReader) init(r fs.Reader) {
+ sr.reset()
+ sr.sr = r.StreamRead()
+ sr.r = r
+}
+
+func (sr *seqReader) mustReadFull(data []byte) {
+ n, err := io.ReadFull(sr.sr, data)
+ if err != nil {
+ if errors.Is(err, io.EOF) {
+ return
+ }
+ logger.Panicf("cannot read data: %v", err)
+ }
+ if n != len(data) {
+ logger.Panicf("cannot read full data: %d/%d", n, len(data))
+ }
+ sr.bytesRead += uint64(n)
+}
+
+func generateSeqReader() *seqReader {
+ if v := seqReaderPool.Get(); v != nil {
+ return v.(*seqReader)
+ }
+ return &seqReader{}
+}
+
+func releaseSeqReader(sr *seqReader) {
+ sr.reset()
+ seqReaderPool.Put(sr)
+}
+
+var seqReaderPool sync.Pool
+
+type seqReaders struct {
+ tagFamilyMetadata map[string]*seqReader
+ tagFamilies map[string]*seqReader
+ primary seqReader
+ timestamps seqReader
+ elementIDs seqReader
+}
+
+func (sr *seqReaders) reset() {
+ sr.primary.reset()
+ sr.timestamps.reset()
+ sr.elementIDs.reset()
+ if sr.tagFamilyMetadata != nil {
+ for k, r := range sr.tagFamilyMetadata {
+ releaseSeqReader(r)
+ delete(sr.tagFamilyMetadata, k)
+ }
+ }
+ if sr.tagFamilies != nil {
+ for k, r := range sr.tagFamilies {
+ releaseSeqReader(r)
+ delete(sr.tagFamilies, k)
+ }
+ }
}
-func (r *reader) Path() string {
- return r.r.Path()
+func (sr *seqReaders) init(p *part) {
+ sr.reset()
+ sr.primary.init(p.primary)
+ sr.timestamps.init(p.timestamps)
+ sr.elementIDs.init(p.elementIDs)
+ if sr.tagFamilies == nil {
+ sr.tagFamilies = make(map[string]*seqReader)
+ sr.tagFamilyMetadata = make(map[string]*seqReader)
+ }
+
+ for k, r := range p.tagFamilies {
+ sr.tagFamilies[k] = generateSeqReader()
+ sr.tagFamilies[k].init(r)
+ sr.tagFamilyMetadata[k] = generateSeqReader()
+ sr.tagFamilyMetadata[k].init(p.tagFamilyMetadata[k])
+ }
+}
+
+type blockReader struct {
+ err error
+ block *blockPointer
+ pih partMergeIterHeap
+ nextBlockNoop bool
+}
+
+func (br *blockReader) reset() {
+ br.block = nil
+ for i := range br.pih {
+ br.pih[i] = nil
+ }
+ br.pih = br.pih[:0]
+ br.nextBlockNoop = false
+ br.err = nil
+}
+
+func (br *blockReader) init(pii []*partMergeIter) {
+ br.reset()
+ for _, pi := range pii {
+ if pi.nextBlockMetadata() {
+ br.pih = append(br.pih, pi)
+ continue
+ }
+ if err := pi.error(); err != nil {
+ br.err = fmt.Errorf("can't get the block to merge: %w", err)
+ return
+ }
+ }
+ if len(br.pih) == 0 {
+ br.err = io.EOF
+ return
+ }
+ heap.Init(&br.pih)
+ br.block = &br.pih[0].block
+ br.nextBlockNoop = true
+}
+
+func (br *blockReader) nextBlockMetadata() bool {
+ if br.err != nil {
+ return false
+ }
+ if br.nextBlockNoop {
+ br.nextBlockNoop = false
+ return true
+ }
+
+ br.err = br.nextMetadata()
+ if br.err != nil {
+ if errors.Is(br.err, io.EOF) {
+ return false
+ }
+ br.err = fmt.Errorf("can't get the block to merge: %w", br.err)
+ return false
+ }
+ return true
+}
+
+func (br *blockReader) nextMetadata() error {
+ head := br.pih[0]
+ if head.nextBlockMetadata() {
+ heap.Fix(&br.pih, 0)
+ br.block = &br.pih[0].block
+ return nil
+ }
+
+ if err := head.error(); err != nil {
+ br.block = nil
+ return err
+ }
+
+ heap.Pop(&br.pih)
+
+ if len(br.pih) == 0 {
+ br.block = nil
+ return io.EOF
+ }
+
+ br.block = &br.pih[0].block
+ return nil
+}
+
+func (br *blockReader) loadBlockData(decoder *encoding.BytesBlockDecoder) {
+ br.pih[0].mustLoadBlockData(decoder, br.block)
+}
+
+func (br *blockReader) error() error {
+ if errors.Is(br.err, io.EOF) {
+ return nil
+ }
+ return br.err
+}
+
+var blockReaderPool sync.Pool
+
+func generateBlockReader() *blockReader {
+ if v := blockReaderPool.Get(); v != nil {
+ return v.(*blockReader)
+ }
+ return &blockReader{}
}
-func (r *reader) Read(p []byte) (int, error) {
- n, err := r.r.Read(0, p)
- r.bytesRead += uint64(n)
- return n, err
+func releaseBlockReader(br *blockReader) {
+ br.reset()
+ blockReaderPool.Put(br)
}
diff --git a/banyand/stream/block_reader_test.go b/banyand/stream/block_reader_test.go
new file mode 100644
index 00000000..84078c70
--- /dev/null
+++ b/banyand/stream/block_reader_test.go
@@ -0,0 +1,165 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package stream
+
+import (
+ "errors"
+ "sort"
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+ "github.com/stretchr/testify/require"
+
+ "github.com/apache/skywalking-banyandb/pkg/fs"
+ "github.com/apache/skywalking-banyandb/pkg/test"
+)
+
+func Test_blockReader_nextBlock(t *testing.T) {
+ tests := []struct {
+ wantErr error
+ name string
+ dpsList []*elements
+ want []blockMetadata
+ }{
+ {
+ name: "Test with no data points",
+ dpsList: []*elements{},
+ },
+ {
+ name: "Test with single part",
+ dpsList: []*elements{esTS1},
+ want: []blockMetadata{
+ {seriesID: 1, count: 1, uncompressedSizeBytes: 881},
+ {seriesID: 2, count: 1, uncompressedSizeBytes: 55},
+ {seriesID: 3, count: 1, uncompressedSizeBytes: 8},
+ },
+ },
+ {
+ name: "Test with multiple parts with different ts",
+ dpsList: []*elements{esTS1, esTS2},
+ want: []blockMetadata{
+ {seriesID: 1, count: 1, uncompressedSizeBytes: 881},
+ {seriesID: 1, count: 1, uncompressedSizeBytes: 881},
+ {seriesID: 2, count: 1, uncompressedSizeBytes: 55},
+ {seriesID: 2, count: 1, uncompressedSizeBytes: 55},
+ {seriesID: 3, count: 1, uncompressedSizeBytes: 8},
+ {seriesID: 3, count: 1, uncompressedSizeBytes: 8},
+ },
+ },
+ {
+ name: "Test with multiple parts with same ts",
+ dpsList: []*elements{esTS1, esTS1},
+ want: []blockMetadata{
+ {seriesID: 1, count: 1, uncompressedSizeBytes: 881},
+ {seriesID: 1, count: 1, uncompressedSizeBytes: 881},
+ {seriesID: 2, count: 1, uncompressedSizeBytes: 55},
+ {seriesID: 2, count: 1, uncompressedSizeBytes: 55},
+ {seriesID: 3, count: 1, uncompressedSizeBytes: 8},
+ {seriesID: 3, count: 1, uncompressedSizeBytes: 8},
+ },
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ verify := func(pp []*part) {
+ var pii []*partMergeIter
+ for _, p := range pp {
+ pmi := &partMergeIter{}
+ pmi.mustInitFromPart(p)
+ pii = append(pii, pmi)
+ }
+ reader := generateBlockReader()
+ defer releaseBlockReader(reader)
+ reader.init(pii)
+ var got []blockMetadata
+ for reader.nextBlockMetadata() {
+ if reader.block.bm.seriesID == 0 {
+ t.Errorf("Expected curBlock to be initialized, but it was nil")
+ }
+ var tagFamilyNames []string
+ for _, cf := range reader.block.block.tagFamilies {
+ tagFamilyNames = append(tagFamilyNames, cf.name)
+ }
+ if !sort.IsSorted(sort.StringSlice(tagFamilyNames)) {
+ t.Errorf("Expected tagFamilyNames %q to be sorted, but it was not", tagFamilyNames)
+ }
+ got = append(got, reader.block.bm)
+ }
+
+ if !errors.Is(reader.error(), tt.wantErr) {
+ t.Errorf("Unexpected error: got %v, want %v", reader.err, tt.wantErr)
+ }
+
+ if diff := cmp.Diff(got, tt.want,
+ cmpopts.IgnoreFields(blockMetadata{}, "timestamps"),
+ cmpopts.IgnoreFields(blockMetadata{}, "elementIDs"),
+ cmpopts.IgnoreFields(blockMetadata{}, "tagFamilies"),
+ cmp.AllowUnexported(blockMetadata{}),
+ ); diff != "" {
+ t.Errorf("Unexpected blockMetadata (-got +want):\n%s", diff)
+ }
+ }
+
+ t.Run("memory parts", func(t *testing.T) {
+ var mpp []*memPart
+ defer func() {
+ for _, mp := range mpp {
+ releaseMemPart(mp)
+ }
+ }()
+ var pp []*part
+ for _, dps := range tt.dpsList {
+ mp := generateMemPart()
+ mpp = append(mpp, mp)
+ mp.mustInitFromElements(dps)
+ pp = append(pp, openMemPart(mp))
+ }
+ verify(pp)
+ })
+
+ t.Run("file parts", func(t *testing.T) {
+ var mpp []*memPart
+ var fpp []*partWrapper
+ tmpPath, defFn := test.Space(require.New(t))
+ defer func() {
+ for _, mp := range mpp {
+ releaseMemPart(mp)
+ }
+ for _, pw := range fpp {
+ pw.decRef()
+ }
+ defFn()
+ }()
+ var pp []*part
+ fileSystem := fs.NewLocalFileSystem()
+ for i, dps := range tt.dpsList {
+ mp := generateMemPart()
+ mpp = append(mpp, mp)
+ mp.mustInitFromElements(dps)
+ mp.mustFlush(fileSystem, partPath(tmpPath, uint64(i)))
+ filePW := newPartWrapper(nil, mustOpenFilePart(uint64(i), tmpPath, fileSystem))
+ filePW.p.partMetadata.ID = uint64(i)
+ fpp = append(fpp, filePW)
+ pp = append(pp, filePW.p)
+ }
+ verify(pp)
+ })
+ })
+ }
+}
diff --git a/banyand/stream/block_test.go b/banyand/stream/block_test.go
index bcbbdb51..290e083e 100644
--- a/banyand/stream/block_test.go
+++ b/banyand/stream/block_test.go
@@ -36,6 +36,7 @@ import (
func Test_block_reset(t *testing.T) {
type fields struct {
timestamps []int64
+ elementIDs []string
tagFamilies []tagFamily
}
tests := []struct {
@@ -47,10 +48,12 @@ func Test_block_reset(t *testing.T) {
name: "Test reset",
fields: fields{
timestamps: []int64{1, 2, 3},
+ elementIDs: []string{"0", "1", "2"},
tagFamilies: []tagFamily{{}, {}, {}},
},
want: block{
timestamps: []int64{},
+ elementIDs: []string{},
tagFamilies: []tagFamily{},
},
},
@@ -59,6 +62,7 @@ func Test_block_reset(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
b := &block{
timestamps: tt.fields.timestamps,
+ elementIDs: tt.fields.elementIDs,
tagFamilies: tt.fields.tagFamilies,
}
b.reset()
@@ -236,7 +240,7 @@ func Test_mustWriteAndReadTimestamps(t *testing.T) {
}()
tm := ×tampsMetadata{}
b := &bytes.Buffer{}
- mustWriteTimestampsTo(tm, tt.args, writer{w: b})
+ mustWriteTimestampsTo(tm, tt.args, &writer{w: b})
timestamps := mustReadTimestampsFrom(nil, tm, len(tt.args), b)
if !reflect.DeepEqual(timestamps, tt.args) {
t.Errorf("mustReadTimestampsFrom() = %v, want %v", timestamps, tt.args)
@@ -245,6 +249,43 @@ func Test_mustWriteAndReadTimestamps(t *testing.T) {
}
}
+func Test_mustWriteAndReadElementIDs(t *testing.T) {
+ tests := []struct {
+ name string
+ args []string
+ wantPanic bool
+ wantTM elementIDsMetadata
+ }{
+ {
+ name: "Test mustWriteAndReadElementIDs",
+ args: []string{"0", "1", "2", "3", "4"},
+ wantPanic: false,
+ },
+ // {
+ // name: "Test mustWriteAndReadTimestamps with panic",
+ // args: getBitInt64Arr(),
+ // wantPanic: true,
+ // },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ defer func() {
+ r := recover()
+ if (r != nil) != tt.wantPanic {
+ t.Errorf("mustWriteElementIDs() recover = %v, wantPanic = %v", r, tt.wantPanic)
+ }
+ }()
+ em := &elementIDsMetadata{}
+ b := &bytes.Buffer{}
+ mustWriteElementIDsTo(em, tt.args, &writer{w: b})
+ elementIDs := mustReadElementIDsFrom(nil, em, len(tt.args), b)
+ if !reflect.DeepEqual(elementIDs, tt.args) {
+ t.Errorf("mustReadElementIDsFrom() = %v, want %v", elementIDs, tt.args)
+ }
+ })
+ }
+}
+
func getBitInt64Arr() []int64 {
size := maxTimestampsBlockSize + 1
randSlice := make([]int64, size)
@@ -297,6 +338,26 @@ func Test_marshalAndUnmarshalTagFamily(t *testing.T) {
); diff != "" {
t.Errorf("block.unmarshalTagFamily() (-got +want):\n%s", diff)
}
+
+ unmarshaled2 := generateBlock()
+ defer releaseBlock(unmarshaled2)
+ unmarshaled2.timestamps = make([]int64, len(b.timestamps))
+ unmarshaled2.resizeTagFamilies(1)
+
+ metaReader := generateSeqReader()
+ defer releaseSeqReader(metaReader)
+ metaReader.init(metaBuffer)
+ valueReader := generateSeqReader()
+ defer releaseSeqReader(valueReader)
+ valueReader.init(dataBuffer)
+
+ unmarshaled2.unmarshalTagFamilyFromSeqReaders(decoder, tfIndex, name, bm.getTagFamilyMetadata(name), metaReader, valueReader)
+
+ if diff := cmp.Diff(unmarshaled2.tagFamilies[0], b.tagFamilies[0],
+ cmp.AllowUnexported(tagFamily{}, tag{}),
+ ); diff != "" {
+ t.Errorf("block.unmarshalTagFamilyFromSeqReaders() (-got +want):\n%s", diff)
+ }
}
func Test_marshalAndUnmarshalBlock(t *testing.T) {
@@ -311,6 +372,7 @@ func Test_marshalAndUnmarshalBlock(t *testing.T) {
elementIDsWriter: writer{w: elementIDsBuffer},
}
p := &part{
+ primary: &bytes.Buffer{},
timestamps: timestampBuffer,
elementIDs: elementIDsBuffer,
}
@@ -350,6 +412,17 @@ func Test_marshalAndUnmarshalBlock(t *testing.T) {
if !reflect.DeepEqual(b, unmarshaled) {
t.Errorf("block.mustReadFrom() = %+v, want %+v", unmarshaled, b)
}
+
+ unmarshaled2 := generateBlock()
+ defer releaseBlock(unmarshaled2)
+ var sr seqReaders
+ sr.init(p)
+ defer sr.reset()
+
+ unmarshaled2.mustSeqReadFrom(decoder, &sr, bm)
+ if !reflect.DeepEqual(b, unmarshaled2) {
+ t.Errorf("block.mustSeqReadFrom() = %+v, want %+v", unmarshaled, b)
+ }
}
func Test_findRange(t *testing.T) {
@@ -357,6 +430,8 @@ func Test_findRange(t *testing.T) {
timestamps []int64
min int64
max int64
+ includeMin bool
+ includeMax bool
}
tests := []struct {
name string
@@ -372,8 +447,8 @@ func Test_findRange(t *testing.T) {
min: 1,
max: 10,
},
- wantStart: 0,
- wantEnd: 0,
+ wantStart: -1,
+ wantEnd: -1,
wantExist: false,
},
{
@@ -382,9 +457,11 @@ func Test_findRange(t *testing.T) {
timestamps: []int64{1},
min: 1,
max: 1,
+ includeMin: true,
+ includeMax: true,
},
wantStart: 0,
- wantEnd: 1,
+ wantEnd: 0,
wantExist: true,
},
{
@@ -394,8 +471,8 @@ func Test_findRange(t *testing.T) {
min: 6,
max: 10,
},
- wantStart: 0,
- wantEnd: 0,
+ wantStart: -1,
+ wantEnd: -1,
wantExist: false,
},
{
@@ -404,9 +481,11 @@ func Test_findRange(t *testing.T) {
timestamps: []int64{1, 2, 3, 4, 5},
min: 2,
max: 4,
+ includeMin: true,
+ includeMax: true,
},
wantStart: 1,
- wantEnd: 4,
+ wantEnd: 3,
wantExist: true,
},
{
@@ -415,15 +494,17 @@ func Test_findRange(t *testing.T) {
timestamps: []int64{1, 2, 3, 4, 5},
min: 1,
max: 5,
+ includeMin: true,
+ includeMax: true,
},
wantStart: 0,
- wantEnd: 5,
+ wantEnd: 4,
wantExist: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- gotStart, gotEnd, gotExist := findRange(tt.args.timestamps, tt.args.min, tt.args.max)
+ gotStart, gotEnd, gotExist := findRange(tt.args.timestamps, tt.args.min, tt.args.max, tt.args.includeMin, tt.args.includeMax)
if gotStart != tt.wantStart {
t.Errorf("findRange() gotStart = %v, want %v", gotStart, tt.wantStart)
}
@@ -436,3 +517,319 @@ func Test_findRange(t *testing.T) {
})
}
}
+
+func Test_blockPointer_append(t *testing.T) {
+ type fields struct {
+ timestamps []int64
+ elementIDs []string
+ tagFamilies []tagFamily
+ partID uint64
+ }
+ type args struct {
+ b *blockPointer
+ offset int
+ }
+ tests := []struct {
+ want *blockPointer
+ name string
+ args args
+ fields fields
+ wantPanic bool
+ }{
+ {
+ name: "Test append with empty block",
+ fields: fields{
+ timestamps: []int64{1, 2},
+ elementIDs: []string{"0", "1"},
+ tagFamilies: []tagFamily{
+ {
+ name: "arrTag",
+ tags: []tag{
+ {
+ name: "strArrTag", valueType: pbv1.ValueTypeStrArr,
+ values: [][]byte{marshalStrArr([][]byte{[]byte("value5"), []byte("value6")}), marshalStrArr([][]byte{[]byte("value7"), []byte("value8")})},
+ },
+ },
+ },
+ },
+ },
+ args: args{
+ b: &blockPointer{
+ block: block{
+ timestamps: []int64{},
+ elementIDs: []string{},
+ tagFamilies: []tagFamily{},
+ },
+ idx: 0,
+ },
+ offset: 0,
+ },
+ want: &blockPointer{
+ block: block{
+ timestamps: []int64{1, 2},
+ elementIDs: []string{"0", "1"},
+ tagFamilies: []tagFamily{
+ {
+ name: "arrTag",
+ tags: []tag{
+ {
+ name: "strArrTag", valueType: pbv1.ValueTypeStrArr,
+ values: [][]byte{marshalStrArr([][]byte{[]byte("value5"), []byte("value6")}), marshalStrArr([][]byte{[]byte("value7"), []byte("value8")})},
+ },
+ },
+ },
+ },
+ },
+ idx: 0,
+ },
+ },
+ {
+ name: "Test append to a empty block",
+ fields: fields{
+ timestamps: nil,
+ elementIDs: nil,
+ tagFamilies: nil,
+ partID: 0,
+ },
+ args: args{
+ b: &blockPointer{
+ lastPartID: 2,
+ block: block{
+ timestamps: []int64{4, 5},
+ elementIDs: []string{"3", "4"},
+ tagFamilies: []tagFamily{
+ {
+ name: "arrTag",
+ tags: []tag{
+ {
+ name: "strArrTag", valueType: pbv1.ValueTypeStrArr,
+ values: [][]byte{marshalStrArr([][]byte{[]byte("value9"), []byte("value10")}), marshalStrArr([][]byte{[]byte("value11"), []byte("value12")})},
+ },
+ },
+ },
+ },
+ },
+ idx: 0,
+ },
+ offset: 2,
+ },
+ want: &blockPointer{
+ lastPartID: 2,
+ block: block{
+ timestamps: []int64{4, 5},
+ elementIDs: []string{"3", "4"},
+ tagFamilies: []tagFamily{
+ {
+ name: "arrTag",
+ tags: []tag{
+ {
+ name: "strArrTag", valueType: pbv1.ValueTypeStrArr,
+ values: [][]byte{marshalStrArr([][]byte{[]byte("value9"), []byte("value10")}), marshalStrArr([][]byte{[]byte("value11"), []byte("value12")})},
+ },
+ },
+ },
+ },
+ },
+ idx: 0,
+ },
+ },
+ {
+ name: "Test append with offset equals to the data size. All data",
+ fields: fields{
+ timestamps: []int64{1, 2},
+ elementIDs: []string{"0", "1"},
+ tagFamilies: []tagFamily{
+ {
+ name: "arrTag",
+ tags: []tag{
+ {
+ name: "strArrTag", valueType: pbv1.ValueTypeStrArr,
+ values: [][]byte{marshalStrArr([][]byte{[]byte("value5"), []byte("value6")}), marshalStrArr([][]byte{[]byte("value7"), []byte("value8")})},
+ },
+ },
+ },
+ },
+ partID: 1,
+ },
+ args: args{
+ b: &blockPointer{
+ lastPartID: 2,
+ block: block{
+ timestamps: []int64{4, 5},
+ elementIDs: []string{"3", "4"},
+ tagFamilies: []tagFamily{
+ {
+ name: "arrTag",
+ tags: []tag{
+ {
+ name: "strArrTag", valueType: pbv1.ValueTypeStrArr,
+ values: [][]byte{marshalStrArr([][]byte{[]byte("value9"), []byte("value10")}), marshalStrArr([][]byte{[]byte("value11"), []byte("value12")})},
+ },
+ },
+ },
+ },
+ },
+ idx: 0,
+ },
+ offset: 2,
+ },
+ want: &blockPointer{
+ lastPartID: 2,
+ block: block{
+ timestamps: []int64{1, 2, 4, 5},
+ elementIDs: []string{"0", "1", "3", "4"},
+ tagFamilies: []tagFamily{
+ {
+ name: "arrTag",
+ tags: []tag{
+ {
+ name: "strArrTag", valueType: pbv1.ValueTypeStrArr,
+ values: [][]byte{
+ marshalStrArr([][]byte{[]byte("value5"), []byte("value6")}),
+ marshalStrArr([][]byte{[]byte("value7"), []byte("value8")}), marshalStrArr([][]byte{[]byte("value9"), []byte("value10")}),
+ marshalStrArr([][]byte{[]byte("value11"), []byte("value12")}),
+ },
+ },
+ },
+ },
+ },
+ },
+ idx: 0,
+ },
+ },
+ {
+ name: "Test append with non-empty block and offset less than timestamps",
+ fields: fields{
+ timestamps: []int64{1, 2},
+ elementIDs: []string{"0", "1"},
+ tagFamilies: []tagFamily{
+ {
+ name: "arrTag",
+ tags: []tag{
+ {
+ name: "strArrTag", valueType: pbv1.ValueTypeStrArr,
+ values: [][]byte{marshalStrArr([][]byte{[]byte("value5"), []byte("value6")}), marshalStrArr([][]byte{[]byte("value7"), []byte("value8")})},
+ },
+ },
+ },
+ },
+ },
+ args: args{
+ b: &blockPointer{
+ lastPartID: 2,
+ block: block{
+ timestamps: []int64{4, 5},
+ elementIDs: []string{"3", "4"},
+ tagFamilies: []tagFamily{
+ {
+ name: "arrTag",
+ tags: []tag{
+ {
+ name: "strArrTag", valueType: pbv1.ValueTypeStrArr,
+ values: [][]byte{marshalStrArr([][]byte{[]byte("value9"), []byte("value10")}), marshalStrArr([][]byte{[]byte("value11"), []byte("value12")})},
+ },
+ },
+ },
+ },
+ },
+ idx: 0,
+ },
+ offset: 1,
+ },
+ want: &blockPointer{
+ lastPartID: 2,
+ block: block{
+ timestamps: []int64{1, 2, 4},
+ elementIDs: []string{"0", "1", "3"},
+ tagFamilies: []tagFamily{
+ {
+ name: "arrTag",
+ tags: []tag{
+ {
+ name: "strArrTag", valueType: pbv1.ValueTypeStrArr,
+ values: [][]byte{marshalStrArr([][]byte{[]byte("value5"), []byte("value6")}), marshalStrArr([][]byte{
+ []byte("value7"),
+ []byte("value8"),
+ }), marshalStrArr([][]byte{[]byte("value9"), []byte("value10")})},
+ },
+ },
+ },
+ },
+ },
+ idx: 0,
+ },
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ defer func() {
+ r := recover()
+ if (r != nil) != tt.wantPanic {
+ t.Errorf("blockPointer.append() recover = %v, wantPanic = %v", r, tt.wantPanic)
+ }
+ }()
+ bi := &blockPointer{
+ block: block{
+ timestamps: tt.fields.timestamps,
+ tagFamilies: tt.fields.tagFamilies,
+ elementIDs: tt.fields.elementIDs,
+ },
+ lastPartID: tt.fields.partID,
+ }
+ bi.append(tt.args.b, tt.args.offset)
+ if !reflect.DeepEqual(bi, tt.want) {
+ t.Errorf("blockPointer.append() = %+v, want %+v", bi, tt.want)
+ }
+ })
+ }
+}
+
+func Test_blockPointer_copyFrom(t *testing.T) {
+ type fields struct {
+ bm blockMetadata
+ idx int
+ }
+ type args struct {
+ src *blockPointer
+ }
+ tests := []struct {
+ args args
+ want *blockPointer
+ name string
+ fields fields
+ }{
+ {
+ name: "Test copyFrom",
+ fields: fields{
+ bm: blockMetadata{},
+ idx: 0,
+ },
+ args: args{
+ src: &blockPointer{
+ bm: blockMetadata{count: 1},
+ idx: 0,
+ block: conventionalBlock,
+ lastPartID: 2,
+ },
+ },
+ want: &blockPointer{
+ bm: blockMetadata{count: 1},
+ idx: 0,
+ block: conventionalBlock,
+ lastPartID: 2,
+ },
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ bi := &blockPointer{
+ bm: tt.fields.bm,
+ idx: tt.fields.idx,
+ }
+ bi.copyFrom(tt.args.src)
+ if !reflect.DeepEqual(bi, tt.want) {
+ t.Errorf("blockPointer.copyFrom() = %+v, want %+v", bi, tt.want)
+ }
+ })
+ }
+}
diff --git a/banyand/stream/block_writer.go b/banyand/stream/block_writer.go
index 48644a1c..a653397e 100644
--- a/banyand/stream/block_writer.go
+++ b/banyand/stream/block_writer.go
@@ -18,6 +18,7 @@
package stream
import (
+ "path/filepath"
"sync"
"github.com/apache/skywalking-banyandb/api/common"
@@ -170,6 +171,19 @@ func (bw *blockWriter) MustInitForMemPart(mp *memPart) {
bw.writers.elementIDsWriter.init(&mp.elementIDs)
}
+func (bw *blockWriter) mustInitForFilePart(fileSystem fs.FileSystem, path string) {
+ bw.reset()
+ fileSystem.MkdirPanicIfExist(path, dirPermission)
+ bw.writers.mustCreateTagFamilyWriters = func(name string) (fs.Writer, fs.Writer) {
+ return fs.MustCreateFile(fileSystem, filepath.Join(path, name+tagFamiliesMetadataFilenameExt), filePermission),
+ fs.MustCreateFile(fileSystem, filepath.Join(path, name+tagFamiliesFilenameExt), filePermission)
+ }
+ bw.writers.metaWriter.init(fs.MustCreateFile(fileSystem, filepath.Join(path, metaFilename), filePermission))
+ bw.writers.primaryWriter.init(fs.MustCreateFile(fileSystem, filepath.Join(path, primaryFilename), filePermission))
+ bw.writers.timestampsWriter.init(fs.MustCreateFile(fileSystem, filepath.Join(path, timestampsFilename), filePermission))
+ bw.writers.elementIDsWriter.init(fs.MustCreateFile(fileSystem, filepath.Join(path, elementIDsFilename), filePermission))
+}
+
func (bw *blockWriter) MustWriteElements(sid common.SeriesID, timestamps []int64, elementIDs []string, tagFamilies [][]tagValues) {
if len(timestamps) == 0 {
return
@@ -178,6 +192,10 @@ func (bw *blockWriter) MustWriteElements(sid common.SeriesID, timestamps []int64
b := generateBlock()
defer releaseBlock(b)
b.mustInitFromElements(timestamps, elementIDs, tagFamilies)
+ bw.mustWriteBlock(sid, b)
+}
+
+func (bw *blockWriter) mustWriteBlock(sid common.SeriesID, b *block) {
if b.Len() == 0 {
return
}
@@ -192,9 +210,9 @@ func (bw *blockWriter) MustWriteElements(sid common.SeriesID, timestamps []int64
isSeenSid := sid == bw.sidLast
bw.sidLast = sid
- bh := generateBlockMetadata()
- b.mustWriteTo(sid, bh, &bw.writers)
- th := &bh.timestamps
+ bm := generateBlockMetadata()
+ b.mustWriteTo(sid, bm, &bw.writers)
+ th := &bm.timestamps
if bw.totalCount == 0 || th.min < bw.totalMinTimestamp {
bw.totalMinTimestamp = th.min
}
@@ -212,12 +230,12 @@ func (bw *blockWriter) MustWriteElements(sid common.SeriesID, timestamps []int64
}
bw.minTimestampLast = th.min
- bw.totalUncompressedSizeBytes += bh.uncompressedSizeBytes
- bw.totalCount += bh.count
+ bw.totalUncompressedSizeBytes += bm.uncompressedSizeBytes
+ bw.totalCount += bm.count
bw.totalBlocksCount++
- bw.primaryBlockData = bh.marshal(bw.primaryBlockData)
- releaseBlockMetadata(bh)
+ bw.primaryBlockData = bm.marshal(bw.primaryBlockData)
+ releaseBlockMetadata(bm)
if len(bw.primaryBlockData) > maxUncompressedPrimaryBlockSize {
bw.mustFlushPrimaryBlock(bw.primaryBlockData)
bw.primaryBlockData = bw.primaryBlockData[:0]
@@ -235,12 +253,12 @@ func (bw *blockWriter) mustFlushPrimaryBlock(data []byte) {
bw.sidFirst = 0
}
-func (bw *blockWriter) Flush(ph *partMetadata) {
- ph.UncompressedSizeBytes = bw.totalUncompressedSizeBytes
- ph.TotalCount = bw.totalCount
- ph.BlocksCount = bw.totalBlocksCount
- ph.MinTimestamp = bw.totalMinTimestamp
- ph.MaxTimestamp = bw.totalMaxTimestamp
+func (bw *blockWriter) Flush(pm *partMetadata) {
+ pm.UncompressedSizeBytes = bw.totalUncompressedSizeBytes
+ pm.TotalCount = bw.totalCount
+ pm.BlocksCount = bw.totalBlocksCount
+ pm.MinTimestamp = bw.totalMinTimestamp
+ pm.MaxTimestamp = bw.totalMaxTimestamp
bw.mustFlushPrimaryBlock(bw.primaryBlockData)
@@ -249,7 +267,7 @@ func (bw *blockWriter) Flush(ph *partMetadata) {
bw.writers.metaWriter.MustWrite(bb.Buf)
bigValuePool.Release(bb)
- ph.CompressedSizeBytes = bw.writers.totalBytesWritten()
+ pm.CompressedSizeBytes = bw.writers.totalBytesWritten()
bw.writers.MustClose()
bw.reset()
diff --git a/banyand/stream/elements.go b/banyand/stream/elements.go
index 9b78290e..be6ebe33 100644
--- a/banyand/stream/elements.go
+++ b/banyand/stream/elements.go
@@ -135,6 +135,7 @@ func (e *elements) Less(i, j int) bool {
func (e *elements) Swap(i, j int) {
e.seriesIDs[i], e.seriesIDs[j] = e.seriesIDs[j], e.seriesIDs[i]
e.timestamps[i], e.timestamps[j] = e.timestamps[j], e.timestamps[i]
+ e.elementIDs[i], e.elementIDs[j] = e.elementIDs[j], e.elementIDs[i]
e.tagFamilies[i], e.tagFamilies[j] = e.tagFamilies[j], e.tagFamilies[i]
}
@@ -147,7 +148,7 @@ type elementsInTable struct {
}
type elementsInGroup struct {
- tsdb storage.TSDB[*tsTable]
+ tsdb storage.TSDB[*tsTable, option]
docs index.Documents
tables []*elementsInTable
diff --git a/banyand/stream/flusher.go b/banyand/stream/flusher.go
index 9562f838..c388ee0e 100644
--- a/banyand/stream/flusher.go
+++ b/banyand/stream/flusher.go
@@ -18,22 +18,34 @@
package stream
import (
+ "errors"
+ "math"
+ "time"
+
"github.com/apache/skywalking-banyandb/pkg/watcher"
)
-func (tst *tsTable) flusherLoop(flushCh chan *flusherIntroduction, introducerWatcher watcher.Channel, epoch uint64) {
+func (tst *tsTable) flusherLoop(flushCh chan *flusherIntroduction, mergeCh chan *mergerIntroduction, introducerWatcher, flusherWatcher watcher.Channel, epoch uint64) {
defer tst.loopCloser.Done()
- epochWatcher := introducerWatcher.Add(0, tst.loopCloser.CloseNotify())
+ epochWatcher := introducerWatcher.Add(epoch, tst.loopCloser.CloseNotify())
if epochWatcher == nil {
return
}
+ var flusherWatchers watcher.Epochs
for {
select {
case <-tst.loopCloser.CloseNotify():
return
+ case e := <-flusherWatcher:
+ flusherWatchers.Add(e)
case <-epochWatcher.Watch():
- var curSnapshot *snapshot
+ curSnapshot := tst.currentSnapshot()
+ if curSnapshot != nil {
+ flusherWatchers = tst.pauseFlusherToPileupMemParts(epoch, flusherWatcher, flusherWatchers)
+ curSnapshot.decRef()
+ curSnapshot = nil
+ }
tst.RLock()
if tst.snapshot != nil && tst.snapshot.epoch > epoch {
curSnapshot = tst.snapshot
@@ -41,11 +53,20 @@ func (tst *tsTable) flusherLoop(flushCh chan *flusherIntroduction, introducerWat
}
tst.RUnlock()
if curSnapshot != nil {
- epoch = tst.flush(curSnapshot, flushCh)
- if epoch == 0 {
- return
+ merged, err := tst.mergeMemParts(curSnapshot, mergeCh)
+ if err != nil {
+ tst.l.Logger.Warn().Err(err).Msgf("cannot merge snapshot: %d", curSnapshot.epoch)
+ curSnapshot.decRef()
+ continue
+ }
+ if !merged {
+ tst.flush(curSnapshot, flushCh)
}
- tst.persistSnapshot(curSnapshot)
+ epoch = curSnapshot.epoch
+ // Notify merger to start a new round of merge.
+ // This round might have be triggered in pauseFlusherToPileupMemParts.
+ flusherWatchers.Notify(math.MaxUint64)
+ flusherWatchers = nil
curSnapshot.decRef()
if tst.currentEpoch() != epoch {
continue
@@ -55,12 +76,60 @@ func (tst *tsTable) flusherLoop(flushCh chan *flusherIntroduction, introducerWat
if epochWatcher == nil {
return
}
- tst.gc.clean()
}
}
}
-func (tst *tsTable) flush(snapshot *snapshot, flushCh chan *flusherIntroduction) uint64 {
+// pauseFlusherToPileupMemParts takes a pause to wait for in-memory parts to pile up.
+// If there is no in-memory part, we can skip the pause.
+// When a merging is finished, we can skip the pause.
+func (tst *tsTable) pauseFlusherToPileupMemParts(epoch uint64, flushWatcher watcher.Channel, flusherWatchers watcher.Epochs) watcher.Epochs {
+ curSnapshot := tst.currentSnapshot()
+ if curSnapshot == nil {
+ return flusherWatchers
+ }
+ curSnapshot.decRef()
+ flusherWatchers.Notify(epoch)
+ select {
+ case <-tst.loopCloser.CloseNotify():
+ case <-time.After(tst.option.flushTimeout):
+ case e := <-flushWatcher:
+ flusherWatchers.Add(e)
+ flusherWatchers.Notify(epoch)
+ }
+ return flusherWatchers
+}
+
+func (tst *tsTable) mergeMemParts(snp *snapshot, mergeCh chan *mergerIntroduction) (bool, error) {
+ var memParts []*partWrapper
+ mergedIDs := make(map[uint64]struct{})
+ for i := range snp.parts {
+ if snp.parts[i].mp != nil {
+ memParts = append(memParts, snp.parts[i])
+ mergedIDs[snp.parts[i].ID()] = struct{}{}
+ continue
+ }
+ }
+ if len(memParts) < 2 {
+ return false, nil
+ }
+ // merge memory must not be closed by the tsTable.close
+ closeCh := make(chan struct{})
+ newPart, err := tst.mergePartsThenSendIntroduction(snapshotCreatorMergedFlusher, memParts, mergedIDs, mergeCh, closeCh)
+ close(closeCh)
+ if err != nil {
+ if errors.Is(err, errClosed) {
+ return true, nil
+ }
+ return false, err
+ }
+ if newPart == nil {
+ return false, nil
+ }
+ return true, nil
+}
+
+func (tst *tsTable) flush(snapshot *snapshot, flushCh chan *flusherIntroduction) {
ind := generateFlusherIntroduction()
defer releaseFlusherIntroduction(ind)
for _, pw := range snapshot.parts {
@@ -69,24 +138,22 @@ func (tst *tsTable) flush(snapshot *snapshot, flushCh chan *flusherIntroduction)
}
partPath := partPath(tst.root, pw.ID())
pw.mp.mustFlush(tst.fileSystem, partPath)
- newPW := newPartWrapper(nil, mustOpenFilePart(partPath, tst.fileSystem), tst.fileSystem)
+ newPW := newPartWrapper(nil, mustOpenFilePart(pw.ID(), tst.root, tst.fileSystem))
newPW.p.partMetadata.ID = pw.ID()
ind.flushed[newPW.ID()] = newPW
}
if len(ind.flushed) < 1 {
- return snapshot.epoch
+ return
}
ind.applied = make(chan struct{})
select {
case flushCh <- ind:
case <-tst.loopCloser.CloseNotify():
- return 0
+ return
}
select {
case <-ind.applied:
- return snapshot.epoch
case <-tst.loopCloser.CloseNotify():
- return 0
}
}
@@ -96,5 +163,5 @@ func (tst *tsTable) persistSnapshot(snapshot *snapshot) {
partNames = append(partNames, partName(snapshot.parts[i].ID()))
}
tst.mustWriteSnapshot(snapshot.epoch, partNames)
- tst.gc.registerSnapshot(snapshot.epoch)
+ tst.gc.registerSnapshot(snapshot)
}
diff --git a/banyand/stream/gc.go b/banyand/stream/gc.go
index 941348f7..75476fc2 100644
--- a/banyand/stream/gc.go
+++ b/banyand/stream/gc.go
@@ -18,69 +18,70 @@
package stream
import (
- "errors"
- "path"
- "sort"
-
- "github.com/apache/skywalking-banyandb/pkg/fs"
+ "path/filepath"
)
type garbageCleaner struct {
- parent *tsTable
- snapshots []uint64
- parts []uint64
-}
-
-func (g *garbageCleaner) registerSnapshot(snapshot uint64) {
- g.snapshots = append(g.snapshots, snapshot)
+ parent *tsTable
+ liveParts map[uint64]map[uint64]struct{}
+ knownPartFiles map[uint64]struct{}
+ deletableEpochs []uint64
+ liveEpoch uint64
}
-func (g *garbageCleaner) submitParts(parts ...uint64) {
- g.parts = append(g.parts, parts...)
+func (g *garbageCleaner) init(parent *tsTable) {
+ g.parent = parent
+ g.liveParts = make(map[uint64]map[uint64]struct{})
+ g.knownPartFiles = make(map[uint64]struct{})
}
-func (g garbageCleaner) clean() {
- if len(g.snapshots) > 1 {
- g.cleanSnapshots()
+func (g *garbageCleaner) registerSnapshot(snapshot *snapshot) {
+ parts := make(map[uint64]struct{})
+ for _, part := range snapshot.parts {
+ parts[part.ID()] = struct{}{}
+ g.knownPartFiles[part.ID()] = struct{}{}
}
- if len(g.parts) > 0 {
- g.cleanParts()
+ g.liveParts[snapshot.epoch] = parts
+
+ if g.liveEpoch > 0 {
+ g.deletableEpochs = append(g.deletableEpochs, g.liveEpoch)
}
+ g.liveEpoch = snapshot.epoch
+}
+
+func (g *garbageCleaner) submitParts(partID uint64) {
+ g.knownPartFiles[partID] = struct{}{}
+}
+
+func (g *garbageCleaner) clean() {
+ g.cleanSnapshots()
+ g.cleanParts()
}
func (g *garbageCleaner) cleanSnapshots() {
- if len(g.snapshots) < 2 {
- return
- }
- if !sort.SliceIsSorted(g.snapshots, func(i, j int) bool {
- return g.snapshots[i] < g.snapshots[j]
- }) {
- sort.Slice(g.snapshots, func(i, j int) bool {
- return g.snapshots[i] < g.snapshots[j]
- })
- }
- // keep the latest snapshot
- var remainingSnapshots []uint64
- for i := 0; i < len(g.snapshots)-1; i++ {
- filePath := path.Join(g.parent.root, snapshotName(g.snapshots[i]))
- if err := g.parent.fileSystem.DeleteFile(filePath); err != nil {
- var notExistErr *fs.FileSystemError
- if errors.As(err, ¬ExistErr) && notExistErr.Code != fs.IsNotExistError {
- g.parent.l.Warn().Err(err).Str("path", filePath).Msg("failed to delete snapshot, will retry in next round. Please check manually")
- remainingSnapshots = append(remainingSnapshots, g.snapshots[i])
- }
+ var remainingEpochs []uint64
+ for _, deletableEpoch := range g.deletableEpochs {
+ path := filepath.Join(g.parent.root, snapshotName(deletableEpoch))
+ err := g.parent.fileSystem.DeleteFile(path)
+ if err == nil {
+ delete(g.liveParts, deletableEpoch)
+ continue
}
+ g.parent.l.Warn().Err(err).Msgf("cannot delete snapshot file: %s", path)
+ remainingEpochs = append(remainingEpochs, deletableEpoch)
}
- if remainingSnapshots == nil {
- g.snapshots = g.snapshots[len(g.snapshots)-1:]
- return
- }
- remained := g.snapshots[len(g.snapshots)-1]
- g.snapshots = g.snapshots[:0]
- g.snapshots = append(g.snapshots, remainingSnapshots...)
- g.snapshots = append(g.snapshots, remained)
+ g.deletableEpochs = remainingEpochs
}
func (g garbageCleaner) cleanParts() {
- panic("implement me")
+OUTER:
+ for partID := range g.knownPartFiles {
+ for _, partInSnapshot := range g.liveParts {
+ if _, ok := partInSnapshot[partID]; ok {
+ continue OUTER
+ }
+ }
+ g.parent.fileSystem.MustRMAll(partPath(g.parent.root, partID))
+ delete(g.knownPartFiles, partID)
+ }
}
diff --git a/banyand/stream/gc_test.go b/banyand/stream/gc_test.go
index f73c2e9e..83e8ca18 100644
--- a/banyand/stream/gc_test.go
+++ b/banyand/stream/gc_test.go
@@ -18,88 +18,54 @@
package stream
import (
- "path/filepath"
- "sort"
+ "fmt"
+ "reflect"
"testing"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-
"github.com/apache/skywalking-banyandb/pkg/fs"
- "github.com/apache/skywalking-banyandb/pkg/logger"
- "github.com/apache/skywalking-banyandb/pkg/test"
)
-func TestGarbageCleanerCleanSnapshot(t *testing.T) {
+func TestDeletableEpochs(t *testing.T) {
tests := []struct {
name string
- snapshotsOnDisk []uint64
- snapshotsInGC []uint64
- wantOnDisk []uint64
- wantInGC []uint64
+ knownEpochs []uint64
+ deletableEpochs []uint64
+ n int
}{
{
- name: "Test with no snapshots on disk and no snapshots in GC",
- },
- {
- name: "Test with some snapshots on disk and no snapshots in GC",
- snapshotsInGC: nil,
- snapshotsOnDisk: []uint64{1, 2, 3},
- wantInGC: nil,
- wantOnDisk: []uint64{1, 2, 3},
- },
- {
- name: "Test with no snapshots on disk and some snapshots in GC",
- snapshotsOnDisk: nil,
- snapshotsInGC: []uint64{1, 2, 3},
- wantOnDisk: nil,
- // gc won't fix the miss match between inmemory and disk
- wantInGC: []uint64{3},
+ name: "empty",
+ n: 1,
+ knownEpochs: nil,
+ deletableEpochs: nil,
},
{
- name: "Test with some snapshots on disk and some snapshots in GC",
- snapshotsOnDisk: []uint64{1, 2, 3, 4, 5},
- snapshotsInGC: []uint64{1, 3, 5},
- wantOnDisk: []uint64{2, 4, 5},
- wantInGC: []uint64{5},
+ name: "one",
+ n: 1,
+ knownEpochs: []uint64{1},
+ deletableEpochs: nil,
},
{
- name: "Test with unsorted",
- snapshotsOnDisk: []uint64{1, 3, 2, 5, 4},
- snapshotsInGC: []uint64{5, 1, 3},
- wantOnDisk: []uint64{2, 4, 5},
- wantInGC: []uint64{5},
+ name: "many",
+ n: 1,
+ knownEpochs: []uint64{1, 2, 3, 4},
+ deletableEpochs: []uint64{1, 2, 3},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- root, defFn := test.Space(require.New(t))
- defer defFn()
- parent := &tsTable{
- root: root,
- l: logger.GetLogger("test"),
+
+ for _, test := range tests {
+ test := test
+ t.Run(fmt.Sprintf("%s-%d", test.name, test.n), func(t *testing.T) {
+ var gc garbageCleaner
+ gc.init(&tsTable{
fileSystem: fs.NewLocalFileSystem(),
+ })
+
+ for _, epoch := range test.knownEpochs {
+ gc.registerSnapshot(&snapshot{epoch: epoch})
}
- for i := range tt.snapshotsOnDisk {
- filePath := filepath.Join(parent.root, snapshotName(tt.snapshotsOnDisk[i]))
- file, err := parent.fileSystem.CreateFile(filePath, filePermission)
- require.NoError(t, err)
- require.NoError(t, file.Close())
- }
- g := &garbageCleaner{
- parent: parent,
- snapshots: tt.snapshotsInGC,
- }
- g.cleanSnapshots()
- var got []uint64
- for _, de := range parent.fileSystem.ReadDir(parent.root) {
- s, err := parseSnapshot(de.Name())
- require.NoError(t, err, "failed to parse snapshot name:%s", de.Name())
- got = append(got, s)
+ if !reflect.DeepEqual(gc.deletableEpochs, test.deletableEpochs) {
+ t.Errorf("expected deletable: %#v, got %#v", test.deletableEpochs, gc.deletableEpochs)
}
- sort.Slice(got, func(i, j int) bool { return got[i] < got[j] })
- assert.Equal(t, tt.wantOnDisk, got)
- assert.Equal(t, tt.wantInGC, g.snapshots)
})
}
}
diff --git a/banyand/stream/index.go b/banyand/stream/index.go
index 087b93f8..8da5bc97 100644
--- a/banyand/stream/index.go
+++ b/banyand/stream/index.go
@@ -18,14 +18,16 @@
package stream
import (
+ "container/heap"
"context"
"path"
+ "sort"
+ "github.com/apache/skywalking-banyandb/api/common"
databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
"github.com/apache/skywalking-banyandb/pkg/index"
"github.com/apache/skywalking-banyandb/pkg/index/inverted"
- "github.com/apache/skywalking-banyandb/pkg/index/posting"
"github.com/apache/skywalking-banyandb/pkg/logger"
pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
)
@@ -61,8 +63,8 @@ func (e *elementIndex) Write(docs index.Documents) error {
return e.store.Batch(docs)
}
-func (e *elementIndex) Search(_ context.Context, seriesList pbv1.SeriesList, filter index.Filter) (posting.List, error) {
- var pls posting.List
+func (e *elementIndex) Search(_ context.Context, seriesList pbv1.SeriesList, filter index.Filter) ([]elementRef, error) {
+ pm := make(map[common.SeriesID][]uint64)
for _, series := range seriesList {
pl, err := filter.Execute(func(ruleType databasev1.IndexRule_Type) (index.Searcher, error) {
return e.store, nil
@@ -70,14 +72,77 @@ func (e *elementIndex) Search(_ context.Context, seriesList pbv1.SeriesList, fil
if err != nil {
return nil, err
}
- err = pls.Union(pl)
- if err != nil {
- return nil, err
- }
+ timestamps := pl.ToSlice()
+ sort.Slice(timestamps, func(i, j int) bool {
+ return timestamps[i] < timestamps[j]
+ })
+ pm[series.ID] = timestamps
}
- return pls, nil
+ return merge(pm), nil
}
func (e *elementIndex) Close() error {
return e.store.Close()
}
+
+type elementRef struct {
+ seriesID common.SeriesID
+ timestamp uint64
+}
+
+type indexedElementRef struct {
+ elementRef
+ elemIdx int
+}
+
+type priorityQueue []*indexedElementRef
+
+func (pq priorityQueue) Len() int { return len(pq) }
+
+func (pq priorityQueue) Less(i, j int) bool {
+ return pq[i].timestamp < pq[j].timestamp
+}
+
+func (pq priorityQueue) Swap(i, j int) {
+ pq[i], pq[j] = pq[j], pq[i]
+}
+
+func (pq *priorityQueue) Push(x interface{}) {
+ item := x.(*indexedElementRef)
+ *pq = append(*pq, item)
+}
+
+func (pq *priorityQueue) Pop() interface{} {
+ old := *pq
+ n := len(old)
+ item := old[n-1]
+ *pq = old[0 : n-1]
+ return item
+}
+
+func merge(postingMap map[common.SeriesID][]uint64) []elementRef {
+ var result []elementRef
+ pq := make(priorityQueue, 0)
+ heap.Init(&pq)
+
+ for seriesID, timestamps := range postingMap {
+ if len(timestamps) > 0 {
+ er := elementRef{seriesID: seriesID, timestamp: timestamps[0]}
+ item := &indexedElementRef{elementRef: er, elemIdx: 0}
+ heap.Push(&pq, item)
+ }
+ }
+ for pq.Len() > 0 {
+ item := heap.Pop(&pq).(*indexedElementRef)
+ result = append(result, item.elementRef)
+
+ if item.elemIdx+1 < len(postingMap[item.seriesID]) {
+ nextTS := postingMap[item.seriesID][item.elemIdx+1]
+ nextEr := elementRef{seriesID: item.seriesID, timestamp: nextTS}
+ nextItem := &indexedElementRef{elementRef: nextEr, elemIdx: item.elemIdx + 1}
+ heap.Push(&pq, nextItem)
+ }
+ }
+
+ return result
+}
diff --git a/banyand/stream/introducer.go b/banyand/stream/introducer.go
index 0cb81cce..6365beb3 100644
--- a/banyand/stream/introducer.go
+++ b/banyand/stream/introducer.go
@@ -77,7 +77,38 @@ func releaseFlusherIntroduction(i *flusherIntroduction) {
flusherIntroductionPool.Put(i)
}
-func (tst *tsTable) introducerLoop(flushCh chan *flusherIntroduction, watcherCh watcher.Channel, epoch uint64) {
+type mergerIntroduction struct {
+ merged map[uint64]struct{}
+ newPart *partWrapper
+ applied chan struct{}
+ creator snapshotCreator
+}
+
+func (i *mergerIntroduction) reset() {
+ for k := range i.merged {
+ delete(i.merged, k)
+ }
+ i.newPart = nil
+ i.applied = nil
+ i.creator = 0
+}
+
+var mergerIntroductionPool = sync.Pool{}
+
+func generateMergerIntroduction() *mergerIntroduction {
+ v := mergerIntroductionPool.Get()
+ if v == nil {
+ return &mergerIntroduction{}
+ }
+ return v.(*mergerIntroduction)
+}
+
+func releaseMergerIntroduction(i *mergerIntroduction) {
+ i.reset()
+ mergerIntroductionPool.Put(i)
+}
+
+func (tst *tsTable) introducerLoop(flushCh chan *flusherIntroduction, mergeCh chan *mergerIntroduction, watcherCh watcher.Channel, epoch uint64) {
var introducerWatchers watcher.Epochs
defer tst.loopCloser.Done()
for {
@@ -89,6 +120,11 @@ func (tst *tsTable) introducerLoop(flushCh chan *flusherIntroduction, watcherCh
epoch++
case next := <-flushCh:
tst.introduceFlushed(next, epoch)
+ tst.gc.cleanSnapshots()
+ epoch++
+ case next := <-mergeCh:
+ tst.introduceMerged(next, epoch)
+ tst.gc.clean()
epoch++
case epochWatcher := <-watcherCh:
introducerWatchers.Add(epochWatcher)
@@ -108,8 +144,8 @@ func (tst *tsTable) introduceMemPart(nextIntroduction *introduction, epoch uint6
next := nextIntroduction.memPart
nextSnp := cur.copyAllTo(epoch)
- next.p.partMetadata.ID = epoch
nextSnp.parts = append(nextSnp.parts, next)
+ nextSnp.creator = snapshotCreatorMemPart
tst.replaceSnapshot(&nextSnp)
if nextIntroduction.applied != nil {
close(nextIntroduction.applied)
@@ -123,7 +159,26 @@ func (tst *tsTable) introduceFlushed(nextIntroduction *flusherIntroduction, epoc
}
defer cur.decRef()
nextSnp := cur.merge(epoch, nextIntroduction.flushed)
+ nextSnp.creator = snapshotCreatorFlusher
+ tst.replaceSnapshot(&nextSnp)
+ tst.persistSnapshot(&nextSnp)
+ if nextIntroduction.applied != nil {
+ close(nextIntroduction.applied)
+ }
+}
+
+func (tst *tsTable) introduceMerged(nextIntroduction *mergerIntroduction, epoch uint64) {
+ cur := tst.currentSnapshot()
+ if cur == nil {
+ tst.l.Panic().Msg("current snapshot is nil")
+ return
+ }
+ defer cur.decRef()
+ nextSnp := cur.remove(epoch, nextIntroduction.merged)
+ nextSnp.parts = append(nextSnp.parts, nextIntroduction.newPart)
+ nextSnp.creator = nextIntroduction.creator
tst.replaceSnapshot(&nextSnp)
+ tst.persistSnapshot(&nextSnp)
if nextIntroduction.applied != nil {
close(nextIntroduction.applied)
}
diff --git a/banyand/stream/iter.go b/banyand/stream/iter.go
index 545b0934..0c9d2662 100644
--- a/banyand/stream/iter.go
+++ b/banyand/stream/iter.go
@@ -26,7 +26,7 @@ import (
)
type searcherIterator struct {
- indexFilter index.Filter
+ indexFilter filterFn
timeFilter filterFn
fieldIterator index.FieldIterator
cur posting.Iterator
@@ -38,7 +38,7 @@ type searcherIterator struct {
}
func newSearcherIterator(l *logger.Logger, fieldIterator index.FieldIterator, table *tsTable,
- seriesID common.SeriesID, indexFilter index.Filter, timeFilter filterFn, tagProjection []pbv1.TagProjection,
+ seriesID common.SeriesID, indexFilter filterFn, timeFilter filterFn, tagProjection []pbv1.TagProjection,
) *searcherIterator {
return &searcherIterator{
fieldIterator: fieldIterator,
@@ -65,6 +65,9 @@ func (s *searcherIterator) Next() bool {
if !s.timeFilter(s.Val()) {
return s.Next()
}
+ if s.indexFilter != nil && !s.indexFilter(s.Val()) {
+ return s.Next()
+ }
if e := s.l.Debug(); e.Enabled() {
e.Uint64("series_id", uint64(s.seriesID)).Uint64("item_id", uint64(s.Val().ID())).Msg("got an item")
}
diff --git a/banyand/stream/iter_builder.go b/banyand/stream/iter_builder.go
index 4f498578..b94c33af 100644
--- a/banyand/stream/iter_builder.go
+++ b/banyand/stream/iter_builder.go
@@ -65,6 +65,20 @@ func (s *seekerBuilder) buildSeriesByIndex() (series []*searcherIterator, err er
return valid
}
for _, tw := range s.seriesSpan.tableWrappers {
+ indexFilter := func(item item) bool {
+ if s.indexFilter == nil {
+ return true
+ }
+ pl, err := s.indexFilter.Execute(func(ruleType databasev1.IndexRule_Type) (index.Searcher, error) {
+ return tw.Table().Index().store, nil
+ }, s.seriesSpan.seriesID)
+ if err != nil {
+ return false
+ }
+ valid := pl.Contains(item.Time())
+ s.seriesSpan.l.Trace().Int("valid_item_num", pl.Len()).Bool("valid", valid).Msg("filter item by index")
+ return valid
+ }
var inner index.FieldIterator
var err error
fieldKey := index.FieldKey{
@@ -85,7 +99,7 @@ func (s *seekerBuilder) buildSeriesByIndex() (series []*searcherIterator, err er
}
if inner != nil {
series = append(series, newSearcherIterator(s.seriesSpan.l, inner, tw.Table(),
- s.seriesSpan.seriesID, s.indexFilter, timeFilter, s.tagProjection))
+ s.seriesSpan.seriesID, indexFilter, timeFilter, s.tagProjection))
}
}
return
diff --git a/banyand/stream/merger.go b/banyand/stream/merger.go
new file mode 100644
index 00000000..6894f0fe
--- /dev/null
+++ b/banyand/stream/merger.go
@@ -0,0 +1,378 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package stream
+
+import (
+ "errors"
+ "fmt"
+ "sync/atomic"
+ "time"
+
+ "github.com/dustin/go-humanize"
+
+ "github.com/apache/skywalking-banyandb/pkg/encoding"
+ "github.com/apache/skywalking-banyandb/pkg/fs"
+ "github.com/apache/skywalking-banyandb/pkg/watcher"
+)
+
+func (tst *tsTable) mergeLoop(merges chan *mergerIntroduction, flusherNotifier watcher.Channel) {
+ defer tst.loopCloser.Done()
+
+ var epoch uint64
+
+ ew := flusherNotifier.Add(0, tst.loopCloser.CloseNotify())
+ if ew == nil {
+ return
+ }
+
+ for {
+ select {
+ case <-tst.loopCloser.CloseNotify():
+ return
+ case <-ew.Watch():
+ curSnapshot := tst.currentSnapshot()
+ if curSnapshot == nil {
+ continue
+ }
+ if curSnapshot.epoch != epoch {
+ if err := tst.mergeSnapshot(curSnapshot, merges); err != nil {
+ if errors.Is(err, errClosed) {
+ curSnapshot.decRef()
+ return
+ }
+ tst.l.Logger.Warn().Err(err).Msgf("cannot merge snapshot: %d", curSnapshot.epoch)
+ curSnapshot.decRef()
+ continue
+ }
+ epoch = curSnapshot.epoch
+ }
+ curSnapshot.decRef()
+ ew = flusherNotifier.Add(epoch, tst.loopCloser.CloseNotify())
+ if ew == nil {
+ return
+ }
+ }
+ }
+}
+
+func (tst *tsTable) mergeSnapshot(curSnapshot *snapshot, merges chan *mergerIntroduction) error {
+ maxFanOut := tst.freeDiskSpace(tst.root) / 2
+ partsToMerge, toBeMerged := tst.getPartsToMerge(curSnapshot, maxFanOut)
+ if len(partsToMerge) < 2 {
+ return nil
+ }
+ if _, err := tst.mergePartsThenSendIntroduction(snapshotCreatorMerger, partsToMerge,
+ toBeMerged, merges, tst.loopCloser.CloseNotify()); err != nil {
+ return err
+ }
+ return nil
+}
+
+func (tst *tsTable) mergePartsThenSendIntroduction(creator snapshotCreator, parts []*partWrapper, merged map[uint64]struct{}, merges chan *mergerIntroduction,
+ closeCh <-chan struct{},
+) (*partWrapper, error) {
+ reservedSpace := tst.reserveSpace(parts)
+ defer releaseDiskSpace(reservedSpace)
+ start := time.Now()
+ newPart, err := mergeParts(tst.fileSystem, closeCh, parts, atomic.AddUint64(&tst.curPartID, 1), tst.root)
+ if err != nil {
+ return nil, err
+ }
+ elapsed := time.Since(start)
+ if elapsed > 30*time.Second {
+ var totalCount uint64
+ for _, pw := range parts {
+ totalCount += pw.p.partMetadata.TotalCount
+ }
+ tst.l.Warn().
+ Uint64("beforeTotalCount", totalCount).
+ Uint64("afterTotalCount", newPart.p.partMetadata.TotalCount).
+ Int("beforePartCount", len(parts)).
+ Dur("elapsed", elapsed).
+ Msg("background merger takes too long")
+ } else if snapshotCreatorMerger == creator && tst.l.Info().Enabled() && len(parts) > 2 {
+ var minSize, maxSize, totalSize, totalCount uint64
+ for _, pw := range parts {
+ totalCount += pw.p.partMetadata.TotalCount
+ totalSize += pw.p.partMetadata.CompressedSizeBytes
+ if minSize == 0 || minSize > pw.p.partMetadata.CompressedSizeBytes {
+ minSize = pw.p.partMetadata.CompressedSizeBytes
+ }
+ if maxSize < pw.p.partMetadata.CompressedSizeBytes {
+ maxSize = pw.p.partMetadata.CompressedSizeBytes
+ }
+ }
+ if totalSize > 10<<20 && minSize*uint64(len(parts)) < maxSize {
+ // it's a unbalanced merge. but it's ok when the size is small.
+ tst.l.Info().
+ Str("beforeTotalCount", humanize.Comma(int64(totalCount))).
+ Str("afterTotalCount", humanize.Comma(int64(newPart.p.partMetadata.TotalCount))).
+ Int("beforePartCount", len(parts)).
+ Str("minSize", humanize.IBytes(minSize)).
+ Str("maxSize", humanize.IBytes(maxSize)).
+ Dur("elapsedMS", elapsed).
+ Msg("background merger merges unbalanced parts")
+ }
+ }
+ mi := generateMergerIntroduction()
+ defer releaseMergerIntroduction(mi)
+ mi.creator = creator
+ mi.newPart = newPart
+ mi.merged = merged
+ mi.applied = make(chan struct{})
+ select {
+ case merges <- mi:
+ case <-tst.loopCloser.CloseNotify():
+ return newPart, errClosed
+ }
+ select {
+ case <-mi.applied:
+ case <-tst.loopCloser.CloseNotify():
+ return newPart, errClosed
+ }
+ return newPart, nil
+}
+
+func (tst *tsTable) freeDiskSpace(path string) uint64 {
+ free := tst.fileSystem.MustGetFreeSpace(path)
+ reserved := atomic.LoadUint64(&reservedDiskSpace)
+ if free < reserved {
+ return 0
+ }
+ return free - reserved
+}
+
+func (tst *tsTable) tryReserveDiskSpace(n uint64) bool {
+ available := tst.fileSystem.MustGetFreeSpace(tst.root)
+ reserved := reserveDiskSpace(n)
+ if available > reserved {
+ return true
+ }
+ releaseDiskSpace(n)
+ return false
+}
+
+func reserveDiskSpace(n uint64) uint64 {
+ return atomic.AddUint64(&reservedDiskSpace, n)
+}
+
+func releaseDiskSpace(n uint64) {
+ atomic.AddUint64(&reservedDiskSpace, ^(n - 1))
+}
+
+var reservedDiskSpace uint64
+
+func (tst *tsTable) getPartsToMerge(snapshot *snapshot, maxFanOut uint64) ([]*partWrapper, map[uint64]struct{}) {
+ var parts []*partWrapper
+ toBeMerged := make(map[uint64]struct{})
+ for _, pw := range snapshot.parts {
+ if pw.mp != nil || pw.p.partMetadata.TotalCount < 1 || pw.p.partMetadata.CompressedSizeBytes > maxFanOut {
+ continue
+ }
+ parts = append(parts, pw)
+ toBeMerged[pw.ID()] = struct{}{}
+ }
+ // TODO: select proper parts to generate the lowest write amplification
+ return parts, toBeMerged
+}
+
+func (tst *tsTable) reserveSpace(parts []*partWrapper) uint64 {
+ var needSize uint64
+ for i := range parts {
+ needSize = +parts[i].p.partMetadata.CompressedSizeBytes
+ }
+ if tst.tryReserveDiskSpace(needSize) {
+ return needSize
+ }
+ return 0
+}
+
+var errNoPartToMerge = fmt.Errorf("no part to merge")
+
+func mergeParts(fileSystem fs.FileSystem, closeCh <-chan struct{}, parts []*partWrapper, partID uint64, root string) (*partWrapper, error) {
+ if len(parts) == 0 {
+ return nil, errNoPartToMerge
+ }
+ dstPath := partPath(root, partID)
+ pii := make([]*partMergeIter, 0, len(parts))
+ for i := range parts {
+ pmi := generatePartMergeIter()
+ pmi.mustInitFromPart(parts[i].p)
+ pii = append(pii, pmi)
+ }
+ br := generateBlockReader()
+ br.init(pii)
+ bw := generateBlockWriter()
+ bw.mustInitForFilePart(fileSystem, dstPath)
+
+ pm, err := mergeBlocks(closeCh, bw, br)
+ releaseBlockWriter(bw)
+ releaseBlockReader(br)
+ for i := range pii {
+ releasePartMergeIter(pii[i])
+ }
+ if err != nil {
+ return nil, err
+ }
+ pm.mustWriteMetadata(fileSystem, dstPath)
+ fileSystem.SyncPath(dstPath)
+ p := mustOpenFilePart(partID, root, fileSystem)
+ return newPartWrapper(nil, p), nil
+}
+
+var errClosed = fmt.Errorf("the merger is closed")
+
+func mergeBlocks(closeCh <-chan struct{}, bw *blockWriter, br *blockReader) (*partMetadata, error) {
+ pendingBlockIsEmpty := true
+ pendingBlock := &blockPointer{}
+ var tmpBlock, tmpBlock2 *blockPointer
+ var decoder *encoding.BytesBlockDecoder
+ getDecoder := func() *encoding.BytesBlockDecoder {
+ if decoder == nil {
+ decoder = generateColumnValuesDecoder()
+ }
+ return decoder
+ }
+ releaseDecoder := func() {
+ if decoder != nil {
+ releaseColumnValuesDecoder(decoder)
+ decoder = nil
+ }
+ }
+ for br.nextBlockMetadata() {
+ select {
+ case <-closeCh:
+ return nil, errClosed
+ default:
+ }
+ b := br.block
+
+ if pendingBlockIsEmpty {
+ br.loadBlockData(getDecoder())
+ pendingBlock.copyFrom(b)
+ pendingBlockIsEmpty = false
+ continue
+ }
+
+ if pendingBlock.bm.seriesID != b.bm.seriesID ||
+ (pendingBlock.isFull() && pendingBlock.bm.timestamps.max <= b.bm.timestamps.min) {
+ bw.mustWriteBlock(pendingBlock.bm.seriesID, &pendingBlock.block)
+ releaseDecoder()
+ pendingBlock.reset()
+ br.loadBlockData(getDecoder())
+ pendingBlock.copyFrom(b)
+ continue
+ }
+
+ if tmpBlock == nil {
+ tmpBlock = &blockPointer{}
+ }
+ tmpBlock.reset()
+ tmpBlock.bm.seriesID = b.bm.seriesID
+ br.loadBlockData(getDecoder())
+ mergeTwoBlocks(tmpBlock, pendingBlock, b)
+ if len(tmpBlock.timestamps) <= maxBlockLength && tmpBlock.uncompressedSizeBytes() <= maxUncompressedBlockSize {
+ if len(tmpBlock.timestamps) == 0 {
+ pendingBlockIsEmpty = true
+ }
+ pendingBlock, tmpBlock = tmpBlock, pendingBlock
+ continue
+ }
+
+ if len(tmpBlock.timestamps) <= maxBlockLength {
+ bw.mustWriteBlock(tmpBlock.bm.seriesID, &tmpBlock.block)
+ releaseDecoder()
+ continue
+ }
+ tmpBlock.idx = maxBlockLength
+ pendingBlock.copyFrom(tmpBlock)
+ l := tmpBlock.idx
+ tmpBlock.idx = 0
+ if tmpBlock2 == nil {
+ tmpBlock2 = &blockPointer{}
+ }
+ tmpBlock2.reset()
+ tmpBlock2.append(tmpBlock, l)
+ bw.mustWriteBlock(tmpBlock.bm.seriesID, &tmpBlock2.block)
+ releaseDecoder()
+ }
+ if err := br.error(); err != nil {
+ return nil, fmt.Errorf("cannot read block to merge: %w", err)
+ }
+ if !pendingBlockIsEmpty {
+ bw.mustWriteBlock(pendingBlock.bm.seriesID, &pendingBlock.block)
+ }
+ releaseDecoder()
+ var result partMetadata
+ bw.Flush(&result)
+ return &result, nil
+}
+
+func mergeTwoBlocks(target, left, right *blockPointer) {
+ appendIfEmpty := func(ib1, ib2 *blockPointer) bool {
+ if ib1.idx >= len(ib1.timestamps) {
+ target.appendAll(ib2)
+ return true
+ }
+ return false
+ }
+
+ defer target.updateMetadata()
+
+ if left.bm.timestamps.max < right.bm.timestamps.min {
+ target.appendAll(left)
+ target.appendAll(right)
+ return
+ }
+ if right.bm.timestamps.max < left.bm.timestamps.min {
+ target.appendAll(right)
+ target.appendAll(left)
+ return
+ }
+ if appendIfEmpty(left, right) || appendIfEmpty(right, left) {
+ return
+ }
+
+ for {
+ i := left.idx
+ ts2 := right.timestamps[right.idx]
+ for i < len(left.timestamps) && left.timestamps[i] <= ts2 {
+ i++
+ }
+ if left.timestamps[i-1] == ts2 {
+ if left.lastPartID >= right.lastPartID {
+ target.append(left, i)
+ } else {
+ target.append(left, i-1) // skip left
+ target.append(right, right.idx+1)
+ }
+ left.idx = i
+ right.idx++ // skip right
+ if appendIfEmpty(right, left) {
+ return
+ }
+ } else {
+ target.append(left, i)
+ left.idx = i
+ }
+ if appendIfEmpty(left, right) {
+ return
+ }
+ left, right = right, left
+ }
+}
diff --git a/banyand/stream/merger_test.go b/banyand/stream/merger_test.go
new file mode 100644
index 00000000..8442b555
--- /dev/null
+++ b/banyand/stream/merger_test.go
@@ -0,0 +1,328 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package stream
+
+import (
+ "errors"
+ "reflect"
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+ "github.com/stretchr/testify/require"
+
+ "github.com/apache/skywalking-banyandb/pkg/fs"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
+ "github.com/apache/skywalking-banyandb/pkg/test"
+)
+
+func Test_mergeTwoBlocks(t *testing.T) {
+ tests := []struct {
+ left *blockPointer
+ right *blockPointer
+ want *blockPointer
+ name string
+ }{
+ {
+ name: "Merge two empty blocks",
+ left: &blockPointer{},
+ right: &blockPointer{},
+ want: &blockPointer{},
+ },
+ {
+ name: "Merge left is non-empty right is empty",
+ left: &blockPointer{block: conventionalBlock},
+ right: &blockPointer{},
+ want: &blockPointer{block: conventionalBlock, bm: blockMetadata{timestamps: timestampsMetadata{min: 1, max: 2}}},
+ },
+ {
+ name: "Merge left is empty right is non-empty",
+ left: &blockPointer{},
+ right: &blockPointer{block: conventionalBlock},
+ want: &blockPointer{block: conventionalBlock, bm: blockMetadata{timestamps: timestampsMetadata{min: 1, max: 2}}},
+ },
+ {
+ name: "Merge two non-empty blocks without overlap",
+ left: &blockPointer{
+ block: block{
+ timestamps: []int64{1, 2},
+ elementIDs: []string{"0", "1"},
+ tagFamilies: []tagFamily{
+ {
+ name: "arrTag",
+ tags: []tag{
+ {
+ name: "strArrTag", valueType: pbv1.ValueTypeStrArr,
+ values: [][]byte{marshalStrArr([][]byte{[]byte("value1"), []byte("value2")}), marshalStrArr([][]byte{[]byte("value3"), []byte("value4")})},
+ },
+ },
+ },
+ },
+ },
+ },
+ right: &blockPointer{
+ block: block{
+ timestamps: []int64{3, 4},
+ elementIDs: []string{"2", "3"},
+ tagFamilies: []tagFamily{
+ {
+ name: "arrTag",
+ tags: []tag{
+ {
+ name: "strArrTag", valueType: pbv1.ValueTypeStrArr,
+ values: [][]byte{marshalStrArr([][]byte{[]byte("value5"), []byte("value6")}), marshalStrArr([][]byte{[]byte("value7"), []byte("value8")})},
+ },
+ },
+ },
+ },
+ },
+ },
+ want: &blockPointer{block: mergedBlock, bm: blockMetadata{timestamps: timestampsMetadata{min: 1, max: 4}}},
+ },
+ {
+ name: "Merge two non-empty blocks without duplicated timestamps",
+ left: &blockPointer{
+ block: block{
+ timestamps: []int64{1, 3},
+ elementIDs: []string{"0", "2"},
+ tagFamilies: []tagFamily{
+ {
+ name: "arrTag",
+ tags: []tag{
+ {
+ name: "strArrTag", valueType: pbv1.ValueTypeStrArr,
+ values: [][]byte{marshalStrArr([][]byte{[]byte("value1"), []byte("value2")}), marshalStrArr([][]byte{[]byte("value5"), []byte("value6")})},
+ },
+ },
+ },
+ },
+ },
+ },
+ right: &blockPointer{
+ block: block{
+ timestamps: []int64{2, 4},
+ elementIDs: []string{"1", "3"},
+ tagFamilies: []tagFamily{
+ {
+ name: "arrTag",
+ tags: []tag{
+ {
+ name: "strArrTag", valueType: pbv1.ValueTypeStrArr,
+ values: [][]byte{marshalStrArr([][]byte{[]byte("value3"), []byte("value4")}), marshalStrArr([][]byte{[]byte("value7"), []byte("value8")})},
+ },
+ },
+ },
+ },
+ },
+ },
+ want: &blockPointer{block: mergedBlock, bm: blockMetadata{timestamps: timestampsMetadata{min: 1, max: 4}}},
+ },
+ {
+ name: "Merge two non-empty blocks with duplicated timestamps",
+ left: &blockPointer{
+ lastPartID: 1, // the less partID will be skipped
+ block: block{
+ timestamps: []int64{1, 2, 3},
+ elementIDs: []string{"0", "1", "2"},
+ tagFamilies: []tagFamily{
+ {
+ name: "arrTag",
+ tags: []tag{
+ {
+ name: "strArrTag", valueType: pbv1.ValueTypeStrArr,
+ values: [][]byte{
+ marshalStrArr([][]byte{[]byte("value1"), []byte("value2")}),
+ marshalStrArr([][]byte{[]byte("duplicated1")}),
+ marshalStrArr([][]byte{[]byte("duplicated2")}),
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ right: &blockPointer{
+ lastPartID: 2, // the greater partID will be appended
+ block: block{
+ timestamps: []int64{2, 3, 4},
+ elementIDs: []string{"1", "2", "3"},
+ tagFamilies: []tagFamily{
+ {
+ name: "arrTag",
+ tags: []tag{
+ {
+ name: "strArrTag", valueType: pbv1.ValueTypeStrArr,
+ values: [][]byte{
+ marshalStrArr([][]byte{[]byte("value3"), []byte("value4")}), marshalStrArr([][]byte{[]byte("value5"), []byte("value6")}),
+ marshalStrArr([][]byte{[]byte("value7"), []byte("value8")}),
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ want: &blockPointer{block: mergedBlock, lastPartID: 2, bm: blockMetadata{timestamps: timestampsMetadata{min: 1, max: 4}}},
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ target := &blockPointer{}
+ mergeTwoBlocks(target, tt.left, tt.right)
+ if !reflect.DeepEqual(target, tt.want) {
+ t.Errorf("mergeTwoBlocks() = %v, want %v", target, tt.want)
+ }
+ })
+ }
+}
+
+var mergedBlock = block{
+ timestamps: []int64{1, 2, 3, 4},
+ elementIDs: []string{"0", "1", "2", "3"},
+ tagFamilies: []tagFamily{
+ {
+ name: "arrTag",
+ tags: []tag{
+ {
+ name: "strArrTag", valueType: pbv1.ValueTypeStrArr,
+ values: [][]byte{
+ marshalStrArr([][]byte{[]byte("value1"), []byte("value2")}), marshalStrArr([][]byte{[]byte("value3"), []byte("value4")}),
+ marshalStrArr([][]byte{[]byte("value5"), []byte("value6")}), marshalStrArr([][]byte{[]byte("value7"), []byte("value8")}),
+ },
+ },
+ },
+ },
+ },
+}
+
+func Test_mergeParts(t *testing.T) {
+ tests := []struct {
+ wantErr error
+ name string
+ dpsList []*elements
+ want []blockMetadata
+ }{
+ {
+ name: "Test with no data points",
+ dpsList: []*elements{},
+ wantErr: errNoPartToMerge,
+ },
+ {
+ name: "Test with single part",
+ dpsList: []*elements{esTS1},
+ want: []blockMetadata{
+ {seriesID: 1, count: 1, uncompressedSizeBytes: 881},
+ {seriesID: 2, count: 1, uncompressedSizeBytes: 55},
+ {seriesID: 3, count: 1, uncompressedSizeBytes: 8},
+ },
+ },
+ {
+ name: "Test with multiple parts with different ts",
+ dpsList: []*elements{esTS1, esTS2, esTS2},
+ want: []blockMetadata{
+ {seriesID: 1, count: 2, uncompressedSizeBytes: 1762},
+ {seriesID: 2, count: 2, uncompressedSizeBytes: 110},
+ {seriesID: 3, count: 2, uncompressedSizeBytes: 16},
+ },
+ },
+ {
+ name: "Test with multiple parts with same ts",
+ dpsList: []*elements{esTS1, esTS1, esTS1},
+ want: []blockMetadata{
+ {seriesID: 1, count: 1, uncompressedSizeBytes: 881},
+ {seriesID: 2, count: 1, uncompressedSizeBytes: 55},
+ {seriesID: 3, count: 1, uncompressedSizeBytes: 8},
+ },
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ verify := func(t *testing.T, pp []*partWrapper, fileSystem fs.FileSystem, root string, partID uint64) {
+ closeCh := make(chan struct{})
+ defer close(closeCh)
+ p, err := mergeParts(fileSystem, closeCh, pp, partID, root)
+ if tt.wantErr != nil {
+ if !errors.Is(err, tt.wantErr) {
+ t.Fatalf("Unexpected error: got %v, want %v", err, tt.wantErr)
+ }
+ return
+ }
+ defer p.decRef()
+ pmi := &partMergeIter{}
+ pmi.mustInitFromPart(p.p)
+ reader := &blockReader{}
+ reader.init([]*partMergeIter{pmi})
+ var got []blockMetadata
+ for reader.nextBlockMetadata() {
+ got = append(got, reader.block.bm)
+ }
+ require.NoError(t, reader.error())
+
+ if diff := cmp.Diff(got, tt.want,
+ cmpopts.IgnoreFields(blockMetadata{}, "timestamps"),
+ cmpopts.IgnoreFields(blockMetadata{}, "elementIDs"),
+ cmpopts.IgnoreFields(blockMetadata{}, "tagFamilies"),
+ cmp.AllowUnexported(blockMetadata{}),
+ ); diff != "" {
+ t.Errorf("Unexpected blockMetadata (-got +want):\n%s", diff)
+ }
+ }
+
+ t.Run("memory parts", func(t *testing.T) {
+ var pp []*partWrapper
+ tmpPath, defFn := test.Space(require.New(t))
+ defer func() {
+ for _, pw := range pp {
+ pw.decRef()
+ }
+ defFn()
+ }()
+ for _, dps := range tt.dpsList {
+ mp := generateMemPart()
+ mp.mustInitFromElements(dps)
+ pp = append(pp, newPartWrapper(mp, openMemPart(mp)))
+ }
+ verify(t, pp, fs.NewLocalFileSystem(), tmpPath, 1)
+ })
+
+ t.Run("file parts", func(t *testing.T) {
+ var fpp []*partWrapper
+ tmpPath, defFn := test.Space(require.New(t))
+ defer func() {
+ for _, pw := range fpp {
+ pw.decRef()
+ }
+ defFn()
+ }()
+ fileSystem := fs.NewLocalFileSystem()
+ for i, dps := range tt.dpsList {
+ mp := generateMemPart()
+ mp.mustInitFromElements(dps)
+ mp.mustFlush(fileSystem, partPath(tmpPath, uint64(i)))
+ filePW := newPartWrapper(nil, mustOpenFilePart(uint64(i), tmpPath, fileSystem))
+ filePW.p.partMetadata.ID = uint64(i)
+ fpp = append(fpp, filePW)
+ releaseMemPart(mp)
+ }
+ verify(t, fpp, fileSystem, tmpPath, uint64(len(tt.dpsList)))
+ })
+ })
+ }
+}
diff --git a/banyand/stream/metadata.go b/banyand/stream/metadata.go
index 0446f2c3..bdd239f9 100644
--- a/banyand/stream/metadata.go
+++ b/banyand/stream/metadata.go
@@ -200,12 +200,12 @@ func (sr *schemaRepo) loadStream(metadata *commonv1.Metadata) (*stream, bool) {
return s, ok
}
-func (sr *schemaRepo) loadTSDB(groupName string) (storage.TSDB[*tsTable], error) {
+func (sr *schemaRepo) loadTSDB(groupName string) (storage.TSDB[*tsTable, option], error) {
g, ok := sr.LoadGroup(groupName)
if !ok {
return nil, fmt.Errorf("group %s not found", groupName)
}
- return g.SupplyTSDB().(storage.TSDB[*tsTable]), nil
+ return g.SupplyTSDB().(storage.TSDB[*tsTable, option]), nil
}
var _ resourceSchema.ResourceSupplier = (*supplier)(nil)
@@ -215,6 +215,7 @@ type supplier struct {
pipeline queue.Queue
l *logger.Logger
path string
+ option option
}
func newSupplier(path string, svc *service) *supplier {
@@ -223,6 +224,7 @@ func newSupplier(path string, svc *service) *supplier {
metadata: svc.metadata,
l: svc.l,
pipeline: svc.localPipeline,
+ option: svc.option,
}
}
@@ -241,12 +243,13 @@ func (s *supplier) ResourceSchema(md *commonv1.Metadata) (resourceSchema.Resourc
}
func (s *supplier) OpenDB(groupSchema *commonv1.Group) (io.Closer, error) {
- opts := storage.TSDBOpts[*tsTable]{
+ opts := storage.TSDBOpts[*tsTable, option]{
ShardNum: groupSchema.ResourceOpts.ShardNum,
Location: path.Join(s.path, groupSchema.Metadata.Name),
TSTableCreator: newTSTable,
SegmentInterval: storage.MustToIntervalRule(groupSchema.ResourceOpts.SegmentInterval),
TTL: storage.MustToIntervalRule(groupSchema.ResourceOpts.Ttl),
+ Option: s.option,
}
name := groupSchema.Metadata.Name
return storage.OpenTSDB(
diff --git a/banyand/stream/part.go b/banyand/stream/part.go
index 5eab3b08..8a559cc5 100644
--- a/banyand/stream/part.go
+++ b/banyand/stream/part.go
@@ -47,6 +47,35 @@ const (
tagFamiliesFilenameExt = ".tf"
)
+type columnElements struct {
+ elementID []string
+ // TODO: change it to 1d array after refactoring low-level query
+ tagFamilies [][]pbv1.TagFamily
+ timestamp []int64
+}
+
+func newColumnElements() *columnElements {
+ ces := &columnElements{}
+ ces.elementID = make([]string, 0)
+ ces.tagFamilies = make([][]pbv1.TagFamily, 0)
+ ces.timestamp = make([]int64, 0)
+ return ces
+}
+
+func (ces *columnElements) Pull() *pbv1.StreamColumnResult {
+ r := &pbv1.StreamColumnResult{}
+ r.Timestamps = make([]int64, 0)
+ r.ElementIDs = make([]string, 0)
+ r.TagFamilies = make([][]pbv1.TagFamily, len(ces.tagFamilies))
+ r.Timestamps = append(r.Timestamps, ces.timestamp...)
+ r.ElementIDs = append(r.ElementIDs, ces.elementID...)
+ for i, tfs := range ces.tagFamilies {
+ r.TagFamilies[i] = make([]pbv1.TagFamily, 0)
+ r.TagFamilies[i] = append(r.TagFamilies[i], tfs...)
+ }
+ return r
+}
+
type element struct {
elementID string
tagFamilies []*tagFamily
@@ -89,10 +118,15 @@ func (p *part) String() string {
func (p *part) getElement(seriesID common.SeriesID, timestamp common.ItemID, tagProjection []pbv1.TagProjection) (*element, error) {
// TODO: refactor to column-based query
// TODO: cache blocks
- for _, primaryMeta := range p.primaryBlockMetadata {
- if !primaryMeta.mightContainElement(seriesID, timestamp) {
+ for i, primaryMeta := range p.primaryBlockMetadata {
+ if i != len(p.primaryBlockMetadata)-1 && seriesID >= p.primaryBlockMetadata[i+1].seriesID {
continue
}
+ if seriesID < p.primaryBlockMetadata[i].seriesID ||
+ timestamp < common.ItemID(p.primaryBlockMetadata[i].minTimestamp) ||
+ timestamp > common.ItemID(p.primaryBlockMetadata[i].maxTimestamp) {
+ break
+ }
compressedPrimaryBuf := make([]byte, primaryMeta.size)
fs.MustReadData(p.primary, int64(primaryMeta.offset), compressedPrimaryBuf)
@@ -107,17 +141,24 @@ func (p *part) getElement(seriesID common.SeriesID, timestamp common.ItemID, tag
if err != nil {
return nil, fmt.Errorf("cannot unmarshal index block: %w", err)
}
+ var targetBlockMetadata blockMetadata
+ for _, blockMetadata := range bm {
+ if blockMetadata.seriesID == seriesID {
+ targetBlockMetadata = blockMetadata
+ break
+ }
+ }
timestamps := make([]int64, 0)
- timestamps = mustReadTimestampsFrom(timestamps, &bm[0].timestamps, int(bm[0].count), p.timestamps)
+ timestamps = mustReadTimestampsFrom(timestamps, &targetBlockMetadata.timestamps, int(targetBlockMetadata.count), p.timestamps)
for i, ts := range timestamps {
if timestamp == common.ItemID(ts) {
elementIDs := make([]string, 0)
- elementIDs = mustReadElementIDsFrom(elementIDs, &bm[0].elementIDs, int(bm[0].count), p.elementIDs)
+ elementIDs = mustReadElementIDsFrom(elementIDs, &targetBlockMetadata.elementIDs, int(targetBlockMetadata.count), p.elementIDs)
tfs := make([]*tagFamily, 0)
for j := range tagProjection {
name := tagProjection[j].Family
- block, ok := bm[0].tagFamilies[name]
+ block, ok := targetBlockMetadata.tagFamilies[name]
if !ok {
continue
}
@@ -176,7 +217,7 @@ func openMemPart(mp *memPart) *part {
var p part
p.partMetadata = mp.partMetadata
- p.primaryBlockMetadata = mustReadPrimaryBlockMetadata(p.primaryBlockMetadata[:0], newReader(&mp.meta))
+ p.primaryBlockMetadata = mustReadPrimaryBlockMetadata(p.primaryBlockMetadata[:0], &mp.meta)
// Open data files
p.meta = &mp.meta
@@ -226,6 +267,7 @@ func (mp *memPart) reset() {
mp.meta.Reset()
mp.primary.Reset()
mp.timestamps.Reset()
+ mp.elementIDs.Reset()
if mp.tagFamilies != nil {
for _, tf := range mp.tagFamilies {
tf.Reset()
@@ -318,15 +360,13 @@ func releaseMemPart(mp *memPart) {
var memPartPool sync.Pool
type partWrapper struct {
- fileSystem fs.FileSystem
- mp *memPart
- p *part
- ref int32
- mustBeDeleted uint32
+ mp *memPart
+ p *part
+ ref int32
}
-func newPartWrapper(mp *memPart, p *part, fileSystem fs.FileSystem) *partWrapper {
- return &partWrapper{mp: mp, p: p, fileSystem: fileSystem, ref: 1}
+func newPartWrapper(mp *memPart, p *part) *partWrapper {
+ return &partWrapper{mp: mp, p: p, ref: 1}
}
func (pw *partWrapper) incRef() {
@@ -345,24 +385,22 @@ func (pw *partWrapper) decRef() {
return
}
pw.p.close()
- if atomic.LoadUint32(&pw.mustBeDeleted) == 0 {
- return
- }
- pw.fileSystem.MustRMAll(pw.p.path)
}
func (pw *partWrapper) ID() uint64 {
return pw.p.partMetadata.ID
}
-func mustOpenFilePart(partPath string, fileSystem fs.FileSystem) *part {
+func mustOpenFilePart(id uint64, root string, fileSystem fs.FileSystem) *part {
var p part
+ partPath := partPath(root, id)
p.path = partPath
p.partMetadata.mustReadMetadata(fileSystem, partPath)
+ p.partMetadata.ID = id
metaPath := path.Join(partPath, metaFilename)
pr := mustOpenReader(metaPath, fileSystem)
- p.primaryBlockMetadata = mustReadPrimaryBlockMetadata(p.primaryBlockMetadata[:0], newReader(pr))
+ p.primaryBlockMetadata = mustReadPrimaryBlockMetadata(p.primaryBlockMetadata[:0], pr)
fs.MustClose(pr)
p.primary = mustOpenReader(path.Join(partPath, primaryFilename), fileSystem)
diff --git a/banyand/stream/part_iter.go b/banyand/stream/part_iter.go
index b106640a..7080eb1c 100644
--- a/banyand/stream/part_iter.go
+++ b/banyand/stream/part_iter.go
@@ -22,10 +22,12 @@ import (
"fmt"
"io"
"sort"
+ "sync"
"github.com/apache/skywalking-banyandb/api/common"
"github.com/apache/skywalking-banyandb/pkg/bytes"
"github.com/apache/skywalking-banyandb/pkg/compress/zstd"
+ "github.com/apache/skywalking-banyandb/pkg/encoding"
"github.com/apache/skywalking-banyandb/pkg/fs"
"github.com/apache/skywalking-banyandb/pkg/logger"
)
@@ -233,3 +235,147 @@ func (pi *partIter) findBlock() bool {
pi.bms = nil
return false
}
+
+type partMergeIter struct {
+ seqReaders seqReaders
+ err error
+ primaryBlockMetadata []primaryBlockMetadata
+ compressedPrimaryBuf []byte
+ primaryBuf []byte
+ block blockPointer
+ partID uint64
+ primaryMetadataIdx int
+}
+
+func (pmi *partMergeIter) reset() {
+ pmi.err = nil
+ pmi.seqReaders.reset()
+ pmi.primaryBlockMetadata = nil
+ pmi.primaryMetadataIdx = 0
+ pmi.partID = 0
+ pmi.primaryBuf = pmi.primaryBuf[:0]
+ pmi.compressedPrimaryBuf = pmi.compressedPrimaryBuf[:0]
+ pmi.block.reset()
+}
+
+func (pmi *partMergeIter) mustInitFromPart(p *part) {
+ pmi.reset()
+ pmi.seqReaders.init(p)
+ pmi.primaryBlockMetadata = p.primaryBlockMetadata
+ pmi.partID = p.partMetadata.ID
+}
+
+func (pmi *partMergeIter) error() error {
+ if pmi.err == nil || errors.Is(pmi.err, io.EOF) {
+ return nil
+ }
+ return pmi.err
+}
+
+func (pmi *partMergeIter) nextBlockMetadata() bool {
+ if pmi.err != nil {
+ return false
+ }
+ pmi.block.reset()
+ if len(pmi.primaryBuf) == 0 {
+ if err := pmi.loadPrimaryBuf(); err != nil {
+ pmi.err = err
+ return false
+ }
+ }
+ if err := pmi.loadBlockMetadata(); err != nil {
+ pmi.err = err
+ return false
+ }
+ return true
+}
+
+func (pmi *partMergeIter) loadPrimaryBuf() error {
+ if pmi.primaryMetadataIdx >= len(pmi.primaryBlockMetadata) {
+ return io.EOF
+ }
+ pm := pmi.primaryBlockMetadata[pmi.primaryMetadataIdx]
+ pmi.compressedPrimaryBuf = bytes.ResizeOver(pmi.compressedPrimaryBuf, int(pm.size))
+ pmi.seqReaders.primary.mustReadFull(pmi.compressedPrimaryBuf)
+ var err error
+ pmi.primaryBuf, err = zstd.Decompress(pmi.primaryBuf[:0], pmi.compressedPrimaryBuf)
+ if err != nil {
+ return fmt.Errorf("cannot decompress primary block: %w", err)
+ }
+ pmi.primaryMetadataIdx++
+ return nil
+}
+
+func (pmi *partMergeIter) loadBlockMetadata() error {
+ pmi.block.reset()
+ var err error
+ pmi.primaryBuf, err = pmi.block.bm.unmarshal(pmi.primaryBuf)
+ if err != nil {
+ pm := pmi.primaryBlockMetadata[pmi.primaryMetadataIdx-1]
+ return fmt.Errorf("can't read block metadata from primary at %d: %w", pm.offset, err)
+ }
+
+ pmi.block.lastPartID = pmi.partID
+ return nil
+}
+
+func (pmi *partMergeIter) mustLoadBlockData(decoder *encoding.BytesBlockDecoder, block *blockPointer) {
+ block.block.mustSeqReadFrom(decoder, &pmi.seqReaders, pmi.block.bm)
+}
+
+func generatePartMergeIter() *partMergeIter {
+ v := pmiPool.Get()
+ if v == nil {
+ return &partMergeIter{}
+ }
+ return v.(*partMergeIter)
+}
+
+func releasePartMergeIter(pmi *partMergeIter) {
+ pmi.reset()
+ pmiPool.Put(pmi)
+}
+
+var pmiPool sync.Pool
+
+type partMergeIterHeap []*partMergeIter
+
+func (pih *partMergeIterHeap) Len() int {
+ return len(*pih)
+}
+
+func (pih *partMergeIterHeap) Less(i, j int) bool {
+ x := *pih
+ return x[i].block.bm.less(x[j].block.bm)
+}
+
+func (pih *partMergeIterHeap) Swap(i, j int) {
+ x := *pih
+ x[i], x[j] = x[j], x[i]
+}
+
+func (pih *partMergeIterHeap) Push(x interface{}) {
+ *pih = append(*pih, x.(*partMergeIter))
+}
+
+func (pih *partMergeIterHeap) Pop() interface{} {
+ a := *pih
+ v := a[len(a)-1]
+ *pih = a[:len(a)-1]
+ return v
+}
+
+func generateColumnValuesDecoder() *encoding.BytesBlockDecoder {
+ v := columnValuesDecoderPool.Get()
+ if v == nil {
+ return &encoding.BytesBlockDecoder{}
+ }
+ return v.(*encoding.BytesBlockDecoder)
+}
+
+func releaseColumnValuesDecoder(d *encoding.BytesBlockDecoder) {
+ d.Reset()
+ columnValuesDecoderPool.Put(d)
+}
+
+var columnValuesDecoderPool sync.Pool
diff --git a/banyand/stream/part_iter_test.go b/banyand/stream/part_iter_test.go
index 03e7a213..b4aac1ac 100644
--- a/banyand/stream/part_iter_test.go
+++ b/banyand/stream/part_iter_test.go
@@ -26,6 +26,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/apache/skywalking-banyandb/api/common"
+ "github.com/apache/skywalking-banyandb/pkg/encoding"
"github.com/apache/skywalking-banyandb/pkg/fs"
"github.com/apache/skywalking-banyandb/pkg/test"
)
@@ -150,8 +151,73 @@ func Test_partIter_nextBlock(t *testing.T) {
partPath := partPath(tmpDir, epoch)
fileSystem := fs.NewLocalFileSystem()
mp.mustFlush(fileSystem, partPath)
- p = mustOpenFilePart(partPath, fileSystem)
+ p = mustOpenFilePart(epoch, tmpDir, fileSystem)
verifyPart(p)
})
}
}
+
+func Test_partMergeIter_nextBlock(t *testing.T) {
+ tests := []struct {
+ wantErr error
+ name string
+ dps *elements
+ want []blockMetadata
+ }{
+ {
+ name: "Test with all data",
+ dps: es,
+ want: []blockMetadata{
+ {seriesID: 1, count: 2}, {seriesID: 2, count: 2}, {seriesID: 3, count: 2},
+ },
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ verifyPart := func(p *part, decoder *encoding.BytesBlockDecoder) {
+ defer p.close()
+ pi := generatePartMergeIter()
+ defer releasePartMergeIter(pi)
+ pi.mustInitFromPart(p)
+ var got []blockMetadata
+ for pi.nextBlockMetadata() {
+ got = append(got, pi.block.bm)
+ require.Nil(t, pi.block.bm.tagProjection)
+ pi.mustLoadBlockData(decoder, &pi.block)
+ require.Equal(t, len(pi.block.bm.tagFamilies), len(pi.block.tagFamilies))
+ }
+
+ if !errors.Is(pi.error(), tt.wantErr) {
+ t.Errorf("Unexpected error: got %v, want %v", pi.err, tt.wantErr)
+ }
+
+ if diff := cmp.Diff(got, tt.want,
+ cmpopts.IgnoreFields(blockMetadata{}, "uncompressedSizeBytes"),
+ cmpopts.IgnoreFields(blockMetadata{}, "timestamps"),
+ cmpopts.IgnoreFields(blockMetadata{}, "elementIDs"),
+ cmpopts.IgnoreFields(blockMetadata{}, "tagFamilies"),
+ cmp.AllowUnexported(blockMetadata{}),
+ ); diff != "" {
+ t.Errorf("Unexpected blockMetadata (-got +want):\n%s", diff)
+ }
+ }
+ mp := generateMemPart()
+ releaseMemPart(mp)
+ mp.mustInitFromElements(tt.dps)
+
+ decoder := generateColumnValuesDecoder()
+ defer releaseColumnValuesDecoder(decoder)
+ p := openMemPart(mp)
+ verifyPart(p, decoder)
+ tmpDir, defFn := test.Space(require.New(t))
+ defer defFn()
+ epoch := uint64(1)
+ partPath := partPath(tmpDir, epoch)
+ fileSystem := fs.NewLocalFileSystem()
+ mp.mustFlush(fileSystem, partPath)
+ p = mustOpenFilePart(epoch, tmpDir, fileSystem)
+ verifyPart(p, decoder)
+ })
+ }
+}
diff --git a/banyand/stream/part_metadata.go b/banyand/stream/part_metadata.go
index 09394841..6ad86923 100644
--- a/banyand/stream/part_metadata.go
+++ b/banyand/stream/part_metadata.go
@@ -26,14 +26,14 @@ import (
)
type partMetadata struct {
- CompressedSizeBytes uint64
- UncompressedSizeBytes uint64
- TotalCount uint64
- BlocksCount uint64
- MinTimestamp int64
- MaxTimestamp int64
+ CompressedSizeBytes uint64 `json:"compressedSizeBytes"`
+ UncompressedSizeBytes uint64 `json:"uncompressedSizeBytes"`
+ TotalCount uint64 `json:"totalCount"`
+ BlocksCount uint64 `json:"blocksCount"`
+ MinTimestamp int64 `json:"minTimestamp"`
+ MaxTimestamp int64 `json:"maxTimestamp"`
// TODO: remove this?
- ID uint64
+ ID uint64 `json:"-"`
}
func (pm *partMetadata) reset() {
diff --git a/banyand/stream/part_test.go b/banyand/stream/part_test.go
index 670396d4..25c80908 100644
--- a/banyand/stream/part_test.go
+++ b/banyand/stream/part_test.go
@@ -97,7 +97,7 @@ func TestMustInitFromElements(t *testing.T) {
fileSystem := fs.NewLocalFileSystem()
mp.mustFlush(fileSystem, path)
- p := mustOpenFilePart(path, fileSystem)
+ p := mustOpenFilePart(epoch, tmpPath, fileSystem)
defer p.close()
assert.Equal(t, tt.want.BlocksCount, p.partMetadata.BlocksCount)
assert.Equal(t, tt.want.MinTimestamp, p.partMetadata.MinTimestamp)
diff --git a/banyand/stream/primary_metadata.go b/banyand/stream/primary_metadata.go
index d43b26c7..411dca83 100644
--- a/banyand/stream/primary_metadata.go
+++ b/banyand/stream/primary_metadata.go
@@ -24,6 +24,7 @@ import (
"github.com/apache/skywalking-banyandb/api/common"
"github.com/apache/skywalking-banyandb/pkg/compress/zstd"
"github.com/apache/skywalking-banyandb/pkg/encoding"
+ "github.com/apache/skywalking-banyandb/pkg/fs"
"github.com/apache/skywalking-banyandb/pkg/logger"
)
@@ -34,10 +35,6 @@ type primaryBlockMetadata struct {
dataBlock
}
-func (ph *primaryBlockMetadata) mightContainElement(seriesID common.SeriesID, timestamp common.ItemID) bool {
- return seriesID == ph.seriesID && timestamp >= common.ItemID(ph.minTimestamp) && timestamp <= common.ItemID(ph.maxTimestamp)
-}
-
// reset resets ih for subsequent re-use.
func (ph *primaryBlockMetadata) reset() {
ph.seriesID = 0
@@ -85,8 +82,8 @@ func (ph *primaryBlockMetadata) unmarshal(src []byte) ([]byte, error) {
return src[8:], nil
}
-func mustReadPrimaryBlockMetadata(dst []primaryBlockMetadata, r *reader) []primaryBlockMetadata {
- data, err := io.ReadAll(r)
+func mustReadPrimaryBlockMetadata(dst []primaryBlockMetadata, r fs.Reader) []primaryBlockMetadata {
+ data, err := io.ReadAll(r.StreamRead())
if err != nil {
logger.Panicf("cannot read primaryBlockMetadata entries from %s: %s", r.Path(), err)
}
diff --git a/banyand/stream/query.go b/banyand/stream/query.go
index 432f4b59..ce7a3325 100644
--- a/banyand/stream/query.go
+++ b/banyand/stream/query.go
@@ -31,6 +31,8 @@ type queryOptions struct {
pbv1.StreamQueryOptions
minTimestamp int64
maxTimestamp int64
+ includeMin bool
+ includeMax bool
}
func mustDecodeTagValue(valueType pbv1.ValueType, value []byte) *modelv1.TagValue {
@@ -166,7 +168,7 @@ func (qr *queryResult) Pull() *pbv1.Result {
if len(qr.data) == 1 {
r := &pbv1.Result{}
bc := qr.data[0]
- bc.copyAllTo(r)
+ bc.copyAllTo(r, qr.orderByTimestampDesc())
qr.data = qr.data[:0]
return r
}
@@ -229,14 +231,9 @@ func (qr *queryResult) merge() *pbv1.Result {
step = -1
}
result := &pbv1.Result{}
- var lastSid common.SeriesID
for qr.Len() > 0 {
topBC := qr.data[0]
- if lastSid != 0 && topBC.bm.seriesID != lastSid {
- return result
- }
- lastSid = topBC.bm.seriesID
topBC.copyTo(result)
topBC.idx += step
diff --git a/banyand/stream/service.go b/banyand/stream/service.go
index 0cecd2cc..0fa29a90 100644
--- a/banyand/stream/service.go
+++ b/banyand/stream/service.go
@@ -59,6 +59,7 @@ type service struct {
localPipeline queue.Queue
l *logger.Logger
root string
+ option option
}
func (s *service) Stream(metadata *commonv1.Metadata) (Stream, error) {
@@ -76,6 +77,7 @@ func (s *service) LoadGroup(name string) (resourceSchema.Group, bool) {
func (s *service) FlagSet() *run.FlagSet {
flagS := run.NewFlagSet("storage")
flagS.StringVar(&s.root, "stream-root-path", "/tmp", "the root path of database")
+ flagS.DurationVar(&s.option.flushTimeout, "stream-flush-timeout", defaultFlushTimeout, "the memory data timeout of stream")
return flagS
}
diff --git a/banyand/stream/snapshot.go b/banyand/stream/snapshot.go
index a271263d..266e089e 100644
--- a/banyand/stream/snapshot.go
+++ b/banyand/stream/snapshot.go
@@ -35,18 +35,28 @@ func (tst *tsTable) currentSnapshot() *snapshot {
return s
}
+type snapshotCreator rune
+
+const (
+ snapshotCreatorMemPart = iota
+ snapshotCreatorFlusher
+ snapshotCreatorMerger
+ snapshotCreatorMergedFlusher
+)
+
type snapshot struct {
- parts []*partWrapper
- epoch uint64
+ parts []*partWrapper
+ epoch uint64
+ creator snapshotCreator
ref int32
}
-func (s *snapshot) getParts(dst []*part, opts queryOptions) ([]*part, int) {
+func (s *snapshot) getParts(dst []*part, minTimestamp, maxTimestamp int64) ([]*part, int) {
var count int
for _, p := range s.parts {
pm := p.p.partMetadata
- if opts.maxTimestamp < pm.MinTimestamp || opts.minTimestamp > pm.MaxTimestamp {
+ if maxTimestamp < pm.MinTimestamp || minTimestamp > pm.MaxTimestamp {
continue
}
dst = append(dst, p.p)
@@ -96,6 +106,19 @@ func (s *snapshot) merge(nextEpoch uint64, nextParts map[uint64]*partWrapper) sn
return result
}
+func (s *snapshot) remove(nextEpoch uint64, merged map[uint64]struct{}) snapshot {
+ var result snapshot
+ result.epoch = nextEpoch
+ result.ref = 1
+ for i := 0; i < len(s.parts); i++ {
+ if _, ok := merged[s.parts[i].ID()]; !ok {
+ s.parts[i].incRef()
+ result.parts = append(result.parts, s.parts[i])
+ }
+ }
+ return result
+}
+
func snapshotName(snapshot uint64) string {
return fmt.Sprintf("%016x%s", snapshot, snapshotSuffix)
}
diff --git a/banyand/stream/snapshot_test.go b/banyand/stream/snapshot_test.go
index e2073374..0d8b80ce 100644
--- a/banyand/stream/snapshot_test.go
+++ b/banyand/stream/snapshot_test.go
@@ -21,6 +21,8 @@ import (
"testing"
"github.com/stretchr/testify/assert"
+
+ "github.com/apache/skywalking-banyandb/pkg/bytes"
)
func TestSnapshotGetParts(t *testing.T) {
@@ -110,7 +112,7 @@ func TestSnapshotGetParts(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- result, count := tt.snapshot.getParts(tt.dst, tt.opts)
+ result, count := tt.snapshot.getParts(tt.dst, tt.opts.minTimestamp, tt.opts.maxTimestamp)
assert.Equal(t, tt.expected, result)
assert.Equal(t, tt.count, count)
})
@@ -285,3 +287,137 @@ func TestSnapshotMerge(t *testing.T) {
})
}
}
+
+func TestSnapshotRemove(t *testing.T) {
+ tests := []struct {
+ snapshot *snapshot
+ mergedParts map[uint64]struct{}
+ name string
+ expected snapshot
+ nextEpoch uint64
+ closePrev bool
+ }{
+ {
+ name: "Test with empty snapshot and no parts to remove",
+ snapshot: &snapshot{
+ parts: []*partWrapper{},
+ },
+ nextEpoch: 1,
+ mergedParts: map[uint64]struct{}{},
+ expected: snapshot{
+ epoch: 1,
+ ref: 1,
+ parts: nil,
+ },
+ },
+ {
+ name: "Test with non-empty snapshot and no parts to remove",
+ snapshot: &snapshot{
+ parts: []*partWrapper{
+ {p: &part{partMetadata: partMetadata{ID: 1}}, ref: 1},
+ {p: &part{partMetadata: partMetadata{ID: 2}}, ref: 2},
+ },
+ },
+ nextEpoch: 1,
+ mergedParts: map[uint64]struct{}{},
+ expected: snapshot{
+ epoch: 1,
+ ref: 1,
+ parts: []*partWrapper{
+ {p: &part{partMetadata: partMetadata{ID: 1}}, ref: 2},
+ {p: &part{partMetadata: partMetadata{ID: 2}}, ref: 3},
+ },
+ },
+ },
+ {
+ name: "Test with non-empty snapshot and some parts to remove",
+ snapshot: &snapshot{
+ parts: []*partWrapper{
+ {p: &part{partMetadata: partMetadata{ID: 1}}, ref: 1},
+ {p: &part{partMetadata: partMetadata{ID: 2}}, ref: 2},
+ },
+ },
+ nextEpoch: 1,
+ mergedParts: map[uint64]struct{}{
+ 1: {},
+ },
+ expected: snapshot{
+ epoch: 1,
+ ref: 1,
+ parts: []*partWrapper{
+ {p: &part{partMetadata: partMetadata{ID: 2}}, ref: 3},
+ },
+ },
+ },
+ {
+ name: "Test with empty snapshot, no parts to remove, and closePrev=true",
+ snapshot: &snapshot{
+ parts: []*partWrapper{},
+ },
+ nextEpoch: 1,
+ mergedParts: map[uint64]struct{}{},
+ expected: snapshot{
+ epoch: 1,
+ ref: 1,
+ parts: nil,
+ },
+ closePrev: true,
+ },
+ {
+ name: "Test with non-empty snapshot, no parts to remove, and closePrev=true",
+ snapshot: &snapshot{
+ parts: []*partWrapper{
+ {p: &part{partMetadata: partMetadata{ID: 1}}, ref: 1},
+ {p: &part{partMetadata: partMetadata{ID: 2}}, ref: 2},
+ },
+ },
+ nextEpoch: 1,
+ mergedParts: map[uint64]struct{}{},
+ expected: snapshot{
+ epoch: 1,
+ ref: 1,
+ parts: []*partWrapper{
+ {p: &part{partMetadata: partMetadata{ID: 1}}, ref: 1},
+ {p: &part{partMetadata: partMetadata{ID: 2}}, ref: 2},
+ },
+ },
+ closePrev: true,
+ },
+ {
+ name: "Test with non-empty snapshot, some parts to remove, and closePrev=true",
+ snapshot: &snapshot{
+ parts: []*partWrapper{
+ {p: &part{
+ partMetadata: partMetadata{ID: 1},
+ timestamps: &bytes.Buffer{},
+ elementIDs: &bytes.Buffer{},
+ primary: &bytes.Buffer{},
+ }, ref: 1},
+ {p: &part{partMetadata: partMetadata{ID: 2}}, ref: 2},
+ },
+ },
+ nextEpoch: 1,
+ mergedParts: map[uint64]struct{}{
+ 1: {},
+ },
+ expected: snapshot{
+ epoch: 1,
+ ref: 1,
+ parts: []*partWrapper{
+ {p: &part{partMetadata: partMetadata{ID: 2}}, ref: 2},
+ },
+ },
+ closePrev: true,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ result := tt.snapshot.remove(tt.nextEpoch, tt.mergedParts)
+ if tt.closePrev {
+ tt.snapshot.decRef()
+ }
+ assert.Equal(t, tt.expected, result)
+ })
+ }
+}
diff --git a/banyand/stream/stream.go b/banyand/stream/stream.go
index 1235e7a7..ac35df34 100644
--- a/banyand/stream/stream.go
+++ b/banyand/stream/stream.go
@@ -28,13 +28,11 @@ import (
"time"
"go.uber.org/multierr"
- "google.golang.org/protobuf/types/known/timestamppb"
"github.com/apache/skywalking-banyandb/api/common"
commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
- streamv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/stream/v1"
"github.com/apache/skywalking-banyandb/banyand/internal/storage"
"github.com/apache/skywalking-banyandb/banyand/tsdb"
itersort "github.com/apache/skywalking-banyandb/pkg/iter/sort"
@@ -52,9 +50,14 @@ const (
maxUncompressedBlockSize = 2 * 1024 * 1024
maxUncompressedPrimaryBlockSize = 128 * 1024
- maxBlockLength = 8 * 1024
+ maxBlockLength = 8 * 1024
+ defaultFlushTimeout = 5 * time.Second
)
+type option struct {
+ flushTimeout time.Duration
+}
+
// Query allow to retrieve elements in a series of streams.
type Query interface {
LoadGroup(name string) (schema.Group, bool)
@@ -67,7 +70,8 @@ type Stream interface {
GetSchema() *databasev1.Stream
GetIndexRules() []*databasev1.IndexRule
Query(ctx context.Context, opts pbv1.StreamQueryOptions) (pbv1.StreamQueryResult, error)
- Sort(ctx context.Context, opts pbv1.StreamSortOptions) ([]*streamv1.Element, error)
+ Sort(ctx context.Context, opts pbv1.StreamSortOptions) (pbv1.StreamSortResult, error)
+ Filter(ctx context.Context, opts pbv1.StreamFilterOptions) (pbv1.StreamFilterResult, error)
}
var _ Stream = (*stream)(nil)
@@ -120,15 +124,86 @@ func NewItemIter(iters []*searcherIterator, s modelv1.Sort) itersort.Iterator[it
return itersort.NewItemIter[item](ii, false)
}
-// TODO: refactor to column-based query.
-func (s *stream) Sort(ctx context.Context, sqo pbv1.StreamSortOptions) (elems []*streamv1.Element, err error) {
+func (s *stream) Filter(ctx context.Context, sqo pbv1.StreamFilterOptions) (sfr pbv1.StreamFilterResult, err error) {
+ if sqo.TimeRange == nil || sqo.Entities == nil {
+ return nil, errors.New("invalid query options: timeRange and series are required")
+ }
+ if len(sqo.TagProjection) == 0 {
+ return nil, errors.New("invalid query options: tagProjection is required")
+ }
+ tsdb := s.databaseSupplier.SupplyTSDB().(storage.TSDB[*tsTable, option])
+ tabWrappers := tsdb.SelectTSTables(*sqo.TimeRange)
+ sort.Slice(tabWrappers, func(i, j int) bool {
+ return tabWrappers[i].GetTimeRange().Start.Before(tabWrappers[j].GetTimeRange().Start)
+ })
+ defer func() {
+ for i := range tabWrappers {
+ tabWrappers[i].DecRef()
+ }
+ }()
+
+ var seriesList pbv1.SeriesList
+ for _, entity := range sqo.Entities {
+ sl, lookupErr := tsdb.Lookup(ctx, &pbv1.Series{Subject: sqo.Name, EntityValues: entity})
+ if lookupErr != nil {
+ return nil, lookupErr
+ }
+ seriesList = seriesList.Merge(sl)
+ }
+
+ ces := newColumnElements()
+ for _, tw := range tabWrappers {
+ if len(ces.timestamp) >= sqo.MaxElementSize {
+ break
+ }
+ index := tw.Table().Index()
+ erl, err := index.Search(ctx, seriesList, sqo.Filter)
+ if err != nil {
+ return nil, err
+ }
+ if len(ces.timestamp)+len(erl) > sqo.MaxElementSize {
+ erl = erl[:sqo.MaxElementSize-len(ces.timestamp)]
+ }
+ for _, er := range erl {
+ e, err := tw.Table().getElement(er.seriesID, common.ItemID(er.timestamp), sqo.TagProjection)
+ if err != nil {
+ return nil, err
+ }
+ tagFamilies := make([]pbv1.TagFamily, 0)
+ for i, tf := range e.tagFamilies {
+ tagFamily := pbv1.TagFamily{
+ Name: tf.name,
+ }
+ for j, t := range tf.tags {
+ tag := pbv1.Tag{
+ Name: t.name,
+ }
+ if tag.Name == "" {
+ tag.Name = sqo.TagProjection[i].Names[j]
+ tag.Values = append(tag.Values, pbv1.NullTagValue)
+ } else {
+ tag.Values = append(tag.Values, mustDecodeTagValue(t.valueType, t.values[e.index]))
+ }
+ tagFamily.Tags = append(tagFamily.Tags, tag)
+ }
+ tagFamilies = append(tagFamilies, tagFamily)
+ }
+ ces.tagFamilies = append(ces.tagFamilies, tagFamilies)
+ ces.elementID = append(ces.elementID, e.elementID)
+ ces.timestamp = append(ces.timestamp, e.timestamp)
+ }
+ }
+ return ces, nil
+}
+
+func (s *stream) Sort(ctx context.Context, sqo pbv1.StreamSortOptions) (ssr pbv1.StreamSortResult, err error) {
if sqo.TimeRange == nil || sqo.Entities == nil {
return nil, errors.New("invalid query options: timeRange and series are required")
}
if len(sqo.TagProjection) == 0 {
return nil, errors.New("invalid query options: tagProjection is required")
}
- tsdb := s.databaseSupplier.SupplyTSDB().(storage.TSDB[*tsTable])
+ tsdb := s.databaseSupplier.SupplyTSDB().(storage.TSDB[*tsTable, option])
tabWrappers := tsdb.SelectTSTables(*sqo.TimeRange)
defer func() {
for i := range tabWrappers {
@@ -160,39 +235,48 @@ func (s *stream) Sort(ctx context.Context, sqo pbv1.StreamSortOptions) (elems []
}
if len(iters) == 0 {
- return elems, nil
+ return ssr, nil
}
it := NewItemIter(iters, sqo.Order.Sort)
defer func() {
err = multierr.Append(err, it.Close())
}()
+
+ ces := newColumnElements()
for it.Next() {
nextItem := it.Val()
e, err := nextItem.Element()
if err != nil {
return nil, err
}
- elem := &streamv1.Element{
- Timestamp: timestamppb.New(time.Unix(0, e.timestamp)),
- ElementId: e.elementID,
- }
- for _, tf := range e.tagFamilies {
- tagFamily := &modelv1.TagFamily{
+ tagFamilies := make([]pbv1.TagFamily, 0)
+ for i, tf := range e.tagFamilies {
+ tagFamily := pbv1.TagFamily{
Name: tf.name,
}
- elem.TagFamilies = append(elem.TagFamilies, tagFamily)
- for _, t := range tf.tags {
- tagFamily.Tags = append(tagFamily.Tags, &modelv1.Tag{
- Key: t.name,
- Value: mustDecodeTagValue(t.valueType, t.values[e.index]),
- })
+ for j, t := range tf.tags {
+ tag := pbv1.Tag{
+ Name: t.name,
+ }
+ if tag.Name == "" {
+ tag.Name = sqo.TagProjection[i].Names[j]
+ tag.Values = append(tag.Values, pbv1.NullTagValue)
+ } else {
+ tag.Values = append(tag.Values, mustDecodeTagValue(t.valueType, t.values[e.index]))
+ }
+ tagFamily.Tags = append(tagFamily.Tags, tag)
}
+ tagFamilies = append(tagFamilies, tagFamily)
+ }
+ ces.tagFamilies = append(ces.tagFamilies, tagFamilies)
+ ces.elementID = append(ces.elementID, e.elementID)
+ ces.timestamp = append(ces.timestamp, e.timestamp)
+ if len(ces.timestamp) >= sqo.MaxElementSize {
+ break
}
- elems = append(elems, elem)
- // TODO: break if reach limit
}
- return elems, nil
+ return ces, nil
}
func (s *stream) Query(ctx context.Context, sqo pbv1.StreamQueryOptions) (pbv1.StreamQueryResult, error) {
@@ -202,7 +286,7 @@ func (s *stream) Query(ctx context.Context, sqo pbv1.StreamQueryOptions) (pbv1.S
if len(sqo.TagProjection) == 0 {
return nil, errors.New("invalid query options: tagProjection is required")
}
- tsdb := s.databaseSupplier.SupplyTSDB().(storage.TSDB[*tsTable])
+ tsdb := s.databaseSupplier.SupplyTSDB().(storage.TSDB[*tsTable, option])
tabWrappers := tsdb.SelectTSTables(*sqo.TimeRange)
defer func() {
for i := range tabWrappers {
@@ -227,6 +311,8 @@ func (s *stream) Query(ctx context.Context, sqo pbv1.StreamQueryOptions) (pbv1.S
StreamQueryOptions: sqo,
minTimestamp: sqo.TimeRange.Start.UnixNano(),
maxTimestamp: sqo.TimeRange.End.UnixNano(),
+ includeMin: sqo.TimeRange.IncludeStart,
+ includeMax: sqo.TimeRange.IncludeEnd,
}
var n int
for i := range tabWrappers {
@@ -234,7 +320,7 @@ func (s *stream) Query(ctx context.Context, sqo pbv1.StreamQueryOptions) (pbv1.S
if s == nil {
continue
}
- parts, n = s.getParts(parts, qo)
+ parts, n = s.getParts(parts, qo.minTimestamp, qo.maxTimestamp)
if n < 1 {
s.decRef()
continue
diff --git a/banyand/stream/tag.go b/banyand/stream/tag.go
index 00584c7f..f9ad8740 100644
--- a/banyand/stream/tag.go
+++ b/banyand/stream/tag.go
@@ -91,6 +91,29 @@ func (t *tag) mustReadValues(decoder *encoding.BytesBlockDecoder, reader fs.Read
}
}
+func (t *tag) mustSeqReadValues(decoder *encoding.BytesBlockDecoder, reader *seqReader, cm tagMetadata, count uint64) {
+ t.name = cm.name
+ t.valueType = cm.valueType
+ if cm.offset != reader.bytesRead {
+ logger.Panicf("%s: offset mismatch: %d vs %d", reader.Path(), cm.offset, reader.bytesRead)
+ }
+ valuesSize := cm.size
+ if valuesSize > maxValuesBlockSize {
+ logger.Panicf("%s: block size cannot exceed %d bytes; got %d bytes", reader.Path(), maxValuesBlockSize, valuesSize)
+ }
+
+ bb := bigValuePool.Generate()
+ defer bigValuePool.Release(bb)
+
+ bb.Buf = bytes.ResizeOver(bb.Buf, int(valuesSize))
+ reader.mustReadFull(bb.Buf)
+ var err error
+ t.values, err = decoder.Decode(t.values[:0], bb.Buf, count)
+ if err != nil {
+ logger.Panicf("%s: cannot decode values: %v", reader.Path(), err)
+ }
+}
+
var bigValuePool bytes.BufferPool
type tagFamily struct {
diff --git a/banyand/stream/tstable.go b/banyand/stream/tstable.go
index 3b6302a9..d49a50fa 100644
--- a/banyand/stream/tstable.go
+++ b/banyand/stream/tstable.go
@@ -28,6 +28,7 @@ import (
"sort"
"strconv"
"sync"
+ "sync/atomic"
"time"
"github.com/apache/skywalking-banyandb/api/common"
@@ -52,8 +53,11 @@ type tsTable struct {
snapshot *snapshot
introductions chan *introduction
loopCloser *run.Closer
+ p common.Position
root string
gc garbageCleaner
+ curPartID uint64
+ option option
sync.RWMutex
}
@@ -62,21 +66,25 @@ func (tst *tsTable) loadSnapshot(epoch uint64, loadedParts []uint64) {
snp := snapshot{
epoch: epoch,
}
- for _, partName := range loadedParts {
+ for _, id := range loadedParts {
var find bool
for j := range parts {
- if partName == parts[j] {
+ if id == parts[j] {
find = true
break
}
}
if !find {
- tst.gc.submitParts(partName)
+ tst.gc.submitParts(id)
+ }
+ p := mustOpenFilePart(id, tst.root, tst.fileSystem)
+ p.partMetadata.ID = id
+ snp.parts = append(snp.parts, newPartWrapper(nil, p))
+ if tst.curPartID < id {
+ tst.curPartID = id
}
- p := mustOpenFilePart(partPath(tst.root, partName), tst.fileSystem)
- p.partMetadata.ID = partName
- snp.parts = append(snp.parts, newPartWrapper(nil, p, tst.fileSystem))
}
+ tst.gc.registerSnapshot(&snp)
tst.gc.clean()
if len(snp.parts) < 1 {
return
@@ -86,12 +94,15 @@ func (tst *tsTable) loadSnapshot(epoch uint64, loadedParts []uint64) {
}
func (tst *tsTable) startLoop(cur uint64) {
- tst.loopCloser = run.NewCloser(3)
+ tst.loopCloser = run.NewCloser(1 + 3)
tst.introductions = make(chan *introduction)
flushCh := make(chan *flusherIntroduction)
+ mergeCh := make(chan *mergerIntroduction)
introducerWatcher := make(watcher.Channel, 1)
- go tst.introducerLoop(flushCh, introducerWatcher, cur+1)
- go tst.flusherLoop(flushCh, introducerWatcher, cur)
+ flusherWatcher := make(watcher.Channel, 1)
+ go tst.introducerLoop(flushCh, mergeCh, introducerWatcher, cur+1)
+ go tst.flusherLoop(flushCh, mergeCh, introducerWatcher, flusherWatcher, cur)
+ go tst.mergeLoop(mergeCh, flusherWatcher)
}
func parseEpoch(epochStr string) (uint64, error) {
@@ -142,17 +153,22 @@ func (tst *tsTable) mustReadSnapshot(snapshot uint64) []uint64 {
return result
}
-func newTSTable(fileSystem fs.FileSystem, rootPath string, _ common.Position, l *logger.Logger, _ timestamp.TimeRange) (*tsTable, error) {
- var tst tsTable
+func newTSTable(fileSystem fs.FileSystem, rootPath string, p common.Position,
+ l *logger.Logger, _ timestamp.TimeRange, option option,
+) (*tsTable, error) {
index, err := newElementIndex(context.TODO(), rootPath)
if err != nil {
return nil, err
}
- tst.index = index
- tst.fileSystem = fileSystem
- tst.root = rootPath
- tst.l = l
- tst.gc.parent = &tst
+ tst := tsTable{
+ index: index,
+ fileSystem: fileSystem,
+ root: rootPath,
+ option: option,
+ l: l,
+ p: p,
+ }
+ tst.gc.init(&tst)
ee := fileSystem.ReadDir(rootPath)
if len(ee) == 0 {
t := &tst
@@ -183,7 +199,6 @@ func newTSTable(fileSystem fs.FileSystem, rootPath string, _ common.Position, l
continue
}
loadedSnapshots = append(loadedSnapshots, snapshot)
- tst.gc.registerSnapshot(snapshot)
}
for i := range needToDelete {
if err := fileSystem.DeleteFile(filepath.Join(rootPath, needToDelete[i])); err != nil {
@@ -216,9 +231,11 @@ func (tst *tsTable) Close() error {
}
tst.RLock()
defer tst.RUnlock()
- if tst.snapshot != nil {
- tst.snapshot.decRef()
+ if tst.snapshot == nil {
+ return tst.index.Close()
}
+ tst.snapshot.decRef()
+ tst.snapshot = nil
return tst.index.Close()
}
@@ -234,7 +251,8 @@ func (tst *tsTable) mustAddElements(es *elements) {
ind := generateIntroduction()
defer releaseIntroduction(ind)
ind.applied = make(chan struct{})
- ind.memPart = newPartWrapper(mp, p, tst.fileSystem)
+ ind.memPart = newPartWrapper(mp, p)
+ ind.memPart.p.partMetadata.ID = atomic.AddUint64(&tst.curPartID, 1)
select {
case tst.introductions <- ind:
case <-tst.loopCloser.CloseNotify():
diff --git a/banyand/stream/tstable_test.go b/banyand/stream/tstable_test.go
index 866da4e5..b0b4d303 100644
--- a/banyand/stream/tstable_test.go
+++ b/banyand/stream/tstable_test.go
@@ -39,15 +39,15 @@ import (
"github.com/apache/skywalking-banyandb/pkg/watcher"
)
-func Test_tsTable_mustAddElements(t *testing.T) {
+func Test_tsTable_mustAddDataPoints(t *testing.T) {
tests := []struct {
- name string
- esList []*elements
- want int
+ name string
+ dpsList []*elements
+ want int
}{
{
name: "Test with empty elements",
- esList: []*elements{
+ dpsList: []*elements{
{
timestamps: []int64{},
elementIDs: []string{},
@@ -59,10 +59,10 @@ func Test_tsTable_mustAddElements(t *testing.T) {
},
{
name: "Test with one item in elements",
- esList: []*elements{
+ dpsList: []*elements{
{
timestamps: []int64{1},
- elementIDs: []string{"1"},
+ elementIDs: []string{"0"},
seriesIDs: []common.SeriesID{1},
tagFamilies: [][]tagValues{
{
@@ -79,8 +79,8 @@ func Test_tsTable_mustAddElements(t *testing.T) {
want: 1,
},
{
- name: "Test with multiple calls to mustAddElements",
- esList: []*elements{
+ name: "Test with multiple calls to mustAddDataPoints",
+ dpsList: []*elements{
esTS1,
esTS2,
},
@@ -92,16 +92,17 @@ func Test_tsTable_mustAddElements(t *testing.T) {
tmpPath, _ := test.Space(require.New(t))
index, _ := newElementIndex(context.TODO(), tmpPath)
tst := &tsTable{
+ index: index,
loopCloser: run.NewCloser(2),
introductions: make(chan *introduction),
- index: index,
}
flushCh := make(chan *flusherIntroduction)
+ mergeCh := make(chan *mergerIntroduction)
introducerWatcher := make(watcher.Channel, 1)
- go tst.introducerLoop(flushCh, introducerWatcher, 1)
+ go tst.introducerLoop(flushCh, mergeCh, introducerWatcher, 1)
defer tst.Close()
- for _, es := range tt.esList {
- tst.mustAddElements(es)
+ for _, dps := range tt.dpsList {
+ tst.mustAddElements(dps)
time.Sleep(100 * time.Millisecond)
}
s := tst.currentSnapshot()
@@ -124,142 +125,264 @@ func Test_tsTable_mustAddElements(t *testing.T) {
}
func Test_tstIter(t *testing.T) {
- tests := []struct {
+ type testCtx struct {
wantErr error
name string
- esList []*elements
+ dpsList []*elements
sids []common.SeriesID
want []blockMetadata
minTimestamp int64
maxTimestamp int64
- }{
- {
- name: "Test with no elements",
- esList: []*elements{},
- sids: []common.SeriesID{1, 2, 3},
- minTimestamp: 1,
- maxTimestamp: 1,
- },
- {
- name: "Test with single part",
- esList: []*elements{esTS1},
- sids: []common.SeriesID{1, 2, 3},
- minTimestamp: 1,
- maxTimestamp: 1,
- want: []blockMetadata{
- {seriesID: 1, count: 1, uncompressedSizeBytes: 881},
- {seriesID: 2, count: 1, uncompressedSizeBytes: 55},
- {seriesID: 3, count: 1, uncompressedSizeBytes: 8},
- },
- },
- {
- name: "Test with multiple parts with different ts",
- esList: []*elements{esTS1, esTS2},
- sids: []common.SeriesID{1, 2, 3},
- minTimestamp: 1,
- maxTimestamp: 2,
- want: []blockMetadata{
- {seriesID: 1, count: 1, uncompressedSizeBytes: 881},
- {seriesID: 1, count: 1, uncompressedSizeBytes: 881},
- {seriesID: 2, count: 1, uncompressedSizeBytes: 55},
- {seriesID: 2, count: 1, uncompressedSizeBytes: 55},
- {seriesID: 3, count: 1, uncompressedSizeBytes: 8},
- {seriesID: 3, count: 1, uncompressedSizeBytes: 8},
- },
- },
- {
- name: "Test with multiple parts with same ts",
- esList: []*elements{esTS1, esTS1},
- sids: []common.SeriesID{1, 2, 3},
- minTimestamp: 1,
- maxTimestamp: 2,
- want: []blockMetadata{
- {seriesID: 1, count: 1, uncompressedSizeBytes: 881},
- {seriesID: 1, count: 1, uncompressedSizeBytes: 881},
- {seriesID: 2, count: 1, uncompressedSizeBytes: 55},
- {seriesID: 2, count: 1, uncompressedSizeBytes: 55},
- {seriesID: 3, count: 1, uncompressedSizeBytes: 8},
- {seriesID: 3, count: 1, uncompressedSizeBytes: 8},
- },
- },
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- verify := func(tst *tsTable) uint64 {
- defer tst.Close()
- s := tst.currentSnapshot()
- if s == nil {
- s = new(snapshot)
- }
- defer s.decRef()
- pp, n := s.getParts(nil, queryOptions{
- minTimestamp: tt.minTimestamp,
- maxTimestamp: tt.maxTimestamp,
- })
- require.Equal(t, len(s.parts), n)
- ti := &tstIter{}
- ti.init(pp, tt.sids, tt.minTimestamp, tt.maxTimestamp)
- var got []blockMetadata
- for ti.nextBlock() {
- if ti.piHeap[0].curBlock.seriesID == 0 {
- t.Errorf("Expected curBlock to be initialized, but it was nil")
- }
- got = append(got, ti.piHeap[0].curBlock)
- }
- if !errors.Is(ti.Error(), tt.wantErr) {
- t.Errorf("Unexpected error: got %v, want %v", ti.err, tt.wantErr)
- }
-
- if diff := cmp.Diff(got, tt.want,
- cmpopts.IgnoreFields(blockMetadata{}, "timestamps"),
- cmpopts.IgnoreFields(blockMetadata{}, "elementIDs"),
- cmpopts.IgnoreFields(blockMetadata{}, "tagFamilies"),
- cmp.AllowUnexported(blockMetadata{}),
- ); diff != "" {
- t.Errorf("Unexpected blockMetadata (-got +want):\n%s", diff)
- }
- return s.epoch
+ verify := func(t *testing.T, tt testCtx, tst *tsTable) uint64 {
+ defer tst.Close()
+ s := tst.currentSnapshot()
+ if s == nil {
+ s = new(snapshot)
+ }
+ defer s.decRef()
+ pp, n := s.getParts(nil, tt.minTimestamp, tt.maxTimestamp)
+ require.Equal(t, len(s.parts), n)
+ ti := &tstIter{}
+ ti.init(pp, tt.sids, tt.minTimestamp, tt.maxTimestamp)
+ var got []blockMetadata
+ for ti.nextBlock() {
+ if ti.piHeap[0].curBlock.seriesID == 0 {
+ t.Errorf("Expected curBlock to be initialized, but it was nil")
}
+ got = append(got, ti.piHeap[0].curBlock)
+ }
+
+ if !errors.Is(ti.Error(), tt.wantErr) {
+ t.Errorf("Unexpected error: got %v, want %v", ti.err, tt.wantErr)
+ }
- t.Run("memory snapshot", func(t *testing.T) {
+ if diff := cmp.Diff(got, tt.want,
+ cmpopts.IgnoreFields(blockMetadata{}, "timestamps"),
+ cmpopts.IgnoreFields(blockMetadata{}, "elementIDs"),
+ cmpopts.IgnoreFields(blockMetadata{}, "tagFamilies"),
+ cmp.AllowUnexported(blockMetadata{}),
+ ); diff != "" {
+ t.Errorf("Unexpected blockMetadata (-got +want):\n%s", diff)
+ }
+ return s.epoch
+ }
+
+ t.Run("memory snapshot", func(t *testing.T) {
+ tests := []testCtx{
+ {
+ name: "Test with no data points",
+ dpsList: []*elements{},
+ sids: []common.SeriesID{1, 2, 3},
+ minTimestamp: 1,
+ maxTimestamp: 1,
+ },
+ {
+ name: "Test with single part",
+ dpsList: []*elements{esTS1},
+ sids: []common.SeriesID{1, 2, 3},
+ minTimestamp: 1,
+ maxTimestamp: 1,
+ want: []blockMetadata{
+ {seriesID: 1, count: 1, uncompressedSizeBytes: 881},
+ {seriesID: 2, count: 1, uncompressedSizeBytes: 55},
+ {seriesID: 3, count: 1, uncompressedSizeBytes: 8},
+ },
+ },
+ {
+ name: "Test with multiple parts with different ts",
+ dpsList: []*elements{esTS1, esTS2},
+ sids: []common.SeriesID{1, 2, 3},
+ minTimestamp: 1,
+ maxTimestamp: 2,
+ want: []blockMetadata{
+ {seriesID: 1, count: 1, uncompressedSizeBytes: 881},
+ {seriesID: 1, count: 1, uncompressedSizeBytes: 881},
+ {seriesID: 2, count: 1, uncompressedSizeBytes: 55},
+ {seriesID: 2, count: 1, uncompressedSizeBytes: 55},
+ {seriesID: 3, count: 1, uncompressedSizeBytes: 8},
+ {seriesID: 3, count: 1, uncompressedSizeBytes: 8},
+ },
+ },
+ {
+ name: "Test with multiple parts with same ts",
+ dpsList: []*elements{esTS1, esTS1},
+ sids: []common.SeriesID{1, 2, 3},
+ minTimestamp: 1,
+ maxTimestamp: 2,
+ want: []blockMetadata{
+ {seriesID: 1, count: 1, uncompressedSizeBytes: 881},
+ {seriesID: 1, count: 1, uncompressedSizeBytes: 881},
+ {seriesID: 2, count: 1, uncompressedSizeBytes: 55},
+ {seriesID: 2, count: 1, uncompressedSizeBytes: 55},
+ {seriesID: 3, count: 1, uncompressedSizeBytes: 8},
+ {seriesID: 3, count: 1, uncompressedSizeBytes: 8},
+ },
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ tmpPath, defFn := test.Space(require.New(t))
+ index, _ := newElementIndex(context.TODO(), tmpPath)
+ defer defFn()
tst := &tsTable{
+ index: index,
loopCloser: run.NewCloser(2),
introductions: make(chan *introduction),
+ fileSystem: fs.NewLocalFileSystem(),
+ root: tmpPath,
}
- tmpPath, _ := test.Space(require.New(t))
- index, _ := newElementIndex(context.TODO(), tmpPath)
- tst.index = index
+ tst.gc.init(tst)
flushCh := make(chan *flusherIntroduction)
+ mergeCh := make(chan *mergerIntroduction)
introducerWatcher := make(watcher.Channel, 1)
- go tst.introducerLoop(flushCh, introducerWatcher, 1)
- for _, es := range tt.esList {
- tst.mustAddElements(es)
+ go tst.introducerLoop(flushCh, mergeCh, introducerWatcher, 1)
+ for _, dps := range tt.dpsList {
+ tst.mustAddElements(dps)
time.Sleep(100 * time.Millisecond)
}
- verify(tst)
+ verify(t, tt, tst)
})
+ }
+ })
- t.Run("file snapshot", func(t *testing.T) {
- tmpPath, defFn := test.Space(require.New(t))
- fileSystem := fs.NewLocalFileSystem()
- defer defFn()
+ t.Run("file snapshot", func(t *testing.T) {
+ tests := []testCtx{
+ {
+ name: "Test with no data points",
+ dpsList: []*elements{},
+ sids: []common.SeriesID{1, 2, 3},
+ minTimestamp: 1,
+ maxTimestamp: 1,
+ },
+ {
+ name: "Test with single part",
+ dpsList: []*elements{esTS1},
+ sids: []common.SeriesID{1, 2, 3},
+ minTimestamp: 1,
+ maxTimestamp: 1,
+ want: []blockMetadata{
+ {seriesID: 1, count: 1, uncompressedSizeBytes: 881},
+ {seriesID: 2, count: 1, uncompressedSizeBytes: 55},
+ {seriesID: 3, count: 1, uncompressedSizeBytes: 8},
+ },
+ },
+ {
+ name: "Test with multiple parts with different ts, the block will be merged",
+ dpsList: []*elements{esTS1, esTS2, esTS2},
+ sids: []common.SeriesID{1, 2, 3},
+ minTimestamp: 1,
+ maxTimestamp: 2,
+ want: []blockMetadata{
+ {seriesID: 1, count: 2, uncompressedSizeBytes: 1762},
+ {seriesID: 2, count: 2, uncompressedSizeBytes: 110},
+ {seriesID: 3, count: 2, uncompressedSizeBytes: 16},
+ },
+ },
+ {
+ name: "Test with multiple parts with same ts, duplicated blocks will be merged",
+ dpsList: []*elements{esTS1, esTS1},
+ sids: []common.SeriesID{1, 2, 3},
+ minTimestamp: 1,
+ maxTimestamp: 2,
+ want: []blockMetadata{
+ {seriesID: 1, count: 1, uncompressedSizeBytes: 881},
+ {seriesID: 2, count: 1, uncompressedSizeBytes: 55},
+ {seriesID: 3, count: 1, uncompressedSizeBytes: 8},
+ },
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ t.Run("merging on the fly", func(t *testing.T) {
+ tmpPath, defFn := test.Space(require.New(t))
+ fileSystem := fs.NewLocalFileSystem()
+ defer defFn()
- tst, err := newTSTable(fileSystem, tmpPath, common.Position{}, logger.GetLogger("test"), timestamp.TimeRange{})
- require.NoError(t, err)
- for _, es := range tt.esList {
- tst.mustAddElements(es)
- time.Sleep(100 * time.Millisecond)
- }
- epoch := verify(tst)
+ tst, err := newTSTable(fileSystem, tmpPath, common.Position{},
+ logger.GetLogger("test"), timestamp.TimeRange{}, option{flushTimeout: 0})
+ require.NoError(t, err)
+ for i, dps := range tt.dpsList {
+ tst.mustAddElements(dps)
+ for {
+ snp := tst.currentSnapshot()
+ if snp == nil {
+ t.Logf("waiting for snapshot %d to be introduced", i)
+ time.Sleep(100 * time.Millisecond)
+ continue
+ }
+ if snp.creator != snapshotCreatorMemPart {
+ snp.decRef()
+ break
+ }
+ t.Logf("waiting for snapshot %d to be flushed or merged: current creator:%d, parts: %+v",
+ i, snp.creator, snp.parts)
+ snp.decRef()
+ time.Sleep(100 * time.Millisecond)
+ }
+ }
+ // wait until some parts are merged
+ if len(tt.dpsList) > 0 {
+ for {
+ snp := tst.currentSnapshot()
+ if snp == nil {
+ time.Sleep(100 * time.Millisecond)
+ continue
+ }
+ if len(snp.parts) == 1 || len(snp.parts) < len(tt.dpsList) {
+ snp.decRef()
+ break
+ }
+ t.Logf("waiting for snapshot to be merged: current creator:%d, parts: %+v", snp.creator, snp.parts)
+ snp.decRef()
+ time.Sleep(100 * time.Millisecond)
+ }
+ }
+ verify(t, tt, tst)
+ })
- // reopen the table
- tst, err = newTSTable(fileSystem, tmpPath, common.Position{}, logger.GetLogger("test"), timestamp.TimeRange{})
- require.NoError(t, err)
- assert.Equal(t, epoch, verify(tst))
+ t.Run("merging on close", func(t *testing.T) {
+ t.Skip("the test is flaky due to unpredictable merge loop schedule.")
+ tmpPath, defFn := test.Space(require.New(t))
+ fileSystem := fs.NewLocalFileSystem()
+ defer defFn()
+
+ tst, err := newTSTable(fileSystem, tmpPath, common.Position{},
+ logger.GetLogger("test"), timestamp.TimeRange{}, option{flushTimeout: defaultFlushTimeout})
+ require.NoError(t, err)
+ for _, dps := range tt.dpsList {
+ tst.mustAddElements(dps)
+ time.Sleep(100 * time.Millisecond)
+ }
+ // wait until the introducer is done
+ if len(tt.dpsList) > 0 {
+ for {
+ snp := tst.currentSnapshot()
+ if snp == nil {
+ time.Sleep(100 * time.Millisecond)
+ continue
+ }
+ if len(snp.parts) == len(tt.dpsList) {
+ snp.decRef()
+ tst.Close()
+ break
+ }
+ snp.decRef()
+ time.Sleep(100 * time.Millisecond)
+ }
+ } else {
+ tst.Close()
+ }
+ // reopen the table
+ tst, err = newTSTable(fileSystem, tmpPath, common.Position{},
+ logger.GetLogger("test"), timestamp.TimeRange{}, option{flushTimeout: defaultFlushTimeout})
+ require.NoError(t, err)
+ verify(t, tt, tst)
+ })
})
- })
- }
+ }
+ })
}
// nolint: unused
diff --git a/banyand/stream/write.go b/banyand/stream/write.go
index 9029aaac..af02f2a2 100644
--- a/banyand/stream/write.go
+++ b/banyand/stream/write.go
@@ -112,6 +112,7 @@ func (w *writeCallback) handle(dst map[string]*elementsInGroup, writeEvent *stre
et.elements.seriesIDs = append(et.elements.seriesIDs, series.ID)
tagFamilies := make([]tagValues, len(stm.schema.TagFamilies))
+ tagFamiliesForIndexWrite := make([]tagValues, len(stm.schema.TagFamilies))
for i := range stm.GetSchema().GetTagFamilies() {
var tagFamily *modelv1.TagFamilyForWrite
if len(req.Element.TagFamilies) <= i {
@@ -128,6 +129,14 @@ func (w *writeCallback) handle(dst map[string]*elementsInGroup, writeEvent *stre
} else {
tagValue = tagFamily.Tags[j]
}
+ tagFamiliesForIndexWrite[i].values = append(tagFamiliesForIndexWrite[i].values, encodeTagValue(
+ tagFamilySpec.Tags[j].Name,
+ tagFamilySpec.Tags[j].Type,
+ tagValue,
+ ))
+ if tagFamilySpec.Tags[j].IndexedOnly {
+ continue
+ }
tagFamilies[i].values = append(tagFamilies[i].values, encodeTagValue(
tagFamilySpec.Tags[j].Name,
tagFamilySpec.Tags[j].Type,
@@ -138,16 +147,16 @@ func (w *writeCallback) handle(dst map[string]*elementsInGroup, writeEvent *stre
et.elements.tagFamilies = append(et.elements.tagFamilies, tagFamilies)
var fields []index.Field
- for _, ruleIndex := range stm.indexRuleLocators {
- tv := getIndexValue(ruleIndex, tagFamilies)
+ for _, indexRule := range stm.indexRuleLocators {
+ tv := getIndexValue(indexRule, tagFamiliesForIndexWrite)
if tv == nil {
continue
}
if tv.value != nil {
fields = append(fields, index.Field{
Key: index.FieldKey{
- IndexRuleID: ruleIndex.Rule.GetMetadata().GetId(),
- Analyzer: ruleIndex.Rule.Analyzer,
+ IndexRuleID: indexRule.Rule.GetMetadata().GetId(),
+ Analyzer: indexRule.Rule.Analyzer,
SeriesID: series.ID,
},
Term: tv.value,
@@ -155,7 +164,7 @@ func (w *writeCallback) handle(dst map[string]*elementsInGroup, writeEvent *stre
continue
}
for _, val := range tv.valueArr {
- rule := ruleIndex.Rule
+ rule := indexRule.Rule
fields = append(fields, index.Field{
Key: index.FieldKey{
IndexRuleID: rule.GetMetadata().GetId(),
diff --git a/pkg/bytes/buffer.go b/pkg/bytes/buffer.go
index 713632f3..be4db36b 100644
--- a/pkg/bytes/buffer.go
+++ b/pkg/bytes/buffer.go
@@ -62,11 +62,41 @@ func (b *Buffer) Read(offset int64, buffer []byte) (int, error) {
return n, err
}
+// StreamRead implements fs.Reader.
+func (b *Buffer) StreamRead() io.Reader {
+ return &reader{bb: b}
+}
+
// Reset resets the buffer.
func (b *Buffer) Reset() {
b.Buf = b.Buf[:0]
}
+type reader struct {
+ bb *Buffer
+
+ readOffset int
+}
+
+func (r *reader) Path() string {
+ return r.bb.Path()
+}
+
+func (r *reader) Read(p []byte) (int, error) {
+ var err error
+ n := copy(p, r.bb.Buf[r.readOffset:])
+ if n < len(p) {
+ err = io.EOF
+ }
+ r.readOffset += n
+ return n, err
+}
+
+func (r *reader) MustClose() {
+ r.bb = nil
+ r.readOffset = 0
+}
+
// BufferPool is a pool of Buffer.
type BufferPool struct {
p sync.Pool
diff --git a/pkg/fs/file_system.go b/pkg/fs/file_system.go
index da483846..4efca46d 100644
--- a/pkg/fs/file_system.go
+++ b/pkg/fs/file_system.go
@@ -19,7 +19,7 @@
package fs
import (
- "bufio"
+ "io"
"github.com/apache/skywalking-banyandb/pkg/logger"
)
@@ -29,13 +29,6 @@ const moduleName string = "filesystem"
// Mode contains permission of file and directory.
type Mode uint64
-// Iter is used for streaming read.
-type Iter struct {
- fileName string
- reader *bufio.Reader
- buffer []byte
-}
-
// Writer allows writing data to a file.
type Writer interface {
// Append mode, which adds new data to the end of a file.
@@ -50,6 +43,8 @@ type Writer interface {
type Reader interface {
// Read the entire file using streaming read.
Read(offset int64, buffer []byte) (int, error)
+ // Read the entire file using streaming read.
+ StreamRead() io.Reader
// Returns the absolute path of the file.
Path() string
// Close File.
@@ -72,8 +67,6 @@ type File interface {
Writev(iov *[][]byte) (int, error)
// Reading contiguous regions of a file and dispersing them into discontinuous buffers.
Readv(offset int64, iov *[][]byte) (int, error)
- // Read the entire file using streaming read.
- StreamRead(buffer []byte) (*Iter, error)
// Get the file written data's size and return an error if the file does not exist. The unit of file size is Byte.
Size() (int64, error)
// Returns the absolute path of the file.
@@ -108,6 +101,8 @@ type FileSystem interface {
MustRMAll(path string)
// SyncPath the directory of file.
SyncPath(path string)
+ // MustGetFreeSpace returns the free space of the file system.
+ MustGetFreeSpace(path string) uint64
}
// DirEntry is the interface that wraps the basic information about a file or directory.
@@ -119,6 +114,15 @@ type DirEntry interface {
IsDir() bool
}
+// MustCreateFile creates a new file with the specified name and permission.
+func MustCreateFile(fs FileSystem, path string, permission Mode) File {
+ f, err := fs.CreateFile(path, permission)
+ if err != nil {
+ logger.GetLogger().Panic().Err(err).Str("path", path).Msg("cannot create file")
+ }
+ return f
+}
+
// MustFlush flushes all data to one file and panics if it cannot write all data.
func MustFlush(fs FileSystem, buffer []byte, name string, permission Mode) {
n, err := fs.Write(buffer, name, permission)
diff --git a/pkg/fs/local_file_system.go b/pkg/fs/local_file_system.go
index 006d7245..b3438265 100644
--- a/pkg/fs/local_file_system.go
+++ b/pkg/fs/local_file_system.go
@@ -27,6 +27,8 @@ import (
"path/filepath"
"time"
+ "github.com/shirou/gopsutil/v3/disk"
+
"github.com/apache/skywalking-banyandb/pkg/logger"
)
@@ -281,8 +283,17 @@ func (fs *localFileSystem) MustRMAll(path string) {
fs.logger.Panic().Str("path", path).Msg("failed to remove all files under path")
}
+func (fs *localFileSystem) MustGetFreeSpace(path string) uint64 {
+ usage, err := disk.Usage(path)
+ if err != nil {
+ fs.logger.Panic().Str("path", path).Err(err).Msg("failed to get disk usage")
+ }
+ return usage.Free
+}
+
// Write adds new data to the end of a file.
func (file *LocalFile) Write(buffer []byte) (int, error) {
+ // TODO: use bufio.Writer to optimize performance.
size, err := file.file.Write(buffer)
switch {
case err == nil:
@@ -363,9 +374,9 @@ func (file *LocalFile) Readv(offset int64, iov *[][]byte) (int, error) {
}
// StreamRead is used to read the entire file using streaming read.
-func (file *LocalFile) StreamRead(buffer []byte) (*Iter, error) {
+func (file *LocalFile) StreamRead() io.Reader {
reader := bufio.NewReader(file.file)
- return &Iter{reader: reader, buffer: buffer, fileName: file.file.Name()}, nil
+ return &seqReader{reader: reader, fileName: file.file.Name()}
}
// Size is used to get the file written data's size and return an error if the file does not exist. The unit of file size is Byte.
@@ -414,11 +425,15 @@ func (file *LocalFile) Close() error {
return nil
}
-// Next is used to obtain the next data of stream.
-func (iter *Iter) Next() (int, error) {
- rsize, err := iter.reader.Read(iter.buffer)
+type seqReader struct {
+ reader *bufio.Reader
+ fileName string
+}
+
+func (i *seqReader) Read(p []byte) (int, error) {
+ rsize, err := i.reader.Read(p)
if err != nil {
- return readErrorHandle("ReadStream operation", err, iter.fileName, rsize)
+ return readErrorHandle("ReadStream operation", err, i.fileName, rsize)
}
return rsize, nil
}
diff --git a/pkg/fs/local_file_system_test.go b/pkg/fs/local_file_system_test.go
index 2340f1d5..1c96a127 100644
--- a/pkg/fs/local_file_system_test.go
+++ b/pkg/fs/local_file_system_test.go
@@ -109,10 +109,9 @@ var _ = ginkgo.Describe("Loacl File System", func() {
gomega.Expect(size == len(data)).To(gomega.BeTrue())
buffer := make([]byte, len(data))
- iter, err := file.StreamRead(buffer)
- gomega.Expect(err).ToNot(gomega.HaveOccurred())
+ iter := file.StreamRead()
for {
- size, err := iter.Next()
+ size, err := iter.Read(buffer)
if err == nil {
gomega.Expect(size == len(data)).To(gomega.BeTrue())
} else {
diff --git a/pkg/index/inverted/inverted.go b/pkg/index/inverted/inverted.go
index 1a2224d9..1a578aad 100644
--- a/pkg/index/inverted/inverted.go
+++ b/pkg/index/inverted/inverted.go
@@ -106,9 +106,9 @@ func (s *store) Batch(docs index.Documents) error {
// NewStore create a new inverted index repository.
func NewStore(opts StoreOpts) (index.SeriesStore, error) {
indexConfig := blugeIndex.DefaultConfig(opts.Path)
- indexConfig.MergePlanOptions.MaxSegmentsPerTier = 1
- indexConfig.MergePlanOptions.MaxSegmentSize = 500000
- indexConfig.MergePlanOptions.SegmentsPerMergeTask = 20
+ // TODO:// parameterize the following options
+ // WithUnsafeBatches().
+ // WithPersisterNapTimeMSec(60 * 1000)
config := bluge.DefaultConfigWithIndexConfig(indexConfig)
config.DefaultSearchAnalyzer = analyzers[databasev1.IndexRule_ANALYZER_KEYWORD]
config.Logger = log.New(opts.Logger, opts.Logger.Module(), 0)
@@ -381,7 +381,7 @@ func (s *store) run() {
}
size++
- batch.Insert(doc)
+ batch.Update(doc.ID(), doc)
}
if isBatch || size >= batchSize {
flush()
diff --git a/pkg/index/inverted/inverted_test.go b/pkg/index/inverted/inverted_test.go
index d352302c..bfa2f276 100644
--- a/pkg/index/inverted/inverted_test.go
+++ b/pkg/index/inverted/inverted_test.go
@@ -171,7 +171,7 @@ func TestStore_SeriesMatch(t *testing.T) {
},
{
matches: []string{"root"},
- want: roaring.NewPostingListWithInitialData(2),
+ want: roaring.DummyPostingList,
},
}
for _, tt := range tests {
@@ -182,9 +182,9 @@ func TestStore_SeriesMatch(t *testing.T) {
tester.Error(err)
return
}
- tester.NoError(err, name)
- tester.NotNil(list, name)
- tester.Equal(tt.want, list, name)
+ require.NoError(t, err)
+ require.NotNil(t, list)
+ assert.Equal(t, tt.want, list)
})
}
}
diff --git a/pkg/pb/v1/metadata.go b/pkg/pb/v1/metadata.go
index dcb48f26..4beef39c 100644
--- a/pkg/pb/v1/metadata.go
+++ b/pkg/pb/v1/metadata.go
@@ -110,6 +110,13 @@ type Result struct {
SID common.SeriesID
}
+// StreamColumnResult is the result of a stream sort or filter.
+type StreamColumnResult struct {
+ TagFamilies [][]TagFamily
+ Timestamps []int64
+ ElementIDs []string
+}
+
// TagProjection is the projection of a tag family and its tags.
type TagProjection struct {
Family string
@@ -128,12 +135,24 @@ type StreamQueryOptions struct {
// StreamSortOptions is the options of a stream sort.
type StreamSortOptions struct {
- Name string
- TimeRange *timestamp.TimeRange
- Entities [][]*modelv1.TagValue
- Filter index.Filter
- Order *OrderBy
- TagProjection []TagProjection
+ Name string
+ TimeRange *timestamp.TimeRange
+ Entities [][]*modelv1.TagValue
+ Filter index.Filter
+ Order *OrderBy
+ TagProjection []TagProjection
+ MaxElementSize int
+}
+
+// StreamFilterOptions is the options of a stream filter.
+type StreamFilterOptions struct {
+ Name string
+ TimeRange *timestamp.TimeRange
+ Entities [][]*modelv1.TagValue
+ Filter index.Filter
+ Order *OrderBy
+ TagProjection []TagProjection
+ MaxElementSize int
}
// StreamQueryResult is the result of a stream query.
@@ -142,6 +161,16 @@ type StreamQueryResult interface {
Release()
}
+// StreamSortResult is the result of a stream sort.
+type StreamSortResult interface {
+ Pull() *StreamColumnResult
+}
+
+// StreamFilterResult is the result of a stream filter.
+type StreamFilterResult interface {
+ Pull() *StreamColumnResult
+}
+
// MeasureQueryOptions is the options of a measure query.
type MeasureQueryOptions struct {
Name string
diff --git a/pkg/query/executor/interface.go b/pkg/query/executor/interface.go
index 22457331..40b73dde 100644
--- a/pkg/query/executor/interface.go
+++ b/pkg/query/executor/interface.go
@@ -31,7 +31,8 @@ import (
// StreamExecutionContext allows retrieving data through the stream module.
type StreamExecutionContext interface {
Query(ctx context.Context, opts pbv1.StreamQueryOptions) (pbv1.StreamQueryResult, error)
- Sort(ctx context.Context, opts pbv1.StreamSortOptions) ([]*streamv1.Element, error)
+ Sort(ctx context.Context, opts pbv1.StreamSortOptions) (pbv1.StreamSortResult, error)
+ Filter(ctx context.Context, opts pbv1.StreamFilterOptions) (pbv1.StreamFilterResult, error)
}
// StreamExecutionContextKey is the key of stream execution context in context.Context.
diff --git a/pkg/query/logical/index_filter.go b/pkg/query/logical/index_filter.go
index e9c5ab92..98d0b98c 100644
--- a/pkg/query/logical/index_filter.go
+++ b/pkg/query/logical/index_filter.go
@@ -80,7 +80,7 @@ func BuildLocalFilterDeprecated(criteria *modelv1.Criteria, schema Schema, entit
} else if mandatoryIndexRule {
return nil, nil, errors.Wrapf(errUnsupportedConditionOp, "mandatory index rule conf:%s", cond)
}
- return eNode, []tsdb.Entity{entity}, nil
+ return Enode, []tsdb.Entity{entity}, nil
case *modelv1.Criteria_Le:
le := criteria.GetLe()
if le.GetLeft() == nil && le.GetRight() == nil {
@@ -323,22 +323,11 @@ func BuildLocalFilter(criteria *modelv1.Criteria, schema Schema, entityDict map[
return nil, parsedEntity, nil
}
if ok, indexRule := schema.IndexDefined(cond.Name); ok {
- if indexRule.Location == databasev1.IndexRule_LOCATION_GLOBAL {
- switch cond.Op {
- case modelv1.Condition_BINARY_OP_EQ, modelv1.Condition_BINARY_OP_IN:
- return nil, nil, GlobalIndexError{
- IndexRule: indexRule,
- Expr: expr,
- }
- default:
- return nil, nil, errors.Wrapf(errUnsupportedConditionOp, "gobal index conf:%s", cond)
- }
- }
return parseCondition(cond, indexRule, expr, entity)
} else if mandatoryIndexRule {
return nil, nil, errors.Wrapf(errUnsupportedConditionOp, "mandatory index rule conf:%s", cond)
}
- return eNode, [][]*modelv1.TagValue{entity}, nil
+ return Enode, [][]*modelv1.TagValue{entity}, nil
case *modelv1.Criteria_Le:
le := criteria.GetLe()
if le.GetLeft() == nil && le.GetRight() == nil {
@@ -904,7 +893,8 @@ func jsonToString(marshaler json.Marshaler) string {
}
var (
- eNode = new(emptyNode)
+ // Enode is an empty node.
+ Enode = new(emptyNode)
bList = new(bypassList)
)
diff --git a/pkg/query/logical/stream/stream_plan_indexscan_local.go b/pkg/query/logical/stream/stream_plan_indexscan_local.go
index a44cae15..cf553a55 100644
--- a/pkg/query/logical/stream/stream_plan_indexscan_local.go
+++ b/pkg/query/logical/stream/stream_plan_indexscan_local.go
@@ -72,18 +72,82 @@ func (i *localIndexScan) Execute(ctx context.Context) (elements []*streamv1.Elem
}
ec := executor.FromStreamExecutionContext(ctx)
if i.order != nil && i.order.Index != nil {
- es, err := ec.Sort(ctx, pbv1.StreamSortOptions{
- Name: i.metadata.GetName(),
- TimeRange: &i.timeRange,
- Entities: i.entities,
- Filter: i.filter,
- Order: orderBy,
- TagProjection: i.projectionTags,
+ ssr, err := ec.Sort(ctx, pbv1.StreamSortOptions{
+ Name: i.metadata.GetName(),
+ TimeRange: &i.timeRange,
+ Entities: i.entities,
+ Filter: i.filter,
+ Order: orderBy,
+ TagProjection: i.projectionTags,
+ MaxElementSize: i.maxElementSize,
+ })
+ if err != nil {
+ return nil, err
+ }
+ if ssr == nil {
+ return elements, nil
+ }
+ r := ssr.Pull()
+ for i := range r.Timestamps {
+ e := &streamv1.Element{
+ Timestamp: timestamppb.New(time.Unix(0, r.Timestamps[i])),
+ ElementId: r.ElementIDs[i],
+ }
+
+ for _, tf := range r.TagFamilies[i] {
+ tagFamily := &modelv1.TagFamily{
+ Name: tf.Name,
+ }
+ e.TagFamilies = append(e.TagFamilies, tagFamily)
+ for _, t := range tf.Tags {
+ tagFamily.Tags = append(tagFamily.Tags, &modelv1.Tag{
+ Key: t.Name,
+ Value: t.Values[0],
+ })
+ }
+ }
+ elements = append(elements, e)
+ }
+ return elements, nil
+ }
+
+ if i.filter != nil && i.filter != logical.Enode {
+ sfr, err := ec.Filter(ctx, pbv1.StreamFilterOptions{
+ Name: i.metadata.GetName(),
+ TimeRange: &i.timeRange,
+ Entities: i.entities,
+ Filter: i.filter,
+ Order: orderBy,
+ TagProjection: i.projectionTags,
+ MaxElementSize: i.maxElementSize,
})
if err != nil {
return nil, err
}
- elements = append(elements, es...)
+ if sfr == nil {
+ return elements, nil
+ }
+ r := sfr.Pull()
+ for i := range r.Timestamps {
+ e := &streamv1.Element{
+ Timestamp: timestamppb.New(time.Unix(0, r.Timestamps[i])),
+ ElementId: r.ElementIDs[i],
+ }
+
+ for _, tf := range r.TagFamilies[i] {
+ tagFamily := &modelv1.TagFamily{
+ Name: tf.Name,
+ }
+ e.TagFamilies = append(e.TagFamilies, tagFamily)
+ for _, t := range tf.Tags {
+ tagFamily.Tags = append(tagFamily.Tags, &modelv1.Tag{
+ Key: t.Name,
+ Value: t.Values[0],
+ })
+ }
+ }
+ elements = append(elements, e)
+ }
return elements, nil
}
@@ -98,7 +162,7 @@ func (i *localIndexScan) Execute(ctx context.Context) (elements []*streamv1.Elem
TagProjection: i.projectionTags,
})
if err != nil {
- return nil, fmt.Errorf("failed to query measure: %w", err)
+ return nil, fmt.Errorf("failed to query stream: %w", err)
}
results = append(results, result)
diff --git a/pkg/watcher/watcher.go b/pkg/watcher/watcher.go
index eaedaa38..8babba9c 100644
--- a/pkg/watcher/watcher.go
+++ b/pkg/watcher/watcher.go
@@ -27,6 +27,9 @@ type Epoch struct {
// Watch returns a channel that will be notified when the epoch is reached.
func (e *Epoch) Watch() <-chan struct{} {
+ if e == nil {
+ return nil
+ }
return e.ch
}
diff --git a/scripts/build/version.mk b/scripts/build/version.mk
index 9f0e32c6..18bb0bd0 100644
--- a/scripts/build/version.mk
+++ b/scripts/build/version.mk
@@ -29,4 +29,4 @@ LICENSE_EYE_VERSION := a790ab8dd23a7f861c18bd6aaa9b012e3a234bce
MOCKGEN_VERSION := v0.3.0
-GINKGO_VERSION := v2.13.0
\ No newline at end of file
+GINKGO_VERSION := v2.13.2
\ No newline at end of file
diff --git a/test/cases/stream/stream.go b/test/cases/stream/stream.go
index 4512ae2d..f1212cef 100644
--- a/test/cases/stream/stream.go
+++ b/test/cases/stream/stream.go
@@ -49,23 +49,28 @@ var _ = g.DescribeTable("Scanning Streams", func(args helpers.Args) {
g.Entry("all elements", helpers.Args{Input: "all", Duration: 1 * time.Hour}),
g.Entry("limit", helpers.Args{Input: "limit", Duration: 1 * time.Hour}),
g.Entry("offset", helpers.Args{Input: "offset", Duration: 1 * time.Hour}),
- g.Entry("nothing", helpers.Args{Input: "all", WantEmpty: true}),
+ g.Entry("nothing", helpers.Args{
+ Input: "all",
+ Begin: timestamppb.New(time.Unix(0, 0).Truncate(time.Millisecond)),
+ End: timestamppb.New(time.Unix(0, 1).Truncate(time.Millisecond)),
+ WantEmpty: true,
+ }),
g.Entry("invalid time range", helpers.Args{
Input: "all",
Begin: timestamppb.New(time.Unix(0, int64(math.MinInt64+time.Millisecond)).Truncate(time.Millisecond)),
End: timestamppb.New(time.Unix(0, math.MaxInt64).Truncate(time.Millisecond)),
}),
g.Entry("sort desc", helpers.Args{Input: "sort_desc", Duration: 1 * time.Hour}),
- // g.Entry("global index", helpers.Args{Input: "global_index", Duration: 1 * time.Hour}),
- // g.Entry("multi-global index", helpers.Args{Input: "global_indices", Duration: 1 * time.Hour}),
- // g.Entry("filter by non-indexed tag", helpers.Args{Input: "filter_tag", Duration: 1 * time.Hour}),
- // g.Entry("get empty result by non-indexed tag", helpers.Args{Input: "filter_tag_empty", Duration: 1 * time.Hour, WantEmpty: true}),
- // g.Entry("numeric local index: less", helpers.Args{Input: "less", Duration: 1 * time.Hour}),
- // g.Entry("numeric local index: less and eq", helpers.Args{Input: "less_eq", Duration: 1 * time.Hour}),
- // g.Entry("logical expression", helpers.Args{Input: "logical", Duration: 1 * time.Hour}),
- // g.Entry("having", helpers.Args{Input: "having", Duration: 1 * time.Hour}),
- // g.Entry("having non indexed", helpers.Args{Input: "having_non_indexed", Duration: 1 * time.Hour}),
- // g.Entry("having non indexed array", helpers.Args{Input: "having_non_indexed_arr", Duration: 1 * time.Hour}),
- // g.Entry("full text searching", helpers.Args{Input: "search", Duration: 1 * time.Hour}),
- // g.Entry("indexed only tags", helpers.Args{Input: "indexed_only", Duration: 1 * time.Hour}),
+ g.Entry("global index", helpers.Args{Input: "global_index", Duration: 1 * time.Hour}),
+ g.Entry("multi-global index", helpers.Args{Input: "global_indices", Duration: 1 * time.Hour}),
+ g.Entry("filter by non-indexed tag", helpers.Args{Input: "filter_tag", Duration: 1 * time.Hour}),
+ g.Entry("get empty result by non-indexed tag", helpers.Args{Input: "filter_tag_empty", Duration: 1 * time.Hour, WantEmpty: true}),
+ g.Entry("numeric local index: less", helpers.Args{Input: "less", Duration: 1 * time.Hour}),
+ g.Entry("numeric local index: less and eq", helpers.Args{Input: "less_eq", Duration: 1 * time.Hour}),
+ g.Entry("logical expression", helpers.Args{Input: "logical", Duration: 1 * time.Hour}),
+ g.Entry("having", helpers.Args{Input: "having", Duration: 1 * time.Hour}),
+ g.Entry("having non indexed", helpers.Args{Input: "having_non_indexed", Duration: 1 * time.Hour}),
+ g.Entry("having non indexed array", helpers.Args{Input: "having_non_indexed_arr", Duration: 1 * time.Hour}),
+ g.Entry("full text searching", helpers.Args{Input: "search", Duration: 1 * time.Hour}),
+ g.Entry("indexed only tags", helpers.Args{Input: "indexed_only", Duration: 1 * time.Hour}),
)