You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by wu...@apache.org on 2023/12/20 00:59:04 UTC
(skywalking-banyandb) branch main updated: Measure column-based storage and new TSDB (#358)
This is an automated email from the ASF dual-hosted git repository.
wusheng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
The following commit(s) were added to refs/heads/main by this push:
new fe2eae7b Measure column-based storage and new TSDB (#358)
fe2eae7b is described below
commit fe2eae7b18115df28c0ede8a056ba09d53365ac3
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Wed Dec 20 08:58:59 2023 +0800
Measure column-based storage and new TSDB (#358)
---
CHANGES.md | 8 +-
api/common/id.go | 6 +
api/proto/banyandb/cluster/v1/rpc.proto | 1 +
banyand/internal/storage/index.go | 216 ++++++++
banyand/internal/storage/index_test.go | 142 ++++++
.../internal/storage/retention.go | 46 +-
banyand/internal/storage/segment.go | 382 ++++++++++++++
banyand/internal/storage/shard.go | 84 ++++
banyand/internal/storage/storage.go | 155 ++++++
banyand/internal/storage/tsdb.go | 192 +++++++
banyand/liaison/grpc/measure.go | 11 +-
banyand/liaison/grpc/stream.go | 2 +-
banyand/measure/block.go | 550 +++++++++++++++++++++
banyand/measure/block_metadata.go | 270 ++++++++++
banyand/measure/block_metadata_test.go | 348 +++++++++++++
.../{field_flag_test.go => block_reader.go} | 33 +-
banyand/measure/block_test.go | 466 +++++++++++++++++
banyand/measure/block_writer.go | 278 +++++++++++
banyand/measure/column.go | 119 +++++
banyand/measure/column_metadata.go | 159 ++++++
banyand/measure/column_metadata_test.go | 189 +++++++
banyand/measure/column_test.go | 121 +++++
banyand/measure/datapoints.go | 154 ++++++
banyand/measure/measure.go | 50 +-
banyand/measure/measure_query.go | 181 -------
banyand/measure/measure_write.go | 195 --------
banyand/measure/metadata.go | 107 ++--
banyand/measure/part.go | 196 ++++++++
banyand/measure/part_iter.go | 235 +++++++++
banyand/measure/part_iter_test.go | 143 ++++++
.../rpc.proto => banyand/measure/part_metadata.go | 38 +-
banyand/measure/part_test.go | 190 +++++++
banyand/measure/primary_metadata.go | 132 +++++
banyand/measure/query.go | 457 +++++++++++++++++
banyand/measure/query_test.go | 433 ++++++++++++++++
banyand/measure/service.go | 32 +-
banyand/measure/{measure_topn.go => topn.go} | 86 ++--
banyand/measure/tstable.go | 296 ++++++-----
banyand/measure/tstable_test.go | 329 ++++++++++++
banyand/measure/write.go | 331 +++++++++++++
banyand/queue/local.go | 24 +-
banyand/queue/pub/pub.go | 1 +
banyand/queue/sub/sub.go | 35 +-
banyand/stream/metadata.go | 18 +-
banyand/stream/stream_write.go | 38 +-
docs/api-reference.md | 1 +
go.mod | 2 +-
pkg/bus/bus.go | 17 +-
pkg/bytes/buffer.go | 88 ++++
.../cluster/v1/rpc.proto => pkg/bytes/resize.go | 37 +-
pkg/compress/zstd/zstd.go | 92 ++++
pkg/compress/zstd/zstd_test.go | 79 +++
pkg/encoding/bytes.go | 300 +++++++++++
pkg/encoding/bytes_test.go | 62 +++
pkg/encoding/delta.go | 118 +++++
pkg/encoding/encoding.go | 13 +
pkg/encoding/int.go | 276 +++++++++++
pkg/encoding/int_list.go | 178 +++++++
pkg/encoding/int_list_test.go | 77 +++
pkg/encoding/int_test.go | 248 ++++++++++
pkg/fs/file_system.go | 90 +++-
pkg/fs/local_file_system.go | 99 +++-
pkg/index/index.go | 25 +
pkg/index/inverted/inverted.go | 91 ++--
pkg/index/inverted/inverted_series.go | 123 +++++
pkg/index/inverted/inverted_series_test.go | 212 ++++++++
pkg/index/inverted/inverted_test.go | 2 +-
pkg/index/lsm/lsm.go | 5 +
pkg/logger/suger.go | 49 ++
pkg/pb/v1/metadata.go | 90 ++++
pkg/pb/v1/series.go | 156 ++++++
pkg/pb/v1/series_test.go | 120 +++++
pkg/pb/v1/value.go | 221 +++++++++
pkg/pb/v1/value_test.go | 76 +++
pkg/query/executor/interface.go | 4 +-
pkg/query/logical/common.go | 8 +-
pkg/query/logical/expr.go | 2 +-
pkg/query/logical/index_filter.go | 295 ++++++++++-
.../logical/measure/measure_plan_distributed.go | 3 -
.../measure/measure_plan_indexscan_local.go | 255 ++++------
pkg/query/logical/measure/topn_plan_localscan.go | 143 ++----
pkg/query/logical/schema.go | 4 +-
pkg/query/logical/stream/stream_plan_tag_filter.go | 2 +-
pkg/schema/metadata.go | 7 +-
pkg/schema/schema.go | 11 +-
test/cases/measure/data/want/all_only_fields.yaml | 66 ---
test/cases/measure/measure.go | 2 +-
test/cases/topn/data/input/null_group.yaml | 3 +-
.../query_ondisk/query_ondisk_suite_test.go | 1 +
89 files changed, 10034 insertions(+), 1198 deletions(-)
diff --git a/CHANGES.md b/CHANGES.md
index c504fc6b..ad38d375 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -8,13 +8,17 @@ Release Notes.
- Support etcd client authentication.
- Implement Local file system.
-- Add health check command for bydbctl
+- Add health check command for bydbctl.
+- Implement Inverted Index for SeriesDatabase.
+- Remove Block Level from TSDB.
+- Remove primary index.
+- Measure column-based storage:
+ - Data ingestion and retrieval.
### Bugs
- Fix the bug that property merge new tags failed.
-
### Chores
- Bump go to 1.21.
diff --git a/api/common/id.go b/api/common/id.go
index 074060cd..bac12211 100644
--- a/api/common/id.go
+++ b/api/common/id.go
@@ -25,6 +25,7 @@ import (
"strings"
"github.com/apache/skywalking-banyandb/pkg/convert"
+ "github.com/apache/skywalking-banyandb/pkg/encoding"
"github.com/apache/skywalking-banyandb/pkg/host"
)
@@ -44,6 +45,11 @@ func (s SeriesID) Marshal() []byte {
return convert.Uint64ToBytes(uint64(s))
}
+// AppendToBytes appends series id to bytes.
+func (s SeriesID) AppendToBytes(dest []byte) []byte {
+ return encoding.Uint64ToBytes(dest, uint64(s))
+}
+
// positionKey is a context key to store the module position.
var positionKey = contextPositionKey{}
diff --git a/api/proto/banyandb/cluster/v1/rpc.proto b/api/proto/banyandb/cluster/v1/rpc.proto
index 732bbe1f..8259717e 100644
--- a/api/proto/banyandb/cluster/v1/rpc.proto
+++ b/api/proto/banyandb/cluster/v1/rpc.proto
@@ -27,6 +27,7 @@ message SendRequest {
string topic = 1;
uint64 message_id = 2;
google.protobuf.Any body = 3;
+ bool batch_mod = 4;
}
message SendResponse {
diff --git a/banyand/internal/storage/index.go b/banyand/internal/storage/index.go
new file mode 100644
index 00000000..eaef4967
--- /dev/null
+++ b/banyand/internal/storage/index.go
@@ -0,0 +1,216 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package storage
+
+import (
+ "context"
+ "path"
+
+ "github.com/pkg/errors"
+ "go.uber.org/multierr"
+
+ "github.com/apache/skywalking-banyandb/api/common"
+ databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ "github.com/apache/skywalking-banyandb/pkg/index"
+ "github.com/apache/skywalking-banyandb/pkg/index/inverted"
+ "github.com/apache/skywalking-banyandb/pkg/index/posting"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
+)
+
+func (d *database[T]) IndexDB() IndexDB {
+ return d.index
+}
+
+func (d *database[T]) Lookup(ctx context.Context, series *pbv1.Series) (pbv1.SeriesList, error) {
+ return d.index.searchPrimary(ctx, series)
+}
+
+type seriesIndex struct {
+ store index.SeriesStore
+ l *logger.Logger
+}
+
+func newSeriesIndex(ctx context.Context, root string) (*seriesIndex, error) {
+ si := &seriesIndex{
+ l: logger.Fetch(ctx, "series_index"),
+ }
+ var err error
+ if si.store, err = inverted.NewStore(inverted.StoreOpts{
+ Path: path.Join(root, "idx"),
+ Logger: si.l,
+ }); err != nil {
+ return nil, err
+ }
+ return si, nil
+}
+
+func (s *seriesIndex) Write(docs index.Documents) error {
+ return s.store.Batch(docs)
+}
+
+var rangeOpts = index.RangeOpts{}
+
+func (s *seriesIndex) searchPrimary(_ context.Context, series *pbv1.Series) (pbv1.SeriesList, error) {
+ var hasAny, hasWildcard bool
+ var prefixIndex int
+
+ for i, tv := range series.EntityValues {
+ if tv == nil {
+ return nil, errors.New("nil tag value")
+ }
+ if tv == pbv1.AnyTagValue {
+ if !hasAny {
+ hasAny = true
+ prefixIndex = i
+ }
+ continue
+ }
+ if hasAny {
+ hasWildcard = true
+ break
+ }
+ }
+
+ var err error
+
+ if hasAny {
+ var ss []index.Series
+ if hasWildcard {
+ if err = series.Marshal(); err != nil {
+ return nil, err
+ }
+ ss, err = s.store.SearchWildcard(series.Buffer)
+ if err != nil {
+ return nil, err
+ }
+ return convertIndexSeriesToSeriesList(ss)
+ }
+ series.EntityValues = series.EntityValues[:prefixIndex]
+ if err = series.Marshal(); err != nil {
+ return nil, err
+ }
+ ss, err = s.store.SearchPrefix(series.Buffer)
+ if err != nil {
+ return nil, err
+ }
+ return convertIndexSeriesToSeriesList(ss)
+ }
+ if err = series.Marshal(); err != nil {
+ return nil, err
+ }
+ var seriesID common.SeriesID
+ seriesID, err = s.store.Search(series.Buffer)
+ if err != nil {
+ return nil, err
+ }
+ if seriesID > 0 {
+ series.ID = seriesID
+ return pbv1.SeriesList{series}, nil
+ }
+ return nil, nil
+}
+
+func convertIndexSeriesToSeriesList(indexSeries []index.Series) (pbv1.SeriesList, error) {
+ seriesList := make(pbv1.SeriesList, 0, len(indexSeries))
+ for _, s := range indexSeries {
+ var series pbv1.Series
+ series.ID = s.ID
+ if err := series.Unmarshal(s.EntityValues); err != nil {
+ return nil, err
+ }
+ seriesList = append(seriesList, &series)
+ }
+ return seriesList, nil
+}
+
+func (s *seriesIndex) Search(ctx context.Context, series *pbv1.Series, filter index.Filter, order *pbv1.OrderBy) (pbv1.SeriesList, error) {
+ seriesList, err := s.searchPrimary(ctx, series)
+ if err != nil {
+ return nil, err
+ }
+
+ pl := seriesList.ToList()
+ if filter != nil {
+ var plFilter posting.List
+ plFilter, err = filter.Execute(func(ruleType databasev1.IndexRule_Type) (index.Searcher, error) {
+ return s.store, nil
+ }, 0)
+ if err != nil {
+ return nil, err
+ }
+ if err = pl.Intersect(plFilter); err != nil {
+ return nil, err
+ }
+ }
+
+ if order == nil || order.Index == nil {
+ return filterSeriesList(seriesList, pl), nil
+ }
+
+ fieldKey := index.FieldKey{
+ IndexRuleID: order.Index.GetMetadata().Id,
+ }
+ iter, err := s.store.Iterator(fieldKey, rangeOpts, order.Sort)
+ if err != nil {
+ return nil, err
+ }
+ defer func() {
+ err = multierr.Append(err, iter.Close())
+ }()
+
+ var sortedSeriesList pbv1.SeriesList
+ for iter.Next() {
+ pv := iter.Val().Value
+ if err = pv.Intersect(pl); err != nil {
+ return nil, err
+ }
+ if pv.IsEmpty() {
+ continue
+ }
+ sortedSeriesList = appendSeriesList(sortedSeriesList, seriesList, pv)
+ if err != nil {
+ return nil, err
+ }
+ }
+ return sortedSeriesList, err
+}
+
+func filterSeriesList(seriesList pbv1.SeriesList, filter posting.List) pbv1.SeriesList {
+ for i := 0; i < len(seriesList); i++ {
+ if !filter.Contains(uint64(seriesList[i].ID)) {
+ seriesList = append(seriesList[:i], seriesList[i+1:]...)
+ i--
+ }
+ }
+ return seriesList
+}
+
+func appendSeriesList(dest, src pbv1.SeriesList, filter posting.List) pbv1.SeriesList {
+ for i := 0; i < len(src); i++ {
+ if !filter.Contains(uint64(src[i].ID)) {
+ continue
+ }
+ dest = append(dest, src[i])
+ }
+ return dest
+}
+
+func (s *seriesIndex) Close() error {
+ return s.store.Close()
+}
diff --git a/banyand/internal/storage/index_test.go b/banyand/internal/storage/index_test.go
new file mode 100644
index 00000000..f1fd0a42
--- /dev/null
+++ b/banyand/internal/storage/index_test.go
@@ -0,0 +1,142 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package storage
+
+import (
+ "context"
+ "fmt"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+ "github.com/apache/skywalking-banyandb/pkg/index"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
+ "github.com/apache/skywalking-banyandb/pkg/test"
+ "github.com/apache/skywalking-banyandb/pkg/test/flags"
+)
+
+var testSeriesPool pbv1.SeriesPool
+
+func TestSeriesIndex_Primary(t *testing.T) {
+ ctx := context.Background()
+ path, fn := setUp(require.New(t))
+ si, err := newSeriesIndex(ctx, path)
+ require.NoError(t, err)
+ defer func() {
+ require.NoError(t, si.Close())
+ fn()
+ }()
+ var docs index.Documents
+ for i := 0; i < 100; i++ {
+ series := testSeriesPool.Generate()
+ series.Subject = "service_instance_latency"
+ series.EntityValues = []*modelv1.TagValue{
+ {Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: fmt.Sprintf("svc_%d", i)}}},
+ {Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: fmt.Sprintf("svc_%d_instance_%d", i, i)}}},
+ }
+
+ // Initialize test data
+ if err = series.Marshal(); err != nil {
+ t.Fatalf("Failed to marshal series: %v", err)
+ }
+ require.True(t, series.ID > 0)
+ doc := index.Document{
+ DocID: uint64(series.ID),
+ EntityValues: make([]byte, len(series.Buffer)),
+ }
+ copy(doc.EntityValues, series.Buffer)
+ docs = append(docs, doc)
+ testSeriesPool.Release(series)
+ }
+ require.NoError(t, si.Write(docs))
+ // Restart the index
+ require.NoError(t, si.Close())
+ si, err = newSeriesIndex(ctx, path)
+ require.NoError(t, err)
+ tests := []struct {
+ name string
+ subject string
+ entityValues []*modelv1.TagValue
+ expected []*modelv1.TagValue
+ }{
+ {
+ name: "Search",
+ subject: "service_instance_latency",
+ entityValues: []*modelv1.TagValue{
+ {Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "svc_1"}}},
+ {Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "svc_1_instance_1"}}},
+ },
+ expected: []*modelv1.TagValue{
+ {Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "svc_1"}}},
+ {Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "svc_1_instance_1"}}},
+ },
+ },
+ {
+ name: "Prefix",
+ subject: "service_instance_latency",
+ entityValues: []*modelv1.TagValue{
+ {Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "svc_1"}}},
+ pbv1.AnyTagValue,
+ },
+ expected: []*modelv1.TagValue{
+ {Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "svc_1"}}},
+ {Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "svc_1_instance_1"}}},
+ },
+ },
+ {
+ name: "Wildcard",
+ subject: "service_instance_latency",
+ entityValues: []*modelv1.TagValue{
+ pbv1.AnyTagValue,
+ {Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "svc_1_instance_1"}}},
+ },
+ expected: []*modelv1.TagValue{
+ {Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "svc_1"}}},
+ {Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "svc_1_instance_1"}}},
+ },
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ seriesQuery := testSeriesPool.Generate()
+ defer testSeriesPool.Release(seriesQuery)
+ seriesQuery.Subject = tt.subject
+ seriesQuery.EntityValues = tt.entityValues
+ sl, err := si.searchPrimary(ctx, seriesQuery)
+ require.NoError(t, err)
+ require.Equal(t, 1, len(sl))
+ assert.Equal(t, tt.subject, sl[0].Subject)
+ assert.Equal(t, tt.expected[0].GetStr().GetValue(), sl[0].EntityValues[0].GetStr().GetValue())
+ assert.Equal(t, tt.expected[1].GetStr().GetValue(), sl[0].EntityValues[1].GetStr().GetValue())
+ assert.True(t, sl[0].ID > 0)
+ })
+ }
+}
+
+func setUp(t *require.Assertions) (tempDir string, deferFunc func()) {
+ t.NoError(logger.Init(logger.Logging{
+ Env: "dev",
+ Level: flags.LogLevel,
+ }))
+ tempDir, deferFunc = test.Space(t)
+ return tempDir, deferFunc
+}
diff --git a/api/proto/banyandb/cluster/v1/rpc.proto b/banyand/internal/storage/retention.go
similarity index 50%
copy from api/proto/banyandb/cluster/v1/rpc.proto
copy to banyand/internal/storage/retention.go
index 732bbe1f..ec76f1c1 100644
--- a/api/proto/banyandb/cluster/v1/rpc.proto
+++ b/banyand/internal/storage/retention.go
@@ -15,26 +15,44 @@
// specific language governing permissions and limitations
// under the License.
-syntax = "proto3";
+package storage
-package banyandb.cluster.v1;
+import (
+ "time"
-import "google/protobuf/any.proto";
+ "github.com/robfig/cron/v3"
-option go_package = "github.com/apache/skywalking-banyandb/api/proto/banyandb/cluster/v1";
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+)
-message SendRequest {
- string topic = 1;
- uint64 message_id = 2;
- google.protobuf.Any body = 3;
+type retentionTask[T TSTable] struct {
+ segment *segmentController[T]
+ expr string
+ option cron.ParseOption
+ duration time.Duration
}
-message SendResponse {
- uint64 message_id = 1;
- string error = 2;
- google.protobuf.Any body = 3;
+func newRetentionTask[T TSTable](segment *segmentController[T], ttl IntervalRule) *retentionTask[T] {
+ var expr string
+ switch ttl.Unit {
+ case HOUR:
+ // Every hour on the 5th minute
+ expr = "5 *"
+ case DAY:
+ // Every day on 00:05
+ expr = "5 0"
+ }
+ return &retentionTask[T]{
+ segment: segment,
+ option: cron.Minute | cron.Hour,
+ expr: expr,
+ duration: ttl.estimatedDuration(),
+ }
}
-service Service {
- rpc Send(stream SendRequest) returns (stream SendResponse);
+func (rc *retentionTask[T]) run(now time.Time, l *logger.Logger) bool {
+ if err := rc.segment.remove(now.Add(-rc.duration)); err != nil {
+ l.Error().Err(err)
+ }
+ return true
}
diff --git a/banyand/internal/storage/segment.go b/banyand/internal/storage/segment.go
new file mode 100644
index 00000000..a74da252
--- /dev/null
+++ b/banyand/internal/storage/segment.go
@@ -0,0 +1,382 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package storage
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "path"
+ "sort"
+ "strconv"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "github.com/apache/skywalking-banyandb/api/common"
+ "github.com/apache/skywalking-banyandb/banyand/tsdb/bucket"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+ "github.com/apache/skywalking-banyandb/pkg/timestamp"
+)
+
+var errEndOfSegment = errors.New("reached the end of the segment")
+
+type segment[T TSTable] struct {
+ bucket.Reporter
+ tsTable T
+ l *logger.Logger
+ position common.Position
+ timestamp.TimeRange
+ path string
+ suffix string
+ refCount int32
+ mustBeDeleted uint32
+ id segmentID
+}
+
+func openSegment[T TSTable](ctx context.Context, startTime, endTime time.Time, path, suffix string,
+ segmentSize IntervalRule, scheduler *timestamp.Scheduler, tsTable T,
+) (s *segment[T], err error) {
+ suffixInteger, err := strconv.Atoi(suffix)
+ if err != nil {
+ return nil, err
+ }
+ id := generateSegID(segmentSize.Unit, suffixInteger)
+ timeRange := timestamp.NewSectionTimeRange(startTime, endTime)
+ s = &segment[T]{
+ id: id,
+ path: path,
+ suffix: suffix,
+ TimeRange: timeRange,
+ position: common.GetPosition(ctx),
+ tsTable: tsTable,
+ refCount: 1,
+ }
+ l := logger.Fetch(ctx, s.String())
+ s.l = l
+ clock, _ := timestamp.GetClock(ctx)
+ s.Reporter = bucket.NewTimeBasedReporter(s.String(), timeRange, clock, scheduler)
+ return s, nil
+}
+
+func (s *segment[T]) incRef() {
+ atomic.AddInt32(&s.refCount, 1)
+}
+
+func (s *segment[T]) DecRef() {
+ n := atomic.AddInt32(&s.refCount, -1)
+ if n > 0 {
+ return
+ }
+
+ deletePath := ""
+ if atomic.LoadUint32(&s.mustBeDeleted) != 0 {
+ deletePath = s.path
+ }
+
+ if err := s.tsTable.Close(); err != nil {
+ s.l.Panic().Err(err).Msg("failed to close tsTable")
+ }
+
+ if deletePath != "" {
+ lfs.MustRMAll(deletePath)
+ }
+}
+
+func (s *segment[T]) Table() T {
+ return s.tsTable
+}
+
+func (s *segment[T]) GetTimeRange() timestamp.TimeRange {
+ return s.TimeRange
+}
+
+func (s *segment[T]) delete() {
+ atomic.StoreUint32(&s.mustBeDeleted, 1)
+ s.DecRef()
+}
+
+func (s *segment[T]) String() string {
+ return "SegID-" + s.suffix
+}
+
+type segmentController[T TSTable] struct {
+ clock timestamp.Clock
+ scheduler *timestamp.Scheduler
+ l *logger.Logger
+ tsTableCreator TSTableCreator[T]
+ position common.Position
+ location string
+ lst []*segment[T]
+ segmentSize IntervalRule
+ sync.RWMutex
+}
+
+func newSegmentController[T TSTable](ctx context.Context, location string,
+ segmentSize IntervalRule, l *logger.Logger, scheduler *timestamp.Scheduler,
+ tsTableCreator TSTableCreator[T],
+) *segmentController[T] {
+ clock, _ := timestamp.GetClock(ctx)
+ return &segmentController[T]{
+ location: location,
+ segmentSize: segmentSize,
+ l: l,
+ clock: clock,
+ scheduler: scheduler,
+ position: common.GetPosition(ctx),
+ tsTableCreator: tsTableCreator,
+ }
+}
+
+func (sc *segmentController[T]) selectTSTables(timeRange timestamp.TimeRange) (tt []TSTableWrapper[T]) {
+ sc.RLock()
+ defer sc.RUnlock()
+ last := len(sc.lst) - 1
+ for i := range sc.lst {
+ s := sc.lst[last-i]
+ if s.Overlapping(timeRange) {
+ s.incRef()
+ tt = append(tt, s)
+ }
+ }
+ return tt
+}
+
+func (sc *segmentController[T]) createTSTable(ts time.Time) (TSTableWrapper[T], error) {
+ s, err := sc.create(ts)
+ if err != nil {
+ return nil, err
+ }
+ s.incRef()
+ return s, nil
+}
+
+func (sc *segmentController[T]) segments() (ss []*segment[T]) {
+ sc.RLock()
+ defer sc.RUnlock()
+ r := make([]*segment[T], len(sc.lst))
+ for i := range sc.lst {
+ sc.lst[i].incRef()
+ r[i] = sc.lst[i]
+ }
+ return r
+}
+
+func (sc *segmentController[T]) Current() (bucket.Reporter, error) {
+ now := sc.Standard(sc.clock.Now())
+ ns := uint64(now.UnixNano())
+ if b := func() bucket.Reporter {
+ sc.RLock()
+ defer sc.RUnlock()
+ for _, s := range sc.lst {
+ if s.Contains(ns) {
+ return s
+ }
+ }
+ return nil
+ }(); b != nil {
+ return b, nil
+ }
+ return sc.create(now)
+}
+
+func (sc *segmentController[T]) Next() (bucket.Reporter, error) {
+ c, err := sc.Current()
+ if err != nil {
+ return nil, err
+ }
+ seg := c.(*segment[T])
+ reporter, err := sc.create(sc.segmentSize.nextTime(seg.Start))
+ if errors.Is(err, errEndOfSegment) {
+ return nil, bucket.ErrNoMoreBucket
+ }
+ return reporter, err
+}
+
+func (sc *segmentController[T]) OnMove(prev bucket.Reporter, next bucket.Reporter) {
+ event := sc.l.Info()
+ if prev != nil {
+ event.Stringer("prev", prev)
+ }
+ if next != nil {
+ event.Stringer("next", next)
+ }
+ event.Msg("move to the next segment")
+}
+
+func (sc *segmentController[T]) Standard(t time.Time) time.Time {
+ switch sc.segmentSize.Unit {
+ case HOUR:
+ return time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), 0, 0, 0, t.Location())
+ case DAY:
+ return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location())
+ }
+ panic("invalid interval unit")
+}
+
+func (sc *segmentController[T]) Format(tm time.Time) string {
+ switch sc.segmentSize.Unit {
+ case HOUR:
+ return tm.Format(hourFormat)
+ case DAY:
+ return tm.Format(dayFormat)
+ }
+ panic("invalid interval unit")
+}
+
+func (sc *segmentController[T]) Parse(value string) (time.Time, error) {
+ switch sc.segmentSize.Unit {
+ case HOUR:
+ return time.ParseInLocation(hourFormat, value, time.Local)
+ case DAY:
+ return time.ParseInLocation(dayFormat, value, time.Local)
+ }
+ panic("invalid interval unit")
+}
+
+func (sc *segmentController[T]) open() error {
+ sc.Lock()
+ defer sc.Unlock()
+ return loadSegments(sc.location, segPathPrefix, sc, sc.segmentSize, func(start, end time.Time) error {
+ _, err := sc.load(start, end, sc.location)
+ if errors.Is(err, errEndOfSegment) {
+ return nil
+ }
+ return err
+ })
+}
+
+func (sc *segmentController[T]) create(start time.Time) (*segment[T], error) {
+ sc.Lock()
+ defer sc.Unlock()
+ start = sc.Standard(start)
+ var next *segment[T]
+ for _, s := range sc.lst {
+ if s.Contains(uint64(start.UnixNano())) {
+ return s, nil
+ }
+ if next == nil && s.Start.After(start) {
+ next = s
+ }
+ }
+ stdEnd := sc.segmentSize.nextTime(start)
+ var end time.Time
+ if next != nil && next.Start.Before(stdEnd) {
+ end = next.Start
+ } else {
+ end = stdEnd
+ }
+ lfs.MkdirPanicIfExist(path.Join(sc.location, fmt.Sprintf(segTemplate, sc.Format(start))), dirPerm)
+ return sc.load(start, end, sc.location)
+}
+
+func (sc *segmentController[T]) sortLst() {
+ sort.Slice(sc.lst, func(i, j int) bool {
+ return sc.lst[i].id < sc.lst[j].id
+ })
+}
+
+func (sc *segmentController[T]) load(start, end time.Time, root string) (seg *segment[T], err error) {
+ var tsTable T
+ if tsTable, err = sc.tsTableCreator(sc.location, sc.position, sc.l, timestamp.NewSectionTimeRange(start, end)); err != nil {
+ return nil, err
+ }
+ suffix := sc.Format(start)
+ ctx := context.WithValue(context.Background(), logger.ContextKey, sc.l)
+ seg, err = openSegment[T](common.SetPosition(ctx, func(p common.Position) common.Position {
+ p.Segment = suffix
+ return p
+ }), start, end, path.Join(root, fmt.Sprintf(segTemplate, suffix)), suffix, sc.segmentSize, sc.scheduler, tsTable)
+ if err != nil {
+ return nil, err
+ }
+ sc.lst = append(sc.lst, seg)
+ sc.sortLst()
+ return seg, nil
+}
+
+func (sc *segmentController[T]) remove(deadline time.Time) (err error) {
+ sc.l.Info().Time("deadline", deadline).Msg("start to remove before deadline")
+ for _, s := range sc.segments() {
+ if s.End.Before(deadline) || s.Contains(uint64(deadline.UnixNano())) {
+ if e := sc.l.Debug(); e.Enabled() {
+ e.Stringer("segment", s).Msg("start to remove data in a segment")
+ }
+ if s.End.Before(deadline) {
+ s.delete()
+ sc.Lock()
+ sc.removeSeg(s.id)
+ sc.Unlock()
+ }
+ }
+ s.DecRef()
+ }
+ return err
+}
+
+func (sc *segmentController[T]) removeSeg(segID segmentID) {
+ for i, b := range sc.lst {
+ if b.id == segID {
+ sc.lst = append(sc.lst[:i], sc.lst[i+1:]...)
+ break
+ }
+ }
+}
+
+func (sc *segmentController[T]) close() {
+ sc.Lock()
+ defer sc.Unlock()
+ for _, s := range sc.lst {
+ s.DecRef()
+ }
+ sc.lst = sc.lst[:0]
+}
+
+type parser interface {
+ Parse(value string) (time.Time, error)
+}
+
+func loadSegments(root, prefix string, parser parser, intervalRule IntervalRule, loadFn func(start, end time.Time) error) error {
+ var startTimeLst []time.Time
+ if err := walkDir(
+ root,
+ prefix,
+ func(suffix string) error {
+ startTime, err := parser.Parse(suffix)
+ if err != nil {
+ return err
+ }
+ startTimeLst = append(startTimeLst, startTime)
+ return nil
+ }); err != nil {
+ return err
+ }
+ sort.Slice(startTimeLst, func(i, j int) bool { return i < j })
+ for i, start := range startTimeLst {
+ var end time.Time
+ if i < len(startTimeLst)-1 {
+ end = startTimeLst[i+1]
+ } else {
+ end = intervalRule.nextTime(start)
+ }
+ if err := loadFn(start, end); err != nil {
+ return err
+ }
+ }
+ return nil
+}
diff --git a/banyand/internal/storage/shard.go b/banyand/internal/storage/shard.go
new file mode 100644
index 00000000..cdec81ae
--- /dev/null
+++ b/banyand/internal/storage/shard.go
@@ -0,0 +1,84 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package storage
+
+import (
+ "context"
+ "fmt"
+ "path"
+ "strconv"
+ "sync"
+
+ "github.com/apache/skywalking-banyandb/api/common"
+ "github.com/apache/skywalking-banyandb/banyand/tsdb/bucket"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+ "github.com/apache/skywalking-banyandb/pkg/timestamp"
+)
+
+type shard[T TSTable] struct {
+ l *logger.Logger
+ segmentController *segmentController[T]
+ segmentManageStrategy *bucket.Strategy
+ scheduler *timestamp.Scheduler
+ position common.Position
+ closeOnce sync.Once
+ id common.ShardID
+}
+
+func (d *database[T]) openShard(ctx context.Context, id common.ShardID) (*shard[T], error) {
+ location := path.Join(d.location, fmt.Sprintf(shardTemplate, int(id)))
+ lfs.MkdirIfNotExist(location, dirPerm)
+ l := logger.Fetch(ctx, "shard"+strconv.Itoa(int(id)))
+ l.Info().Int("shard_id", int(id)).Str("path", location).Msg("creating a shard")
+ shardCtx := context.WithValue(ctx, logger.ContextKey, l)
+ shardCtx = common.SetPosition(shardCtx, func(p common.Position) common.Position {
+ p.Shard = strconv.Itoa(int(id))
+ return p
+ })
+ clock, _ := timestamp.GetClock(shardCtx)
+
+ scheduler := timestamp.NewScheduler(l, clock)
+ s := &shard[T]{
+ id: id,
+ l: l,
+ scheduler: scheduler,
+ position: common.GetPosition(shardCtx),
+ segmentController: newSegmentController[T](shardCtx, location, d.opts.SegmentInterval, l, scheduler, d.opts.TSTableCreator),
+ }
+ var err error
+ if err = s.segmentController.open(); err != nil {
+ return nil, err
+ }
+ if s.segmentManageStrategy, err = bucket.NewStrategy(s.segmentController, bucket.WithLogger(s.l)); err != nil {
+ return nil, err
+ }
+ s.segmentManageStrategy.Run()
+ retentionTask := newRetentionTask(s.segmentController, d.opts.TTL)
+ if err := scheduler.Register("retention", retentionTask.option, retentionTask.expr, retentionTask.run); err != nil {
+ return nil, err
+ }
+ return s, nil
+}
+
+func (s *shard[T]) closer() {
+ s.closeOnce.Do(func() {
+ s.scheduler.Close()
+ s.segmentManageStrategy.Close()
+ s.segmentController.close()
+ })
+}
diff --git a/banyand/internal/storage/storage.go b/banyand/internal/storage/storage.go
new file mode 100644
index 00000000..cfda39bf
--- /dev/null
+++ b/banyand/internal/storage/storage.go
@@ -0,0 +1,155 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Package storage implements a time-series-based storage engine.
+// It provides:
+// - Partition data based on a time axis.
+// - Sharding data based on a series id which represents a unique entity of stream/measure
+// - Retrieving data based on index.Filter.
+// - Cleaning expired data, or the data retention.
+package storage
+
+import (
+ "context"
+ "io"
+ "time"
+
+ "github.com/pkg/errors"
+
+ "github.com/apache/skywalking-banyandb/api/common"
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ "github.com/apache/skywalking-banyandb/pkg/fs"
+ "github.com/apache/skywalking-banyandb/pkg/index"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
+ "github.com/apache/skywalking-banyandb/pkg/timestamp"
+)
+
+const (
+ shardPathPrefix = "shard"
+ shardTemplate = shardPathPrefix + "-%d"
+ metadataPath = "metadata"
+ segTemplate = "seg-%s"
+ segPathPrefix = "seg"
+
+ hourFormat = "2006010215"
+ dayFormat = "20060102"
+
+ dirPerm = 0o700
+)
+
+var (
+ // ErrUnknownShard indicates that the shard is not found.
+ ErrUnknownShard = errors.New("unknown shard")
+ errOpenDatabase = errors.New("fails to open the database")
+
+ lfs = fs.NewLocalFileSystemWithLogger(logger.GetLogger("storage"))
+)
+
+// SupplyTSDB allows getting a tsdb's runtime.
+type SupplyTSDB[T TSTable] func() T
+
+// IndexDB is the interface of index database.
+type IndexDB interface {
+ Write(docs index.Documents) error
+ Search(ctx context.Context, series *pbv1.Series, filter index.Filter, order *pbv1.OrderBy) (pbv1.SeriesList, error)
+}
+
+// TSDB allows listing and getting shard details.
+type TSDB[T TSTable] interface {
+ io.Closer
+ Lookup(ctx context.Context, series *pbv1.Series) (pbv1.SeriesList, error)
+ CreateTSTableIfNotExist(shardID common.ShardID, ts time.Time) (TSTableWrapper[T], error)
+ SelectTSTables(timeRange timestamp.TimeRange) []TSTableWrapper[T]
+ IndexDB() IndexDB
+}
+
+// TSTable is time series table.
+type TSTable interface {
+ io.Closer
+}
+
+// TSTableWrapper is a wrapper of TSTable.
+// It is used to manage the reference count of TSTable.
+type TSTableWrapper[T TSTable] interface {
+ DecRef()
+ Table() T
+ GetTimeRange() timestamp.TimeRange
+}
+
+// TSTableCreator creates a TSTable.
+type TSTableCreator[T TSTable] func(root string, position common.Position,
+ l *logger.Logger, timeRange timestamp.TimeRange) (T, error)
+
+// IntervalUnit denotes the unit of a time point.
+type IntervalUnit int
+
+// Available IntervalUnits. HOUR and DAY are adequate for the APM scenario.
+const (
+ HOUR IntervalUnit = iota
+ DAY
+)
+
+func (iu IntervalUnit) String() string {
+ switch iu {
+ case HOUR:
+ return "hour"
+ case DAY:
+ return "day"
+ }
+ panic("invalid interval unit")
+}
+
+// IntervalRule defines a length of two points in time.
+type IntervalRule struct {
+ Unit IntervalUnit
+ Num int
+}
+
+func (ir IntervalRule) nextTime(current time.Time) time.Time {
+ switch ir.Unit {
+ case HOUR:
+ return current.Add(time.Hour * time.Duration(ir.Num))
+ case DAY:
+ return current.AddDate(0, 0, ir.Num)
+ }
+ panic("invalid interval unit")
+}
+
+func (ir IntervalRule) estimatedDuration() time.Duration {
+ switch ir.Unit {
+ case HOUR:
+ return time.Hour * time.Duration(ir.Num)
+ case DAY:
+ return 24 * time.Hour * time.Duration(ir.Num)
+ }
+ panic("invalid interval unit")
+}
+
+// MustToIntervalRule converts a commonv1.IntervalRule to IntervalRule.
+func MustToIntervalRule(ir *commonv1.IntervalRule) (result IntervalRule) {
+ switch ir.Unit {
+ case commonv1.IntervalRule_UNIT_DAY:
+ result.Unit = DAY
+ case commonv1.IntervalRule_UNIT_HOUR:
+ result.Unit = HOUR
+ default:
+ logger.Panicf("unknown interval rule:%v", ir)
+ }
+ result.Num = int(ir.Num)
+ return result
+}
diff --git a/banyand/internal/storage/tsdb.go b/banyand/internal/storage/tsdb.go
new file mode 100644
index 00000000..a425609e
--- /dev/null
+++ b/banyand/internal/storage/tsdb.go
@@ -0,0 +1,192 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package storage
+
+import (
+ "context"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "github.com/pkg/errors"
+
+ "github.com/apache/skywalking-banyandb/api/common"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+ "github.com/apache/skywalking-banyandb/pkg/timestamp"
+)
+
+// IndexGranularity denotes the granularity of the local index.
+type IndexGranularity int
+
+// The options of the local index granularity.
+const (
+ IndexGranularityBlock IndexGranularity = iota
+ IndexGranularitySeries
+)
+
+// TSDBOpts wraps options to create a tsdb.
+type TSDBOpts[T TSTable] struct {
+ TSTableCreator TSTableCreator[T]
+ Location string
+ SegmentInterval IntervalRule
+ TTL IntervalRule
+ ShardNum uint32
+}
+
+type (
+ segmentID uint32
+)
+
+func generateSegID(unit IntervalUnit, suffix int) segmentID {
+ return segmentID(unit)<<31 | ((segmentID(suffix) << 1) >> 1)
+}
+
+type database[T TSTable] struct {
+ logger *logger.Logger
+ index *seriesIndex
+ location string
+ sLst []*shard[T]
+ opts TSDBOpts[T]
+ sync.RWMutex
+ sLen uint32
+}
+
+func (d *database[T]) Close() error {
+ d.Lock()
+ defer d.Unlock()
+ for _, s := range d.sLst {
+ s.closer()
+ }
+ return d.index.Close()
+}
+
+// OpenTSDB returns a new tsdb runtime. This constructor will create a new database if it's absent,
+// or load an existing one.
+func OpenTSDB[T TSTable](ctx context.Context, opts TSDBOpts[T]) (TSDB[T], error) {
+ if opts.SegmentInterval.Num == 0 {
+ return nil, errors.Wrap(errOpenDatabase, "segment interval is absent")
+ }
+ if opts.TTL.Num == 0 {
+ return nil, errors.Wrap(errOpenDatabase, "ttl is absent")
+ }
+ p := common.GetPosition(ctx)
+ location := filepath.Clean(opts.Location)
+ lfs.MkdirIfNotExist(location, dirPerm)
+ si, err := newSeriesIndex(ctx, location)
+ if err != nil {
+ return nil, errors.Wrap(errOpenDatabase, errors.WithMessage(err, "create series index failed").Error())
+ }
+ db := &database[T]{
+ location: location,
+ logger: logger.Fetch(ctx, p.Database),
+ index: si,
+ opts: opts,
+ }
+ db.logger.Info().Str("path", opts.Location).Msg("initialized")
+ if err = db.loadDatabase(); err != nil {
+ return nil, errors.Wrap(errOpenDatabase, errors.WithMessage(err, "load database failed").Error())
+ }
+ return db, nil
+}
+
+func (d *database[T]) CreateTSTableIfNotExist(shardID common.ShardID, ts time.Time) (TSTableWrapper[T], error) {
+ id := uint32(shardID)
+ if id >= atomic.LoadUint32(&d.sLen) {
+ return func() (TSTableWrapper[T], error) {
+ d.Lock()
+ defer d.Unlock()
+ if int(id) >= len(d.sLst) {
+ for i := len(d.sLst); i <= int(id); i++ {
+ d.logger.Info().Int("shard_id", i).Msg("creating a shard")
+ if err := d.registerShard(i); err != nil {
+ return nil, err
+ }
+ }
+ }
+ return d.createTSTTable(shardID, ts)
+ }()
+ }
+ d.RLock()
+ defer d.RUnlock()
+ return d.createTSTTable(shardID, ts)
+}
+
+func (d *database[T]) createTSTTable(shardID common.ShardID, ts time.Time) (TSTableWrapper[T], error) {
+ timeRange := timestamp.NewInclusiveTimeRange(ts, ts)
+ ss := d.sLst[shardID].segmentController.selectTSTables(timeRange)
+ if len(ss) > 0 {
+ return ss[0], nil
+ }
+ return d.sLst[shardID].segmentController.createTSTable(ts)
+}
+
+func (d *database[T]) SelectTSTables(timeRange timestamp.TimeRange) []TSTableWrapper[T] {
+ var result []TSTableWrapper[T]
+ d.RLock()
+ for i := range d.sLst {
+ result = append(result, d.sLst[i].segmentController.selectTSTables(timeRange)...)
+ }
+ d.RUnlock()
+ return result
+}
+
+func (d *database[T]) registerShard(id int) error {
+ ctx := context.WithValue(context.Background(), logger.ContextKey, d.logger)
+ so, err := d.openShard(ctx, common.ShardID(id))
+ if err != nil {
+ return err
+ }
+ d.sLst = append(d.sLst, so)
+ d.sLen++
+ return nil
+}
+
+func (d *database[T]) loadDatabase() error {
+ d.Lock()
+ defer d.Unlock()
+ return walkDir(d.location, shardPathPrefix, func(suffix string) error {
+ shardID, err := strconv.Atoi(suffix)
+ if err != nil {
+ return err
+ }
+ if shardID >= int(d.opts.ShardNum) {
+ return nil
+ }
+ d.logger.Info().Int("shard_id", shardID).Msg("opening a existed shard")
+ return d.registerShard(shardID)
+ })
+}
+
+type walkFn func(suffix string) error
+
+func walkDir(root, prefix string, wf walkFn) error {
+ for _, f := range lfs.ReadDir(root) {
+ if !f.IsDir() || !strings.HasPrefix(f.Name(), prefix) {
+ continue
+ }
+ segs := strings.Split(f.Name(), "-")
+ errWalk := wf(segs[len(segs)-1])
+ if errWalk != nil {
+ return errors.WithMessagef(errWalk, "failed to load: %s", f.Name())
+ }
+ }
+ return nil
+}
diff --git a/banyand/liaison/grpc/measure.go b/banyand/liaison/grpc/measure.go
index ebb5c58f..9517b6cb 100644
--- a/banyand/liaison/grpc/measure.go
+++ b/banyand/liaison/grpc/measure.go
@@ -113,10 +113,11 @@ func (ms *measureService) Write(measure measurev1.MeasureService_WriteServer) er
}
}
iwr := &measurev1.InternalWriteRequest{
- Request: writeRequest,
- ShardId: uint32(shardID),
- SeriesHash: tsdb.HashEntity(entity),
- EntityValues: tagValues.Encode(),
+ Request: writeRequest,
+ ShardId: uint32(shardID),
+ SeriesHash: tsdb.HashEntity(entity),
+ // TODO: remove the first value (measure name) of tagValues
+ EntityValues: tagValues[1:].Encode(),
}
nodeID, errPickNode := ms.nodeRegistry.Locate(writeRequest.GetMetadata().GetGroup(), writeRequest.GetMetadata().GetName(), uint32(shardID))
if errPickNode != nil {
@@ -124,7 +125,7 @@ func (ms *measureService) Write(measure measurev1.MeasureService_WriteServer) er
reply(writeRequest.GetMetadata(), modelv1.Status_STATUS_INTERNAL_ERROR, writeRequest.GetMessageId(), measure, ms.sampled)
continue
}
- message := bus.NewMessageWithNode(bus.MessageID(time.Now().UnixNano()), nodeID, iwr)
+ message := bus.NewBatchMessageWithNode(bus.MessageID(time.Now().UnixNano()), nodeID, iwr)
_, errWritePub := publisher.Publish(data.TopicMeasureWrite, message)
if errWritePub != nil {
ms.sampled.Error().Err(errWritePub).RawJSON("written", logger.Proto(writeRequest)).Str("nodeID", nodeID).Msg("failed to send a message")
diff --git a/banyand/liaison/grpc/stream.go b/banyand/liaison/grpc/stream.go
index d2aca976..37dc44ea 100644
--- a/banyand/liaison/grpc/stream.go
+++ b/banyand/liaison/grpc/stream.go
@@ -126,7 +126,7 @@ func (s *streamService) Write(stream streamv1.StreamService_WriteServer) error {
reply(writeEntity.GetMetadata(), modelv1.Status_STATUS_INTERNAL_ERROR, writeEntity.GetMessageId(), stream, s.sampled)
continue
}
- message := bus.NewMessageWithNode(bus.MessageID(time.Now().UnixNano()), nodeID, iwr)
+ message := bus.NewBatchMessageWithNode(bus.MessageID(time.Now().UnixNano()), nodeID, iwr)
_, errWritePub := publisher.Publish(data.TopicStreamWrite, message)
if errWritePub != nil {
s.sampled.Error().Err(errWritePub).RawJSON("written", logger.Proto(writeEntity)).Str("nodeID", nodeID).Msg("failed to send a message")
diff --git a/banyand/measure/block.go b/banyand/measure/block.go
new file mode 100644
index 00000000..d3f10459
--- /dev/null
+++ b/banyand/measure/block.go
@@ -0,0 +1,550 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package measure
+
+import (
+ "sort"
+ "sync"
+
+ "github.com/apache/skywalking-banyandb/api/common"
+ "github.com/apache/skywalking-banyandb/pkg/bytes"
+ "github.com/apache/skywalking-banyandb/pkg/encoding"
+ "github.com/apache/skywalking-banyandb/pkg/fs"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
+)
+
+type block struct {
+ timestamps []int64
+
+ tagFamilies []columnFamily
+
+ field columnFamily
+}
+
+func (b *block) reset() {
+ b.timestamps = b.timestamps[:0]
+
+ tff := b.tagFamilies
+ for i := range tff {
+ tff[i].reset()
+ }
+ b.tagFamilies = tff[:0]
+ b.field.reset()
+}
+
+func (b *block) mustInitFromDataPoints(timestamps []int64, tagFamilies [][]nameValues, fields []nameValues) {
+ b.reset()
+ size := len(timestamps)
+ if size == 0 {
+ return
+ }
+ if size != len(tagFamilies) {
+ logger.Panicf("the number of timestamps %d must match the number of tagFamilies %d", size, len(tagFamilies))
+ }
+ if size != len(fields) {
+ logger.Panicf("the number of timestamps %d must match the number of fields %d", size, len(fields))
+ }
+
+ assertTimestampsSorted(timestamps)
+ b.timestamps = append(b.timestamps, timestamps...)
+ b.mustInitFromTagsAndFields(tagFamilies, fields)
+}
+
+func assertTimestampsSorted(timestamps []int64) {
+ for i := range timestamps {
+ if i > 0 && timestamps[i-1] > timestamps[i] {
+ logger.Panicf("log entries must be sorted by timestamp; got the previous entry with bigger timestamp %d than the current entry with timestamp %d",
+ timestamps[i-1], timestamps[i])
+ }
+ }
+}
+
+func (b *block) mustInitFromTagsAndFields(tagFamilies [][]nameValues, fields []nameValues) {
+ dataPointsLen := len(tagFamilies)
+ if dataPointsLen == 0 {
+ return
+ }
+ for i, tff := range tagFamilies {
+ b.processTagFamilies(tff, i, dataPointsLen)
+ }
+ for i, f := range fields {
+ columns := b.field.resizeColumns(len(f.values))
+ for j, t := range f.values {
+ columns[j].name = t.name
+ columns[j].resizeValues(dataPointsLen)
+ columns[j].valueType = t.valueType
+ columns[j].values[i] = t.marshal()
+ }
+ }
+}
+
+func (b *block) processTagFamilies(tff []nameValues, i int, dataPointsLen int) {
+ tagFamilies := b.resizeTagFamilies(len(tff))
+ for j, tf := range tff {
+ tagFamilies[j].name = tf.name
+ b.processTags(tf, j, i, dataPointsLen)
+ }
+}
+
+func (b *block) processTags(tf nameValues, columnFamilyIdx, i int, dataPointsLen int) {
+ columns := b.tagFamilies[columnFamilyIdx].resizeColumns(len(tf.values))
+ for j, t := range tf.values {
+ columns[j].name = t.name
+ columns[j].resizeValues(dataPointsLen)
+ columns[j].valueType = t.valueType
+ columns[j].values[i] = t.marshal()
+ }
+}
+
+func (b *block) resizeTagFamilies(tagFamiliesLen int) []columnFamily {
+ tff := b.tagFamilies[:0]
+ if n := tagFamiliesLen - cap(tff); n > 0 {
+ tff = append(tff[:cap(tff)], make([]columnFamily, n)...)
+ }
+ tff = tff[:tagFamiliesLen]
+ b.tagFamilies = tff
+ return tff
+}
+
+func (b *block) Len() int {
+ return len(b.timestamps)
+}
+
+func (b *block) mustWriteTo(sid common.SeriesID, bm *blockMetadata, ww *writers) {
+ b.validate()
+ bm.reset()
+
+ bm.seriesID = sid
+ bm.uncompressedSizeBytes = b.uncompressedSizeBytes()
+ bm.count = uint64(b.Len())
+
+ mustWriteTimestampsTo(&bm.timestamps, b.timestamps, ww.timestampsWriter)
+
+ for ti := range b.tagFamilies {
+ b.marshalTagFamily(b.tagFamilies[ti], bm, ww)
+ }
+
+ f := b.field
+ cc := f.columns
+ cmm := bm.field.resizeColumnMetadata(len(cc))
+ for i := range cc {
+ cc[i].mustWriteTo(&cmm[i], &ww.fieldValuesWriter)
+ }
+}
+
+func (b *block) validate() {
+ timestamps := b.timestamps
+ for i := 1; i < len(timestamps); i++ {
+ if timestamps[i-1] > timestamps[i] {
+ logger.Panicf("log entries must be sorted by timestamp; got the previous entry with bigger timestamp %d than the current entry with timestamp %d",
+ timestamps[i-1], timestamps[i])
+ }
+ }
+
+ itemsCount := len(timestamps)
+ tff := b.tagFamilies
+ for _, tf := range tff {
+ for _, c := range tf.columns {
+ if len(c.values) != itemsCount {
+ logger.Panicf("unexpected number of values for tags %q: got %d; want %d", c.name, len(c.values), itemsCount)
+ }
+ }
+ }
+ ff := b.field
+ for _, f := range ff.columns {
+ if len(f.values) != itemsCount {
+ logger.Panicf("unexpected number of values for fields %q: got %d; want %d", f.name, len(f.values), itemsCount)
+ }
+ }
+}
+
+func (b *block) marshalTagFamily(tf columnFamily, bm *blockMetadata, ww *writers) {
+ hw, w := ww.getColumnMetadataWriterAndColumnWriter(tf.name)
+ cc := tf.columns
+ cfm := generateColumnFamilyMetadata()
+ cmm := cfm.resizeColumnMetadata(len(cc))
+ for i := range cc {
+ cc[i].mustWriteTo(&cmm[i], w)
+ }
+ bb := bigValuePool.Generate()
+ defer bigValuePool.Release(bb)
+ bb.Buf = cfm.marshal(bb.Buf)
+ releaseColumnFamilyMetadata(cfm)
+ tfm := bm.getTagFamilyMetadata(tf.name)
+ tfm.offset = hw.bytesWritten
+ tfm.size = uint64(len(bb.Buf))
+ if tfm.size > maxTagFamiliesMetadataSize {
+ logger.Panicf("too big columnFamilyMetadataSize: %d bytes; mustn't exceed %d bytes", tfm.size, maxTagFamiliesMetadataSize)
+ }
+ hw.MustWrite(bb.Buf)
+}
+
+func (b *block) unmarshalTagFamily(decoder *encoding.BytesBlockDecoder, tfIndex int, name string,
+ columnFamilyMetadataBlock *dataBlock, tagProjection []string, metaReader, valueReader fs.Reader,
+) {
+ if len(tagProjection) < 1 {
+ return
+ }
+ bb := bigValuePool.Generate()
+ bb.Buf = bytes.ResizeExact(bb.Buf, int(columnFamilyMetadataBlock.size))
+ fs.MustReadData(metaReader, int64(columnFamilyMetadataBlock.offset), bb.Buf)
+ cfm := generateColumnFamilyMetadata()
+ defer releaseColumnFamilyMetadata(cfm)
+ _, err := cfm.unmarshal(bb.Buf)
+ if err != nil {
+ logger.Panicf("%s: cannot unmarshal columnFamilyMetadata: %v", metaReader.Path(), err)
+ }
+ bigValuePool.Release(bb)
+ b.tagFamilies[tfIndex].name = name
+ cc := b.tagFamilies[tfIndex].resizeColumns(len(tagProjection))
+
+ for j := range tagProjection {
+ for i := range cfm.columnMetadata {
+ if tagProjection[j] == cfm.columnMetadata[i].name {
+ cc[j].mustReadValues(decoder, valueReader, cfm.columnMetadata[i], uint64(b.Len()))
+ break
+ }
+ }
+ }
+}
+
+func (b *block) uncompressedSizeBytes() uint64 {
+ dataPointsCount := uint64(b.Len())
+
+ n := dataPointsCount * 8
+
+ tff := b.tagFamilies
+ for i := range tff {
+ tf := tff[i]
+ nameLen := uint64(len(tf.name))
+ for _, c := range tf.columns {
+ nameLen += uint64(len(c.name))
+ for _, v := range c.values {
+ if len(v) > 0 {
+ n += nameLen + uint64(len(v))
+ }
+ }
+ }
+ }
+
+ ff := b.field
+ for i := range ff.columns {
+ c := ff.columns[i]
+ nameLen := uint64(len(c.name))
+ for _, v := range c.values {
+ if len(v) > 0 {
+ n += nameLen + uint64(len(v))
+ }
+ }
+ }
+ return n
+}
+
+func (b *block) mustReadFrom(decoder *encoding.BytesBlockDecoder, p *part, bm blockMetadata) {
+ b.reset()
+
+ b.timestamps = mustReadTimestampsFrom(b.timestamps, &bm.timestamps, int(bm.count), p.timestamps)
+
+ _ = b.resizeTagFamilies(len(bm.tagProjection))
+ for i := range bm.tagProjection {
+ name := bm.tagProjection[i].Family
+ block, ok := bm.tagFamilies[name]
+ if !ok {
+ continue
+ }
+ b.unmarshalTagFamily(decoder, i, name, block, bm.tagProjection[i].Names, p.tagFamilyMetadata[name], p.tagFamilies[name])
+ }
+ cc := b.field.resizeColumns(len(bm.field.columnMetadata))
+ for i := range cc {
+ cc[i].mustReadValues(decoder, p.fieldValues, bm.field.columnMetadata[i], bm.count)
+ }
+}
+
+// For testing purpose only.
+func (b *block) sortTagFamilies() {
+ sort.Slice(b.tagFamilies, func(i, j int) bool {
+ return b.tagFamilies[i].name < b.tagFamilies[j].name
+ })
+}
+
+func mustWriteTimestampsTo(tm *timestampsMetadata, timestamps []int64, timestampsWriter writer) {
+ tm.reset()
+
+ bb := bigValuePool.Generate()
+ defer bigValuePool.Release(bb)
+ bb.Buf, tm.encodeType, tm.min = encoding.Int64ListToBytes(bb.Buf[:0], timestamps)
+ if len(bb.Buf) > maxTimestampsBlockSize {
+ logger.Panicf("too big block with timestamps: %d bytes; the maximum supported size is %d bytes", len(bb.Buf), maxTimestampsBlockSize)
+ }
+ tm.max = timestamps[len(timestamps)-1]
+ tm.offset = timestampsWriter.bytesWritten
+ tm.size = uint64(len(bb.Buf))
+ timestampsWriter.MustWrite(bb.Buf)
+}
+
+func mustReadTimestampsFrom(dst []int64, tm *timestampsMetadata, count int, reader fs.Reader) []int64 {
+ bb := bigValuePool.Generate()
+ defer bigValuePool.Release(bb)
+ bb.Buf = bytes.ResizeExact(bb.Buf, int(tm.size))
+ fs.MustReadData(reader, int64(tm.offset), bb.Buf)
+ var err error
+ dst, err = encoding.BytesToInt64List(dst, bb.Buf, tm.encodeType, tm.min, count)
+ if err != nil {
+ logger.Panicf("%s: cannot unmarshal timestamps: %v", reader.Path(), err)
+ }
+ return dst
+}
+
+func generateBlock() *block {
+ v := blockPool.Get()
+ if v == nil {
+ return &block{}
+ }
+ return v.(*block)
+}
+
+func releaseBlock(b *block) {
+ b.reset()
+ blockPool.Put(b)
+}
+
+var blockPool sync.Pool
+
+type blockCursor struct {
+ p *part
+ fields columnFamily
+ timestamps []int64
+ tagFamilies []columnFamily
+ columnValuesDecoder encoding.BytesBlockDecoder
+ tagProjection []pbv1.TagProjection
+ fieldProjection []string
+ bm blockMetadata
+ idx int
+ minTimestamp int64
+ maxTimestamp int64
+}
+
+func (bc *blockCursor) reset() {
+ bc.idx = 0
+ bc.p = nil
+ bc.bm = blockMetadata{}
+ bc.minTimestamp = 0
+ bc.maxTimestamp = 0
+ bc.tagProjection = bc.tagProjection[:0]
+ bc.fieldProjection = bc.fieldProjection[:0]
+
+ bc.timestamps = bc.timestamps[:0]
+
+ tff := bc.tagFamilies
+ for i := range tff {
+ tff[i].reset()
+ }
+ bc.tagFamilies = tff[:0]
+ bc.fields.reset()
+}
+
+func (bc *blockCursor) init(p *part, bm blockMetadata, queryOpts queryOptions) {
+ bc.reset()
+ bc.p = p
+ bc.bm = bm
+ bc.minTimestamp = queryOpts.minTimestamp
+ bc.maxTimestamp = queryOpts.maxTimestamp
+ bc.tagProjection = queryOpts.TagProjection
+ bc.fieldProjection = queryOpts.FieldProjection
+}
+
+func (bc *blockCursor) copyAllTo(r *pbv1.Result) {
+ r.SID = bc.bm.seriesID
+ r.Timestamps = append(r.Timestamps, bc.timestamps...)
+ for _, cf := range bc.tagFamilies {
+ tf := pbv1.TagFamily{
+ Name: cf.name,
+ }
+ for _, c := range cf.columns {
+ t := pbv1.Tag{
+ Name: c.name,
+ }
+ for _, v := range c.values {
+ t.Values = append(t.Values, mustDecodeTagValue(c.valueType, v))
+ }
+ tf.Tags = append(tf.Tags, t)
+ }
+ r.TagFamilies = append(r.TagFamilies, tf)
+ }
+ for _, c := range bc.fields.columns {
+ f := pbv1.Field{
+ Name: c.name,
+ }
+ for _, v := range c.values {
+ f.Values = append(f.Values, mustDecodeFieldValue(c.valueType, v))
+ }
+ r.Fields = append(r.Fields, f)
+ }
+}
+
+func (bc *blockCursor) copyTo(r *pbv1.Result) {
+ r.SID = bc.bm.seriesID
+ r.Timestamps = append(r.Timestamps, bc.timestamps[bc.idx])
+ if len(r.TagFamilies) != len(bc.tagProjection) {
+ for _, tp := range bc.tagProjection {
+ tf := pbv1.TagFamily{
+ Name: tp.Family,
+ }
+ for _, n := range tp.Names {
+ t := pbv1.Tag{
+ Name: n,
+ }
+ tf.Tags = append(tf.Tags, t)
+ }
+ r.TagFamilies = append(r.TagFamilies, tf)
+ }
+ }
+ if len(bc.tagFamilies) != len(r.TagFamilies) {
+ logger.Panicf("unexpected number of tag families: got %d; want %d", len(bc.tagFamilies), len(r.TagFamilies))
+ }
+ for i, cf := range bc.tagFamilies {
+ if len(r.TagFamilies[i].Tags) != len(cf.columns) {
+ logger.Panicf("unexpected number of tags: got %d; want %d", len(r.TagFamilies[i].Tags), len(bc.tagProjection[i].Names))
+ }
+ for i2, c := range cf.columns {
+ r.TagFamilies[i].Tags[i2].Values = append(r.TagFamilies[i].Tags[i2].Values, mustDecodeTagValue(c.valueType, c.values[bc.idx]))
+ }
+ }
+
+ if len(r.Fields) != len(bc.fieldProjection) {
+ for _, n := range bc.fieldProjection {
+ f := pbv1.Field{
+ Name: n,
+ }
+ r.Fields = append(r.Fields, f)
+ }
+ }
+ for i, c := range bc.fields.columns {
+ r.Fields[i].Values = append(r.Fields[i].Values, mustDecodeFieldValue(c.valueType, c.values[bc.idx]))
+ }
+}
+
+func (bc *blockCursor) loadData(tmpBlock *block) bool {
+ tmpBlock.reset()
+ cfm := make([]columnMetadata, 0, len(bc.fieldProjection))
+ for j := range bc.fieldProjection {
+ for i := range bc.bm.field.columnMetadata {
+ if bc.bm.field.columnMetadata[i].name == bc.fieldProjection[j] {
+ cfm = append(cfm, bc.bm.field.columnMetadata[i])
+ }
+ }
+ }
+ bc.bm.field.columnMetadata = cfm
+ bc.bm.tagProjection = bc.tagProjection
+ tf := make(map[string]*dataBlock, len(bc.tagProjection))
+ for i := range bc.tagProjection {
+ for tfName, block := range bc.bm.tagFamilies {
+ if bc.tagProjection[i].Family == tfName {
+ tf[tfName] = block
+ }
+ }
+ }
+ bc.bm.tagFamilies = tf
+ tmpBlock.mustReadFrom(&bc.columnValuesDecoder, bc.p, bc.bm)
+
+ start, end, ok := findRange(tmpBlock.timestamps, bc.minTimestamp, bc.maxTimestamp)
+ if !ok {
+ return false
+ }
+ bc.timestamps = append(bc.timestamps, tmpBlock.timestamps[start:end]...)
+
+ for _, cf := range tmpBlock.tagFamilies {
+ tf := columnFamily{
+ name: cf.name,
+ }
+ for i := range cf.columns {
+ column := column{
+ name: cf.columns[i].name,
+ valueType: cf.columns[i].valueType,
+ }
+ if len(cf.columns[i].values) == 0 {
+ continue
+ }
+ if len(cf.columns[i].values) != len(tmpBlock.timestamps) {
+ logger.Panicf("unexpected number of values for tags %q: got %d; want %d", cf.columns[i].name, len(cf.columns[i].values), len(tmpBlock.timestamps))
+ }
+ column.values = append(column.values, cf.columns[i].values[start:end]...)
+ tf.columns = append(tf.columns, column)
+ }
+ bc.tagFamilies = append(bc.tagFamilies, tf)
+ }
+ bc.fields.name = tmpBlock.field.name
+ for i := range tmpBlock.field.columns {
+ if len(tmpBlock.field.columns[i].values) == 0 {
+ continue
+ }
+ if len(tmpBlock.field.columns[i].values) != len(tmpBlock.timestamps) {
+ logger.Panicf("unexpected number of values for fields %q: got %d; want %d",
+ tmpBlock.field.columns[i].name, len(tmpBlock.field.columns[i].values), len(tmpBlock.timestamps))
+ }
+ c := column{
+ name: tmpBlock.field.columns[i].name,
+ valueType: tmpBlock.field.columns[i].valueType,
+ }
+ c.values = append(c.values, tmpBlock.field.columns[i].values[start:end]...)
+ bc.fields.columns = append(bc.fields.columns, c)
+ }
+ return true
+}
+
+func findRange(timestamps []int64, min int64, max int64) (int, int, bool) {
+ l := len(timestamps)
+ start, end := -1, -1
+
+ for i := 0; i < l; i++ {
+ if timestamps[i] > max || timestamps[l-i-1] < min {
+ break
+ }
+ if timestamps[i] >= min && start == -1 {
+ start = i
+ }
+ if timestamps[l-i-1] <= max && end == -1 {
+ end = l - i
+ }
+ if start != -1 && end != -1 {
+ break
+ }
+ }
+
+ if start == -1 || end == -1 || start >= end {
+ return 0, 0, false
+ }
+
+ return start, end, true
+}
+
+var blockCursorPool sync.Pool
+
+func generateBlockCursor() *blockCursor {
+ v := blockCursorPool.Get()
+ if v == nil {
+ return &blockCursor{}
+ }
+ return v.(*blockCursor)
+}
+
+func releaseBlockCursor(bc *blockCursor) {
+ bc.reset()
+ blockCursorPool.Put(bc)
+}
diff --git a/banyand/measure/block_metadata.go b/banyand/measure/block_metadata.go
new file mode 100644
index 00000000..e239162c
--- /dev/null
+++ b/banyand/measure/block_metadata.go
@@ -0,0 +1,270 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package measure
+
+import (
+ "errors"
+ "fmt"
+ "sync"
+
+ "github.com/apache/skywalking-banyandb/api/common"
+ "github.com/apache/skywalking-banyandb/pkg/convert"
+ "github.com/apache/skywalking-banyandb/pkg/encoding"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
+)
+
+type dataBlock struct {
+ offset uint64
+ size uint64
+}
+
+func (h *dataBlock) reset() {
+ h.offset = 0
+ h.size = 0
+}
+
+func (h *dataBlock) copyFrom(src *dataBlock) {
+ h.offset = src.offset
+ h.size = src.size
+}
+
+func (h *dataBlock) marshal(dst []byte) []byte {
+ dst = encoding.VarUint64ToBytes(dst, h.offset)
+ dst = encoding.VarUint64ToBytes(dst, h.size)
+ return dst
+}
+
+func (h *dataBlock) unmarshal(src []byte) ([]byte, error) {
+ src, n, err := encoding.BytesToVarUint64(src)
+ if err != nil {
+ return nil, fmt.Errorf("cannot unmarshal offset: %w", err)
+ }
+ h.offset = n
+
+ src, n, err = encoding.BytesToVarUint64(src)
+ if err != nil {
+ return nil, fmt.Errorf("cannot unmarshal size: %w", err)
+ }
+ h.size = n
+ return src, nil
+}
+
+type blockMetadata struct {
+ tagFamilies map[string]*dataBlock
+ field columnFamilyMetadata
+ tagProjection []pbv1.TagProjection
+ timestamps timestampsMetadata
+ seriesID common.SeriesID
+ uncompressedSizeBytes uint64
+ count uint64
+}
+
+func (bh *blockMetadata) getTagFamilyMetadata(name string) *dataBlock {
+ if bh.tagFamilies == nil {
+ bh.tagFamilies = make(map[string]*dataBlock)
+ }
+ tf, ok := bh.tagFamilies[name]
+ if !ok {
+ tf = &dataBlock{}
+ bh.tagFamilies[name] = tf
+ }
+ return tf
+}
+
+func (bh *blockMetadata) reset() {
+ bh.seriesID = 0
+ bh.uncompressedSizeBytes = 0
+ bh.count = 0
+ bh.timestamps.reset()
+ bh.field.reset()
+ for k := range bh.tagFamilies {
+ bh.tagFamilies[k].reset()
+ delete(bh.tagFamilies, k)
+ }
+ bh.tagProjection = bh.tagProjection[:0]
+}
+
+func (bh *blockMetadata) marshal(dst []byte) []byte {
+ dst = bh.seriesID.AppendToBytes(dst)
+ dst = encoding.VarUint64ToBytes(dst, bh.uncompressedSizeBytes)
+ dst = encoding.VarUint64ToBytes(dst, bh.count)
+ dst = bh.timestamps.marshal(dst)
+ dst = encoding.VarUint64ToBytes(dst, uint64(len(bh.tagFamilies)))
+ for name, cf := range bh.tagFamilies {
+ dst = encoding.EncodeBytes(dst, convert.StringToBytes(name))
+ dst = cf.marshal(dst)
+ }
+ return bh.field.marshal(dst)
+}
+
+func (bh *blockMetadata) unmarshal(src []byte) ([]byte, error) {
+ if len(src) < 8 {
+ return nil, errors.New("cannot unmarshal blockMetadata from less than 8 bytes")
+ }
+ bh.seriesID = common.SeriesID(encoding.BytesToUint64(src))
+ src = src[8:]
+ src, n, err := encoding.BytesToVarUint64(src)
+ if err != nil {
+ return nil, fmt.Errorf("cannot unmarshal uncompressedSizeBytes: %w", err)
+ }
+ bh.uncompressedSizeBytes = n
+
+ src, n, err = encoding.BytesToVarUint64(src)
+ if err != nil {
+ return nil, fmt.Errorf("cannot unmarshal count: %w", err)
+ }
+ bh.count = n
+ src, err = bh.timestamps.unmarshal(src)
+ if err != nil {
+ return nil, fmt.Errorf("cannot unmarshal timestampsMetadata: %w", err)
+ }
+ src, n, err = encoding.BytesToVarUint64(src)
+ if err != nil {
+ return nil, fmt.Errorf("cannot unmarshal tagFamilies count: %w", err)
+ }
+ if n > 0 {
+ if bh.tagFamilies == nil {
+ bh.tagFamilies = make(map[string]*dataBlock, n)
+ }
+ var nameBytes []byte
+ for i := uint64(0); i < n; i++ {
+ src, nameBytes, err = encoding.DecodeBytes(src)
+ if err != nil {
+ return nil, fmt.Errorf("cannot unmarshal tagFamily name: %w", err)
+ }
+ // TODO: cache dataBlock
+ tf := &dataBlock{}
+ src, err = tf.unmarshal(src)
+ if err != nil {
+ return nil, fmt.Errorf("cannot unmarshal tagFamily dataBlock: %w", err)
+ }
+ bh.tagFamilies[convert.BytesToString(nameBytes)] = tf
+ }
+ }
+ src, err = bh.field.unmarshal(src)
+ if err != nil {
+ return nil, fmt.Errorf("cannot unmarshal columnFamilyMetadata: %w", err)
+ }
+ return src, nil
+}
+
+func (bh blockMetadata) less(other blockMetadata) bool {
+ if bh.seriesID == other.seriesID {
+ return bh.timestamps.min < other.timestamps.min
+ }
+ return bh.seriesID < other.seriesID
+}
+
+func generateBlockMetadata() *blockMetadata {
+ v := blockMetadataPool.Get()
+ if v == nil {
+ return &blockMetadata{}
+ }
+ return v.(*blockMetadata)
+}
+
+func releaseBlockMetadata(bh *blockMetadata) {
+ bh.reset()
+ blockMetadataPool.Put(bh)
+}
+
+var blockMetadataPool sync.Pool
+
+type timestampsMetadata struct {
+ dataBlock
+ min int64
+ max int64
+ encodeType encoding.EncodeType
+}
+
+func (th *timestampsMetadata) reset() {
+ th.dataBlock.reset()
+ th.min = 0
+ th.max = 0
+ th.encodeType = 0
+}
+
+func (th *timestampsMetadata) copyFrom(src *timestampsMetadata) {
+ th.dataBlock.copyFrom(&src.dataBlock)
+ th.min = src.min
+ th.max = src.max
+ th.encodeType = src.encodeType
+}
+
+func (th *timestampsMetadata) marshal(dst []byte) []byte {
+ dst = th.dataBlock.marshal(dst)
+ dst = encoding.Uint64ToBytes(dst, uint64(th.min))
+ dst = encoding.Uint64ToBytes(dst, uint64(th.max))
+ dst = append(dst, byte(th.encodeType))
+ return dst
+}
+
+func (th *timestampsMetadata) unmarshal(src []byte) ([]byte, error) {
+ src, err := th.dataBlock.unmarshal(src)
+ if err != nil {
+ return nil, fmt.Errorf("cannot unmarshal dataBlock: %w", err)
+ }
+ th.min = int64(encoding.BytesToUint64(src))
+ src = src[8:]
+ th.max = int64(encoding.BytesToUint64(src))
+ src = src[8:]
+ th.encodeType = encoding.EncodeType(src[0])
+ return src[1:], nil
+}
+
+func unmarshalBlockMetadata(dst []blockMetadata, src []byte) ([]blockMetadata, error) {
+ dstOrig := dst
+ var pre *blockMetadata
+ for len(src) > 0 {
+ if len(dst) < cap(dst) {
+ dst = dst[:len(dst)+1]
+ } else {
+ dst = append(dst, blockMetadata{})
+ }
+ bm := &dst[len(dst)-1]
+ tail, err := bm.unmarshal(src)
+ if err != nil {
+ return dstOrig, fmt.Errorf("cannot unmarshal blockMetadata entries: %w", err)
+ }
+ src = tail
+
+ // Validate the order of blockMetadata during unmarshalling
+ if pre != nil {
+ if err := validateBlockMetadataOrder(pre, bm); err != nil {
+ return dstOrig, err
+ }
+ }
+ pre = bm
+ }
+ return dst, nil
+}
+
+func validateBlockMetadataOrder(pre, cur *blockMetadata) error {
+ if cur.seriesID < pre.seriesID {
+ return fmt.Errorf("unexpected blockMetadata with smaller seriesID=%d after bigger seriesID=%d", cur.seriesID, pre.seriesID)
+ }
+ if cur.seriesID != pre.seriesID {
+ return nil
+ }
+ tmCur := cur.timestamps
+ tmPre := pre.timestamps
+ if tmCur.min < tmPre.min {
+ return fmt.Errorf("unexpected blockMetadata with smaller timestamp=%d after bigger timestamp=%d", tmCur.min, tmPre.min)
+ }
+ return nil
+}
diff --git a/banyand/measure/block_metadata_test.go b/banyand/measure/block_metadata_test.go
new file mode 100644
index 00000000..81da440b
--- /dev/null
+++ b/banyand/measure/block_metadata_test.go
@@ -0,0 +1,348 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package measure
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "github.com/apache/skywalking-banyandb/api/common"
+ "github.com/apache/skywalking-banyandb/pkg/encoding"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
+)
+
+func Test_dataBlock_reset(t *testing.T) {
+ h := &dataBlock{
+ offset: 1,
+ size: 1,
+ }
+
+ h.reset()
+
+ assert.Equal(t, uint64(0), h.offset)
+ assert.Equal(t, uint64(0), h.size)
+}
+
+func Test_dataBlock_copyFrom(t *testing.T) {
+ src := &dataBlock{
+ offset: 1,
+ size: 1,
+ }
+
+ dest := &dataBlock{
+ offset: 2,
+ size: 2,
+ }
+
+ dest.copyFrom(src)
+
+ assert.Equal(t, src.offset, dest.offset)
+ assert.Equal(t, src.size, dest.size)
+}
+
+func Test_dataBlock_marshal_unmarshal(t *testing.T) {
+ original := &dataBlock{
+ offset: 1,
+ size: 1,
+ }
+
+ marshaled := original.marshal(nil)
+
+ unmarshaled := &dataBlock{}
+
+ _, err := unmarshaled.unmarshal(marshaled)
+ require.NoError(t, err)
+
+ assert.Equal(t, original.offset, unmarshaled.offset)
+ assert.Equal(t, original.size, unmarshaled.size)
+}
+
+func Test_timestampsMetadata_reset(t *testing.T) {
+ th := ×tampsMetadata{
+ dataBlock: dataBlock{
+ offset: 1,
+ size: 1,
+ },
+ min: 1,
+ max: 1,
+ encodeType: encoding.EncodeTypeConst,
+ }
+
+ th.reset()
+
+ assert.Equal(t, uint64(0), th.dataBlock.offset)
+ assert.Equal(t, uint64(0), th.dataBlock.size)
+ assert.Equal(t, int64(0), th.min)
+ assert.Equal(t, int64(0), th.max)
+ assert.Equal(t, encoding.EncodeTypeUnknown, th.encodeType)
+}
+
+func Test_timestampsMetadata_copyFrom(t *testing.T) {
+ src := ×tampsMetadata{
+ dataBlock: dataBlock{
+ offset: 1,
+ size: 1,
+ },
+ min: 1,
+ max: 1,
+ encodeType: encoding.EncodeTypeConst,
+ }
+
+ dest := ×tampsMetadata{
+ dataBlock: dataBlock{
+ offset: 2,
+ size: 2,
+ },
+ min: 2,
+ max: 2,
+ encodeType: encoding.EncodeTypeDelta,
+ }
+
+ dest.copyFrom(src)
+
+ assert.Equal(t, src.dataBlock.offset, dest.dataBlock.offset)
+ assert.Equal(t, src.dataBlock.size, dest.dataBlock.size)
+ assert.Equal(t, src.min, dest.min)
+ assert.Equal(t, src.max, dest.max)
+ assert.Equal(t, src.encodeType, dest.encodeType)
+}
+
+func Test_timestampsMetadata_marshal_unmarshal(t *testing.T) {
+ original := ×tampsMetadata{
+ dataBlock: dataBlock{
+ offset: 1,
+ size: 1,
+ },
+ min: 1,
+ max: 1,
+ encodeType: encoding.EncodeTypeConst,
+ }
+
+ marshaled := original.marshal(nil)
+
+ unmarshaled := ×tampsMetadata{}
+
+ _, err := unmarshaled.unmarshal(marshaled)
+ require.NoError(t, err)
+
+ assert.Equal(t, original.dataBlock.offset, unmarshaled.dataBlock.offset)
+ assert.Equal(t, original.dataBlock.size, unmarshaled.dataBlock.size)
+ assert.Equal(t, original.min, unmarshaled.min)
+ assert.Equal(t, original.max, unmarshaled.max)
+ assert.Equal(t, original.encodeType, unmarshaled.encodeType)
+}
+
+func Test_blockMetadata_marshal_unmarshal(t *testing.T) {
+ testCases := []struct {
+ original *blockMetadata
+ name string
+ }{
+ {
+ name: "Zero values",
+ original: &blockMetadata{
+ seriesID: common.SeriesID(0),
+ uncompressedSizeBytes: 0,
+ count: 0,
+ timestamps: timestampsMetadata{},
+ tagFamilies: make(map[string]*dataBlock),
+ field: columnFamilyMetadata{},
+ },
+ },
+ {
+ name: "Non-zero values",
+ original: &blockMetadata{
+ seriesID: common.SeriesID(1),
+ uncompressedSizeBytes: 1,
+ count: 1,
+ timestamps: timestampsMetadata{
+ dataBlock: dataBlock{
+ offset: 1,
+ size: 1,
+ },
+ min: 1,
+ max: 1,
+ encodeType: encoding.EncodeTypeConst,
+ },
+ tagFamilies: map[string]*dataBlock{
+ "tag1": {
+ offset: 1,
+ size: 1,
+ },
+ },
+ field: columnFamilyMetadata{
+ columnMetadata: []columnMetadata{
+ {
+ dataBlock: dataBlock{
+ offset: 1,
+ size: 1,
+ },
+ name: "field1",
+ valueType: pbv1.ValueTypeInt64,
+ },
+ },
+ },
+ },
+ },
+ {
+ name: "Multiple tagFamilies and columnMetadata",
+ original: &blockMetadata{
+ seriesID: common.SeriesID(2),
+ uncompressedSizeBytes: 2,
+ count: 2,
+ timestamps: timestampsMetadata{
+ dataBlock: dataBlock{
+ offset: 2,
+ size: 2,
+ },
+ min: 2,
+ max: 2,
+ encodeType: encoding.EncodeTypeConst,
+ },
+ tagFamilies: map[string]*dataBlock{
+ "tag1": {
+ offset: 2,
+ size: 2,
+ },
+ "tag2": {
+ offset: 3,
+ size: 3,
+ },
+ },
+ field: columnFamilyMetadata{
+ columnMetadata: []columnMetadata{
+ {
+ dataBlock: dataBlock{
+ offset: 2,
+ size: 2,
+ },
+ name: "field1",
+ valueType: pbv1.ValueTypeInt64,
+ },
+ {
+ dataBlock: dataBlock{
+ offset: 3,
+ size: 3,
+ },
+ name: "field2",
+ valueType: pbv1.ValueTypeInt64,
+ },
+ },
+ },
+ },
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ marshaled := tc.original.marshal(nil)
+
+ unmarshaled := blockMetadata{
+ tagFamilies: make(map[string]*dataBlock),
+ }
+
+ _, err := unmarshaled.unmarshal(marshaled)
+ require.NoError(t, err)
+
+ assert.Equal(t, tc.original.seriesID, unmarshaled.seriesID)
+ assert.Equal(t, tc.original.uncompressedSizeBytes, unmarshaled.uncompressedSizeBytes)
+ assert.Equal(t, tc.original.count, unmarshaled.count)
+ assert.Equal(t, tc.original.timestamps, unmarshaled.timestamps)
+ assert.Equal(t, tc.original.tagFamilies, unmarshaled.tagFamilies)
+ assert.Equal(t, tc.original.field, unmarshaled.field)
+ })
+ }
+}
+
+func Test_unmarshalBlockMetadata(t *testing.T) {
+ t.Run("unmarshal valid blockMetadata", func(t *testing.T) {
+ original := []blockMetadata{
+ {
+ seriesID: common.SeriesID(1),
+ timestamps: timestampsMetadata{
+ dataBlock: dataBlock{
+ offset: 1,
+ size: 1,
+ },
+ min: 1,
+ max: 1,
+ encodeType: encoding.EncodeTypeConst,
+ },
+ },
+ {
+ seriesID: common.SeriesID(2),
+ timestamps: timestampsMetadata{
+ dataBlock: dataBlock{
+ offset: 2,
+ size: 2,
+ },
+ min: 2,
+ max: 2,
+ encodeType: encoding.EncodeTypeConst,
+ },
+ },
+ }
+
+ var marshaled []byte
+ for _, bm := range original {
+ marshaled = bm.marshal(marshaled)
+ }
+
+ unmarshaled, err := unmarshalBlockMetadata(nil, marshaled)
+ require.NoError(t, err)
+ require.Equal(t, original, unmarshaled)
+ })
+
+ t.Run("unmarshal invalid blockMetadata", func(t *testing.T) {
+ original := []blockMetadata{
+ {
+ seriesID: common.SeriesID(2),
+ timestamps: timestampsMetadata{
+ dataBlock: dataBlock{
+ offset: 2,
+ size: 2,
+ },
+ min: 2,
+ max: 2,
+ encodeType: encoding.EncodeTypeConst,
+ },
+ },
+ {
+ seriesID: common.SeriesID(1),
+ timestamps: timestampsMetadata{
+ dataBlock: dataBlock{
+ offset: 1,
+ size: 1,
+ },
+ min: 1,
+ max: 1,
+ encodeType: encoding.EncodeTypeConst,
+ },
+ },
+ }
+
+ var marshaled []byte
+ for _, bm := range original {
+ marshaled = bm.marshal(marshaled)
+ }
+
+ _, err := unmarshalBlockMetadata(nil, marshaled)
+ require.Error(t, err)
+ })
+}
diff --git a/banyand/measure/field_flag_test.go b/banyand/measure/block_reader.go
similarity index 51%
rename from banyand/measure/field_flag_test.go
rename to banyand/measure/block_reader.go
index c47e457f..de5c5c2f 100644
--- a/banyand/measure/field_flag_test.go
+++ b/banyand/measure/block_reader.go
@@ -17,24 +17,23 @@
package measure
-import (
- "testing"
- "time"
+import "github.com/apache/skywalking-banyandb/pkg/fs"
- "github.com/stretchr/testify/assert"
+type reader struct {
+ r fs.Reader
+ bytesRead uint64
+}
+
+func newReader(r fs.Reader) *reader {
+ return &reader{r: r}
+}
- databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
- pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
-)
+func (r *reader) Path() string {
+ return r.r.Path()
+}
-func TestEncodeFieldFlag(t *testing.T) {
- flag := pbv1.EncoderFieldFlag(&databasev1.FieldSpec{
- EncodingMethod: databasev1.EncodingMethod_ENCODING_METHOD_GORILLA,
- CompressionMethod: databasev1.CompressionMethod_COMPRESSION_METHOD_ZSTD,
- }, time.Minute)
- fieldSpec, interval, err := pbv1.DecodeFieldFlag(flag)
- assert.NoError(t, err)
- assert.Equal(t, databasev1.EncodingMethod_ENCODING_METHOD_GORILLA, fieldSpec.EncodingMethod)
- assert.Equal(t, databasev1.CompressionMethod_COMPRESSION_METHOD_ZSTD, fieldSpec.CompressionMethod)
- assert.Equal(t, time.Minute, interval)
+func (r *reader) Read(p []byte) (int, error) {
+ n, err := r.r.Read(0, p)
+ r.bytesRead += uint64(n)
+ return n, err
}
diff --git a/banyand/measure/block_test.go b/banyand/measure/block_test.go
new file mode 100644
index 00000000..e7b776c1
--- /dev/null
+++ b/banyand/measure/block_test.go
@@ -0,0 +1,466 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package measure
+
+import (
+ "crypto/rand"
+ "encoding/binary"
+ "reflect"
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+
+ "github.com/apache/skywalking-banyandb/api/common"
+ "github.com/apache/skywalking-banyandb/pkg/bytes"
+ "github.com/apache/skywalking-banyandb/pkg/convert"
+ "github.com/apache/skywalking-banyandb/pkg/encoding"
+ "github.com/apache/skywalking-banyandb/pkg/fs"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
+)
+
+func Test_block_reset(t *testing.T) {
+ type fields struct {
+ timestamps []int64
+ tagFamilies []columnFamily
+ field columnFamily
+ }
+ tests := []struct {
+ name string
+ fields fields
+ want block
+ }{
+ {
+ name: "Test reset",
+ fields: fields{
+ timestamps: []int64{1, 2, 3},
+ tagFamilies: []columnFamily{{}, {}, {}},
+ field: columnFamily{columns: []column{{}, {}}},
+ },
+ want: block{
+ timestamps: []int64{},
+ tagFamilies: []columnFamily{},
+ field: columnFamily{columns: []column{}},
+ },
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ b := &block{
+ timestamps: tt.fields.timestamps,
+ tagFamilies: tt.fields.tagFamilies,
+ field: tt.fields.field,
+ }
+ b.reset()
+ if !reflect.DeepEqual(*b, tt.want) {
+ t.Errorf("block.reset() = %+v, want %+v", *b, tt.want)
+ }
+ })
+ }
+}
+
+func toTagProjection(b block) map[string][]string {
+ result := make(map[string][]string, len(b.tagFamilies))
+ for i := range b.tagFamilies {
+ names := make([]string, len(b.tagFamilies[i].columns))
+ for i2 := range b.tagFamilies[i].columns {
+ names[i2] = b.tagFamilies[i].columns[i2].name
+ }
+ result[b.tagFamilies[i].name] = names
+ }
+ return result
+}
+
+var conventionalBlock = block{
+ timestamps: []int64{1, 2},
+ tagFamilies: []columnFamily{
+ {
+ name: "arrTag",
+ columns: []column{
+ {
+ name: "strArrTag", valueType: pbv1.ValueTypeStrArr,
+ values: [][]byte{marshalStrArr([][]byte{[]byte("value1"), []byte("value2")}), marshalStrArr([][]byte{[]byte("value3"), []byte("value4")})},
+ },
+ {
+ name: "intArrTag", valueType: pbv1.ValueTypeInt64Arr,
+ values: [][]byte{
+ marshalIntArr([][]byte{convert.Int64ToBytes(25), convert.Int64ToBytes(30)}),
+ marshalIntArr([][]byte{convert.Int64ToBytes(50), convert.Int64ToBytes(60)}),
+ },
+ },
+ },
+ },
+ {
+ name: "binaryTag",
+ columns: []column{
+ {name: "binaryTag", valueType: pbv1.ValueTypeBinaryData, values: [][]byte{longText, longText}},
+ },
+ },
+ {
+ name: "singleTag",
+ columns: []column{
+ {name: "strTag", valueType: pbv1.ValueTypeStr, values: [][]byte{[]byte("value1"), []byte("value2")}},
+ {name: "intTag", valueType: pbv1.ValueTypeInt64, values: [][]byte{convert.Int64ToBytes(10), convert.Int64ToBytes(20)}},
+ },
+ },
+ },
+ field: columnFamily{
+ columns: []column{
+ {name: "strField", valueType: pbv1.ValueTypeStr, values: [][]byte{[]byte("field1"), []byte("field2")}},
+ {name: "intField", valueType: pbv1.ValueTypeInt64, values: [][]byte{convert.Int64ToBytes(1110), convert.Int64ToBytes(2220)}},
+ {name: "floatField", valueType: pbv1.ValueTypeFloat64, values: [][]byte{convert.Float64ToBytes(1221233.343), convert.Float64ToBytes(2442466.686)}},
+ {name: "binaryField", valueType: pbv1.ValueTypeBinaryData, values: [][]byte{longText, longText}},
+ },
+ },
+}
+
+func Test_block_mustInitFromDataPoints(t *testing.T) {
+ type args struct {
+ timestamps []int64
+ tagFamilies [][]nameValues
+ fields []nameValues
+ }
+ tests := []struct {
+ name string
+ args args
+ want block
+ }{
+ {
+ name: "Test mustInitFromDataPoints",
+ args: args{
+ timestamps: []int64{1, 2},
+ tagFamilies: [][]nameValues{
+ {
+ {
+ "arrTag", []*nameValue{
+ {name: "strArrTag", valueType: pbv1.ValueTypeStrArr, value: nil, valueArr: [][]byte{[]byte("value1"), []byte("value2")}},
+ {name: "intArrTag", valueType: pbv1.ValueTypeInt64Arr, value: nil, valueArr: [][]byte{convert.Int64ToBytes(25), convert.Int64ToBytes(30)}},
+ },
+ },
+ {
+ "binaryTag", []*nameValue{
+ {name: "binaryTag", valueType: pbv1.ValueTypeBinaryData, value: longText, valueArr: nil},
+ },
+ },
+ {
+ "singleTag", []*nameValue{
+ {name: "strTag", valueType: pbv1.ValueTypeStr, value: []byte("value1"), valueArr: nil},
+ {name: "intTag", valueType: pbv1.ValueTypeInt64, value: convert.Int64ToBytes(10), valueArr: nil},
+ },
+ },
+ },
+ {
+ {
+ "arrTag", []*nameValue{
+ {name: "strArrTag", valueType: pbv1.ValueTypeStrArr, value: nil, valueArr: [][]byte{[]byte("value3"), []byte("value4")}},
+ {name: "intArrTag", valueType: pbv1.ValueTypeInt64Arr, value: nil, valueArr: [][]byte{convert.Int64ToBytes(50), convert.Int64ToBytes(60)}},
+ },
+ },
+ {
+ "binaryTag", []*nameValue{
+ {name: "binaryTag", valueType: pbv1.ValueTypeBinaryData, value: longText, valueArr: nil},
+ },
+ },
+ {
+ "singleTag", []*nameValue{
+ {name: "strTag", valueType: pbv1.ValueTypeStr, value: []byte("value2"), valueArr: nil},
+ {name: "intTag", valueType: pbv1.ValueTypeInt64, value: convert.Int64ToBytes(20), valueArr: nil},
+ },
+ },
+ },
+ },
+ fields: []nameValues{
+ {
+ "skipped", []*nameValue{
+ {name: "strField", valueType: pbv1.ValueTypeStr, value: []byte("field1"), valueArr: nil},
+ {name: "intField", valueType: pbv1.ValueTypeInt64, value: convert.Int64ToBytes(1110), valueArr: nil},
+ {name: "floatField", valueType: pbv1.ValueTypeFloat64, value: convert.Float64ToBytes(1221233.343), valueArr: nil},
+ {name: "binaryField", valueType: pbv1.ValueTypeBinaryData, value: longText, valueArr: nil},
+ },
+ },
+ {
+ "skipped", []*nameValue{
+ {name: "strField", valueType: pbv1.ValueTypeStr, value: []byte("field2"), valueArr: nil},
+ {name: "intField", valueType: pbv1.ValueTypeInt64, value: convert.Int64ToBytes(2220), valueArr: nil},
+ {name: "floatField", valueType: pbv1.ValueTypeFloat64, value: convert.Float64ToBytes(2442466.686), valueArr: nil},
+ {name: "binaryField", valueType: pbv1.ValueTypeBinaryData, value: longText, valueArr: nil},
+ },
+ },
+ },
+ },
+ want: conventionalBlock,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ b := &block{}
+ b.mustInitFromDataPoints(tt.args.timestamps, tt.args.tagFamilies, tt.args.fields)
+ if !reflect.DeepEqual(*b, tt.want) {
+ t.Errorf("block.mustInitFromDataPoints() = %+v, want %+v", *b, tt.want)
+ }
+ })
+ }
+}
+
+var longText = []byte(`
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec a diam lectus. Sed sit amet ipsum mauris.
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec a diam lectus. Sed sit amet ipsum mauris.
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec a diam lectus. Sed sit amet ipsum mauris.
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec a diam lectus. Sed sit amet ipsum mauris.
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec a diam lectus. Sed sit amet ipsum mauris.
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec a diam lectus. Sed sit amet ipsum mauris.
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec a diam lectus. Sed sit amet ipsum mauris.
+`)
+
+func marshalStrArr(arr [][]byte) []byte {
+ nv := &nameValue{valueType: pbv1.ValueTypeStrArr, valueArr: arr}
+ return nv.marshal()
+}
+
+func marshalIntArr(arr [][]byte) []byte {
+ nv := &nameValue{valueType: pbv1.ValueTypeInt64Arr, valueArr: arr}
+ return nv.marshal()
+}
+
+func Test_mustWriteAndReadTimestamps(t *testing.T) {
+ tests := []struct {
+ name string
+ args []int64
+ wantPanic bool
+ wantTM timestampsMetadata
+ }{
+ {
+ name: "Test mustWriteAndReadTimestamps",
+ args: []int64{1, 2, 3, 4, 5},
+ wantPanic: false,
+ },
+ {
+ name: "Test mustWriteAndReadTimestamps with panic",
+ args: getBitInt64Arr(),
+ wantPanic: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ defer func() {
+ r := recover()
+ if (r != nil) != tt.wantPanic {
+ t.Errorf("mustWriteTimestampsTo() recover = %v, wantPanic = %v", r, tt.wantPanic)
+ }
+ }()
+ tm := ×tampsMetadata{}
+ b := &bytes.Buffer{}
+ mustWriteTimestampsTo(tm, tt.args, writer{w: b})
+ timestamps := mustReadTimestampsFrom(nil, tm, len(tt.args), b)
+ if !reflect.DeepEqual(timestamps, tt.args) {
+ t.Errorf("mustReadTimestampsFrom() = %v, want %v", timestamps, tt.args)
+ }
+ })
+ }
+}
+
+func getBitInt64Arr() []int64 {
+ size := maxTimestampsBlockSize + 1
+ randSlice := make([]int64, size)
+ for i := range randSlice {
+ b := make([]byte, 8)
+ _, _ = rand.Read(b)
+ randSlice[i] = int64(binary.BigEndian.Uint64(b))
+ }
+ return randSlice
+}
+
+func Test_marshalAndUnmarshalTagFamily(t *testing.T) {
+ metaBuffer, dataBuffer := &bytes.Buffer{}, &bytes.Buffer{}
+ ww := &writers{
+ mustCreateTagFamilyWriters: func(name string) (fs.Writer, fs.Writer) {
+ return metaBuffer, dataBuffer
+ },
+ tagFamilyMetadataWriters: make(map[string]*writer),
+ tagFamilyWriters: make(map[string]*writer),
+ }
+ b := &conventionalBlock
+ tagProjection := toTagProjection(*b)
+ tfIndex := 0
+ name := "arrTag"
+ decoder := &encoding.BytesBlockDecoder{}
+ bm := &blockMetadata{}
+
+ b.marshalTagFamily(b.tagFamilies[tfIndex], bm, ww)
+
+ metaWriter, ok1 := ww.tagFamilyMetadataWriters[name]
+ valueWriter, ok2 := ww.tagFamilyWriters[name]
+ if !ok1 || !ok2 {
+ t.Fatalf("Writers not correctly added to maps")
+ }
+ if metaWriter.w != metaBuffer || valueWriter.w != dataBuffer {
+ t.Fatalf("Writers not correctly added to maps")
+ }
+
+ unmarshaled := generateBlock()
+ defer releaseBlock(unmarshaled)
+ // set the timestamps to the same length as the original block
+ // the data size in a block depends on the timestamps length
+ unmarshaled.timestamps = make([]int64, len(b.timestamps))
+ unmarshaled.resizeTagFamilies(1)
+
+ unmarshaled.unmarshalTagFamily(decoder, tfIndex, name, bm.getTagFamilyMetadata(name), tagProjection[name], metaBuffer, dataBuffer)
+
+ if diff := cmp.Diff(unmarshaled.tagFamilies[0], b.tagFamilies[0],
+ cmp.AllowUnexported(columnFamily{}, column{}),
+ ); diff != "" {
+ t.Errorf("block.unmarshalTagFamily() (-got +want):\n%s", diff)
+ }
+}
+
+func Test_marshalAndUnmarshalBlock(t *testing.T) {
+ timestampBuffer, fieldBuffer := &bytes.Buffer{}, &bytes.Buffer{}
+ ww := &writers{
+ mustCreateTagFamilyWriters: func(name string) (fs.Writer, fs.Writer) {
+ return &bytes.Buffer{}, &bytes.Buffer{}
+ },
+ tagFamilyMetadataWriters: make(map[string]*writer),
+ tagFamilyWriters: make(map[string]*writer),
+ timestampsWriter: writer{w: timestampBuffer},
+ fieldValuesWriter: writer{w: fieldBuffer},
+ }
+ p := &part{
+ timestamps: timestampBuffer,
+ fieldValues: fieldBuffer,
+ }
+ b := &conventionalBlock
+ tagProjection := toTagProjection(*b)
+ decoder := &encoding.BytesBlockDecoder{}
+ sid := common.SeriesID(1)
+ bm := blockMetadata{}
+
+ b.mustWriteTo(sid, &bm, ww)
+
+ tagFamilyMetadataReaders := make(map[string]fs.Reader)
+ tagFamilyReaders := make(map[string]fs.Reader)
+
+ for k, w := range ww.tagFamilyMetadataWriters {
+ tagFamilyMetadataReaders[k] = w.w.(*bytes.Buffer)
+ tagFamilyReaders[k] = ww.tagFamilyWriters[k].w.(*bytes.Buffer)
+ }
+ p.tagFamilyMetadata = tagFamilyMetadataReaders
+ p.tagFamilies = tagFamilyReaders
+
+ unmarshaled := generateBlock()
+ defer releaseBlock(unmarshaled)
+
+ var tp []pbv1.TagProjection
+ for family, names := range tagProjection {
+ tp = append(tp, pbv1.TagProjection{
+ Family: family,
+ Names: names,
+ })
+ }
+ bm.tagProjection = tp
+ unmarshaled.mustReadFrom(decoder, p, bm)
+ // blockMetadata is using a map, so the order of tag families is not guaranteed
+ unmarshaled.sortTagFamilies()
+
+ if !reflect.DeepEqual(b, unmarshaled) {
+ t.Errorf("block.mustReadFrom() = %+v, want %+v", unmarshaled, b)
+ }
+}
+
+func Test_findRange(t *testing.T) {
+ type args struct {
+ timestamps []int64
+ min int64
+ max int64
+ }
+ tests := []struct {
+ name string
+ args args
+ wantStart int
+ wantEnd int
+ wantExist bool
+ }{
+ {
+ name: "Test with empty timestamps",
+ args: args{
+ timestamps: []int64{},
+ min: 1,
+ max: 10,
+ },
+ wantStart: 0,
+ wantEnd: 0,
+ wantExist: false,
+ },
+ {
+ name: "Test with single timestamp",
+ args: args{
+ timestamps: []int64{1},
+ min: 1,
+ max: 1,
+ },
+ wantStart: 0,
+ wantEnd: 1,
+ wantExist: true,
+ },
+ {
+ name: "Test with range not in timestamps",
+ args: args{
+ timestamps: []int64{1, 2, 3, 4, 5},
+ min: 6,
+ max: 10,
+ },
+ wantStart: 0,
+ wantEnd: 0,
+ wantExist: false,
+ },
+ {
+ name: "Test with range in timestamps",
+ args: args{
+ timestamps: []int64{1, 2, 3, 4, 5},
+ min: 2,
+ max: 4,
+ },
+ wantStart: 1,
+ wantEnd: 4,
+ wantExist: true,
+ },
+ {
+ name: "Test with range as timestamps",
+ args: args{
+ timestamps: []int64{1, 2, 3, 4, 5},
+ min: 1,
+ max: 5,
+ },
+ wantStart: 0,
+ wantEnd: 5,
+ wantExist: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ gotStart, gotEnd, gotExist := findRange(tt.args.timestamps, tt.args.min, tt.args.max)
+ if gotStart != tt.wantStart {
+ t.Errorf("findRange() gotStart = %v, want %v", gotStart, tt.wantStart)
+ }
+ if gotEnd != tt.wantEnd {
+ t.Errorf("findRange() gotEnd = %v, want %v", gotEnd, tt.wantEnd)
+ }
+ if gotExist != tt.wantExist {
+ t.Errorf("findRange() gotExist = %v, want %v", gotExist, tt.wantExist)
+ }
+ })
+ }
+}
diff --git a/banyand/measure/block_writer.go b/banyand/measure/block_writer.go
new file mode 100644
index 00000000..9fcc6e9a
--- /dev/null
+++ b/banyand/measure/block_writer.go
@@ -0,0 +1,278 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package measure
+
+import (
+ "sync"
+ "time"
+
+ "github.com/apache/skywalking-banyandb/api/common"
+ "github.com/apache/skywalking-banyandb/pkg/compress/zstd"
+ "github.com/apache/skywalking-banyandb/pkg/fs"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+)
+
+type writer struct {
+ w fs.Writer
+ bytesWritten uint64
+}
+
+func (w *writer) reset() {
+ w.w = nil
+ w.bytesWritten = 0
+}
+
+func (w *writer) init(wc fs.Writer) {
+ w.reset()
+
+ w.w = wc
+}
+
+func (w *writer) MustWrite(data []byte) {
+ fs.MustWriteData(w.w, data)
+ w.bytesWritten += uint64(len(data))
+}
+
+func (w *writer) MustClose() {
+ fs.MustClose(w.w)
+ w.reset()
+}
+
+type mustCreateTagFamilyWriters func(name string) (fs.Writer, fs.Writer)
+
+type writers struct {
+ mustCreateTagFamilyWriters mustCreateTagFamilyWriters
+ metaWriter writer
+ primaryWriter writer
+ tagFamilyMetadataWriters map[string]*writer
+ tagFamilyWriters map[string]*writer
+ timestampsWriter writer
+ fieldValuesWriter writer
+}
+
+func (sw *writers) reset() {
+ sw.mustCreateTagFamilyWriters = nil
+ sw.metaWriter.reset()
+ sw.primaryWriter.reset()
+ sw.timestampsWriter.reset()
+ sw.fieldValuesWriter.reset()
+
+ for i, w := range sw.tagFamilyMetadataWriters {
+ w.reset()
+ delete(sw.tagFamilyMetadataWriters, i)
+ }
+ for i, w := range sw.tagFamilyWriters {
+ w.reset()
+ delete(sw.tagFamilyWriters, i)
+ }
+}
+
+func (sw *writers) totalBytesWritten() uint64 {
+ n := sw.metaWriter.bytesWritten + sw.primaryWriter.bytesWritten +
+ sw.timestampsWriter.bytesWritten + sw.fieldValuesWriter.bytesWritten
+ for _, w := range sw.tagFamilyMetadataWriters {
+ n += w.bytesWritten
+ }
+ for _, w := range sw.tagFamilyWriters {
+ n += w.bytesWritten
+ }
+ return n
+}
+
+func (sw *writers) MustClose() {
+ sw.metaWriter.MustClose()
+ sw.primaryWriter.MustClose()
+ sw.timestampsWriter.MustClose()
+ sw.fieldValuesWriter.MustClose()
+
+ for _, w := range sw.tagFamilyMetadataWriters {
+ w.MustClose()
+ }
+ for _, w := range sw.tagFamilyWriters {
+ w.MustClose()
+ }
+}
+
+func (sw *writers) getColumnMetadataWriterAndColumnWriter(columnName string) (*writer, *writer) {
+ chw, ok := sw.tagFamilyMetadataWriters[columnName]
+ cw := sw.tagFamilyWriters[columnName]
+ if ok {
+ return chw, cw
+ }
+ hw, w := sw.mustCreateTagFamilyWriters(columnName)
+ chw = &writer{
+ w: hw,
+ }
+ cw = &writer{
+ w: w,
+ }
+ sw.tagFamilyMetadataWriters[columnName] = chw
+ sw.tagFamilyWriters[columnName] = cw
+ return chw, cw
+}
+
+type blockWriter struct {
+ writers writers
+ metaData []byte
+ primaryBlockData []byte
+ primaryBlockMetadata primaryBlockMetadata
+ totalBlocksCount uint64
+ maxTimestamp int64
+ totalUncompressedSizeBytes uint64
+ totalCount uint64
+ minTimestamp int64
+ totalMinTimestamp int64
+ totalMaxTimestamp int64
+ minTimestampLast int64
+ sidFirst common.SeriesID
+ sidLast common.SeriesID
+ hasWrittenBlocks bool
+}
+
+func (bw *blockWriter) reset() {
+ bw.writers.reset()
+ bw.sidLast = 0
+ bw.sidFirst = 0
+ bw.minTimestampLast = 0
+ bw.minTimestamp = 0
+ bw.maxTimestamp = 0
+ bw.hasWrittenBlocks = false
+ bw.totalUncompressedSizeBytes = 0
+ bw.totalCount = 0
+ bw.totalBlocksCount = 0
+ bw.totalMinTimestamp = 0
+ bw.totalMaxTimestamp = 0
+ bw.primaryBlockData = bw.primaryBlockData[:0]
+ bw.metaData = bw.metaData[:0]
+ bw.primaryBlockMetadata.reset()
+}
+
+func (bw *blockWriter) MustInitForMemPart(mp *memPart) {
+ bw.reset()
+ bw.writers.mustCreateTagFamilyWriters = mp.mustCreateMemTagFamilyWriters
+ bw.writers.metaWriter.init(&mp.meta)
+ bw.writers.primaryWriter.init(&mp.primary)
+ bw.writers.timestampsWriter.init(&mp.timestamps)
+ bw.writers.fieldValuesWriter.init(&mp.fieldValues)
+}
+
+func (bw *blockWriter) MustWriteDataPoints(sid common.SeriesID, timestamps []int64, tagFamilies [][]nameValues, fields []nameValues) {
+ if len(timestamps) == 0 {
+ return
+ }
+
+ b := generateBlock()
+ defer releaseBlock(b)
+ b.mustInitFromDataPoints(timestamps, tagFamilies, fields)
+ if b.Len() == 0 {
+ return
+ }
+ if sid < bw.sidLast {
+ logger.Panicf("the sid=%d cannot be smaller than the previously written sid=%d", sid, &bw.sidLast)
+ }
+ hasWrittenBlocks := bw.hasWrittenBlocks
+ if !hasWrittenBlocks {
+ bw.sidFirst = sid
+ bw.hasWrittenBlocks = true
+ }
+ isSeenSid := sid == bw.sidLast
+ bw.sidLast = sid
+
+ bh := generateBlockMetadata()
+ b.mustWriteTo(sid, bh, &bw.writers)
+ th := &bh.timestamps
+ if bw.totalCount == 0 || th.min < bw.totalMinTimestamp {
+ bw.totalMinTimestamp = th.min
+ }
+ if bw.totalCount == 0 || th.max > bw.totalMaxTimestamp {
+ bw.totalMaxTimestamp = th.max
+ }
+ if !hasWrittenBlocks || th.min < bw.minTimestamp {
+ bw.minTimestamp = th.min
+ }
+ if !hasWrittenBlocks || th.max > bw.maxTimestamp {
+ bw.maxTimestamp = th.max
+ }
+ if isSeenSid && th.min < bw.minTimestampLast {
+ logger.Panicf("the block for sid=%d cannot contain timestamp smaller than %d, but it contains timestamp %d", sid, bw.minTimestampLast, th.min)
+ }
+ bw.minTimestampLast = th.min
+
+ bw.totalUncompressedSizeBytes += bh.uncompressedSizeBytes
+ bw.totalCount += bh.count
+ bw.totalBlocksCount++
+
+ bw.primaryBlockData = bh.marshal(bw.primaryBlockData)
+ releaseBlockMetadata(bh)
+ if len(bw.primaryBlockData) > maxUncompressedPrimaryBlockSize {
+ bw.mustFlushPrimaryBlock(bw.primaryBlockData)
+ bw.primaryBlockData = bw.primaryBlockData[:0]
+ }
+}
+
+func (bw *blockWriter) mustFlushPrimaryBlock(data []byte) {
+ if len(data) > 0 {
+ bw.primaryBlockMetadata.mustWriteBlock(data, bw.sidFirst, bw.minTimestamp, bw.maxTimestamp, &bw.writers)
+ bw.metaData = bw.primaryBlockMetadata.marshal(bw.metaData)
+ }
+ bw.hasWrittenBlocks = false
+ bw.minTimestamp = 0
+ bw.maxTimestamp = 0
+ bw.sidFirst = 0
+}
+
+func (bw *blockWriter) Flush(ph *partMetadata) {
+ ph.UncompressedSizeBytes = bw.totalUncompressedSizeBytes
+ ph.TotalCount = bw.totalCount
+ ph.BlocksCount = bw.totalBlocksCount
+ ph.MinTimestamp = bw.totalMinTimestamp
+ ph.MaxTimestamp = bw.totalMaxTimestamp
+ ph.Version = time.Now().UnixNano() // TODO: use a global version
+
+ bw.mustFlushPrimaryBlock(bw.primaryBlockData)
+
+ bb := bigValuePool.Generate()
+ bb.Buf = zstd.Compress(bb.Buf[:0], bw.metaData, 1)
+ bw.writers.metaWriter.MustWrite(bb.Buf)
+ bigValuePool.Release(bb)
+
+ ph.CompressedSizeBytes = bw.writers.totalBytesWritten()
+
+ bw.writers.MustClose()
+ bw.reset()
+}
+
+func generateBlockWriter() *blockWriter {
+ v := blockWriterPool.Get()
+ if v == nil {
+ return &blockWriter{
+ writers: writers{
+ tagFamilyMetadataWriters: make(map[string]*writer),
+ tagFamilyWriters: make(map[string]*writer),
+ },
+ }
+ }
+ return v.(*blockWriter)
+}
+
+func releaseBlockWriter(bsw *blockWriter) {
+ bsw.reset()
+ blockWriterPool.Put(bsw)
+}
+
+var blockWriterPool sync.Pool
diff --git a/banyand/measure/column.go b/banyand/measure/column.go
new file mode 100644
index 00000000..e84062fc
--- /dev/null
+++ b/banyand/measure/column.go
@@ -0,0 +1,119 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package measure
+
+import (
+ "github.com/apache/skywalking-banyandb/pkg/bytes"
+ "github.com/apache/skywalking-banyandb/pkg/encoding"
+ "github.com/apache/skywalking-banyandb/pkg/fs"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
+)
+
+type column struct {
+ name string
+ values [][]byte
+ valueType pbv1.ValueType
+}
+
+func (c *column) reset() {
+ c.name = ""
+
+ values := c.values
+ for i := range values {
+ values[i] = nil
+ }
+ c.values = values[:0]
+}
+
+func (c *column) resizeValues(valuesLen int) [][]byte {
+ values := c.values
+ if n := valuesLen - cap(values); n > 0 {
+ values = append(values[:cap(values)], make([][]byte, n)...)
+ }
+ values = values[:valuesLen]
+ c.values = values
+ return values
+}
+
+func (c *column) mustWriteTo(ch *columnMetadata, columnWriter *writer) {
+ ch.reset()
+
+ ch.name = c.name
+ ch.valueType = c.valueType
+
+ // TODO: encoding values based on value type
+
+ bb := bigValuePool.Generate()
+ defer bigValuePool.Release(bb)
+
+ // marshal values
+ bb.Buf = encoding.EncodeBytesBlock(bb.Buf[:0], c.values)
+ ch.size = uint64(len(bb.Buf))
+ if ch.size > maxValuesBlockSize {
+ logger.Panicf("too valuesSize: %d bytes; mustn't exceed %d bytes", ch.size, maxValuesBlockSize)
+ }
+ ch.offset = columnWriter.bytesWritten
+ columnWriter.MustWrite(bb.Buf)
+}
+
+func (c *column) mustReadValues(decoder *encoding.BytesBlockDecoder, reader fs.Reader, cm columnMetadata, count uint64) {
+ c.name = cm.name
+ c.valueType = cm.valueType
+
+ bb := bigValuePool.Generate()
+ defer bigValuePool.Release(bb)
+ valuesSize := cm.size
+ if valuesSize > maxValuesBlockSize {
+ logger.Panicf("%s: block size cannot exceed %d bytes; got %d bytes", reader.Path(), maxValuesBlockSize, valuesSize)
+ }
+ bb.Buf = bytes.ResizeOver(bb.Buf, int(valuesSize))
+ fs.MustReadData(reader, int64(cm.offset), bb.Buf)
+ var err error
+ c.values, err = decoder.Decode(c.values[:0], bb.Buf, count)
+ if err != nil {
+ logger.Panicf("%s: cannot decode values: %v", reader.Path(), err)
+ }
+}
+
+var bigValuePool bytes.BufferPool
+
+type columnFamily struct {
+ name string
+ columns []column
+}
+
+func (cf *columnFamily) reset() {
+ cf.name = ""
+
+ columns := cf.columns
+ for i := range columns {
+ columns[i].reset()
+ }
+ cf.columns = columns[:0]
+}
+
+func (cf *columnFamily) resizeColumns(columnsLen int) []column {
+ columns := cf.columns
+ if n := columnsLen - cap(columns); n > 0 {
+ columns = append(columns[:cap(columns)], make([]column, n)...)
+ }
+ columns = columns[:columnsLen]
+ cf.columns = columns
+ return columns
+}
diff --git a/banyand/measure/column_metadata.go b/banyand/measure/column_metadata.go
new file mode 100644
index 00000000..3f1eb957
--- /dev/null
+++ b/banyand/measure/column_metadata.go
@@ -0,0 +1,159 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Licensed to Apacme Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apacme Software Foundation (ASF) licenses this file to you under
+// the Apacme License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apacme.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package measure
+
+import (
+ "fmt"
+ "sync"
+
+ "github.com/apache/skywalking-banyandb/pkg/convert"
+ "github.com/apache/skywalking-banyandb/pkg/encoding"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
+)
+
+type columnMetadata struct {
+ name string
+ dataBlock
+ valueType pbv1.ValueType
+}
+
+func (cm *columnMetadata) reset() {
+ cm.name = ""
+ cm.valueType = 0
+ cm.dataBlock.reset()
+}
+
+func (cm *columnMetadata) copyFrom(src *columnMetadata) {
+ cm.name = src.name
+ cm.valueType = src.valueType
+ cm.dataBlock.copyFrom(&src.dataBlock)
+}
+
+func (cm *columnMetadata) marshal(dst []byte) []byte {
+ dst = encoding.EncodeBytes(dst, convert.StringToBytes(cm.name))
+ dst = append(dst, byte(cm.valueType))
+ dst = cm.dataBlock.marshal(dst)
+ return dst
+}
+
+func (cm *columnMetadata) unmarshal(src []byte) ([]byte, error) {
+ src, nameBytes, err := encoding.DecodeBytes(src)
+ if err != nil {
+ return nil, fmt.Errorf("cannot unmarshal columnMetadata.name: %w", err)
+ }
+ cm.name = string(nameBytes)
+ if len(src) < 1 {
+ return nil, fmt.Errorf("cannot unmarshal columnMetadata.valueType: src is too short")
+ }
+ cm.valueType = pbv1.ValueType(src[0])
+ src = src[1:]
+ src, err = cm.dataBlock.unmarshal(src)
+ if err != nil {
+ return nil, fmt.Errorf("cannot unmarshal columnMetadata.dataBlock: %w", err)
+ }
+ return src, nil
+}
+
+type columnFamilyMetadata struct {
+ columnMetadata []columnMetadata
+}
+
+func (cfm *columnFamilyMetadata) reset() {
+ cms := cfm.columnMetadata
+ for i := range cms {
+ cms[i].reset()
+ }
+ cfm.columnMetadata = cms[:0]
+}
+
+func (cfm *columnFamilyMetadata) copyFrom(src *columnFamilyMetadata) {
+ cfm.reset()
+ cms := cfm.resizeColumnMetadata(len(src.columnMetadata))
+ for i := range src.columnMetadata {
+ cms[i].copyFrom(&src.columnMetadata[i])
+ }
+}
+
+func (cfm *columnFamilyMetadata) resizeColumnMetadata(columnMetadataLen int) []columnMetadata {
+ cms := cfm.columnMetadata
+ if n := columnMetadataLen - cap(cms); n > 0 {
+ cms = append(cms[:cap(cms)], make([]columnMetadata, n)...)
+ }
+ cms = cms[:columnMetadataLen]
+ cfm.columnMetadata = cms
+ return cms
+}
+
+func (cfm *columnFamilyMetadata) marshal(dst []byte) []byte {
+ cms := cfm.columnMetadata
+ dst = encoding.VarUint64ToBytes(dst, uint64(len(cms)))
+ for i := range cms {
+ dst = cms[i].marshal(dst)
+ }
+ return dst
+}
+
+func (cfm *columnFamilyMetadata) unmarshal(src []byte) ([]byte, error) {
+ src, columnMetadataLen, err := encoding.BytesToVarUint64(src)
+ if err != nil {
+ return nil, fmt.Errorf("cannot unmarshal columnMetadataLen: %w", err)
+ }
+ if columnMetadataLen < 1 {
+ return src, nil
+ }
+ cms := cfm.resizeColumnMetadata(int(columnMetadataLen))
+ for i := range cms {
+ src, err = cms[i].unmarshal(src)
+ if err != nil {
+ return nil, fmt.Errorf("cannot unmarshal columnMetadata %d: %w", i, err)
+ }
+ }
+ return src, nil
+}
+
+func generateColumnFamilyMetadata() *columnFamilyMetadata {
+ v := columnFamilyMetadataPool.Get()
+ if v == nil {
+ return &columnFamilyMetadata{}
+ }
+ return v.(*columnFamilyMetadata)
+}
+
+func releaseColumnFamilyMetadata(cfm *columnFamilyMetadata) {
+ cfm.reset()
+ columnFamilyMetadataPool.Put(cfm)
+}
+
+var columnFamilyMetadataPool sync.Pool
diff --git a/banyand/measure/column_metadata_test.go b/banyand/measure/column_metadata_test.go
new file mode 100644
index 00000000..d6eff380
--- /dev/null
+++ b/banyand/measure/column_metadata_test.go
@@ -0,0 +1,189 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Licensed to Apacme Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apacme Software Foundation (ASF) licenses this file to you under
+// the Apacme License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apacme.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package measure
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
+)
+
+func Test_columnMetadata_reset(t *testing.T) {
+ cm := &columnMetadata{
+ name: "test",
+ valueType: pbv1.ValueTypeStr,
+ dataBlock: dataBlock{offset: 1, size: 10},
+ }
+
+ cm.reset()
+
+ assert.Equal(t, "", cm.name)
+ assert.Equal(t, pbv1.ValueType(0), cm.valueType)
+ assert.Equal(t, dataBlock{}, cm.dataBlock)
+}
+
+func Test_columnMetadata_copyFrom(t *testing.T) {
+ src := &columnMetadata{
+ name: "test",
+ valueType: pbv1.ValueTypeStr,
+ dataBlock: dataBlock{offset: 1, size: 10},
+ }
+
+ dest := &columnMetadata{}
+ assert.NotEqual(t, src, dest)
+
+ dest.copyFrom(src)
+ assert.Equal(t, src, dest)
+}
+
+func Test_columnMetadata_marshal(t *testing.T) {
+ original := &columnMetadata{
+ name: "test",
+ valueType: pbv1.ValueTypeStr,
+ dataBlock: dataBlock{offset: 1, size: 10},
+ }
+
+ marshaled := original.marshal(nil)
+ assert.NotNil(t, marshaled)
+
+ unmarshaled := &columnMetadata{}
+ _, err := unmarshaled.unmarshal(marshaled)
+ assert.Nil(t, err)
+
+ assert.Equal(t, original, unmarshaled)
+}
+
+func Test_columnFamilyMetadata_reset(t *testing.T) {
+ cfm := &columnFamilyMetadata{
+ columnMetadata: []columnMetadata{
+ {
+ name: "test1",
+ valueType: pbv1.ValueTypeStr,
+ dataBlock: dataBlock{offset: 1, size: 10},
+ },
+ {
+ name: "test2",
+ valueType: pbv1.ValueTypeInt64,
+ dataBlock: dataBlock{offset: 2, size: 20},
+ },
+ },
+ }
+
+ cfm.reset()
+
+ assert.Equal(t, 0, len(cfm.columnMetadata))
+}
+
+func Test_columnFamilyMetadata_copyFrom(t *testing.T) {
+ src := &columnFamilyMetadata{
+ columnMetadata: []columnMetadata{
+ {
+ name: "test1",
+ valueType: pbv1.ValueTypeStr,
+ dataBlock: dataBlock{offset: 1, size: 10},
+ },
+ {
+ name: "test2",
+ valueType: pbv1.ValueTypeInt64,
+ dataBlock: dataBlock{offset: 2, size: 20},
+ },
+ },
+ }
+
+ dest := &columnFamilyMetadata{}
+ assert.NotEqual(t, src, dest)
+
+ dest.copyFrom(src)
+
+ assert.Equal(t, src, dest)
+}
+
+func Test_columnFamilyMetadata_resizeColumnMetadata(t *testing.T) {
+ cfm := &columnFamilyMetadata{
+ columnMetadata: make([]columnMetadata, 2, 5),
+ }
+
+ cms := cfm.resizeColumnMetadata(3)
+ assert.Equal(t, 3, len(cms))
+ assert.Equal(t, 5, cap(cms))
+
+ cms = cfm.resizeColumnMetadata(6)
+ assert.Equal(t, 6, len(cms))
+ assert.True(t, cap(cms) >= 6) // The capacity is at least 6, but could be more
+}
+
+func Test_columnFamilyMetadata_marshalUnmarshal(t *testing.T) {
+ tests := []struct {
+ original *columnFamilyMetadata
+ name string
+ }{
+ {
+ name: "Non-empty columnMetadata",
+ original: &columnFamilyMetadata{
+ columnMetadata: []columnMetadata{
+ {
+ name: "test1",
+ valueType: pbv1.ValueTypeStr,
+ dataBlock: dataBlock{offset: 1, size: 10},
+ },
+ {
+ name: "test2",
+ valueType: pbv1.ValueTypeInt64,
+ dataBlock: dataBlock{offset: 2, size: 20},
+ },
+ },
+ },
+ },
+ {
+ name: "Empty columnMetadata",
+ original: &columnFamilyMetadata{},
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ marshaled := tt.original.marshal(nil)
+ assert.NotNil(t, marshaled)
+
+ unmarshaled := &columnFamilyMetadata{}
+ _, err := unmarshaled.unmarshal(marshaled)
+ assert.Nil(t, err)
+
+ assert.Equal(t, tt.original, unmarshaled)
+ })
+ }
+}
diff --git a/banyand/measure/column_test.go b/banyand/measure/column_test.go
new file mode 100644
index 00000000..ecf7752f
--- /dev/null
+++ b/banyand/measure/column_test.go
@@ -0,0 +1,121 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package measure
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+
+ "github.com/apache/skywalking-banyandb/pkg/bytes"
+ "github.com/apache/skywalking-banyandb/pkg/encoding"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
+)
+
+func TestColumn_reset(t *testing.T) {
+ c := &column{
+ name: "test",
+ valueType: pbv1.ValueTypeStr,
+ values: [][]byte{[]byte("value1"), []byte("value2")},
+ }
+
+ c.reset()
+
+ assert.Equal(t, "", c.name)
+ assert.Equal(t, 0, len(c.values))
+}
+
+func TestColumn_resizeValues(t *testing.T) {
+ c := &column{
+ values: make([][]byte, 2, 5),
+ }
+
+ values := c.resizeValues(3)
+ assert.Equal(t, 3, len(values))
+ assert.Equal(t, 5, cap(values))
+
+ values = c.resizeValues(6)
+ assert.Equal(t, 6, len(values))
+ assert.True(t, cap(values) >= 6) // The capacity is at least 6, but could be more
+}
+
+func TestColumn_mustWriteTo_mustReadValues(t *testing.T) {
+ original := &column{
+ name: "test",
+ valueType: pbv1.ValueTypeStr,
+ values: [][]byte{[]byte("value1"), nil, []byte("value2"), nil},
+ }
+
+ cm := &columnMetadata{}
+
+ buf := &bytes.Buffer{}
+ w := &writer{w: buf}
+ original.mustWriteTo(cm, w)
+ assert.Equal(t, w.bytesWritten, cm.size)
+ assert.Equal(t, uint64(len(buf.Buf)), cm.size)
+ assert.Equal(t, uint64(0), cm.offset)
+ assert.Equal(t, original.name, cm.name)
+ assert.Equal(t, original.valueType, cm.valueType)
+
+ decoder := &encoding.BytesBlockDecoder{}
+
+ unmarshaled := &column{}
+ unmarshaled.mustReadValues(decoder, buf, *cm, uint64(len(original.values)))
+
+ // Check that the original and new instances are equal
+ assert.Equal(t, original.name, unmarshaled.name)
+ assert.Equal(t, original.valueType, unmarshaled.valueType)
+ assert.Equal(t, original.values, unmarshaled.values)
+}
+
+func TestColumnFamily_reset(t *testing.T) {
+ cf := &columnFamily{
+ name: "test",
+ columns: []column{
+ {
+ name: "test1",
+ valueType: pbv1.ValueTypeStr,
+ values: [][]byte{[]byte("value1"), []byte("value2")},
+ },
+ {
+ name: "test2",
+ valueType: pbv1.ValueTypeInt64,
+ values: [][]byte{[]byte("value3"), []byte("value4")},
+ },
+ },
+ }
+
+ cf.reset()
+
+ assert.Equal(t, "", cf.name)
+ assert.Equal(t, 0, len(cf.columns))
+}
+
+func TestColumnFamily_resizeColumns(t *testing.T) {
+ cf := &columnFamily{
+ columns: make([]column, 2, 5),
+ }
+
+ columns := cf.resizeColumns(3)
+ assert.Equal(t, 3, len(columns))
+ assert.Equal(t, 5, cap(columns))
+
+ columns = cf.resizeColumns(6)
+ assert.Equal(t, 6, len(columns))
+ assert.True(t, cap(columns) >= 6) // The capacity is at least 6, but could be more
+}
diff --git a/banyand/measure/datapoints.go b/banyand/measure/datapoints.go
new file mode 100644
index 00000000..f8136705
--- /dev/null
+++ b/banyand/measure/datapoints.go
@@ -0,0 +1,154 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package measure
+
+import (
+ "bytes"
+
+ "github.com/pkg/errors"
+
+ "github.com/apache/skywalking-banyandb/api/common"
+ "github.com/apache/skywalking-banyandb/banyand/internal/storage"
+ "github.com/apache/skywalking-banyandb/pkg/index"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
+ "github.com/apache/skywalking-banyandb/pkg/timestamp"
+)
+
+type nameValue struct {
+ name string
+ value []byte
+ valueArr [][]byte
+ valueType pbv1.ValueType
+}
+
+func (n *nameValue) size() int {
+ s := len(n.name)
+ if n.value != nil {
+ s += len(n.value)
+ }
+ if n.valueArr != nil {
+ for i := range n.valueArr {
+ s += len(n.valueArr[i])
+ }
+ }
+ return s
+}
+
+func (n *nameValue) marshal() []byte {
+ if n.valueArr != nil {
+ var dst []byte
+ for i := range n.valueArr {
+ if n.valueType == pbv1.ValueTypeInt64Arr {
+ dst = append(dst, n.valueArr[i]...)
+ continue
+ }
+ dst = marshalVarArray(dst, n.valueArr[i])
+ }
+ return dst
+ }
+ return n.value
+}
+
+const (
+ entityDelimiter = '|'
+ escape = '\\'
+)
+
+func marshalVarArray(dest, src []byte) []byte {
+ if bytes.IndexByte(src, entityDelimiter) < 0 && bytes.IndexByte(src, escape) < 0 {
+ dest = append(dest, src...)
+ dest = append(dest, entityDelimiter)
+ return dest
+ }
+ for _, b := range src {
+ if b == entityDelimiter || b == escape {
+ dest = append(dest, escape)
+ }
+ dest = append(dest, b)
+ }
+ dest = append(dest, entityDelimiter)
+ return dest
+}
+
+func unmarshalVarArray(dest, src []byte) ([]byte, []byte, error) {
+ if len(src) == 0 {
+ return nil, nil, errors.New("empty entity value")
+ }
+ if src[0] == entityDelimiter {
+ return dest, src[1:], nil
+ }
+ for len(src) > 0 {
+ if src[0] == escape {
+ if len(src) < 2 {
+ return nil, nil, errors.New("invalid escape character")
+ }
+ src = src[1:]
+ dest = append(dest, src[0])
+ } else if src[0] == entityDelimiter {
+ return dest, src[1:], nil
+ } else {
+ dest = append(dest, src[0])
+ }
+ src = src[1:]
+ }
+ return nil, nil, errors.New("invalid variable array")
+}
+
+type nameValues struct {
+ name string
+ values []*nameValue
+}
+
+type dataPoints struct {
+ seriesIDs []common.SeriesID
+ timestamps []int64
+ tagFamilies [][]nameValues
+ fields []nameValues
+}
+
+func (d *dataPoints) Len() int {
+ return len(d.seriesIDs)
+}
+
+func (d *dataPoints) Less(i, j int) bool {
+ if d.seriesIDs[i] != d.seriesIDs[j] {
+ return d.seriesIDs[i] < d.seriesIDs[j]
+ }
+ return d.timestamps[i] < d.timestamps[j]
+}
+
+func (d *dataPoints) Swap(i, j int) {
+ d.seriesIDs[i], d.seriesIDs[j] = d.seriesIDs[j], d.seriesIDs[i]
+ d.timestamps[i], d.timestamps[j] = d.timestamps[j], d.timestamps[i]
+ d.tagFamilies[i], d.tagFamilies[j] = d.tagFamilies[j], d.tagFamilies[i]
+ d.fields[i], d.fields[j] = d.fields[j], d.fields[i]
+}
+
+type dataPointsInTable struct {
+ timeRange timestamp.TimeRange
+ tsTable storage.TSTableWrapper[*tsTable]
+
+ dataPoints dataPoints
+}
+
+type dataPointsInGroup struct {
+ tsdb storage.TSDB[*tsTable]
+
+ docs index.Documents
+ tables []*dataPointsInTable
+}
diff --git a/banyand/measure/measure.go b/banyand/measure/measure.go
index e5f28956..3f258275 100644
--- a/banyand/measure/measure.go
+++ b/banyand/measure/measure.go
@@ -21,37 +21,40 @@
package measure
import (
- "context"
"time"
commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
"github.com/apache/skywalking-banyandb/banyand/queue"
- "github.com/apache/skywalking-banyandb/banyand/tsdb"
- "github.com/apache/skywalking-banyandb/banyand/tsdb/index"
"github.com/apache/skywalking-banyandb/pkg/logger"
+ "github.com/apache/skywalking-banyandb/pkg/partition"
"github.com/apache/skywalking-banyandb/pkg/query/logical"
+ "github.com/apache/skywalking-banyandb/pkg/schema"
"github.com/apache/skywalking-banyandb/pkg/timestamp"
)
const (
- plainChunkSize = 1 << 20
- intChunkNum = 120
- intChunkSize = 4 * 1024
+ maxValuesBlockSize = 8 * 1024 * 1024
+ maxTimestampsBlockSize = 8 * 1024 * 1024
+ maxTagFamiliesMetadataSize = 8 * 1024 * 1024
+ maxUncompressedBlockSize = 2 * 1024 * 1024
+ maxUncompressedPrimaryBlockSize = 128 * 1024
+
+ maxBlockLength = 8 * 1024
)
type measure struct {
- databaseSupplier tsdb.Supplier
- l *logger.Logger
- schema *databasev1.Measure
- indexWriter *index.Writer
- processorManager *topNProcessorManager
- name string
- group string
- indexRules []*databasev1.IndexRule
- topNAggregations []*databasev1.TopNAggregation
- interval time.Duration
- shardNum uint32
+ databaseSupplier schema.Supplier
+ l *logger.Logger
+ schema *databasev1.Measure
+ processorManager *topNProcessorManager
+ name string
+ group string
+ indexRules []*databasev1.IndexRule
+ indexRuleLocators []*partition.IndexRuleLocator
+ topNAggregations []*databasev1.TopNAggregation
+ interval time.Duration
+ shardNum uint32
}
func (s *measure) startSteamingManager(pipeline queue.Queue) error {
@@ -93,6 +96,8 @@ func (s *measure) parseSpec() (err error) {
if s.schema.Interval != "" {
s.interval, err = timestamp.ParseDuration(s.schema.Interval)
}
+ s.indexRuleLocators = partition.ParseIndexRuleLocators(s.schema.GetTagFamilies(), s.indexRules)
+
return err
}
@@ -102,7 +107,7 @@ type measureSpec struct {
topNAggregations []*databasev1.TopNAggregation
}
-func openMeasure(shardNum uint32, db tsdb.Supplier, spec measureSpec, l *logger.Logger, pipeline queue.Queue,
+func openMeasure(shardNum uint32, db schema.Supplier, spec measureSpec, l *logger.Logger, pipeline queue.Queue,
) (*measure, error) {
m := &measure{
shardNum: shardNum,
@@ -119,18 +124,9 @@ func openMeasure(shardNum uint32, db tsdb.Supplier, spec measureSpec, l *logger.
}
m.databaseSupplier = db
- ctx := context.WithValue(context.Background(), logger.ContextKey, l)
- m.indexWriter = index.NewWriter(ctx, index.WriterOptions{
- DB: db,
- ShardNum: shardNum,
- Families: spec.schema.TagFamilies,
- IndexRules: spec.indexRules,
- })
-
if startErr := m.startSteamingManager(pipeline); startErr != nil {
l.Err(startErr).Str("measure", spec.schema.GetMetadata().GetName()).
Msg("fail to start streaming manager")
}
-
return m, nil
}
diff --git a/banyand/measure/measure_query.go b/banyand/measure/measure_query.go
deleted file mode 100644
index a34c0f7b..00000000
--- a/banyand/measure/measure_query.go
+++ /dev/null
@@ -1,181 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package measure
-
-import (
- "io"
- "time"
-
- "github.com/pkg/errors"
- "google.golang.org/protobuf/proto"
-
- "github.com/apache/skywalking-banyandb/api/common"
- commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
- databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
- measurev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/measure/v1"
- modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
- "github.com/apache/skywalking-banyandb/banyand/tsdb"
- "github.com/apache/skywalking-banyandb/pkg/partition"
- pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
- resourceSchema "github.com/apache/skywalking-banyandb/pkg/schema"
-)
-
-var errTagFamilyNotExist = errors.New("tag family doesn't exist")
-
-// Query allow to retrieve measure data points.
-type Query interface {
- LoadGroup(name string) (resourceSchema.Group, bool)
- Measure(measure *commonv1.Metadata) (Measure, error)
-}
-
-// Measure allows inspecting measure data points' details.
-type Measure interface {
- io.Closer
- Shards(entity tsdb.Entity) ([]tsdb.Shard, error)
- CompanionShards(metadata *commonv1.Metadata) ([]tsdb.Shard, error)
- Shard(id common.ShardID) (tsdb.Shard, error)
- ParseTagFamily(family string, item tsdb.Item) (*modelv1.TagFamily, error)
- ParseField(name string, item tsdb.Item) (*measurev1.DataPoint_Field, error)
- GetSchema() *databasev1.Measure
- GetIndexRules() []*databasev1.IndexRule
- GetInterval() time.Duration
- SetSchema(schema *databasev1.Measure)
-}
-
-var _ Measure = (*measure)(nil)
-
-func (s *measure) SetSchema(schema *databasev1.Measure) {
- s.schema = schema
-}
-
-func (s *measure) GetInterval() time.Duration {
- return s.interval
-}
-
-func (s *measure) Shards(entity tsdb.Entity) ([]tsdb.Shard, error) {
- wrap := func(shards []tsdb.Shard) []tsdb.Shard {
- result := make([]tsdb.Shard, len(shards))
- for i := 0; i < len(shards); i++ {
- result[i] = tsdb.NewScopedShard(tsdb.Entry(s.name), shards[i])
- }
- return result
- }
- db := s.databaseSupplier.SupplyTSDB()
- if len(entity) < 1 {
- return wrap(db.Shards()), nil
- }
- for _, e := range entity {
- if e == nil {
- return wrap(db.Shards()), nil
- }
- }
- shardID, err := partition.ShardID(entity.Prepend(tsdb.Entry(s.name)).Marshal(), s.shardNum)
- if err != nil {
- return nil, err
- }
- shard, err := s.databaseSupplier.SupplyTSDB().Shard(common.ShardID(shardID))
- if err != nil {
- if errors.Is(err, tsdb.ErrUnknownShard) {
- return []tsdb.Shard{}, nil
- }
- return nil, err
- }
- return []tsdb.Shard{tsdb.NewScopedShard(tsdb.Entry(s.name), shard)}, nil
-}
-
-func (s *measure) CompanionShards(metadata *commonv1.Metadata) ([]tsdb.Shard, error) {
- wrap := func(shards []tsdb.Shard) []tsdb.Shard {
- result := make([]tsdb.Shard, len(shards))
- for i := 0; i < len(shards); i++ {
- result[i] = tsdb.NewScopedShard(tsdb.Entry(formatMeasureCompanionPrefix(s.name, metadata.GetName())), shards[i])
- }
- return result
- }
- db := s.databaseSupplier.SupplyTSDB()
- return wrap(db.Shards()), nil
-}
-
-func formatMeasureCompanionPrefix(measureName, name string) string {
- return measureName + "." + name
-}
-
-func (s *measure) Shard(id common.ShardID) (tsdb.Shard, error) {
- shard, err := s.databaseSupplier.SupplyTSDB().Shard(id)
- if err != nil {
- return nil, err
- }
- return tsdb.NewScopedShard(tsdb.Entry(s.name), shard), nil
-}
-
-func (s *measure) ParseTagFamily(family string, item tsdb.Item) (*modelv1.TagFamily, error) {
- fid := familyIdentity(family, pbv1.TagFlag)
- familyRawBytes, err := item.Family(fid)
- if err != nil {
- return nil, errors.Wrapf(err, "measure %s.%s parse family %s", s.name, s.group, family)
- }
- tagFamily := &modelv1.TagFamilyForWrite{}
- err = proto.Unmarshal(familyRawBytes, tagFamily)
- if err != nil {
- return nil, err
- }
- tags := make([]*modelv1.Tag, len(tagFamily.GetTags()))
- var tagSpec []*databasev1.TagSpec
- for _, tf := range s.schema.GetTagFamilies() {
- if tf.GetName() == family {
- tagSpec = tf.GetTags()
- }
- }
- if tagSpec == nil {
- return nil, errTagFamilyNotExist
- }
- for i, tag := range tagFamily.GetTags() {
- tags[i] = &modelv1.Tag{
- Key: tagSpec[i].GetName(),
- Value: &modelv1.TagValue{
- Value: tag.GetValue(),
- },
- }
- }
- return &modelv1.TagFamily{
- Name: family,
- Tags: tags,
- }, err
-}
-
-func (s *measure) ParseField(name string, item tsdb.Item) (*measurev1.DataPoint_Field, error) {
- var fieldSpec *databasev1.FieldSpec
- for _, spec := range s.schema.GetFields() {
- if spec.GetName() == name {
- fieldSpec = spec
- break
- }
- }
- fid := familyIdentity(name, pbv1.EncoderFieldFlag(fieldSpec, s.interval))
- bytes, err := item.Family(fid)
- if err != nil {
- return nil, errors.Wrapf(err, "measure %s.%s parse field %s", s.name, s.group, name)
- }
- fieldValue, err := pbv1.DecodeFieldValue(bytes, fieldSpec)
- if err != nil {
- return nil, err
- }
- return &measurev1.DataPoint_Field{
- Name: name,
- Value: fieldValue,
- }, err
-}
diff --git a/banyand/measure/measure_write.go b/banyand/measure/measure_write.go
deleted file mode 100644
index 5384480f..00000000
--- a/banyand/measure/measure_write.go
+++ /dev/null
@@ -1,195 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package measure
-
-import (
- "bytes"
- "context"
- "time"
-
- "github.com/pkg/errors"
-
- "github.com/apache/skywalking-banyandb/api/common"
- measurev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/measure/v1"
- modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
- "github.com/apache/skywalking-banyandb/banyand/tsdb"
- "github.com/apache/skywalking-banyandb/banyand/tsdb/index"
- "github.com/apache/skywalking-banyandb/pkg/bus"
- "github.com/apache/skywalking-banyandb/pkg/convert"
- "github.com/apache/skywalking-banyandb/pkg/logger"
- pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
- "github.com/apache/skywalking-banyandb/pkg/timestamp"
-)
-
-var errMalformedElement = errors.New("element is malformed")
-
-func (s *measure) write(shardID common.ShardID, entity []byte, entityValues tsdb.EntityValues,
- value *measurev1.DataPointValue,
-) error {
- t := value.GetTimestamp().AsTime().Local()
- if err := timestamp.Check(t); err != nil {
- return errors.WithMessage(err, "writing stream")
- }
- fLen := len(value.GetTagFamilies())
- if fLen < 1 {
- return errors.Wrap(errMalformedElement, "no tag family")
- }
- if fLen > len(s.schema.GetTagFamilies()) {
- return errors.Wrap(errMalformedElement, "tag family number is more than expected")
- }
- shard, err := s.databaseSupplier.SupplyTSDB().CreateShardsAndGetByID(shardID)
- if err != nil {
- return err
- }
- seriesDB := shard.Series()
- series, err := seriesDB.Get(entity, entityValues)
- if err != nil {
- return err
- }
- ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
- defer cancel()
- wp, err := series.Create(ctx, t)
- if err != nil {
- if wp != nil {
- _ = wp.Close()
- }
- return err
- }
- writeFn := func() error {
- builder := wp.WriterBuilder().Time(t)
- for fi, family := range value.GetTagFamilies() {
- spec := s.schema.GetTagFamilies()[fi]
- bb, errMarshal := pbv1.EncodeFamily(spec, family)
- if errMarshal != nil {
- return errMarshal
- }
- builder.Family(familyIdentity(spec.GetName(), pbv1.TagFlag), bb)
- }
- if len(value.GetFields()) > len(s.schema.GetFields()) {
- return errors.Wrap(errMalformedElement, "fields number is more than expected")
- }
- for fi, fieldValue := range value.GetFields() {
- fieldSpec := s.schema.GetFields()[fi]
- fType, isNull := pbv1.FieldValueTypeConv(fieldValue)
- if isNull {
- s.l.Warn().RawJSON("written", logger.Proto(value)).Msg("ignore null field")
- continue
- }
- if fType != fieldSpec.GetFieldType() {
- return errors.Wrapf(errMalformedElement, "field %s type is unexpected", fieldSpec.GetName())
- }
- data := encodeFieldValue(fieldValue)
- if data == nil {
- s.l.Warn().RawJSON("written", logger.Proto(value)).Msg("ignore unknown field")
- continue
- }
- builder.Family(familyIdentity(s.schema.GetFields()[fi].GetName(), pbv1.EncoderFieldFlag(fieldSpec, s.interval)), data)
- }
- writer, errWrite := builder.Build()
- if errWrite != nil {
- return errWrite
- }
- _, errWrite = writer.Write()
- if s.l.Debug().Enabled() {
- s.l.Debug().Time("ts", t).
- Int("ts_nano", t.Nanosecond()).
- RawJSON("data", logger.Proto(value)).
- Uint64("series_id", uint64(series.ID())).
- Stringer("series", series).
- Uint64("item_id", uint64(writer.ItemID().ID)).
- Int("shard_id", int(shardID)).
- Msg("write measure")
- }
- return errWrite
- }
-
- if err = writeFn(); err != nil {
- _ = wp.Close()
- return err
- }
- m := index.Message{
- IndexWriter: tsdb.NewSeriesIndexWriter(series.ID(), seriesDB),
- Value: index.Value{
- TagFamilies: value.GetTagFamilies(),
- Timestamp: t,
- },
- BlockCloser: wp,
- }
- s.indexWriter.Write(m)
- if s.processorManager != nil {
- s.processorManager.onMeasureWrite(&measurev1.InternalWriteRequest{
- Request: &measurev1.WriteRequest{
- Metadata: s.GetSchema().Metadata,
- DataPoint: value,
- MessageId: uint64(time.Now().UnixNano()),
- },
- EntityValues: entityValues[1:],
- })
- }
- return err
-}
-
-type writeCallback struct {
- l *logger.Logger
- schemaRepo *schemaRepo
-}
-
-func setUpWriteCallback(l *logger.Logger, schemaRepo *schemaRepo) bus.MessageListener {
- return &writeCallback{
- l: l,
- schemaRepo: schemaRepo,
- }
-}
-
-func (w *writeCallback) Rev(message bus.Message) (resp bus.Message) {
- writeEvent, ok := message.Data().(*measurev1.InternalWriteRequest)
- if !ok {
- w.l.Warn().Msg("invalid event data type")
- return
- }
-
- stm, ok := w.schemaRepo.loadMeasure(writeEvent.GetRequest().GetMetadata())
- if !ok {
- w.l.Warn().Msg("cannot find measure definition")
- return
- }
- err := stm.write(common.ShardID(writeEvent.GetShardId()),
- writeEvent.SeriesHash, tsdb.DecodeEntityValues(writeEvent.GetEntityValues()), writeEvent.GetRequest().GetDataPoint())
- if err != nil {
- w.l.Error().Err(err).RawJSON("written", logger.Proto(writeEvent)).Msg("fail to write entity")
- }
- return
-}
-
-func familyIdentity(name string, flag []byte) []byte {
- return bytes.Join([][]byte{tsdb.Hash([]byte(name)), flag}, nil)
-}
-
-func encodeFieldValue(fieldValue *modelv1.FieldValue) []byte {
- switch fieldValue.GetValue().(type) {
- case *modelv1.FieldValue_Int:
- return convert.Int64ToBytes(fieldValue.GetInt().GetValue())
- case *modelv1.FieldValue_Float:
- return convert.Float64ToBytes(fieldValue.GetFloat().GetValue())
- case *modelv1.FieldValue_Str:
- return []byte(fieldValue.GetStr().GetValue())
- case *modelv1.FieldValue_BinaryData:
- return bytes.Clone(fieldValue.GetBinaryData())
- }
- return nil
-}
diff --git a/banyand/measure/metadata.go b/banyand/measure/metadata.go
index a63acad2..ba3cafef 100644
--- a/banyand/measure/metadata.go
+++ b/banyand/measure/metadata.go
@@ -31,14 +31,11 @@ import (
"github.com/apache/skywalking-banyandb/api/common"
commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ "github.com/apache/skywalking-banyandb/banyand/internal/storage"
"github.com/apache/skywalking-banyandb/banyand/metadata"
"github.com/apache/skywalking-banyandb/banyand/metadata/schema"
"github.com/apache/skywalking-banyandb/banyand/queue"
- "github.com/apache/skywalking-banyandb/banyand/tsdb"
- "github.com/apache/skywalking-banyandb/pkg/encoding"
"github.com/apache/skywalking-banyandb/pkg/logger"
- pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
- pb_v1 "github.com/apache/skywalking-banyandb/pkg/pb/v1/tsdb"
resourceSchema "github.com/apache/skywalking-banyandb/pkg/schema"
)
@@ -53,16 +50,14 @@ type schemaRepo struct {
metadata metadata.Repo
}
-func newSchemaRepo(path string, metadata metadata.Repo,
- dbOpts tsdb.DatabaseOpts, l *logger.Logger, pipeline queue.Queue, encoderBufferSize, bufferSize int64,
-) schemaRepo {
+func newSchemaRepo(path string, svc *service) schemaRepo {
sr := schemaRepo{
- l: l,
- metadata: metadata,
+ l: svc.l,
+ metadata: svc.metadata,
Repository: resourceSchema.NewRepository(
- metadata,
- l,
- newSupplier(path, metadata, dbOpts, l, pipeline, encoderBufferSize, bufferSize),
+ svc.metadata,
+ svc.l,
+ newSupplier(path, svc),
),
}
sr.start()
@@ -207,7 +202,7 @@ func createOrUpdateTopNMeasure(ctx context.Context, measureSchemaRegistry schema
Interval: sourceMeasureSchema.GetInterval(),
TagFamilies: []*databasev1.TagFamilySpec{
{
- Name: TopNTagFamily,
+ Name: topNTagFamily,
Tags: append([]*databasev1.TagSpec{
{
Name: "measure_id",
@@ -216,7 +211,7 @@ func createOrUpdateTopNMeasure(ctx context.Context, measureSchemaRegistry schema
}, seriesSpecs...),
},
},
- Fields: []*databasev1.FieldSpec{TopNValueFieldSpec},
+ Fields: []*databasev1.FieldSpec{topNValueFieldSpec},
Entity: sourceMeasureSchema.GetEntity(),
}
if oldTopNSchema == nil {
@@ -306,34 +301,35 @@ func (sr *schemaRepo) loadMeasure(metadata *commonv1.Metadata) (*measure, bool)
return s, ok
}
+func (sr *schemaRepo) loadTSDB(groupName string) (storage.TSDB[*tsTable], error) {
+ g, ok := sr.LoadGroup(groupName)
+ if !ok {
+ return nil, fmt.Errorf("group %s not found", groupName)
+ }
+ return g.SupplyTSDB().(storage.TSDB[*tsTable]), nil
+}
+
var _ resourceSchema.ResourceSupplier = (*supplier)(nil)
type supplier struct {
- metadata metadata.Repo
- pipeline queue.Queue
- l *logger.Logger
- path string
- dbOpts tsdb.DatabaseOpts
- encoderBufferSize, bufferSize int64
+ metadata metadata.Repo
+ pipeline queue.Queue
+ l *logger.Logger
+ path string
}
-func newSupplier(path string, metadata metadata.Repo, dbOpts tsdb.DatabaseOpts, l *logger.Logger,
- pipeline queue.Queue, encoderBufferSize, bufferSize int64,
-) *supplier {
+func newSupplier(path string, svc *service) *supplier {
return &supplier{
- path: path,
- dbOpts: dbOpts,
- metadata: metadata,
- l: l,
- pipeline: pipeline,
- encoderBufferSize: encoderBufferSize,
- bufferSize: bufferSize,
+ path: path,
+ metadata: svc.metadata,
+ l: svc.l,
+ pipeline: svc.localPipeline,
}
}
-func (s *supplier) OpenResource(shardNum uint32, db tsdb.Supplier, spec resourceSchema.Resource) (io.Closer, error) {
+func (s *supplier) OpenResource(shardNum uint32, supplier resourceSchema.Supplier, spec resourceSchema.Resource) (io.Closer, error) {
measureSchema := spec.Schema().(*databasev1.Measure)
- return openMeasure(shardNum, db, measureSpec{
+ return openMeasure(shardNum, supplier, measureSpec{
schema: measureSchema,
indexRules: spec.IndexRules(),
topNAggregations: spec.TopN(),
@@ -346,33 +342,16 @@ func (s *supplier) ResourceSchema(md *commonv1.Metadata) (resourceSchema.Resourc
return s.metadata.MeasureRegistry().GetMeasure(ctx, md)
}
-func (s *supplier) OpenDB(groupSchema *commonv1.Group) (tsdb.Database, error) {
- opts := s.dbOpts
- opts.ShardNum = groupSchema.ResourceOpts.ShardNum
- opts.Location = path.Join(s.path, groupSchema.Metadata.Name)
- name := groupSchema.Metadata.Name
- opts.TSTableFactory = &tsTableFactory{
- bufferSize: s.bufferSize,
- encoderBufferSize: s.encoderBufferSize,
- encoderPool: encoding.NewEncoderPool(name, intChunkNum, intervalFn),
- decoderPool: encoding.NewDecoderPool(name, intChunkNum, intervalFn),
- compressionMethod: databasev1.CompressionMethod_COMPRESSION_METHOD_ZSTD,
- encodingChunkSize: intChunkSize,
- plainChunkSize: plainChunkSize,
- }
-
- var err error
- if opts.BlockInterval, err = pb_v1.ToIntervalRule(groupSchema.ResourceOpts.BlockInterval); err != nil {
- return nil, err
- }
- if opts.SegmentInterval, err = pb_v1.ToIntervalRule(groupSchema.ResourceOpts.SegmentInterval); err != nil {
- return nil, err
- }
- if opts.TTL, err = pb_v1.ToIntervalRule(groupSchema.ResourceOpts.Ttl); err != nil {
- return nil, err
+func (s *supplier) OpenDB(groupSchema *commonv1.Group) (io.Closer, error) {
+ opts := storage.TSDBOpts[*tsTable]{
+ ShardNum: groupSchema.ResourceOpts.ShardNum,
+ Location: path.Join(s.path, groupSchema.Metadata.Name),
+ TSTableCreator: newTSTable,
+ SegmentInterval: storage.MustToIntervalRule(groupSchema.ResourceOpts.SegmentInterval),
+ TTL: storage.MustToIntervalRule(groupSchema.ResourceOpts.Ttl),
}
-
- return tsdb.OpenDatabase(
+ name := groupSchema.Metadata.Name
+ return storage.OpenTSDB(
common.SetPosition(context.Background(), func(p common.Position) common.Position {
p.Module = "measure"
p.Database = name
@@ -381,14 +360,6 @@ func (s *supplier) OpenDB(groupSchema *commonv1.Group) (tsdb.Database, error) {
opts)
}
-func intervalFn(key []byte) time.Duration {
- _, interval, err := pbv1.DecodeFieldFlag(key)
- if err != nil {
- panic(err)
- }
- return interval
-}
-
type portableSupplier struct {
metadata metadata.Repo
l *logger.Logger
@@ -407,11 +378,11 @@ func (s *portableSupplier) ResourceSchema(md *commonv1.Metadata) (resourceSchema
return s.metadata.MeasureRegistry().GetMeasure(ctx, md)
}
-func (*portableSupplier) OpenDB(_ *commonv1.Group) (tsdb.Database, error) {
+func (*portableSupplier) OpenDB(_ *commonv1.Group) (io.Closer, error) {
panic("do not support open db")
}
-func (s *portableSupplier) OpenResource(shardNum uint32, _ tsdb.Supplier, spec resourceSchema.Resource) (io.Closer, error) {
+func (s *portableSupplier) OpenResource(shardNum uint32, _ resourceSchema.Supplier, spec resourceSchema.Resource) (io.Closer, error) {
measureSchema := spec.Schema().(*databasev1.Measure)
return openMeasure(shardNum, nil, measureSpec{
schema: measureSchema,
diff --git a/banyand/measure/part.go b/banyand/measure/part.go
new file mode 100644
index 00000000..6760f10e
--- /dev/null
+++ b/banyand/measure/part.go
@@ -0,0 +1,196 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package measure
+
+import (
+ "sort"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "github.com/apache/skywalking-banyandb/api/common"
+ "github.com/apache/skywalking-banyandb/pkg/bytes"
+ "github.com/apache/skywalking-banyandb/pkg/fs"
+)
+
+type part struct {
+ meta fs.Reader
+ primary fs.Reader
+ timestamps fs.Reader
+ fieldValues fs.Reader
+ tagFamilyMetadata map[string]fs.Reader
+ tagFamilies map[string]fs.Reader
+ primaryBlockMetadata []primaryBlockMetadata
+ partMetadata partMetadata
+}
+
+func openMemPart(mp *memPart) *part {
+ var p part
+ p.partMetadata = mp.partMetadata
+
+ p.primaryBlockMetadata = mustReadPrimaryBlockMetadata(p.primaryBlockMetadata[:0], newReader(&mp.meta))
+
+ // Open data files
+ p.meta = &mp.meta
+ p.primary = &mp.primary
+ p.timestamps = &mp.timestamps
+ p.fieldValues = &mp.fieldValues
+ if mp.tagFamilies != nil {
+ p.tagFamilies = make(map[string]fs.Reader)
+ p.tagFamilyMetadata = make(map[string]fs.Reader)
+ for name, tf := range mp.tagFamilies {
+ p.tagFamilies[name] = tf
+ p.tagFamilyMetadata[name] = mp.tagFamilyMetadata[name]
+ }
+ }
+ return &p
+}
+
+type memPart struct {
+ tagFamilyMetadata map[string]*bytes.Buffer
+ tagFamilies map[string]*bytes.Buffer
+ meta bytes.Buffer
+ primary bytes.Buffer
+ timestamps bytes.Buffer
+ fieldValues bytes.Buffer
+ partMetadata partMetadata
+}
+
+func (mp *memPart) mustCreateMemTagFamilyWriters(name string) (fs.Writer, fs.Writer) {
+ if mp.tagFamilies == nil {
+ mp.tagFamilies = make(map[string]*bytes.Buffer)
+ mp.tagFamilyMetadata = make(map[string]*bytes.Buffer)
+ }
+ tf, ok := mp.tagFamilies[name]
+ tfh := mp.tagFamilyMetadata[name]
+ if ok {
+ tf.Reset()
+ tfh.Reset()
+ return tfh, tf
+ }
+ mp.tagFamilies[name] = &bytes.Buffer{}
+ mp.tagFamilyMetadata[name] = &bytes.Buffer{}
+ return mp.tagFamilyMetadata[name], mp.tagFamilies[name]
+}
+
+func (mp *memPart) reset() {
+ mp.partMetadata.reset()
+ mp.meta.Reset()
+ mp.primary.Reset()
+ mp.timestamps.Reset()
+ mp.fieldValues.Reset()
+ if mp.tagFamilies != nil {
+ for _, tf := range mp.tagFamilies {
+ tf.Reset()
+ }
+ }
+ if mp.tagFamilyMetadata != nil {
+ for _, tfh := range mp.tagFamilyMetadata {
+ tfh.Reset()
+ }
+ }
+}
+
+func (mp *memPart) mustInitFromDataPoints(dps *dataPoints) {
+ mp.reset()
+
+ if len(dps.timestamps) == 0 {
+ return
+ }
+
+ sort.Sort(dps)
+
+ bsw := generateBlockWriter()
+ bsw.MustInitForMemPart(mp)
+ var sidPrev common.SeriesID
+ uncompressedBlockSizeBytes := uint64(0)
+ var indexPrev int
+ for i := range dps.timestamps {
+ sid := dps.seriesIDs[i]
+ if sidPrev == 0 {
+ sidPrev = sid
+ }
+
+ if uncompressedBlockSizeBytes >= maxUncompressedBlockSize ||
+ (i-indexPrev) > maxBlockLength || sid != sidPrev {
+ bsw.MustWriteDataPoints(sidPrev, dps.timestamps[indexPrev:i], dps.tagFamilies[indexPrev:i], dps.fields[indexPrev:i])
+ sidPrev = sid
+ indexPrev = i
+ uncompressedBlockSizeBytes = 0
+ }
+ uncompressedBlockSizeBytes += uncompressedDataPointSizeBytes(i, dps)
+ }
+ bsw.MustWriteDataPoints(sidPrev, dps.timestamps[indexPrev:], dps.tagFamilies[indexPrev:], dps.fields[indexPrev:])
+ bsw.Flush(&mp.partMetadata)
+ releaseBlockWriter(bsw)
+}
+
+func uncompressedDataPointSizeBytes(index int, dps *dataPoints) uint64 {
+ n := uint64(len(time.RFC3339Nano))
+ n += uint64(len(dps.fields[index].name))
+ for i := range dps.fields[index].values {
+ n += uint64(dps.fields[index].values[i].size())
+ }
+ for i := range dps.tagFamilies[index] {
+ n += uint64(len(dps.tagFamilies[index][i].name))
+ for j := range dps.tagFamilies[index][i].values {
+ n += uint64(dps.tagFamilies[index][i].values[j].size())
+ }
+ }
+ return n
+}
+
+func generateMemPart() *memPart {
+ v := memPartPool.Get()
+ if v == nil {
+ return &memPart{}
+ }
+ return v.(*memPart)
+}
+
+func releaseMemPart(mp *memPart) {
+ mp.reset()
+ memPartPool.Put(mp)
+}
+
+var memPartPool sync.Pool
+
+type partWrapper struct {
+ mp *memPart
+ p *part
+ ref int32
+}
+
+func newMemPartWrapper(mp *memPart, p *part) *partWrapper {
+ return &partWrapper{mp: mp, p: p, ref: 1}
+}
+
+func (pw *partWrapper) incRef() {
+ atomic.AddInt32(&pw.ref, 1)
+}
+
+func (pw *partWrapper) decRef() {
+ n := atomic.AddInt32(&pw.ref, -1)
+ if n > 0 {
+ return
+ }
+ if pw.mp != nil {
+ releaseMemPart(pw.mp)
+ pw.mp = nil
+ }
+}
diff --git a/banyand/measure/part_iter.go b/banyand/measure/part_iter.go
new file mode 100644
index 00000000..14b4c8c3
--- /dev/null
+++ b/banyand/measure/part_iter.go
@@ -0,0 +1,235 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package measure
+
+import (
+ "errors"
+ "fmt"
+ "io"
+ "sort"
+
+ "github.com/apache/skywalking-banyandb/api/common"
+ "github.com/apache/skywalking-banyandb/pkg/bytes"
+ "github.com/apache/skywalking-banyandb/pkg/compress/zstd"
+ "github.com/apache/skywalking-banyandb/pkg/fs"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+)
+
+type partIter struct {
+ err error
+ p *part
+ sids []common.SeriesID
+ primaryBlockMetadata []primaryBlockMetadata
+ bms []blockMetadata
+ compressedPrimaryBuf []byte
+ primaryBuf []byte
+ curBlock blockMetadata
+ sidIdx int
+ minTimestamp int64
+ maxTimestamp int64
+}
+
+func (pi *partIter) reset() {
+ pi.curBlock = blockMetadata{}
+ pi.p = nil
+ pi.sids = nil
+ pi.sidIdx = 0
+ pi.primaryBlockMetadata = nil
+ pi.bms = nil
+ pi.compressedPrimaryBuf = pi.compressedPrimaryBuf[:0]
+ pi.primaryBuf = pi.primaryBuf[:0]
+ pi.err = nil
+}
+
+func (pi *partIter) init(p *part, sids []common.SeriesID, minTimestamp, maxTimestamp int64) {
+ pi.reset()
+ pi.p = p
+
+ pi.sids = sids
+ pi.minTimestamp = minTimestamp
+ pi.maxTimestamp = maxTimestamp
+
+ pi.primaryBlockMetadata = p.primaryBlockMetadata
+
+ pi.nextSeriesID()
+}
+
+func (pi *partIter) nextBlock() bool {
+ for {
+ if pi.err != nil {
+ return false
+ }
+ if len(pi.bms) == 0 {
+ if !pi.loadNextBlockMetadata() {
+ return false
+ }
+ }
+ if pi.findBlock() {
+ return true
+ }
+ }
+}
+
+func (pi *partIter) error() error {
+ if errors.Is(pi.err, io.EOF) {
+ return nil
+ }
+ return pi.err
+}
+
+func (pi *partIter) nextSeriesID() bool {
+ if pi.sidIdx >= len(pi.sids) {
+ pi.err = io.EOF
+ return false
+ }
+ pi.curBlock.seriesID = pi.sids[pi.sidIdx]
+ pi.sidIdx++
+ return true
+}
+
+func (pi *partIter) searchTargetSeriesID(sid common.SeriesID) bool {
+ if pi.curBlock.seriesID >= sid {
+ return true
+ }
+ if !pi.nextSeriesID() {
+ return false
+ }
+ if pi.curBlock.seriesID >= sid {
+ return true
+ }
+ sids := pi.sids[pi.sidIdx:]
+ pi.sidIdx += sort.Search(len(sids), func(i int) bool {
+ return sid <= sids[i]
+ })
+ if pi.sidIdx >= len(pi.sids) {
+ pi.sidIdx = len(pi.sids)
+ pi.err = io.EOF
+ return false
+ }
+ pi.curBlock.seriesID = pi.sids[pi.sidIdx]
+ pi.sidIdx++
+ return true
+}
+
+func (pi *partIter) loadNextBlockMetadata() bool {
+ for len(pi.primaryBlockMetadata) > 0 {
+ if !pi.searchTargetSeriesID(pi.primaryBlockMetadata[0].seriesID) {
+ return false
+ }
+ pi.primaryBlockMetadata = searchPBM(pi.primaryBlockMetadata, pi.curBlock.seriesID)
+
+ pbm := &pi.primaryBlockMetadata[0]
+ pi.primaryBlockMetadata = pi.primaryBlockMetadata[1:]
+ if pi.curBlock.seriesID < pbm.seriesID {
+ logger.Panicf("invariant violation: pi.curBlock.seriesID cannot be smaller than pbm.seriesID; got %+v vs %+v", &pi.curBlock.seriesID, &pbm.seriesID)
+ }
+
+ if pbm.maxTimestamp < pi.minTimestamp || pbm.minTimestamp > pi.maxTimestamp {
+ continue
+ }
+
+ bm, err := pi.readPrimaryBlock(pbm)
+ if err != nil {
+ pi.err = fmt.Errorf("cannot read primary block for part %q at offset %d with size %d: %w",
+ &pi.p.partMetadata, pbm.offset, pbm.size, err)
+ return false
+ }
+ pi.bms = bm
+ return true
+ }
+ pi.err = io.EOF
+ return false
+}
+
+func searchPBM(pbmIndex []primaryBlockMetadata, sid common.SeriesID) []primaryBlockMetadata {
+ if sid < pbmIndex[0].seriesID {
+ logger.Panicf("invariant violation: sid cannot be smaller than pbmIndex[0]; got %d vs %d", sid, &pbmIndex[0].seriesID)
+ }
+
+ if sid == pbmIndex[0].seriesID {
+ return pbmIndex
+ }
+
+ n := sort.Search(len(pbmIndex), func(i int) bool {
+ return sid <= pbmIndex[i].seriesID
+ })
+ if n == 0 {
+ logger.Panicf("invariant violation: sort.Search returned 0 for sid > pbmIndex[0].seriesID; sid=%+v; pbmIndex[0].seriesID=%+v",
+ sid, &pbmIndex[0].seriesID)
+ }
+ return pbmIndex[n-1:]
+}
+
+func (pi *partIter) readPrimaryBlock(mr *primaryBlockMetadata) ([]blockMetadata, error) {
+ pi.compressedPrimaryBuf = bytes.ResizeOver(pi.compressedPrimaryBuf, int(mr.size))
+ fs.MustReadData(pi.p.primary, int64(mr.offset), pi.compressedPrimaryBuf)
+
+ var err error
+ pi.primaryBuf, err = zstd.Decompress(pi.primaryBuf[:0], pi.compressedPrimaryBuf)
+ if err != nil {
+ return nil, fmt.Errorf("cannot decompress index block: %w", err)
+ }
+ bm := make([]blockMetadata, 0)
+ bm, err = unmarshalBlockMetadata(bm, pi.primaryBuf)
+ if err != nil {
+ return nil, fmt.Errorf("cannot unmarshal index block: %w", err)
+ }
+ return bm, nil
+}
+
+func (pi *partIter) findBlock() bool {
+ bhs := pi.bms
+ for len(bhs) > 0 {
+ sid := pi.curBlock.seriesID
+ if bhs[0].seriesID < sid {
+ n := sort.Search(len(bhs), func(i int) bool {
+ return sid <= bhs[i].seriesID
+ })
+ if n == len(bhs) {
+ break
+ }
+ bhs = bhs[n:]
+ }
+ bm := &bhs[0]
+
+ if bm.seriesID != sid {
+ if !pi.searchTargetSeriesID(bm.seriesID) {
+ return false
+ }
+ continue
+ }
+
+ if bm.timestamps.max < pi.minTimestamp {
+ bhs = bhs[1:]
+ continue
+ }
+ if bm.timestamps.min > pi.maxTimestamp {
+ if !pi.nextSeriesID() {
+ return false
+ }
+ continue
+ }
+
+ pi.curBlock = *bm
+
+ pi.bms = bhs[1:]
+ return true
+ }
+ pi.bms = nil
+ return false
+}
diff --git a/banyand/measure/part_iter_test.go b/banyand/measure/part_iter_test.go
new file mode 100644
index 00000000..37579062
--- /dev/null
+++ b/banyand/measure/part_iter_test.go
@@ -0,0 +1,143 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package measure
+
+import (
+ "errors"
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+
+ "github.com/apache/skywalking-banyandb/api/common"
+)
+
+// TODO: test more scenarios.
+func Test_partIter_nextBlock(t *testing.T) {
+ tests := []struct {
+ wantErr error
+ opt *queryOptions
+ name string
+ sids []common.SeriesID
+ want []blockMetadata
+ }{
+ {
+ name: "Test with all seriesIDs",
+ sids: []common.SeriesID{1, 2, 3},
+ opt: &queryOptions{
+ minTimestamp: 1,
+ maxTimestamp: 220,
+ },
+ want: []blockMetadata{
+ {seriesID: 1, count: 2}, {seriesID: 2, count: 2}, {seriesID: 3, count: 2},
+ },
+ },
+ {
+ name: "Test with no seriesIDs",
+ sids: []common.SeriesID{},
+ opt: &queryOptions{
+ minTimestamp: 1,
+ maxTimestamp: 220,
+ },
+ want: nil,
+ },
+ {
+ name: "Test with a single seriesID",
+ sids: []common.SeriesID{1},
+ opt: &queryOptions{
+ minTimestamp: 1,
+ maxTimestamp: 220,
+ },
+ want: []blockMetadata{{seriesID: 1, count: 2}},
+ },
+ {
+ name: "Test with non-sequential seriesIDs",
+ sids: []common.SeriesID{1, 3},
+ opt: &queryOptions{
+ minTimestamp: 1,
+ maxTimestamp: 220,
+ },
+ want: []blockMetadata{{seriesID: 1, count: 2}, {seriesID: 3, count: 2}},
+ },
+ {
+ name: "Test with seriesID not in data",
+ sids: []common.SeriesID{4},
+ opt: &queryOptions{
+ minTimestamp: 1,
+ maxTimestamp: 220,
+ },
+ want: nil,
+ wantErr: nil,
+ },
+ {
+ name: "Test with multiple seriesIDs not in data",
+ sids: []common.SeriesID{4, 5, 6},
+ opt: &queryOptions{
+ minTimestamp: 1,
+ maxTimestamp: 220,
+ },
+ want: nil,
+ wantErr: nil,
+ },
+ {
+ name: "Test with some seriesIDs in data and some not",
+ sids: []common.SeriesID{1, 4},
+ opt: &queryOptions{
+ minTimestamp: 1,
+ maxTimestamp: 220,
+ },
+ want: []blockMetadata{{seriesID: 1, count: 2}},
+ wantErr: nil,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ mp := generateMemPart()
+ releaseMemPart(mp)
+ mp.mustInitFromDataPoints(dps)
+
+ p := openMemPart(mp)
+
+ pi := partIter{}
+ pi.init(p, tt.sids, tt.opt.minTimestamp, tt.opt.maxTimestamp)
+
+ var got []blockMetadata
+ for pi.nextBlock() {
+ if pi.curBlock.seriesID == 0 {
+ t.Errorf("Expected currBlock to be initialized, but it was nil")
+ }
+ got = append(got, pi.curBlock)
+ }
+
+ if !errors.Is(pi.error(), tt.wantErr) {
+ t.Errorf("Unexpected error: got %v, want %v", pi.err, tt.wantErr)
+ }
+
+ if diff := cmp.Diff(got, tt.want,
+ cmpopts.IgnoreFields(blockMetadata{}, "uncompressedSizeBytes"),
+ cmpopts.IgnoreFields(blockMetadata{}, "timestamps"),
+ cmpopts.IgnoreFields(blockMetadata{}, "field"),
+ cmpopts.IgnoreFields(blockMetadata{}, "tagFamilies"),
+ cmp.AllowUnexported(blockMetadata{}),
+ ); diff != "" {
+ t.Errorf("Unexpected blockMetadata (-got +want):\n%s", diff)
+ }
+ })
+ }
+}
diff --git a/api/proto/banyandb/cluster/v1/rpc.proto b/banyand/measure/part_metadata.go
similarity index 60%
copy from api/proto/banyandb/cluster/v1/rpc.proto
copy to banyand/measure/part_metadata.go
index 732bbe1f..6c6f6e17 100644
--- a/api/proto/banyandb/cluster/v1/rpc.proto
+++ b/banyand/measure/part_metadata.go
@@ -6,7 +6,7 @@
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
-// http://www.apache.org/licenses/LICENSE-2.0
+// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
@@ -15,26 +15,24 @@
// specific language governing permissions and limitations
// under the License.
-syntax = "proto3";
+package measure
-package banyandb.cluster.v1;
-
-import "google/protobuf/any.proto";
-
-option go_package = "github.com/apache/skywalking-banyandb/api/proto/banyandb/cluster/v1";
-
-message SendRequest {
- string topic = 1;
- uint64 message_id = 2;
- google.protobuf.Any body = 3;
-}
-
-message SendResponse {
- uint64 message_id = 1;
- string error = 2;
- google.protobuf.Any body = 3;
+type partMetadata struct {
+ CompressedSizeBytes uint64
+ UncompressedSizeBytes uint64
+ TotalCount uint64
+ BlocksCount uint64
+ MinTimestamp int64
+ MaxTimestamp int64
+ Version int64
}
-service Service {
- rpc Send(stream SendRequest) returns (stream SendResponse);
+func (ph *partMetadata) reset() {
+ ph.CompressedSizeBytes = 0
+ ph.UncompressedSizeBytes = 0
+ ph.TotalCount = 0
+ ph.BlocksCount = 0
+ ph.MinTimestamp = 0
+ ph.MaxTimestamp = 0
+ ph.Version = 0
}
diff --git a/banyand/measure/part_test.go b/banyand/measure/part_test.go
new file mode 100644
index 00000000..29b5fdfe
--- /dev/null
+++ b/banyand/measure/part_test.go
@@ -0,0 +1,190 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package measure
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+
+ "github.com/apache/skywalking-banyandb/api/common"
+ "github.com/apache/skywalking-banyandb/pkg/convert"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
+)
+
+func Test_memPart_mustInitFromDataPoints(t *testing.T) {
+ tests := []struct {
+ dps *dataPoints
+ name string
+ want partMetadata
+ }{
+ {
+ name: "Test with empty dataPoints",
+ dps: &dataPoints{
+ timestamps: []int64{},
+ seriesIDs: []common.SeriesID{},
+ tagFamilies: make([][]nameValues, 0),
+ fields: make([]nameValues, 0),
+ },
+ want: partMetadata{},
+ },
+ {
+ name: "Test with one item in dataPoints",
+ dps: &dataPoints{
+ timestamps: []int64{1},
+ seriesIDs: []common.SeriesID{1},
+ tagFamilies: [][]nameValues{
+ {
+ {
+ "arrTag", []*nameValue{
+ {name: "strArrTag", valueType: pbv1.ValueTypeStrArr, value: nil, valueArr: [][]byte{[]byte("value1"), []byte("value2")}},
+ {name: "intArrTag", valueType: pbv1.ValueTypeInt64Arr, value: nil, valueArr: [][]byte{convert.Int64ToBytes(25), convert.Int64ToBytes(30)}},
+ },
+ },
+ },
+ },
+ fields: []nameValues{
+ {
+ "skipped", []*nameValue{
+ {name: "intField", valueType: pbv1.ValueTypeInt64, value: convert.Int64ToBytes(1110), valueArr: nil},
+ },
+ },
+ },
+ },
+ want: partMetadata{
+ BlocksCount: 1,
+ MinTimestamp: 1,
+ MaxTimestamp: 1,
+ TotalCount: 1,
+ },
+ },
+ {
+ name: "Test with multiple items in dataPoints",
+ dps: dps,
+ want: partMetadata{
+ BlocksCount: 3,
+ MinTimestamp: 1,
+ MaxTimestamp: 220,
+ TotalCount: 6,
+ },
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ mp := &memPart{}
+ mp.mustInitFromDataPoints(tt.dps)
+ assert.Equal(t, tt.want.BlocksCount, mp.partMetadata.BlocksCount)
+ assert.Equal(t, tt.want.MinTimestamp, mp.partMetadata.MinTimestamp)
+ assert.Equal(t, tt.want.MaxTimestamp, mp.partMetadata.MaxTimestamp)
+ assert.Equal(t, tt.want.TotalCount, mp.partMetadata.TotalCount)
+ })
+ }
+}
+
+var dps = &dataPoints{
+ seriesIDs: []common.SeriesID{1, 1, 2, 2, 3, 3},
+ timestamps: []int64{1, 2, 8, 10, 100, 220},
+ tagFamilies: [][]nameValues{
+ {
+ {
+ name: "arrTag", values: []*nameValue{
+ {name: "strArrTag", valueType: pbv1.ValueTypeStrArr, value: nil, valueArr: [][]byte{[]byte("value1"), []byte("value2")}},
+ {name: "intArrTag", valueType: pbv1.ValueTypeInt64Arr, value: nil, valueArr: [][]byte{convert.Int64ToBytes(25), convert.Int64ToBytes(30)}},
+ },
+ },
+ {
+ name: "binaryTag", values: []*nameValue{
+ {name: "binaryTag", valueType: pbv1.ValueTypeBinaryData, value: longText, valueArr: nil},
+ },
+ },
+ {
+ name: "singleTag", values: []*nameValue{
+ {name: "strTag", valueType: pbv1.ValueTypeStr, value: []byte("value1"), valueArr: nil},
+ {name: "intTag", valueType: pbv1.ValueTypeInt64, value: convert.Int64ToBytes(10), valueArr: nil},
+ },
+ },
+ },
+ {
+ {
+ name: "arrTag", values: []*nameValue{
+ {name: "strArrTag", valueType: pbv1.ValueTypeStrArr, value: nil, valueArr: [][]byte{[]byte("value3"), []byte("value4")}},
+ {name: "intArrTag", valueType: pbv1.ValueTypeInt64Arr, value: nil, valueArr: [][]byte{convert.Int64ToBytes(50), convert.Int64ToBytes(60)}},
+ },
+ },
+ {
+ name: "binaryTag", values: []*nameValue{
+ {name: "binaryTag", valueType: pbv1.ValueTypeBinaryData, value: longText, valueArr: nil},
+ },
+ },
+ {
+ name: "singleTag", values: []*nameValue{
+ {name: "strTag", valueType: pbv1.ValueTypeStr, value: []byte("value2"), valueArr: nil},
+ {name: "intTag", valueType: pbv1.ValueTypeInt64, value: convert.Int64ToBytes(20), valueArr: nil},
+ },
+ },
+ },
+ {
+ {
+ name: "singleTag", values: []*nameValue{
+ {name: "strTag", valueType: pbv1.ValueTypeStr, value: []byte("tag1"), valueArr: nil},
+ {name: "strTag", valueType: pbv1.ValueTypeInt64, value: []byte("tag2"), valueArr: nil},
+ },
+ },
+ },
+ {
+ {
+ name: "singleTag", values: []*nameValue{
+ {name: "strTag", valueType: pbv1.ValueTypeStr, value: []byte("tag11"), valueArr: nil},
+ {name: "strTag", valueType: pbv1.ValueTypeInt64, value: []byte("tag22"), valueArr: nil},
+ },
+ },
+ },
+ {},
+ {}, // empty tagFamilies for seriesID 3
+ },
+ fields: []nameValues{
+ {
+ name: "skipped", values: []*nameValue{
+ {name: "strField", valueType: pbv1.ValueTypeStr, value: []byte("field1"), valueArr: nil},
+ {name: "intField", valueType: pbv1.ValueTypeInt64, value: convert.Int64ToBytes(1110), valueArr: nil},
+ {name: "floatField", valueType: pbv1.ValueTypeFloat64, value: convert.Float64ToBytes(1221233.343), valueArr: nil},
+ {name: "binaryField", valueType: pbv1.ValueTypeBinaryData, value: longText, valueArr: nil},
+ },
+ },
+ {
+ name: "skipped", values: []*nameValue{
+ {name: "strField", valueType: pbv1.ValueTypeStr, value: []byte("field2"), valueArr: nil},
+ {name: "intField", valueType: pbv1.ValueTypeInt64, value: convert.Int64ToBytes(2220), valueArr: nil},
+ {name: "floatField", valueType: pbv1.ValueTypeFloat64, value: convert.Float64ToBytes(2442466.686), valueArr: nil},
+ {name: "binaryField", valueType: pbv1.ValueTypeBinaryData, value: longText, valueArr: nil},
+ },
+ },
+ {},
+ {}, // empty fields for seriesID 2
+ {
+ name: "onlyFields", values: []*nameValue{
+ {name: "intField", valueType: pbv1.ValueTypeInt64, value: convert.Int64ToBytes(1110), valueArr: nil},
+ },
+ },
+ {
+ name: "onlyFields", values: []*nameValue{
+ {name: "intField", valueType: pbv1.ValueTypeInt64, value: convert.Int64ToBytes(2220), valueArr: nil},
+ },
+ },
+ },
+}
diff --git a/banyand/measure/primary_metadata.go b/banyand/measure/primary_metadata.go
new file mode 100644
index 00000000..19374283
--- /dev/null
+++ b/banyand/measure/primary_metadata.go
@@ -0,0 +1,132 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package measure
+
+import (
+ "fmt"
+ "io"
+
+ "github.com/apache/skywalking-banyandb/api/common"
+ "github.com/apache/skywalking-banyandb/pkg/compress/zstd"
+ "github.com/apache/skywalking-banyandb/pkg/encoding"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+)
+
+type primaryBlockMetadata struct {
+ seriesID common.SeriesID
+ minTimestamp int64
+ maxTimestamp int64
+ dataBlock
+}
+
+// reset resets ih for subsequent re-use.
+func (ph *primaryBlockMetadata) reset() {
+ ph.seriesID = 0
+ ph.minTimestamp = 0
+ ph.maxTimestamp = 0
+ ph.offset = 0
+ ph.size = 0
+}
+
+func (ph *primaryBlockMetadata) mustWriteBlock(data []byte, sidFirst common.SeriesID, minTimestamp, maxTimestamp int64, sw *writers) {
+ ph.seriesID = sidFirst
+ ph.minTimestamp = minTimestamp
+ ph.maxTimestamp = maxTimestamp
+
+ bb := bigValuePool.Generate()
+ bb.Buf = zstd.Compress(bb.Buf[:0], data, 1)
+ ph.offset = sw.primaryWriter.bytesWritten
+ ph.size = uint64(len(bb.Buf))
+ sw.primaryWriter.MustWrite(bb.Buf)
+ bigValuePool.Release(bb)
+}
+
+func (ph *primaryBlockMetadata) marshal(dst []byte) []byte {
+ dst = ph.seriesID.AppendToBytes(dst)
+ dst = encoding.Uint64ToBytes(dst, uint64(ph.minTimestamp))
+ dst = encoding.Uint64ToBytes(dst, uint64(ph.maxTimestamp))
+ dst = encoding.Uint64ToBytes(dst, ph.offset)
+ dst = encoding.Uint64ToBytes(dst, ph.size)
+ return dst
+}
+
+func (ph *primaryBlockMetadata) unmarshal(src []byte) ([]byte, error) {
+ if len(src) < 40 {
+ return nil, fmt.Errorf("cannot unmarshal primaryBlockMetadata from %d bytes; expect at least 40 bytes", len(src))
+ }
+ ph.seriesID = common.SeriesID(encoding.BytesToUint64(src))
+ src = src[8:]
+ ph.minTimestamp = int64(encoding.BytesToUint64(src))
+ src = src[8:]
+ ph.maxTimestamp = int64(encoding.BytesToUint64(src))
+ src = src[8:]
+ ph.offset = encoding.BytesToUint64(src)
+ src = src[8:]
+ ph.size = encoding.BytesToUint64(src)
+ return src[8:], nil
+}
+
+func mustReadPrimaryBlockMetadata(dst []primaryBlockMetadata, r *reader) []primaryBlockMetadata {
+ data, err := io.ReadAll(r)
+ if err != nil {
+ logger.Panicf("cannot read primaryBlockMetadata entries from %s: %s", r.Path(), err)
+ }
+
+ bb := bigValuePool.Generate()
+ bb.Buf, err = zstd.Decompress(bb.Buf[:0], data)
+ if err != nil {
+ logger.Panicf("cannot decompress indexBlockHeader entries from %s: %s", r.Path(), err)
+ }
+ dst, err = unmarshalPrimaryBlockMetadata(dst, bb.Buf)
+ bigValuePool.Release(bb)
+ if err != nil {
+ logger.Panicf("cannot parse indexBlockHeader entries from %s: %s", r.Path(), err)
+ }
+ return dst
+}
+
+// unmarshalPrimaryBlockMetadata appends unmarshaled from src indexBlockHeader entries to dst and returns the result.
+func unmarshalPrimaryBlockMetadata(dst []primaryBlockMetadata, src []byte) ([]primaryBlockMetadata, error) {
+ dstOrig := dst
+ for len(src) > 0 {
+ if len(dst) < cap(dst) {
+ dst = dst[:len(dst)+1]
+ } else {
+ dst = append(dst, primaryBlockMetadata{})
+ }
+ ih := &dst[len(dst)-1]
+ tail, err := ih.unmarshal(src)
+ if err != nil {
+ return dstOrig, fmt.Errorf("cannot unmarshal indexBlockHeader %d: %w", len(dst)-len(dstOrig), err)
+ }
+ src = tail
+ }
+ if err := validatePrimaryBlockMetadata(dst[len(dstOrig):]); err != nil {
+ return dstOrig, err
+ }
+ return dst, nil
+}
+
+func validatePrimaryBlockMetadata(ihs []primaryBlockMetadata) error {
+ for i := 1; i < len(ihs); i++ {
+ if ihs[i].seriesID < ihs[i-1].seriesID {
+ return fmt.Errorf("unexpected primaryBlockMetadata with smaller seriesID=%d after bigger seriesID=%d", &ihs[i].seriesID, &ihs[i-1].seriesID)
+ }
+ }
+ return nil
+}
diff --git a/banyand/measure/query.go b/banyand/measure/query.go
new file mode 100644
index 00000000..d4ebadab
--- /dev/null
+++ b/banyand/measure/query.go
@@ -0,0 +1,457 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package measure
+
+import (
+ "container/heap"
+ "context"
+ "fmt"
+ "io"
+ "sort"
+
+ "github.com/pkg/errors"
+
+ "github.com/apache/skywalking-banyandb/api/common"
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+ "github.com/apache/skywalking-banyandb/banyand/internal/storage"
+ "github.com/apache/skywalking-banyandb/pkg/convert"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
+ resourceSchema "github.com/apache/skywalking-banyandb/pkg/schema"
+)
+
+// Query allow to retrieve measure data points.
+type Query interface {
+ LoadGroup(name string) (resourceSchema.Group, bool)
+ Measure(measure *commonv1.Metadata) (Measure, error)
+}
+
+// Measure allows inspecting measure data points' details.
+type Measure interface {
+ io.Closer
+ Query(ctx context.Context, opts pbv1.MeasureQueryOptions) (pbv1.MeasureQueryResult, error)
+ GetSchema() *databasev1.Measure
+ GetIndexRules() []*databasev1.IndexRule
+ SetSchema(schema *databasev1.Measure)
+}
+
+var _ Measure = (*measure)(nil)
+
+func (s *measure) SetSchema(schema *databasev1.Measure) {
+ s.schema = schema
+}
+
+type queryOptions struct {
+ pbv1.MeasureQueryOptions
+ minTimestamp int64
+ maxTimestamp int64
+}
+
+func (s *measure) Query(ctx context.Context, mqo pbv1.MeasureQueryOptions) (pbv1.MeasureQueryResult, error) {
+ if mqo.TimeRange == nil || mqo.Entity == nil {
+ return nil, errors.New("invalid query options: timeRange and series are required")
+ }
+ if len(mqo.TagProjection) == 0 && len(mqo.FieldProjection) == 0 {
+ return nil, errors.New("invalid query options: tagProjection or fieldProjection is required")
+ }
+ tsdb := s.databaseSupplier.SupplyTSDB().(storage.TSDB[*tsTable])
+ tabWrappers := tsdb.SelectTSTables(*mqo.TimeRange)
+ sl, err := tsdb.IndexDB().Search(ctx, &pbv1.Series{Subject: mqo.Name, EntityValues: mqo.Entity}, mqo.Filter, mqo.Order)
+ if err != nil {
+ return nil, err
+ }
+
+ var result queryResult
+ if len(sl) < 1 {
+ return &result, nil
+ }
+ var sids []common.SeriesID
+ for i := range sl {
+ sids = append(sids, sl[i].ID)
+ }
+ var pws []*partWrapper
+ var parts []*part
+ qo := queryOptions{
+ MeasureQueryOptions: mqo,
+ minTimestamp: mqo.TimeRange.Start.UnixNano(),
+ maxTimestamp: mqo.TimeRange.End.UnixNano(),
+ }
+ for _, tw := range tabWrappers {
+ pws, parts = tw.Table().getParts(pws, parts, qo)
+ }
+ // TODO: cache tstIter
+ var tstIter tstIter
+ originalSids := make([]common.SeriesID, len(sids))
+ copy(originalSids, sids)
+ sort.Slice(sids, func(i, j int) bool { return sids[i] < sids[j] })
+ tstIter.init(parts, sids, qo.minTimestamp, qo.maxTimestamp)
+ if tstIter.Error() != nil {
+ return nil, fmt.Errorf("cannot init tstIter: %w", tstIter.Error())
+ }
+ for tstIter.nextBlock() {
+ bc := generateBlockCursor()
+ p := tstIter.piHeap[0]
+ bc.init(p.p, p.curBlock, qo)
+ result.data = append(result.data, bc)
+ }
+ if tstIter.Error() != nil {
+ return nil, fmt.Errorf("cannot iterate tstIter: %w", tstIter.Error())
+ }
+ if mqo.Order == nil {
+ result.orderByTS = true
+ result.ascTS = true
+ return &result, nil
+ }
+ if mqo.Order.Index == nil {
+ result.orderByTS = true
+ if mqo.Order.Sort == modelv1.Sort_SORT_ASC || mqo.Order.Sort == modelv1.Sort_SORT_UNSPECIFIED {
+ result.ascTS = true
+ }
+ return &result, nil
+ }
+
+ result.sidToIndex = make(map[common.SeriesID]int)
+ for i, si := range originalSids {
+ result.sidToIndex[si] = i
+ }
+ return &result, nil
+}
+
+func mustDecodeTagValue(valueType pbv1.ValueType, value []byte) *modelv1.TagValue {
+ if value == nil {
+ switch valueType {
+ case pbv1.ValueTypeInt64:
+ logger.Panicf("int64 can be nil")
+ case pbv1.ValueTypeStr:
+ return pbv1.EmptyStrTagValue
+ case pbv1.ValueTypeStrArr:
+ return pbv1.EmptyStrArrTagValue
+ case pbv1.ValueTypeInt64Arr:
+ return pbv1.EmptyIntArrTagValue
+ case pbv1.ValueTypeBinaryData:
+ return pbv1.EmptyBinaryTagValue
+ default:
+ return pbv1.NullTagValue
+ }
+ }
+ switch valueType {
+ case pbv1.ValueTypeInt64:
+ return int64TagValue(convert.BytesToInt64(value))
+ case pbv1.ValueTypeStr:
+ return strTagValue(string(value))
+ case pbv1.ValueTypeBinaryData:
+ return binaryDataTagValue(value)
+ case pbv1.ValueTypeInt64Arr:
+ var values []int64
+ for i := 0; i < len(value); i += 8 {
+ values = append(values, convert.BytesToInt64(value[i:i+8]))
+ }
+ return int64ArrTagValue(values)
+ case pbv1.ValueTypeStrArr:
+ var values []string
+ bb := bigValuePool.Generate()
+ var err error
+ for len(value) > 0 {
+ bb.Buf, value, err = unmarshalVarArray(bb.Buf[:0], value)
+ if err != nil {
+ logger.Panicf("unmarshalVarArray failed: %v", err)
+ }
+ values = append(values, string(bb.Buf))
+ }
+ return strArrTagValue(values)
+ default:
+ logger.Panicf("unsupported value type: %v", valueType)
+ return nil
+ }
+}
+
+func int64TagValue(value int64) *modelv1.TagValue {
+ return &modelv1.TagValue{
+ Value: &modelv1.TagValue_Int{
+ Int: &modelv1.Int{
+ Value: value,
+ },
+ },
+ }
+}
+
+func strTagValue(value string) *modelv1.TagValue {
+ return &modelv1.TagValue{
+ Value: &modelv1.TagValue_Str{
+ Str: &modelv1.Str{
+ Value: value,
+ },
+ },
+ }
+}
+
+func binaryDataTagValue(value []byte) *modelv1.TagValue {
+ data := make([]byte, len(value))
+ copy(data, value)
+ return &modelv1.TagValue{
+ Value: &modelv1.TagValue_BinaryData{
+ BinaryData: data,
+ },
+ }
+}
+
+func int64ArrTagValue(values []int64) *modelv1.TagValue {
+ return &modelv1.TagValue{
+ Value: &modelv1.TagValue_IntArray{
+ IntArray: &modelv1.IntArray{
+ Value: values,
+ },
+ },
+ }
+}
+
+func strArrTagValue(values []string) *modelv1.TagValue {
+ return &modelv1.TagValue{
+ Value: &modelv1.TagValue_StrArray{
+ StrArray: &modelv1.StrArray{
+ Value: values,
+ },
+ },
+ }
+}
+
+func mustDecodeFieldValue(valueType pbv1.ValueType, value []byte) *modelv1.FieldValue {
+ if value == nil {
+ switch valueType {
+ case pbv1.ValueTypeInt64, pbv1.ValueTypeFloat64:
+ logger.Panicf("int64 and float64 can't be nil")
+ case pbv1.ValueTypeStr:
+ return pbv1.EmptyStrFieldValue
+ case pbv1.ValueTypeBinaryData:
+ return pbv1.EmptyBinaryFieldValue
+ default:
+ return pbv1.NullFieldValue
+ }
+ }
+ switch valueType {
+ case pbv1.ValueTypeInt64:
+ return int64FieldValue(convert.BytesToInt64(value))
+ case pbv1.ValueTypeFloat64:
+ return float64FieldValue(convert.BytesToFloat64(value))
+ case pbv1.ValueTypeStr:
+ return strFieldValue(string(value))
+ case pbv1.ValueTypeBinaryData:
+ return binaryDataFieldValue(value)
+ default:
+ logger.Panicf("unsupported value type: %v", valueType)
+ return nil
+ }
+}
+
+func int64FieldValue(value int64) *modelv1.FieldValue {
+ return &modelv1.FieldValue{
+ Value: &modelv1.FieldValue_Int{
+ Int: &modelv1.Int{
+ Value: value,
+ },
+ },
+ }
+}
+
+func float64FieldValue(value float64) *modelv1.FieldValue {
+ return &modelv1.FieldValue{
+ Value: &modelv1.FieldValue_Float{
+ Float: &modelv1.Float{
+ Value: value,
+ },
+ },
+ }
+}
+
+func strFieldValue(value string) *modelv1.FieldValue {
+ return &modelv1.FieldValue{
+ Value: &modelv1.FieldValue_Str{
+ Str: &modelv1.Str{
+ Value: value,
+ },
+ },
+ }
+}
+
+func binaryDataFieldValue(value []byte) *modelv1.FieldValue {
+ data := make([]byte, len(value))
+ copy(data, value)
+ return &modelv1.FieldValue{
+ Value: &modelv1.FieldValue_BinaryData{
+ BinaryData: data,
+ },
+ }
+}
+
+type queryResult struct {
+ sidToIndex map[common.SeriesID]int
+ data []*blockCursor
+ pws []*partWrapper
+ loaded bool
+ orderByTS bool
+ ascTS bool
+}
+
+func (qr *queryResult) Pull() *pbv1.Result {
+ if !qr.loaded {
+ if len(qr.data) == 0 {
+ return nil
+ }
+ // TODO:// Parallel load
+ tmpBlock := generateBlock()
+ defer releaseBlock(tmpBlock)
+ for i := 0; i < len(qr.data); i++ {
+ if !qr.data[i].loadData(tmpBlock) {
+ qr.data = append(qr.data[:i], qr.data[i+1:]...)
+ i--
+ }
+ if qr.orderByTimestampDesc() {
+ qr.data[i].idx = len(qr.data[i].timestamps) - 1
+ }
+ }
+ qr.loaded = true
+ heap.Init(qr)
+ }
+ if len(qr.data) == 0 {
+ return nil
+ }
+ if len(qr.data) == 1 {
+ r := &pbv1.Result{}
+ bc := qr.data[0]
+ bc.copyAllTo(r)
+ qr.data = qr.data[:0]
+ return r
+ }
+ return qr.merge()
+}
+
+func (qr *queryResult) Release() {
+ for i, v := range qr.data {
+ releaseBlockCursor(v)
+ qr.data[i] = nil
+ }
+ qr.data = qr.data[:0]
+ for i := range qr.pws {
+ qr.pws[i].decRef()
+ }
+ qr.pws = qr.pws[:0]
+}
+
+func (qr queryResult) Len() int {
+ return len(qr.data)
+}
+
+func (qr queryResult) Less(i, j int) bool {
+ leftTS := qr.data[i].timestamps[qr.data[i].idx]
+ rightTS := qr.data[j].timestamps[qr.data[j].idx]
+ leftVersion := qr.data[i].p.partMetadata.Version
+ rightVersion := qr.data[j].p.partMetadata.Version
+ if qr.orderByTS {
+ if leftTS == rightTS {
+ if qr.data[i].bm.seriesID == qr.data[j].bm.seriesID {
+ // sort version in descending order if timestamps and seriesID are equal
+ return leftVersion > rightVersion
+ }
+ // sort seriesID in ascending order if timestamps are equal
+ return qr.data[i].bm.seriesID < qr.data[j].bm.seriesID
+ }
+ if qr.ascTS {
+ return leftTS < rightTS
+ }
+ return leftTS > rightTS
+ }
+ leftSIDIndex := qr.sidToIndex[qr.data[i].bm.seriesID]
+ rightSIDIndex := qr.sidToIndex[qr.data[j].bm.seriesID]
+ if leftSIDIndex == rightSIDIndex {
+ if leftTS == rightTS {
+ // sort version in descending order if timestamps and seriesID are equal
+ return leftVersion > rightVersion
+ }
+ // sort timestamps in ascending order if seriesID are equal
+ return leftTS < rightTS
+ }
+ return leftSIDIndex < rightSIDIndex
+}
+
+func (qr queryResult) Swap(i, j int) {
+ qr.data[i], qr.data[j] = qr.data[j], qr.data[i]
+}
+
+func (qr *queryResult) Push(x interface{}) {
+ qr.data = append(qr.data, x.(*blockCursor))
+}
+
+func (qr *queryResult) Pop() interface{} {
+ old := qr.data
+ n := len(old)
+ x := old[n-1]
+ qr.data = old[0 : n-1]
+ return x
+}
+
+func (qr *queryResult) orderByTimestampDesc() bool {
+ return qr.orderByTS && !qr.ascTS
+}
+
+func (qr *queryResult) merge() *pbv1.Result {
+ step := 1
+ if qr.orderByTimestampDesc() {
+ step = -1
+ }
+ result := &pbv1.Result{}
+ var lastPartVersion int64
+ var lastSid common.SeriesID
+
+ for qr.Len() > 0 {
+ topBC := qr.data[0]
+ if lastSid != 0 && topBC.bm.seriesID != lastSid {
+ return result
+ }
+ lastSid = topBC.bm.seriesID
+
+ if len(result.Timestamps) > 0 &&
+ topBC.timestamps[topBC.idx] == result.Timestamps[len(result.Timestamps)-1] {
+ if topBC.p.partMetadata.Version > lastPartVersion {
+ logger.Panicf("following parts version should be less or equal to the previous one")
+ }
+ } else {
+ topBC.copyTo(result)
+ lastPartVersion = topBC.p.partMetadata.Version
+ }
+
+ topBC.idx += step
+
+ if qr.orderByTimestampDesc() {
+ if topBC.idx < 0 {
+ heap.Pop(qr)
+ } else {
+ heap.Fix(qr, 0)
+ }
+ } else {
+ if topBC.idx >= len(topBC.timestamps) {
+ heap.Pop(qr)
+ } else {
+ heap.Fix(qr, 0)
+ }
+ }
+ }
+
+ return result
+}
diff --git a/banyand/measure/query_test.go b/banyand/measure/query_test.go
new file mode 100644
index 00000000..d69f77ad
--- /dev/null
+++ b/banyand/measure/query_test.go
@@ -0,0 +1,433 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package measure
+
+import (
+ "errors"
+ "sort"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "google.golang.org/protobuf/testing/protocmp"
+
+ "github.com/apache/skywalking-banyandb/api/common"
+ modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
+)
+
+func TestQueryResult(t *testing.T) {
+ tests := []struct {
+ wantErr error
+ name string
+ dpsList []*dataPoints
+ sids []common.SeriesID
+ want []pbv1.Result
+ minTimestamp int64
+ maxTimestamp int64
+ orderBySeries bool
+ ascTS bool
+ }{
+ {
+ name: "Test with multiple parts with duplicated data order by TS",
+ dpsList: []*dataPoints{dpsTS1, dpsTS1},
+ sids: []common.SeriesID{1, 2, 3},
+ minTimestamp: 1,
+ maxTimestamp: 1,
+ want: []pbv1.Result{{
+ SID: 1,
+ Timestamps: []int64{1},
+ TagFamilies: []pbv1.TagFamily{
+ {Name: "arrTag", Tags: []pbv1.Tag{
+ {Name: "strArrTag", Values: []*modelv1.TagValue{strArrTagValue([]string{"value1", "value2"})}},
+ {Name: "intArrTag", Values: []*modelv1.TagValue{int64ArrTagValue([]int64{25, 30})}},
+ }},
+ {Name: "binaryTag", Tags: []pbv1.Tag{
+ {Name: "binaryTag", Values: []*modelv1.TagValue{binaryDataTagValue(longText)}},
+ }},
+ {Name: "singleTag", Tags: []pbv1.Tag{
+ {Name: "strTag", Values: []*modelv1.TagValue{strTagValue("value1")}},
+ {Name: "intTag", Values: []*modelv1.TagValue{int64TagValue(10)}},
+ }},
+ },
+ Fields: []pbv1.Field{
+ {Name: "strField", Values: []*modelv1.FieldValue{strFieldValue("field1")}},
+ {Name: "intField", Values: []*modelv1.FieldValue{int64FieldValue(1110)}},
+ {Name: "floatField", Values: []*modelv1.FieldValue{float64FieldValue(1.221233343e+06)}},
+ {Name: "binaryField", Values: []*modelv1.FieldValue{binaryDataFieldValue(longText)}},
+ },
+ }, {
+ SID: 2,
+ Timestamps: []int64{1},
+ TagFamilies: []pbv1.TagFamily{
+ {Name: "singleTag", Tags: []pbv1.Tag{
+ {Name: "strTag1", Values: []*modelv1.TagValue{strTagValue("tag1")}},
+ {Name: "strTag2", Values: []*modelv1.TagValue{strTagValue("tag2")}},
+ }},
+ },
+ Fields: nil,
+ }, {
+ SID: 3,
+ Timestamps: []int64{1},
+ TagFamilies: nil,
+ Fields: []pbv1.Field{
+ {Name: "intField", Values: []*modelv1.FieldValue{int64FieldValue(1110)}},
+ },
+ }},
+ },
+ {
+ name: "Test with multiple parts with multiple data orderBy TS desc",
+ dpsList: []*dataPoints{dpsTS1, dpsTS2},
+ sids: []common.SeriesID{1, 2, 3},
+ minTimestamp: 1,
+ maxTimestamp: 2,
+ want: []pbv1.Result{{
+ SID: 1,
+ Timestamps: []int64{2},
+ TagFamilies: []pbv1.TagFamily{
+ {Name: "arrTag", Tags: []pbv1.Tag{
+ {Name: "strArrTag", Values: []*modelv1.TagValue{strArrTagValue([]string{"value5", "value6"})}},
+ {Name: "intArrTag", Values: []*modelv1.TagValue{int64ArrTagValue([]int64{35, 40})}},
+ }},
+ {Name: "binaryTag", Tags: []pbv1.Tag{
+ {Name: "binaryTag", Values: []*modelv1.TagValue{binaryDataTagValue(longText)}},
+ }},
+ {Name: "singleTag", Tags: []pbv1.Tag{
+ {Name: "strTag", Values: []*modelv1.TagValue{strTagValue("value3")}},
+ {Name: "intTag", Values: []*modelv1.TagValue{int64TagValue(30)}},
+ }},
+ },
+ Fields: []pbv1.Field{
+ {Name: "strField", Values: []*modelv1.FieldValue{strFieldValue("field3")}},
+ {Name: "intField", Values: []*modelv1.FieldValue{int64FieldValue(3330)}},
+ {Name: "floatField", Values: []*modelv1.FieldValue{float64FieldValue(3663699.029)}},
+ {Name: "binaryField", Values: []*modelv1.FieldValue{binaryDataFieldValue(longText)}},
+ },
+ }, {
+ SID: 2,
+ Timestamps: []int64{2},
+ TagFamilies: []pbv1.TagFamily{
+ {Name: "singleTag", Tags: []pbv1.Tag{
+ {Name: "strTag1", Values: []*modelv1.TagValue{strTagValue("tag3")}},
+ {Name: "strTag2", Values: []*modelv1.TagValue{strTagValue("tag4")}},
+ }},
+ },
+ Fields: nil,
+ }, {
+ SID: 3,
+ Timestamps: []int64{2},
+ TagFamilies: nil,
+ Fields: []pbv1.Field{
+ {Name: "intField", Values: []*modelv1.FieldValue{int64FieldValue(4440)}},
+ },
+ }, {
+ SID: 1,
+ Timestamps: []int64{1},
+ TagFamilies: []pbv1.TagFamily{
+ {Name: "arrTag", Tags: []pbv1.Tag{
+ {Name: "strArrTag", Values: []*modelv1.TagValue{strArrTagValue([]string{"value1", "value2"})}},
+ {Name: "intArrTag", Values: []*modelv1.TagValue{int64ArrTagValue([]int64{25, 30})}},
+ }},
+ {Name: "binaryTag", Tags: []pbv1.Tag{
+ {Name: "binaryTag", Values: []*modelv1.TagValue{binaryDataTagValue(longText)}},
+ }},
+ {Name: "singleTag", Tags: []pbv1.Tag{
+ {Name: "strTag", Values: []*modelv1.TagValue{strTagValue("value1")}},
+ {Name: "intTag", Values: []*modelv1.TagValue{int64TagValue(10)}},
+ }},
+ },
+ Fields: []pbv1.Field{
+ {Name: "strField", Values: []*modelv1.FieldValue{strFieldValue("field1")}},
+ {Name: "intField", Values: []*modelv1.FieldValue{int64FieldValue(1110)}},
+ {Name: "floatField", Values: []*modelv1.FieldValue{float64FieldValue(1221233.343)}},
+ {Name: "binaryField", Values: []*modelv1.FieldValue{binaryDataFieldValue(longText)}},
+ },
+ }, {
+ SID: 2,
+ Timestamps: []int64{1},
+ TagFamilies: []pbv1.TagFamily{
+ {Name: "singleTag", Tags: []pbv1.Tag{
+ {Name: "strTag1", Values: []*modelv1.TagValue{strTagValue("tag1")}},
+ {Name: "strTag2", Values: []*modelv1.TagValue{strTagValue("tag2")}},
+ }},
+ },
+ Fields: nil,
+ }, {
+ SID: 3,
+ Timestamps: []int64{1},
+ TagFamilies: nil,
+ Fields: []pbv1.Field{
+ {Name: "intField", Values: []*modelv1.FieldValue{int64FieldValue(1110)}},
+ },
+ }},
+ },
+ {
+ name: "Test with multiple parts with multiple data orderBy TS asc",
+ dpsList: []*dataPoints{dpsTS1, dpsTS2},
+ sids: []common.SeriesID{1, 2, 3},
+ ascTS: true,
+ minTimestamp: 1,
+ maxTimestamp: 2,
+ want: []pbv1.Result{{
+ SID: 1,
+ Timestamps: []int64{1},
+ TagFamilies: []pbv1.TagFamily{
+ {Name: "arrTag", Tags: []pbv1.Tag{
+ {Name: "strArrTag", Values: []*modelv1.TagValue{strArrTagValue([]string{"value1", "value2"})}},
+ {Name: "intArrTag", Values: []*modelv1.TagValue{int64ArrTagValue([]int64{25, 30})}},
+ }},
+ {Name: "binaryTag", Tags: []pbv1.Tag{
+ {Name: "binaryTag", Values: []*modelv1.TagValue{binaryDataTagValue(longText)}},
+ }},
+ {Name: "singleTag", Tags: []pbv1.Tag{
+ {Name: "strTag", Values: []*modelv1.TagValue{strTagValue("value1")}},
+ {Name: "intTag", Values: []*modelv1.TagValue{int64TagValue(10)}},
+ }},
+ },
+ Fields: []pbv1.Field{
+ {Name: "strField", Values: []*modelv1.FieldValue{strFieldValue("field1")}},
+ {Name: "intField", Values: []*modelv1.FieldValue{int64FieldValue(1110)}},
+ {Name: "floatField", Values: []*modelv1.FieldValue{float64FieldValue(1221233.343)}},
+ {Name: "binaryField", Values: []*modelv1.FieldValue{binaryDataFieldValue(longText)}},
+ },
+ }, {
+ SID: 2,
+ Timestamps: []int64{1},
+ TagFamilies: []pbv1.TagFamily{
+ {Name: "singleTag", Tags: []pbv1.Tag{
+ {Name: "strTag1", Values: []*modelv1.TagValue{strTagValue("tag1")}},
+ {Name: "strTag2", Values: []*modelv1.TagValue{strTagValue("tag2")}},
+ }},
+ },
+ Fields: nil,
+ }, {
+ SID: 3,
+ Timestamps: []int64{1},
+ TagFamilies: nil,
+ Fields: []pbv1.Field{
+ {Name: "intField", Values: []*modelv1.FieldValue{int64FieldValue(1110)}},
+ },
+ }, {
+ SID: 1,
+ Timestamps: []int64{2},
+ TagFamilies: []pbv1.TagFamily{
+ {Name: "arrTag", Tags: []pbv1.Tag{
+ {Name: "strArrTag", Values: []*modelv1.TagValue{strArrTagValue([]string{"value5", "value6"})}},
+ {Name: "intArrTag", Values: []*modelv1.TagValue{int64ArrTagValue([]int64{35, 40})}},
+ }},
+ {Name: "binaryTag", Tags: []pbv1.Tag{
+ {Name: "binaryTag", Values: []*modelv1.TagValue{binaryDataTagValue(longText)}},
+ }},
+ {Name: "singleTag", Tags: []pbv1.Tag{
+ {Name: "strTag", Values: []*modelv1.TagValue{strTagValue("value3")}},
+ {Name: "intTag", Values: []*modelv1.TagValue{int64TagValue(30)}},
+ }},
+ },
+ Fields: []pbv1.Field{
+ {Name: "strField", Values: []*modelv1.FieldValue{strFieldValue("field3")}},
+ {Name: "intField", Values: []*modelv1.FieldValue{int64FieldValue(3330)}},
+ {Name: "floatField", Values: []*modelv1.FieldValue{float64FieldValue(3663699.029)}},
+ {Name: "binaryField", Values: []*modelv1.FieldValue{binaryDataFieldValue(longText)}},
+ },
+ }, {
+ SID: 2,
+ Timestamps: []int64{2},
+ TagFamilies: []pbv1.TagFamily{
+ {Name: "singleTag", Tags: []pbv1.Tag{
+ {Name: "strTag1", Values: []*modelv1.TagValue{strTagValue("tag3")}},
+ {Name: "strTag2", Values: []*modelv1.TagValue{strTagValue("tag4")}},
+ }},
+ },
+ Fields: nil,
+ }, {
+ SID: 3,
+ Timestamps: []int64{2},
+ TagFamilies: nil,
+ Fields: []pbv1.Field{
+ {Name: "intField", Values: []*modelv1.FieldValue{int64FieldValue(4440)}},
+ },
+ }},
+ },
+ {
+ name: "Test with multiple parts with duplicated data order by Series",
+ dpsList: []*dataPoints{dpsTS1, dpsTS1},
+ sids: []common.SeriesID{1, 2, 3},
+ orderBySeries: true,
+ minTimestamp: 1,
+ maxTimestamp: 1,
+ want: []pbv1.Result{{
+ SID: 1,
+ Timestamps: []int64{1},
+ TagFamilies: []pbv1.TagFamily{
+ {Name: "arrTag", Tags: []pbv1.Tag{
+ {Name: "strArrTag", Values: []*modelv1.TagValue{strArrTagValue([]string{"value1", "value2"})}},
+ {Name: "intArrTag", Values: []*modelv1.TagValue{int64ArrTagValue([]int64{25, 30})}},
+ }},
+ {Name: "binaryTag", Tags: []pbv1.Tag{
+ {Name: "binaryTag", Values: []*modelv1.TagValue{binaryDataTagValue(longText)}},
+ }},
+ {Name: "singleTag", Tags: []pbv1.Tag{
+ {Name: "strTag", Values: []*modelv1.TagValue{strTagValue("value1")}},
+ {Name: "intTag", Values: []*modelv1.TagValue{int64TagValue(10)}},
+ }},
+ },
+ Fields: []pbv1.Field{
+ {Name: "strField", Values: []*modelv1.FieldValue{strFieldValue("field1")}},
+ {Name: "intField", Values: []*modelv1.FieldValue{int64FieldValue(1110)}},
+ {Name: "floatField", Values: []*modelv1.FieldValue{float64FieldValue(1.221233343e+06)}},
+ {Name: "binaryField", Values: []*modelv1.FieldValue{binaryDataFieldValue(longText)}},
+ },
+ }, {
+ SID: 2,
+ Timestamps: []int64{1},
+ TagFamilies: []pbv1.TagFamily{
+ {Name: "singleTag", Tags: []pbv1.Tag{
+ {Name: "strTag1", Values: []*modelv1.TagValue{strTagValue("tag1")}},
+ {Name: "strTag2", Values: []*modelv1.TagValue{strTagValue("tag2")}},
+ }},
+ },
+ Fields: nil,
+ }, {
+ SID: 3,
+ Timestamps: []int64{1},
+ TagFamilies: nil,
+ Fields: []pbv1.Field{
+ {Name: "intField", Values: []*modelv1.FieldValue{int64FieldValue(1110)}},
+ },
+ }},
+ },
+ {
+ name: "Test with multiple parts with multiple data order by Series",
+ dpsList: []*dataPoints{dpsTS1, dpsTS2},
+ sids: []common.SeriesID{2, 1, 3},
+ orderBySeries: true,
+ minTimestamp: 1,
+ maxTimestamp: 2,
+ want: []pbv1.Result{{
+ SID: 2,
+ Timestamps: []int64{1, 2},
+ TagFamilies: []pbv1.TagFamily{
+ {Name: "singleTag", Tags: []pbv1.Tag{
+ {Name: "strTag1", Values: []*modelv1.TagValue{strTagValue("tag1"), strTagValue("tag3")}},
+ {Name: "strTag2", Values: []*modelv1.TagValue{strTagValue("tag2"), strTagValue("tag4")}},
+ }},
+ },
+ Fields: nil,
+ }, {
+ SID: 1,
+ Timestamps: []int64{1, 2},
+ TagFamilies: []pbv1.TagFamily{
+ {Name: "arrTag", Tags: []pbv1.Tag{
+ {Name: "strArrTag", Values: []*modelv1.TagValue{strArrTagValue([]string{"value1", "value2"}), strArrTagValue([]string{"value5", "value6"})}},
+ {Name: "intArrTag", Values: []*modelv1.TagValue{int64ArrTagValue([]int64{25, 30}), int64ArrTagValue([]int64{35, 40})}},
+ }},
+ {Name: "binaryTag", Tags: []pbv1.Tag{
+ {Name: "binaryTag", Values: []*modelv1.TagValue{binaryDataTagValue(longText), binaryDataTagValue(longText)}},
+ }},
+ {Name: "singleTag", Tags: []pbv1.Tag{
+ {Name: "strTag", Values: []*modelv1.TagValue{strTagValue("value1"), strTagValue("value3")}},
+ {Name: "intTag", Values: []*modelv1.TagValue{int64TagValue(10), int64TagValue(30)}},
+ }},
+ },
+ Fields: []pbv1.Field{
+ {Name: "strField", Values: []*modelv1.FieldValue{strFieldValue("field1"), strFieldValue("field3")}},
+ {Name: "intField", Values: []*modelv1.FieldValue{int64FieldValue(1110), int64FieldValue(3330)}},
+ {Name: "floatField", Values: []*modelv1.FieldValue{float64FieldValue(1.221233343e+06), float64FieldValue(3663699.029)}},
+ {Name: "binaryField", Values: []*modelv1.FieldValue{binaryDataFieldValue(longText), binaryDataFieldValue(longText)}},
+ },
+ }, {
+ SID: 3,
+ Timestamps: []int64{1, 2},
+ TagFamilies: nil,
+ Fields: []pbv1.Field{
+ {Name: "intField", Values: []*modelv1.FieldValue{int64FieldValue(1110), int64FieldValue(4440)}},
+ },
+ }},
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ // Initialize a tstIter object.
+ tst := &tsTable{}
+ defer tst.Close()
+ for _, dps := range tt.dpsList {
+ tst.mustAddDataPoints(dps)
+ time.Sleep(100 * time.Millisecond)
+ }
+ queryOpts := queryOptions{
+ minTimestamp: tt.minTimestamp,
+ maxTimestamp: tt.maxTimestamp,
+ }
+ pws, pp := tst.getParts(nil, nil, queryOpts)
+ defer func() {
+ for _, pw := range pws {
+ pw.decRef()
+ }
+ }()
+ sids := make([]common.SeriesID, len(tt.sids))
+ copy(sids, tt.sids)
+ sort.Slice(sids, func(i, j int) bool {
+ return sids[i] < tt.sids[j]
+ })
+ ti := &tstIter{}
+ ti.init(pp, sids, tt.minTimestamp, tt.maxTimestamp)
+
+ var result queryResult
+ for ti.nextBlock() {
+ bc := generateBlockCursor()
+ p := ti.piHeap[0]
+ opts := queryOpts
+ opts.TagProjection = tagProjections[int(p.curBlock.seriesID)]
+ opts.FieldProjection = fieldProjections[int(p.curBlock.seriesID)]
+ bc.init(p.p, p.curBlock, opts)
+ result.data = append(result.data, bc)
+ }
+ defer result.Release()
+ if tt.orderBySeries {
+ result.sidToIndex = make(map[common.SeriesID]int)
+ for i, si := range tt.sids {
+ result.sidToIndex[si] = i
+ }
+ } else {
+ result.orderByTS = true
+ result.ascTS = tt.ascTS
+ }
+ var got []pbv1.Result
+ for {
+ r := result.Pull()
+ if r == nil {
+ break
+ }
+ sort.Slice(r.TagFamilies, func(i, j int) bool {
+ return r.TagFamilies[i].Name < r.TagFamilies[j].Name
+ })
+ got = append(got, *r)
+ }
+
+ if !errors.Is(ti.Error(), tt.wantErr) {
+ t.Errorf("Unexpected error: got %v, want %v", ti.err, tt.wantErr)
+ }
+
+ if diff := cmp.Diff(got, tt.want,
+ protocmp.IgnoreUnknown(), protocmp.Transform()); diff != "" {
+ t.Errorf("Unexpected []pbv1.Result (-got +want):\n%s", diff)
+ }
+ })
+ }
+}
diff --git a/banyand/measure/service.go b/banyand/measure/service.go
index 98a9e101..9a84f308 100644
--- a/banyand/measure/service.go
+++ b/banyand/measure/service.go
@@ -29,7 +29,6 @@ import (
"github.com/apache/skywalking-banyandb/banyand/metadata"
"github.com/apache/skywalking-banyandb/banyand/observability"
"github.com/apache/skywalking-banyandb/banyand/queue"
- "github.com/apache/skywalking-banyandb/banyand/tsdb"
"github.com/apache/skywalking-banyandb/pkg/bus"
"github.com/apache/skywalking-banyandb/pkg/logger"
"github.com/apache/skywalking-banyandb/pkg/run"
@@ -53,16 +52,13 @@ type Service interface {
var _ Service = (*service)(nil)
type service struct {
- schemaRepo schemaRepo
- writeListener bus.MessageListener
- metadata metadata.Repo
- pipeline queue.Server
- localPipeline queue.Queue
- l *logger.Logger
- root string
- dbOpts tsdb.DatabaseOpts
- BlockEncoderBufferSize run.Bytes
- BlockBufferSize run.Bytes
+ schemaRepo schemaRepo
+ writeListener bus.MessageListener
+ metadata metadata.Repo
+ pipeline queue.Server
+ localPipeline queue.Queue
+ l *logger.Logger
+ root string
}
func (s *service) Measure(metadata *commonv1.Metadata) (Measure, error) {
@@ -80,14 +76,6 @@ func (s *service) LoadGroup(name string) (resourceSchema.Group, bool) {
func (s *service) FlagSet() *run.FlagSet {
flagS := run.NewFlagSet("storage")
flagS.StringVar(&s.root, "measure-root-path", "/tmp", "the root path of database")
- s.BlockEncoderBufferSize = 12 << 20
- s.BlockBufferSize = 4 << 20
- s.dbOpts.SeriesMemSize = 1 << 20
- flagS.Var(&s.BlockEncoderBufferSize, "measure-encoder-buffer-size", "block fields buffer size")
- flagS.Var(&s.BlockBufferSize, "measure-buffer-size", "block buffer size")
- flagS.Var(&s.dbOpts.SeriesMemSize, "measure-seriesmeta-mem-size", "series metadata memory size")
- flagS.Int64Var(&s.dbOpts.BlockInvertedIndex.BatchWaitSec, "measure-idx-batch-wait-sec", 1, "index batch wait in second")
- flagS.BoolVar(&s.dbOpts.EnableWAL, "measure-enable-wal", true, "enable write ahead log")
return flagS
}
@@ -111,8 +99,7 @@ func (s *service) PreRun(_ context.Context) error {
path := path.Join(s.root, s.Name())
observability.UpdatePath(path)
s.localPipeline = queue.Local()
- s.schemaRepo = newSchemaRepo(path, s.metadata, s.dbOpts,
- s.l, s.localPipeline, int64(s.BlockEncoderBufferSize), int64(s.BlockBufferSize))
+ s.schemaRepo = newSchemaRepo(path, s)
// run a serial watcher
s.writeListener = setUpWriteCallback(s.l, &s.schemaRepo)
@@ -137,8 +124,5 @@ func NewService(_ context.Context, metadata metadata.Repo, pipeline queue.Server
return &service{
metadata: metadata,
pipeline: pipeline,
- dbOpts: tsdb.DatabaseOpts{
- IndexGranularity: tsdb.IndexGranularitySeries,
- },
}, nil
}
diff --git a/banyand/measure/measure_topn.go b/banyand/measure/topn.go
similarity index 88%
rename from banyand/measure/measure_topn.go
rename to banyand/measure/topn.go
index 15780b54..80d7145f 100644
--- a/banyand/measure/measure_topn.go
+++ b/banyand/measure/topn.go
@@ -39,7 +39,6 @@ import (
measurev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/measure/v1"
modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
"github.com/apache/skywalking-banyandb/banyand/queue"
- "github.com/apache/skywalking-banyandb/banyand/tsdb"
"github.com/apache/skywalking-banyandb/pkg/bus"
"github.com/apache/skywalking-banyandb/pkg/flow"
"github.com/apache/skywalking-banyandb/pkg/flow/streaming"
@@ -52,8 +51,7 @@ import (
const (
timeBucketFormat = "200601021504"
- // TopNTagFamily is the identity of a tag family which contains the topN calculated result.
- TopNTagFamily = "__topN__"
+ topNTagFamily = "__topN__"
)
var (
@@ -61,8 +59,7 @@ var (
_ io.Closer = (*topNProcessorManager)(nil)
_ flow.Sink = (*topNStreamingProcessor)(nil)
- // TopNValueFieldSpec denotes the field specification of the topN calculated result.
- TopNValueFieldSpec = &databasev1.FieldSpec{
+ topNValueFieldSpec = &databasev1.FieldSpec{
Name: "value",
FieldType: databasev1.FieldType_FIELD_TYPE_INT,
EncodingMethod: databasev1.EncodingMethod_ENCODING_METHOD_GORILLA,
@@ -72,7 +69,7 @@ var (
type dataPointWithEntityValues struct {
*measurev1.DataPointValue
- entityValues tsdb.EntityValues
+ entityValues []*modelv1.TagValue
}
type topNStreamingProcessor struct {
@@ -144,6 +141,8 @@ func (t *topNStreamingProcessor) writeStreamRecord(record flow.StreamRecord) err
eventTime := t.downSampleTimeBucket(record.TimestampMillis())
timeBucket := eventTime.Format(timeBucketFormat)
var err error
+ publisher := t.pipeline.NewBatchPublisher()
+ defer publisher.Close()
for group, tuples := range tuplesGroups {
if e := t.l.Debug(); e.Enabled() {
e.Str("TopN", t.topNSchema.GetMetadata().GetName()).
@@ -154,13 +153,13 @@ func (t *topNStreamingProcessor) writeStreamRecord(record flow.StreamRecord) err
for rankNum, tuple := range tuples {
fieldValue := tuple.V1.(int64)
data := tuple.V2.(flow.StreamRecord).Data().(flow.Data)
- err = multierr.Append(err, t.writeData(eventTime, timeBucket, fieldValue, group, data, rankNum))
+ err = multierr.Append(err, t.writeData(publisher, eventTime, timeBucket, fieldValue, group, data, rankNum))
}
}
return err
}
-func (t *topNStreamingProcessor) writeData(eventTime time.Time, timeBucket string, fieldValue int64,
+func (t *topNStreamingProcessor) writeData(publisher queue.BatchPublisher, eventTime time.Time, timeBucket string, fieldValue int64,
group string, data flow.Data, rankNum int,
) error {
var tagValues []*modelv1.TagValue
@@ -170,7 +169,12 @@ func (t *topNStreamingProcessor) writeData(eventTime time.Time, timeBucket strin
return errors.New("fail to extract tag values from topN result")
}
}
- entity, entityValues, shardID, err := t.locate(tagValues, rankNum)
+ for _, tv := range tagValues {
+ if tv.Value == nil {
+ t.l.Warn().Msg("tag value is nil")
+ }
+ }
+ series, shardID, err := t.locate(tagValues, rankNum)
if err != nil {
return err
}
@@ -197,7 +201,7 @@ func (t *topNStreamingProcessor) writeData(eventTime time.Time, timeBucket strin
},
},
},
- }, data[0].(tsdb.EntityValues)...),
+ }, data[0].([]*modelv1.TagValue)...),
},
},
Fields: []*modelv1.FieldValue{
@@ -211,14 +215,11 @@ func (t *topNStreamingProcessor) writeData(eventTime time.Time, timeBucket strin
},
},
},
- ShardId: uint32(shardID),
- SeriesHash: tsdb.HashEntity(entity),
- }
- if t.l.Debug().Enabled() {
- iwr.EntityValues = entityValues.Encode()
+ EntityValues: series.EntityValues,
+ ShardId: uint32(shardID),
}
- message := bus.NewMessage(bus.MessageID(time.Now().UnixNano()), iwr)
- _, errWritePub := t.pipeline.Publish(apiData.TopicMeasureWrite, message)
+ message := bus.NewBatchMessageWithNode(bus.MessageID(time.Now().UnixNano()), "local", iwr)
+ _, errWritePub := publisher.Publish(apiData.TopicMeasureWrite, message)
return errWritePub
}
@@ -226,34 +227,49 @@ func (t *topNStreamingProcessor) downSampleTimeBucket(eventTimeMillis int64) tim
return time.UnixMilli(eventTimeMillis - eventTimeMillis%t.interval.Milliseconds())
}
-func (t *topNStreamingProcessor) locate(tagValues []*modelv1.TagValue, rankNum int) (tsdb.Entity, tsdb.EntityValues, common.ShardID, error) {
+func (t *topNStreamingProcessor) locate(tagValues []*modelv1.TagValue, rankNum int) (*pbv1.Series, common.ShardID, error) {
if len(tagValues) != 0 && len(t.topNSchema.GetGroupByTagNames()) != len(tagValues) {
- return nil, nil, 0, errors.New("no enough tag values for the entity")
+ return nil, 0, errors.New("no enough tag values for the entity")
}
+ // Subject: topN aggregation Name
+ //
// entity prefix
- // 1) source measure Name + topN aggregation Name
- // 2) sort direction
- // 3) rank number
- // >4) group tag values if needed
- entity := make(tsdb.EntityValues, 1+1+1+len(tagValues))
+ //
+ // 1) sort direction
+ // 2) rank number
+ // >3) group tag values if needed
+ series := &pbv1.Series{
+ Subject: t.topNSchema.GetMetadata().GetName(),
+ EntityValues: make([]*modelv1.TagValue, 1+1+len(tagValues)),
+ }
+
// entity prefix
- entity[0] = tsdb.StrValue(formatMeasureCompanionPrefix(t.topNSchema.GetSourceMeasure().GetName(),
- t.topNSchema.GetMetadata().GetName()))
- entity[1] = tsdb.Int64Value(int64(t.sortDirection.Number()))
- entity[2] = tsdb.Int64Value(int64(rankNum))
+ series.EntityValues[0] = &modelv1.TagValue{
+ Value: &modelv1.TagValue_Int{
+ Int: &modelv1.Int{
+ Value: int64(t.sortDirection),
+ },
+ },
+ }
+ series.EntityValues[1] = &modelv1.TagValue{
+ Value: &modelv1.TagValue_Int{
+ Int: &modelv1.Int{
+ Value: int64(rankNum),
+ },
+ },
+ }
// measureID as sharding key
for idx, tagVal := range tagValues {
- entity[idx+3] = tagVal
+ series.EntityValues[idx+2] = tagVal
}
- e, err := entity.ToEntity()
- if err != nil {
- return nil, nil, 0, err
+ if err := series.Marshal(); err != nil {
+ return nil, 0, fmt.Errorf("fail to marshal series: %w", err)
}
- id, err := partition.ShardID(e.Marshal(), t.m.shardNum)
+ id, err := partition.ShardID(series.Buffer, t.m.shardNum)
if err != nil {
- return nil, nil, 0, err
+ return nil, 0, err
}
- return e, entity, common.ShardID(id), nil
+ return series, common.ShardID(id), nil
}
func (t *topNStreamingProcessor) start() *topNStreamingProcessor {
diff --git a/banyand/measure/tstable.go b/banyand/measure/tstable.go
index 1e0723ad..6ce4adbe 100644
--- a/banyand/measure/tstable.go
+++ b/banyand/measure/tstable.go
@@ -18,200 +18,194 @@
package measure
import (
+ "container/heap"
+ "errors"
"fmt"
"io"
- "path"
- "strings"
"sync"
- "sync/atomic"
- "time"
-
- "github.com/dgraph-io/badger/v3"
- "github.com/dgraph-io/badger/v3/skl"
- "go.uber.org/multierr"
"github.com/apache/skywalking-banyandb/api/common"
- databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
- "github.com/apache/skywalking-banyandb/banyand/kv"
- "github.com/apache/skywalking-banyandb/banyand/tsdb"
- "github.com/apache/skywalking-banyandb/pkg/encoding"
"github.com/apache/skywalking-banyandb/pkg/logger"
- pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
"github.com/apache/skywalking-banyandb/pkg/timestamp"
)
-const (
- defaultNumBufferShards = 2
- defaultWriteConcurrency = 1000
- defaultWriteWal = true
- plain = "tst"
- encoded = "encoded"
-)
-
-var _ tsdb.TSTable = (*tsTable)(nil)
+func newTSTable(_ string, _ common.Position, _ *logger.Logger, _ timestamp.TimeRange) (*tsTable, error) {
+ return &tsTable{}, nil
+}
type tsTable struct {
- encoderSST kv.TimeSeriesStore
- sst kv.TimeSeriesStore
- bufferSupplier *tsdb.BufferSupplier
- l *logger.Logger
- encoderBuffer *tsdb.Buffer
- buffer *tsdb.Buffer
- position common.Position
- id string
- bufferSize int64
- encoderBufferSize int64
- lock sync.RWMutex
-}
-
-func (t *tsTable) SizeOnDisk() int64 {
- return t.encoderSST.SizeOnDisk()
-}
-
-func (t *tsTable) openBuffer() (err error) {
- t.lock.Lock()
- defer t.lock.Unlock()
- if t.encoderBuffer != nil {
- return nil
- }
- bufferSize := int(t.encoderBufferSize / defaultNumBufferShards)
- if t.encoderBuffer, err = t.bufferSupplier.Borrow(encoded, t.id, bufferSize, t.encoderFlush); err != nil {
- return fmt.Errorf("failed to create encoder buffer: %w", err)
- }
- bufferSize = int(t.bufferSize / defaultNumBufferShards)
- if t.buffer, err = t.bufferSupplier.Borrow(plain, t.id, bufferSize, t.flush); err != nil {
- return fmt.Errorf("failed to create buffer: %w", err)
+ memParts []*partWrapper
+ sync.RWMutex
+}
+
+func (tst *tsTable) Close() error {
+ tst.Lock()
+ defer tst.Unlock()
+ for _, p := range tst.memParts {
+ p.decRef()
}
return nil
}
-func (t *tsTable) Close() (err error) {
- t.lock.Lock()
- defer t.lock.Unlock()
- for _, b := range []io.Closer{t.sst, t.encoderSST} {
- if b != nil {
- err = multierr.Append(err, b.Close())
+func (tst *tsTable) mustAddDataPoints(dps *dataPoints) {
+ if len(dps.seriesIDs) == 0 {
+ return
+ }
+
+ mp := generateMemPart()
+ mp.mustInitFromDataPoints(dps)
+ p := openMemPart(mp)
+
+ pw := newMemPartWrapper(mp, p)
+
+ tst.Lock()
+ defer tst.Unlock()
+ tst.memParts = append(tst.memParts, pw)
+}
+
+func (tst *tsTable) getParts(dst []*partWrapper, dstPart []*part, opts queryOptions) ([]*partWrapper, []*part) {
+ tst.RLock()
+ defer tst.RUnlock()
+ for _, p := range tst.memParts {
+ pm := p.mp.partMetadata
+ if opts.maxTimestamp < pm.MinTimestamp || opts.minTimestamp > pm.MaxTimestamp {
+ continue
}
+ p.incRef()
+ dst = append(dst, p)
+ dstPart = append(dstPart, p.p)
}
- for _, b := range []string{encoded, plain} {
- t.bufferSupplier.Return(b, t.id)
+ return dst, dstPart
+}
+
+type tstIter struct {
+ err error
+ parts []*part
+ piPool []partIter
+ piHeap partIterHeap
+ nextBlockNoop bool
+}
+
+func (ti *tstIter) reset() {
+ for i := range ti.parts {
+ ti.parts[i] = nil
+ }
+ ti.parts = ti.parts[:0]
+
+ for i := range ti.piPool {
+ ti.piPool[i].reset()
+ }
+ ti.piPool = ti.piPool[:0]
+
+ for i := range ti.piHeap {
+ ti.piHeap[i] = nil
}
- return err
+ ti.piHeap = ti.piHeap[:0]
+
+ ti.err = nil
+ ti.nextBlockNoop = false
}
-func (t *tsTable) CollectStats() *badger.Statistics {
- mergedMap := &sync.Map{}
- for _, s := range []*badger.Statistics{t.encoderSST.CollectStats(), t.sst.CollectStats()} {
- if s != nil && s.TableBuilderSize != nil {
- s.TableBuilderSize.Range(func(key, value interface{}) bool {
- val := value.(*atomic.Int64)
- if val.Load() > 0 {
- mergedMap.Store(key, val)
- }
- return true
- })
+func (ti *tstIter) init(parts []*part, sids []common.SeriesID, minTimestamp, maxTimestamp int64) {
+ ti.reset()
+ ti.parts = parts
+
+ if n := len(ti.parts) - cap(ti.piPool); n > 0 {
+ ti.piPool = append(ti.piPool[:cap(ti.piPool)], make([]partIter, n)...)
+ }
+ ti.piPool = ti.piPool[:len(ti.parts)]
+ for i, p := range ti.parts {
+ ti.piPool[i].init(p, sids, minTimestamp, maxTimestamp)
+ }
+
+ ti.piHeap = ti.piHeap[:0]
+ for i := range ti.piPool {
+ ps := &ti.piPool[i]
+ if !ps.nextBlock() {
+ if err := ps.error(); err != nil {
+ ti.err = fmt.Errorf("cannot initialize tsTable iteration: %w", err)
+ return
+ }
+ continue
}
+ ti.piHeap = append(ti.piHeap, ps)
}
- return &badger.Statistics{
- TableBuilderSize: mergedMap,
+ if len(ti.piHeap) == 0 {
+ ti.err = io.EOF
+ return
}
+ heap.Init(&ti.piHeap)
+ ti.nextBlockNoop = true
}
-func (t *tsTable) Get(key []byte, ts time.Time) ([]byte, error) {
- if t.toEncode(key) {
- if v, ok := t.encoderBuffer.Read(key, ts); ok {
- return v, nil
- }
- return t.encoderSST.Get(key, uint64(ts.UnixNano()))
+func (ti *tstIter) nextBlock() bool {
+ if ti.err != nil {
+ return false
}
- if v, ok := t.buffer.Read(key, ts); ok {
- return v, nil
+ if ti.nextBlockNoop {
+ ti.nextBlockNoop = false
+ return true
}
- return t.sst.Get(key, uint64(ts.UnixNano()))
+
+ ti.err = ti.next()
+ if ti.err != nil {
+ if errors.Is(ti.err, io.EOF) {
+ ti.err = fmt.Errorf("cannot obtain the next block to search in the partition: %w", ti.err)
+ }
+ return false
+ }
+ return true
}
-func (t *tsTable) Put(key []byte, val []byte, ts time.Time) error {
- t.lock.RLock()
- if t.encoderBuffer != nil {
- defer t.lock.RUnlock()
- return t.writeToBuffer(key, val, ts)
+func (ti *tstIter) next() error {
+ psMin := ti.piHeap[0]
+ if psMin.nextBlock() {
+ heap.Fix(&ti.piHeap, 0)
+ return nil
}
- t.lock.RUnlock()
- if err := t.openBuffer(); err != nil {
+
+ if err := psMin.error(); err != nil {
return err
}
- return t.writeToBuffer(key, val, ts)
-}
-func (t *tsTable) writeToBuffer(key []byte, val []byte, ts time.Time) error {
- if t.toEncode(key) {
- return t.encoderBuffer.Write(key, val, ts)
+ heap.Pop(&ti.piHeap)
+
+ if len(ti.piHeap) == 0 {
+ return io.EOF
}
- return t.buffer.Write(key, val, ts)
+ return nil
}
-func (t *tsTable) encoderFlush(shardIndex int, skl *skl.Skiplist) error {
- t.l.Info().Int("shard", shardIndex).Msg("flushing encoder buffer")
- return t.encoderSST.Handover(skl)
+func (ti *tstIter) Error() error {
+ if errors.Is(ti.err, io.EOF) {
+ return nil
+ }
+ return ti.err
}
-func (t *tsTable) flush(shardIndex int, skl *skl.Skiplist) error {
- t.l.Info().Int("shard", shardIndex).Msg("flushing buffer")
- return t.sst.Handover(skl)
+type partIterHeap []*partIter
+
+func (pih *partIterHeap) Len() int {
+ return len(*pih)
}
-func (t *tsTable) toEncode(key []byte) bool {
- fieldSpec, _, err := pbv1.DecodeFieldFlag(key)
- if err != nil {
- t.l.Err(err).Msg("failed to decode field flag")
- }
- return fieldSpec.EncodingMethod == databasev1.EncodingMethod_ENCODING_METHOD_GORILLA
+func (pih *partIterHeap) Less(i, j int) bool {
+ x := *pih
+ return x[i].curBlock.less(x[j].curBlock)
}
-var _ tsdb.TSTableFactory = (*tsTableFactory)(nil)
+func (pih *partIterHeap) Swap(i, j int) {
+ x := *pih
+ x[i], x[j] = x[j], x[i]
+}
-type tsTableFactory struct {
- encoderPool encoding.SeriesEncoderPool
- decoderPool encoding.SeriesDecoderPool
- bufferSize int64
- encoderBufferSize int64
- plainChunkSize int64
- encodingChunkSize int
- compressionMethod databasev1.CompressionMethod
+func (pih *partIterHeap) Push(x any) {
+ *pih = append(*pih, x.(*partIter))
}
-func (ttf *tsTableFactory) NewTSTable(bufferSupplier *tsdb.BufferSupplier, root string,
- position common.Position, l *logger.Logger, timeRange timestamp.TimeRange,
-) (tsdb.TSTable, error) {
- encoderSST, err := kv.OpenTimeSeriesStore(
- path.Join(root, encoded),
- timeRange,
- kv.TSSWithMemTableSize(ttf.bufferSize),
- kv.TSSWithLogger(l.Named(encoded)),
- kv.TSSWithEncoding(ttf.encoderPool, ttf.decoderPool, ttf.encodingChunkSize),
- )
- if err != nil {
- return nil, fmt.Errorf("failed to create time series table: %w", err)
- }
- sst, err := kv.OpenTimeSeriesStore(
- path.Join(root, plain),
- timeRange,
- kv.TSSWithMemTableSize(ttf.bufferSize),
- kv.TSSWithLogger(l.Named(plain)),
- kv.TSSWithZSTDCompression(int(ttf.plainChunkSize)),
- )
- if err != nil {
- return nil, fmt.Errorf("failed to create time series table: %w", err)
- }
- table := &tsTable{
- bufferSize: ttf.bufferSize,
- encoderBufferSize: ttf.encoderBufferSize,
- l: l,
- position: position,
- encoderSST: encoderSST,
- sst: sst,
- id: strings.Join([]string{position.Segment, position.Block}, "-"),
- bufferSupplier: bufferSupplier,
- }
- return table, nil
+func (pih *partIterHeap) Pop() any {
+ a := *pih
+ v := a[len(a)-1]
+ *pih = a[:len(a)-1]
+ return v
}
diff --git a/banyand/measure/tstable_test.go b/banyand/measure/tstable_test.go
new file mode 100644
index 00000000..fdfb9879
--- /dev/null
+++ b/banyand/measure/tstable_test.go
@@ -0,0 +1,329 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package measure
+
+import (
+ "errors"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "github.com/apache/skywalking-banyandb/api/common"
+ "github.com/apache/skywalking-banyandb/pkg/convert"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
+)
+
+func Test_tsTable_mustAddDataPoints(t *testing.T) {
+ tests := []struct {
+ name string
+ dpsList []*dataPoints
+ want int
+ }{
+ {
+ name: "Test with empty dataPoints",
+ dpsList: []*dataPoints{
+ {
+ timestamps: []int64{},
+ seriesIDs: []common.SeriesID{},
+ tagFamilies: make([][]nameValues, 0),
+ fields: make([]nameValues, 0),
+ },
+ },
+ want: 0,
+ },
+ {
+ name: "Test with one item in dataPoints",
+ dpsList: []*dataPoints{
+ {
+ timestamps: []int64{1},
+ seriesIDs: []common.SeriesID{1},
+ tagFamilies: [][]nameValues{
+ {
+ {
+ name: "arrTag", values: []*nameValue{
+ {name: "strArrTag", valueType: pbv1.ValueTypeStrArr, value: nil, valueArr: [][]byte{[]byte("value1"), []byte("value2")}},
+ {name: "intArrTag", valueType: pbv1.ValueTypeInt64Arr, value: nil, valueArr: [][]byte{convert.Int64ToBytes(25), convert.Int64ToBytes(30)}},
+ },
+ },
+ },
+ },
+ fields: []nameValues{
+ {
+ name: "skipped", values: []*nameValue{
+ {name: "intField", valueType: pbv1.ValueTypeInt64, value: convert.Int64ToBytes(1110), valueArr: nil},
+ },
+ },
+ },
+ },
+ },
+ want: 1,
+ },
+ {
+ name: "Test with multiple calls to mustAddDataPoints",
+ dpsList: []*dataPoints{
+ dpsTS1,
+ dpsTS2,
+ },
+ want: 2,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ tst := &tsTable{}
+ defer tst.Close()
+ for _, dps := range tt.dpsList {
+ tst.mustAddDataPoints(dps)
+ time.Sleep(100 * time.Millisecond)
+ }
+ assert.Equal(t, tt.want, len(tst.memParts))
+ var lastVersion int64
+ for _, pw := range tst.memParts {
+ if lastVersion == 0 {
+ lastVersion = pw.p.partMetadata.Version
+ } else {
+ require.Less(t, lastVersion, pw.p.partMetadata.Version)
+ }
+ }
+ })
+ }
+}
+
+func Test_tstIter(t *testing.T) {
+ tests := []struct {
+ wantErr error
+ name string
+ dpsList []*dataPoints
+ sids []common.SeriesID
+ want []blockMetadata
+ minTimestamp int64
+ maxTimestamp int64
+ }{
+ {
+ name: "Test with no data points",
+ dpsList: []*dataPoints{},
+ sids: []common.SeriesID{1, 2, 3},
+ minTimestamp: 1,
+ maxTimestamp: 1,
+ },
+ {
+ name: "Test with single part",
+ dpsList: []*dataPoints{dpsTS1},
+ sids: []common.SeriesID{1, 2, 3},
+ minTimestamp: 1,
+ maxTimestamp: 1,
+ want: []blockMetadata{
+ {seriesID: 1, count: 1, uncompressedSizeBytes: 1676},
+ {seriesID: 2, count: 1, uncompressedSizeBytes: 55},
+ {seriesID: 3, count: 1, uncompressedSizeBytes: 24},
+ },
+ },
+ {
+ name: "Test with multiple parts with different ts",
+ dpsList: []*dataPoints{dpsTS1, dpsTS2},
+ sids: []common.SeriesID{1, 2, 3},
+ minTimestamp: 1,
+ maxTimestamp: 2,
+ want: []blockMetadata{
+ {seriesID: 1, count: 1, uncompressedSizeBytes: 1676},
+ {seriesID: 1, count: 1, uncompressedSizeBytes: 1676},
+ {seriesID: 2, count: 1, uncompressedSizeBytes: 55},
+ {seriesID: 2, count: 1, uncompressedSizeBytes: 55},
+ {seriesID: 3, count: 1, uncompressedSizeBytes: 24},
+ {seriesID: 3, count: 1, uncompressedSizeBytes: 24},
+ },
+ },
+ {
+ name: "Test with multiple parts with same ts",
+ dpsList: []*dataPoints{dpsTS1, dpsTS1},
+ sids: []common.SeriesID{1, 2, 3},
+ minTimestamp: 1,
+ maxTimestamp: 2,
+ want: []blockMetadata{
+ {seriesID: 1, count: 1, uncompressedSizeBytes: 1676},
+ {seriesID: 1, count: 1, uncompressedSizeBytes: 1676},
+ {seriesID: 2, count: 1, uncompressedSizeBytes: 55},
+ {seriesID: 2, count: 1, uncompressedSizeBytes: 55},
+ {seriesID: 3, count: 1, uncompressedSizeBytes: 24},
+ {seriesID: 3, count: 1, uncompressedSizeBytes: 24},
+ },
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ tst := &tsTable{}
+ defer tst.Close()
+ for _, dps := range tt.dpsList {
+ tst.mustAddDataPoints(dps)
+ time.Sleep(100 * time.Millisecond)
+ }
+ pws, pp := tst.getParts(nil, nil, queryOptions{
+ minTimestamp: tt.minTimestamp,
+ maxTimestamp: tt.maxTimestamp,
+ })
+ defer func() {
+ for _, pw := range pws {
+ pw.decRef()
+ }
+ }()
+ ti := &tstIter{}
+ ti.init(pp, tt.sids, tt.minTimestamp, tt.maxTimestamp)
+ var got []blockMetadata
+ for ti.nextBlock() {
+ if ti.piHeap[0].curBlock.seriesID == 0 {
+ t.Errorf("Expected curBlock to be initialized, but it was nil")
+ }
+ got = append(got, ti.piHeap[0].curBlock)
+ }
+
+ if !errors.Is(ti.Error(), tt.wantErr) {
+ t.Errorf("Unexpected error: got %v, want %v", ti.err, tt.wantErr)
+ }
+
+ if diff := cmp.Diff(got, tt.want,
+ cmpopts.IgnoreFields(blockMetadata{}, "timestamps"),
+ cmpopts.IgnoreFields(blockMetadata{}, "field"),
+ cmpopts.IgnoreFields(blockMetadata{}, "tagFamilies"),
+ cmp.AllowUnexported(blockMetadata{}),
+ ); diff != "" {
+ t.Errorf("Unexpected blockMetadata (-got +want):\n%s", diff)
+ }
+ })
+ }
+}
+
+var tagProjections = map[int][]pbv1.TagProjection{
+ 1: {
+ {Family: "arrTag", Names: []string{"strArrTag", "intArrTag"}},
+ {Family: "binaryTag", Names: []string{"binaryTag"}},
+ {Family: "singleTag", Names: []string{"strTag", "intTag"}},
+ },
+ 2: {
+ {Family: "singleTag", Names: []string{"strTag1", "strTag2"}},
+ },
+}
+
+var fieldProjections = map[int][]string{
+ 1: {"strField", "intField", "floatField", "binaryField"},
+ 3: {"intField"},
+}
+
+var dpsTS1 = &dataPoints{
+ seriesIDs: []common.SeriesID{1, 2, 3},
+ timestamps: []int64{1, 1, 1},
+ tagFamilies: [][]nameValues{
+ {
+ {
+ name: "arrTag", values: []*nameValue{
+ {name: "strArrTag", valueType: pbv1.ValueTypeStrArr, value: nil, valueArr: [][]byte{[]byte("value1"), []byte("value2")}},
+ {name: "intArrTag", valueType: pbv1.ValueTypeInt64Arr, value: nil, valueArr: [][]byte{convert.Int64ToBytes(25), convert.Int64ToBytes(30)}},
+ },
+ },
+ {
+ name: "binaryTag", values: []*nameValue{
+ {name: "binaryTag", valueType: pbv1.ValueTypeBinaryData, value: longText, valueArr: nil},
+ },
+ },
+ {
+ name: "singleTag", values: []*nameValue{
+ {name: "strTag", valueType: pbv1.ValueTypeStr, value: []byte("value1"), valueArr: nil},
+ {name: "intTag", valueType: pbv1.ValueTypeInt64, value: convert.Int64ToBytes(10), valueArr: nil},
+ },
+ },
+ },
+ {
+ {
+ name: "singleTag", values: []*nameValue{
+ {name: "strTag1", valueType: pbv1.ValueTypeStr, value: []byte("tag1"), valueArr: nil},
+ {name: "strTag2", valueType: pbv1.ValueTypeStr, value: []byte("tag2"), valueArr: nil},
+ },
+ },
+ },
+ {}, // empty tagFamilies for seriesID 3
+ },
+ fields: []nameValues{
+ {
+ name: "skipped", values: []*nameValue{
+ {name: "strField", valueType: pbv1.ValueTypeStr, value: []byte("field1"), valueArr: nil},
+ {name: "intField", valueType: pbv1.ValueTypeInt64, value: convert.Int64ToBytes(1110), valueArr: nil},
+ {name: "floatField", valueType: pbv1.ValueTypeFloat64, value: convert.Float64ToBytes(1221233.343), valueArr: nil},
+ {name: "binaryField", valueType: pbv1.ValueTypeBinaryData, value: longText, valueArr: nil},
+ },
+ },
+ {}, // empty fields for seriesID 2
+ {
+ name: "onlyFields", values: []*nameValue{
+ {name: "intField", valueType: pbv1.ValueTypeInt64, value: convert.Int64ToBytes(1110), valueArr: nil},
+ },
+ },
+ },
+}
+
+var dpsTS2 = &dataPoints{
+ seriesIDs: []common.SeriesID{1, 2, 3},
+ timestamps: []int64{2, 2, 2},
+ tagFamilies: [][]nameValues{
+ {
+ {
+ name: "arrTag", values: []*nameValue{
+ {name: "strArrTag", valueType: pbv1.ValueTypeStrArr, value: nil, valueArr: [][]byte{[]byte("value5"), []byte("value6")}},
+ {name: "intArrTag", valueType: pbv1.ValueTypeInt64Arr, value: nil, valueArr: [][]byte{convert.Int64ToBytes(35), convert.Int64ToBytes(40)}},
+ },
+ },
+ {
+ name: "binaryTag", values: []*nameValue{
+ {name: "binaryTag", valueType: pbv1.ValueTypeBinaryData, value: longText, valueArr: nil},
+ },
+ },
+ {
+ name: "singleTag", values: []*nameValue{
+ {name: "strTag", valueType: pbv1.ValueTypeStr, value: []byte("value3"), valueArr: nil},
+ {name: "intTag", valueType: pbv1.ValueTypeInt64, value: convert.Int64ToBytes(30), valueArr: nil},
+ },
+ },
+ },
+ {
+ {
+ name: "singleTag", values: []*nameValue{
+ {name: "strTag1", valueType: pbv1.ValueTypeStr, value: []byte("tag3"), valueArr: nil},
+ {name: "strTag2", valueType: pbv1.ValueTypeStr, value: []byte("tag4"), valueArr: nil},
+ },
+ },
+ },
+ {}, // empty tagFamilies for seriesID 6
+ },
+ fields: []nameValues{
+ {
+ name: "skipped", values: []*nameValue{
+ {name: "strField", valueType: pbv1.ValueTypeStr, value: []byte("field3"), valueArr: nil},
+ {name: "intField", valueType: pbv1.ValueTypeInt64, value: convert.Int64ToBytes(3330), valueArr: nil},
+ {name: "floatField", valueType: pbv1.ValueTypeFloat64, value: convert.Float64ToBytes(3663699.029), valueArr: nil},
+ {name: "binaryField", valueType: pbv1.ValueTypeBinaryData, value: longText, valueArr: nil},
+ },
+ },
+ {}, // empty fields for seriesID 5
+ {
+ name: "onlyFields", values: []*nameValue{
+ {name: "intField", valueType: pbv1.ValueTypeInt64, value: convert.Int64ToBytes(4440), valueArr: nil},
+ },
+ },
+ },
+}
diff --git a/banyand/measure/write.go b/banyand/measure/write.go
new file mode 100644
index 00000000..4ba62d6b
--- /dev/null
+++ b/banyand/measure/write.go
@@ -0,0 +1,331 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package measure
+
+import (
+ "bytes"
+ "fmt"
+ "time"
+
+ "google.golang.org/protobuf/types/known/anypb"
+
+ "github.com/apache/skywalking-banyandb/api/common"
+ databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ measurev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/measure/v1"
+ modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+ "github.com/apache/skywalking-banyandb/pkg/bus"
+ "github.com/apache/skywalking-banyandb/pkg/convert"
+ "github.com/apache/skywalking-banyandb/pkg/index"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+ "github.com/apache/skywalking-banyandb/pkg/partition"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
+ "github.com/apache/skywalking-banyandb/pkg/timestamp"
+)
+
+type writeCallback struct {
+ l *logger.Logger
+ schemaRepo *schemaRepo
+}
+
+func setUpWriteCallback(l *logger.Logger, schemaRepo *schemaRepo) bus.MessageListener {
+ return &writeCallback{
+ l: l,
+ schemaRepo: schemaRepo,
+ }
+}
+
+func (w *writeCallback) handle(dst map[string]*dataPointsInGroup, writeEvent *measurev1.InternalWriteRequest) (map[string]*dataPointsInGroup, error) {
+ req := writeEvent.Request
+ t := req.DataPoint.Timestamp.AsTime().Local()
+ if err := timestamp.Check(t); err != nil {
+ return nil, fmt.Errorf("invalid timestamp: %w", err)
+ }
+ ts := uint64(t.UnixNano())
+
+ gn := req.Metadata.Group
+ tsdb, err := w.schemaRepo.loadTSDB(gn)
+ if err != nil {
+ return nil, fmt.Errorf("cannot load tsdb for group %s: %w", gn, err)
+ }
+ dpg, ok := dst[gn]
+ if !ok {
+ dpg = &dataPointsInGroup{
+ tsdb: tsdb,
+ tables: make([]*dataPointsInTable, 0),
+ }
+ dst[gn] = dpg
+ }
+
+ var dpt *dataPointsInTable
+ for i := range dpg.tables {
+ if dpg.tables[i].timeRange.Contains(ts) {
+ dpt = dpg.tables[i]
+ break
+ }
+ }
+ shardID := common.ShardID(writeEvent.ShardId)
+ if dpt == nil {
+ tstb, err := tsdb.CreateTSTableIfNotExist(shardID, t)
+ if err != nil {
+ return nil, fmt.Errorf("cannot create ts table: %w", err)
+ }
+ dpt = &dataPointsInTable{
+ timeRange: tstb.GetTimeRange(),
+ tsTable: tstb,
+ }
+ dpg.tables = append(dpg.tables, dpt)
+ }
+ dpt.dataPoints.timestamps = append(dpt.dataPoints.timestamps, int64(ts))
+ stm, ok := w.schemaRepo.loadMeasure(writeEvent.GetRequest().GetMetadata())
+ if !ok {
+ return nil, fmt.Errorf("cannot find measure definition: %s", writeEvent.GetRequest().GetMetadata())
+ }
+ fLen := len(req.DataPoint.GetTagFamilies())
+ if fLen < 1 {
+ return nil, fmt.Errorf("%s has no tag family", req.Metadata)
+ }
+ if fLen > len(stm.schema.GetTagFamilies()) {
+ return nil, fmt.Errorf("%s has more tag families than expected", req.Metadata)
+ }
+ series := &pbv1.Series{
+ Subject: req.Metadata.Name,
+ EntityValues: writeEvent.EntityValues,
+ }
+ if err := series.Marshal(); err != nil {
+ return nil, fmt.Errorf("cannot marshal series: %w", err)
+ }
+ dpt.dataPoints.seriesIDs = append(dpt.dataPoints.seriesIDs, series.ID)
+ field := nameValues{}
+ for i := range stm.GetSchema().GetFields() {
+ var v *modelv1.FieldValue
+ if len(req.DataPoint.Fields) <= i {
+ v = pbv1.NullFieldValue
+ } else {
+ v = req.DataPoint.Fields[i]
+ }
+ field.values = append(field.values, encodeFieldValue(
+ stm.GetSchema().GetFields()[i].GetName(),
+ stm.GetSchema().GetFields()[i].FieldType,
+ v,
+ ))
+ }
+ dpt.dataPoints.fields = append(dpt.dataPoints.fields, field)
+ tagFamilies := make([]nameValues, len(stm.schema.TagFamilies))
+ dpt.dataPoints.tagFamilies = append(dpt.dataPoints.tagFamilies, tagFamilies)
+ for i := range stm.GetSchema().GetTagFamilies() {
+ var tagFamily *modelv1.TagFamilyForWrite
+ if len(req.DataPoint.TagFamilies) <= i {
+ tagFamily = pbv1.NullTagFamily
+ } else {
+ tagFamily = req.DataPoint.TagFamilies[i]
+ }
+ tagFamilySpec := stm.GetSchema().GetTagFamilies()[i]
+ tagFamilies[i].name = tagFamilySpec.Name
+ for j := range tagFamilySpec.Tags {
+ var tagValue *modelv1.TagValue
+ if tagFamily == pbv1.NullTagFamily || len(tagFamily.Tags) <= j {
+ tagValue = pbv1.NullTagValue
+ } else {
+ tagValue = tagFamily.Tags[j]
+ }
+ tagFamilies[i].values = append(tagFamilies[i].values, encodeTagValue(
+ tagFamilySpec.Tags[j].Name,
+ tagFamilySpec.Tags[j].Type,
+ tagValue,
+ ))
+ }
+ }
+
+ if stm.processorManager != nil {
+ stm.processorManager.onMeasureWrite(&measurev1.InternalWriteRequest{
+ Request: &measurev1.WriteRequest{
+ Metadata: stm.GetSchema().Metadata,
+ DataPoint: req.DataPoint,
+ MessageId: uint64(time.Now().UnixNano()),
+ },
+ EntityValues: writeEvent.EntityValues,
+ })
+ }
+ var fields []index.Field
+ for _, ruleIndex := range stm.indexRuleLocators {
+ nv := getIndexValue(ruleIndex, tagFamilies)
+ if nv == nil {
+ continue
+ }
+ if nv.value != nil {
+ fields = append(fields, index.Field{
+ Key: index.FieldKey{
+ IndexRuleID: ruleIndex.Rule.GetMetadata().GetId(),
+ Analyzer: ruleIndex.Rule.Analyzer,
+ },
+ Term: nv.value,
+ })
+ continue
+ }
+ for _, val := range nv.valueArr {
+ rule := ruleIndex.Rule
+ fields = append(fields, index.Field{
+ Key: index.FieldKey{
+ IndexRuleID: rule.GetMetadata().GetId(),
+ Analyzer: rule.Analyzer,
+ },
+ Term: val,
+ })
+ }
+ }
+
+ dpg.docs = append(dpg.docs, index.Document{
+ DocID: uint64(series.ID),
+ EntityValues: series.Buffer,
+ Fields: fields,
+ })
+ return dst, nil
+}
+
+func (w *writeCallback) Rev(message bus.Message) (resp bus.Message) {
+ events, ok := message.Data().([]any)
+ if !ok {
+ w.l.Warn().Msg("invalid event data type")
+ return
+ }
+ if len(events) < 1 {
+ w.l.Warn().Msg("empty event")
+ return
+ }
+ groups := make(map[string]*dataPointsInGroup)
+ for i := range events {
+ var writeEvent *measurev1.InternalWriteRequest
+ switch e := events[i].(type) {
+ case *measurev1.InternalWriteRequest:
+ writeEvent = e
+ case *anypb.Any:
+ writeEvent = &measurev1.InternalWriteRequest{}
+ if err := e.UnmarshalTo(writeEvent); err != nil {
+ w.l.Error().Err(err).RawJSON("written", logger.Proto(e)).Msg("fail to unmarshal event")
+ continue
+ }
+ default:
+ w.l.Warn().Msg("invalid event data type")
+ continue
+ }
+ var err error
+ if groups, err = w.handle(groups, writeEvent); err != nil {
+ w.l.Error().Err(err).Msg("cannot handle write event")
+ continue
+ }
+ }
+ for i := range groups {
+ g := groups[i]
+ for j := range g.tables {
+ dps := g.tables[j]
+ dps.tsTable.Table().mustAddDataPoints(&dps.dataPoints)
+ dps.tsTable.DecRef()
+ }
+ if err := g.tsdb.IndexDB().Write(g.docs); err != nil {
+ w.l.Error().Err(err).Msg("cannot write index")
+ }
+ }
+ return
+}
+
+func encodeFieldValue(name string, fieldType databasev1.FieldType, fieldValue *modelv1.FieldValue) *nameValue {
+ nv := &nameValue{name: name}
+ switch fieldType {
+ case databasev1.FieldType_FIELD_TYPE_INT:
+ nv.valueType = pbv1.ValueTypeInt64
+ if fieldValue.GetInt() != nil {
+ nv.value = convert.Int64ToBytes(fieldValue.GetInt().GetValue())
+ }
+ case databasev1.FieldType_FIELD_TYPE_FLOAT:
+ nv.valueType = pbv1.ValueTypeFloat64
+ if fieldValue.GetFloat() != nil {
+ nv.value = convert.Float64ToBytes(fieldValue.GetFloat().GetValue())
+ }
+ case databasev1.FieldType_FIELD_TYPE_STRING:
+ nv.valueType = pbv1.ValueTypeStr
+ if fieldValue.GetStr() != nil {
+ nv.value = []byte(fieldValue.GetStr().GetValue())
+ }
+ case databasev1.FieldType_FIELD_TYPE_DATA_BINARY:
+ nv.valueType = pbv1.ValueTypeBinaryData
+ if fieldValue.GetBinaryData() != nil {
+ nv.value = bytes.Clone(fieldValue.GetBinaryData())
+ }
+ default:
+ logger.Panicf("unsupported field value type: %T", fieldValue.GetValue())
+ }
+ return nv
+}
+
+func encodeTagValue(name string, tagType databasev1.TagType, tagValue *modelv1.TagValue) *nameValue {
+ nv := &nameValue{name: name}
+ switch tagType {
+ case databasev1.TagType_TAG_TYPE_INT:
+ nv.valueType = pbv1.ValueTypeInt64
+ if tagValue.GetInt() != nil {
+ nv.value = convert.Int64ToBytes(tagValue.GetInt().GetValue())
+ }
+ case databasev1.TagType_TAG_TYPE_STRING:
+ nv.valueType = pbv1.ValueTypeStr
+ if tagValue.GetStr() != nil {
+ nv.value = []byte(tagValue.GetStr().GetValue())
+ }
+ case databasev1.TagType_TAG_TYPE_DATA_BINARY:
+ nv.valueType = pbv1.ValueTypeBinaryData
+ if tagValue.GetBinaryData() != nil {
+ nv.value = bytes.Clone(tagValue.GetBinaryData())
+ }
+ case databasev1.TagType_TAG_TYPE_INT_ARRAY:
+ nv.valueType = pbv1.ValueTypeInt64Arr
+ if tagValue.GetIntArray() == nil {
+ return nv
+ }
+ nv.valueArr = make([][]byte, len(tagValue.GetIntArray().Value))
+ for i := range tagValue.GetIntArray().Value {
+ nv.valueArr[i] = convert.Int64ToBytes(tagValue.GetIntArray().Value[i])
+ }
+ case databasev1.TagType_TAG_TYPE_STRING_ARRAY:
+ nv.valueType = pbv1.ValueTypeStrArr
+ if tagValue.GetStrArray() == nil {
+ return nv
+ }
+ nv.valueArr = make([][]byte, len(tagValue.GetStrArray().Value))
+ for i := range tagValue.GetStrArray().Value {
+ nv.valueArr[i] = []byte(tagValue.GetStrArray().Value[i])
+ }
+ default:
+ logger.Panicf("unsupported tag value type: %T", tagValue.GetValue())
+ }
+ return nv
+}
+
+func getIndexValue(ruleIndex *partition.IndexRuleLocator, tagFamilies []nameValues) *nameValue {
+ if len(ruleIndex.TagIndices) != 1 {
+ logger.Panicf("the index rule %s(%v) didn't support composited tags",
+ ruleIndex.Rule.Metadata.Name, ruleIndex.Rule.Tags)
+ }
+ tIndex := ruleIndex.TagIndices[0]
+ if tIndex.FamilyOffset >= len(tagFamilies) {
+ return nil
+ }
+ tf := tagFamilies[tIndex.FamilyOffset]
+ if tIndex.TagOffset >= len(tf.values) {
+ return nil
+ }
+ return tf.values[tIndex.TagOffset]
+}
diff --git a/banyand/queue/local.go b/banyand/queue/local.go
index a6872881..9609ac24 100644
--- a/banyand/queue/local.go
+++ b/banyand/queue/local.go
@@ -89,13 +89,31 @@ func (*local) Register(schema.EventHandler) {
}
type localBatchPublisher struct {
- local *bus.Bus
+ local *bus.Bus
+ topic *bus.Topic
+ messages []any
}
-func (l *localBatchPublisher) Publish(topic bus.Topic, message ...bus.Message) (bus.Future, error) {
- return l.local.Publish(topic, message...)
+func (l *localBatchPublisher) Publish(topic bus.Topic, messages ...bus.Message) (bus.Future, error) {
+ if l.topic == nil {
+ l.topic = &topic
+ }
+ for i := range messages {
+ l.messages = append(l.messages, messages[i].Data())
+ }
+ return nil, nil
}
func (l *localBatchPublisher) Close() error {
+ if l.local == nil || len(l.messages) == 0 {
+ return nil
+ }
+ newMessage := bus.NewMessage(1, l.messages)
+ _, err := l.local.Publish(*l.topic, newMessage)
+ if err != nil {
+ return err
+ }
+ l.messages = nil
+ l.topic = nil
return nil
}
diff --git a/banyand/queue/pub/pub.go b/banyand/queue/pub/pub.go
index 51d56967..73cf9927 100644
--- a/banyand/queue/pub/pub.go
+++ b/banyand/queue/pub/pub.go
@@ -229,6 +229,7 @@ func messageToRequest(topic bus.Topic, m bus.Message) (*clusterv1.SendRequest, e
r := &clusterv1.SendRequest{
Topic: topic.String(),
MessageId: uint64(m.ID()),
+ BatchMod: m.BatchModeEnabled(),
}
message, ok := m.Data().(proto.Message)
if !ok {
diff --git a/banyand/queue/sub/sub.go b/banyand/queue/sub/sub.go
index 021ec877..9846c4db 100644
--- a/banyand/queue/sub/sub.go
+++ b/banyand/queue/sub/sub.go
@@ -44,6 +44,8 @@ func (s *server) Send(stream clusterv1.Service_SendServer) error {
}
ctx := stream.Context()
var topic *bus.Topic
+ var m bus.Message
+ var dataCollection []any
for {
select {
case <-ctx.Done():
@@ -52,6 +54,15 @@ func (s *server) Send(stream clusterv1.Service_SendServer) error {
}
writeEntity, err := stream.Recv()
if errors.Is(err, io.EOF) {
+ if len(dataCollection) < 1 {
+ return nil
+ }
+ listener := s.getListeners(*topic)
+ if listener == nil {
+ reply(writeEntity, err, "no listener found")
+ return nil
+ }
+ _ = listener.Rev(bus.NewMessage(bus.MessageID(0), dataCollection))
return nil
}
if err != nil {
@@ -73,24 +84,34 @@ func (s *server) Send(stream clusterv1.Service_SendServer) error {
reply(writeEntity, err, "topic is empty")
continue
}
- listener := s.getListeners(*topic)
- if listener == nil {
- reply(writeEntity, err, "no listener found")
- continue
- }
- var m bus.Message
+
if reqSupplier, ok := data.TopicRequestMap[*topic]; ok {
req := reqSupplier()
if errUnmarshal := writeEntity.Body.UnmarshalTo(req); errUnmarshal != nil {
reply(writeEntity, errUnmarshal, "failed to unmarshal message")
continue
}
- m = listener.Rev(bus.NewMessage(bus.MessageID(writeEntity.MessageId), req))
+ m = bus.NewMessage(bus.MessageID(writeEntity.MessageId), req)
} else {
reply(writeEntity, err, "unknown topic")
continue
}
+ if writeEntity.BatchMod {
+ dataCollection = append(dataCollection, writeEntity.Body)
+ if errSend := stream.Send(&clusterv1.SendResponse{
+ MessageId: writeEntity.MessageId,
+ }); errSend != nil {
+ s.log.Error().Stringer("written", writeEntity).Err(errSend).Msg("failed to send response")
+ }
+ continue
+ }
+ listener := s.getListeners(*topic)
+ if listener == nil {
+ reply(writeEntity, err, "no listener found")
+ continue
+ }
+ m = listener.Rev(m)
if m.Data() == nil {
if errSend := stream.Send(&clusterv1.SendResponse{
MessageId: writeEntity.MessageId,
diff --git a/banyand/stream/metadata.go b/banyand/stream/metadata.go
index 18d5c3a1..e1273539 100644
--- a/banyand/stream/metadata.go
+++ b/banyand/stream/metadata.go
@@ -223,21 +223,29 @@ func newSupplier(path string, metadata metadata.Repo, bufferSize int64, dbOpts t
}
}
-func (s *supplier) OpenResource(shardNum uint32, db tsdb.Supplier, resource resourceSchema.Resource) (io.Closer, error) {
+func (s *supplier) OpenResource(shardNum uint32, db resourceSchema.Supplier, resource resourceSchema.Resource) (io.Closer, error) {
streamSchema := resource.Schema().(*databasev1.Stream)
- return openStream(shardNum, db, streamSpec{
+ return openStream(shardNum, &tsdbSupplier{db: db.SupplyTSDB().(tsdb.Database)}, streamSpec{
schema: streamSchema,
indexRules: resource.IndexRules(),
}, s.l), nil
}
+type tsdbSupplier struct {
+ db tsdb.Database
+}
+
+func (s *tsdbSupplier) SupplyTSDB() tsdb.Database {
+ return s.db
+}
+
func (s *supplier) ResourceSchema(md *commonv1.Metadata) (resourceSchema.ResourceSchema, error) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
return s.metadata.StreamRegistry().GetStream(ctx, md)
}
-func (s *supplier) OpenDB(groupSchema *commonv1.Group) (tsdb.Database, error) {
+func (s *supplier) OpenDB(groupSchema *commonv1.Group) (io.Closer, error) {
name := groupSchema.Metadata.Name
opts := s.dbOpts
opts.ShardNum = groupSchema.ResourceOpts.ShardNum
@@ -271,11 +279,11 @@ type portableSupplier struct {
l *logger.Logger
}
-func (*portableSupplier) OpenDB(_ *commonv1.Group) (tsdb.Database, error) {
+func (*portableSupplier) OpenDB(_ *commonv1.Group) (io.Closer, error) {
panic("do not support open db")
}
-func (s *portableSupplier) OpenResource(shardNum uint32, _ tsdb.Supplier, resource resourceSchema.Resource) (io.Closer, error) {
+func (s *portableSupplier) OpenResource(shardNum uint32, _ resourceSchema.Supplier, resource resourceSchema.Resource) (io.Closer, error) {
streamSchema := resource.Schema().(*databasev1.Stream)
return openStream(shardNum, nil, streamSpec{
schema: streamSchema,
diff --git a/banyand/stream/stream_write.go b/banyand/stream/stream_write.go
index 8c47dc7e..eb81a3d9 100644
--- a/banyand/stream/stream_write.go
+++ b/banyand/stream/stream_write.go
@@ -24,6 +24,7 @@ import (
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
+ "google.golang.org/protobuf/types/known/anypb"
"github.com/apache/skywalking-banyandb/api/common"
streamv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/stream/v1"
@@ -147,20 +148,41 @@ func setUpWriteCallback(l *logger.Logger, schemaRepo *schemaRepo) *writeCallback
}
func (w *writeCallback) Rev(message bus.Message) (resp bus.Message) {
- writeEvent, ok := message.Data().(*streamv1.InternalWriteRequest)
+ events, ok := message.Data().([]any)
if !ok {
w.l.Warn().Msg("invalid event data type")
return
}
- stm, ok := w.schemaRepo.loadStream(writeEvent.GetRequest().GetMetadata())
- if !ok {
- w.l.Warn().Msg("cannot find stream definition")
+ if len(events) < 1 {
+ w.l.Warn().Msg("empty event")
return
}
- err := stm.write(common.ShardID(writeEvent.GetShardId()), writeEvent.SeriesHash,
- tsdb.DecodeEntityValues(writeEvent.GetEntityValues()), writeEvent.GetRequest().GetElement())
- if err != nil {
- w.l.Error().Err(err).RawJSON("written", logger.Proto(writeEvent)).Msg("fail to write entity")
+ for _, event := range events {
+ var writeEvent *streamv1.InternalWriteRequest
+ switch e := event.(type) {
+ case *streamv1.InternalWriteRequest:
+ writeEvent = e
+ case *anypb.Any:
+ writeEvent = &streamv1.InternalWriteRequest{}
+ if err := e.UnmarshalTo(writeEvent); err != nil {
+ w.l.Error().Err(err).RawJSON("written", logger.Proto(e)).Msg("fail to unmarshal event")
+ continue
+ }
+ default:
+ w.l.Warn().Msg("invalid event data type")
+ continue
+ }
+ stm, ok := w.schemaRepo.loadStream(writeEvent.GetRequest().GetMetadata())
+ if !ok {
+ w.l.Warn().Msg("cannot find stream definition")
+ continue
+ }
+ err := stm.write(common.ShardID(writeEvent.GetShardId()), writeEvent.SeriesHash,
+ tsdb.DecodeEntityValues(writeEvent.GetEntityValues()), writeEvent.GetRequest().GetElement())
+ if err != nil {
+ w.l.Error().Err(err).RawJSON("written", logger.Proto(writeEvent)).Msg("fail to write entity")
+ }
}
+
return
}
diff --git a/docs/api-reference.md b/docs/api-reference.md
index 24550676..0d7875e7 100644
--- a/docs/api-reference.md
+++ b/docs/api-reference.md
@@ -236,6 +236,7 @@
| topic | [string](#string) | | |
| message_id | [uint64](#uint64) | | |
| body | [google.protobuf.Any](#google-protobuf-Any) | | |
+| batch_mod | [bool](#bool) | | |
diff --git a/go.mod b/go.mod
index 59edbba8..16395f0d 100644
--- a/go.mod
+++ b/go.mod
@@ -88,7 +88,7 @@ require (
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jonboulle/clockwork v0.4.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
- github.com/klauspost/compress v1.17.2 // indirect
+ github.com/klauspost/compress v1.17.2
github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed // indirect
github.com/magiconair/properties v1.8.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
diff --git a/pkg/bus/bus.go b/pkg/bus/bus.go
index 5df38130..26c5be17 100644
--- a/pkg/bus/bus.go
+++ b/pkg/bus/bus.go
@@ -44,9 +44,10 @@ type (
// Message is send on the bus to all subscribed listeners.
type Message struct {
- payload payload
- node string
- id MessageID
+ payload payload
+ node string
+ id MessageID
+ batchMode bool
}
// ID outputs the MessageID of the Message.
@@ -64,11 +65,21 @@ func (m Message) Node() string {
return m.node
}
+// BatchModeEnabled returns whether the Message is sent in batch mode.
+func (m Message) BatchModeEnabled() bool {
+ return m.batchMode
+}
+
// NewMessage returns a new Message with a MessageID and embed data.
func NewMessage(id MessageID, data interface{}) Message {
return Message{id: id, node: "local", payload: data}
}
+// NewBatchMessageWithNode returns a new Message with a MessageID and NodeID and embed data.
+func NewBatchMessageWithNode(id MessageID, node string, data interface{}) Message {
+ return Message{id: id, node: node, payload: data, batchMode: true}
+}
+
// NewMessageWithNode returns a new Message with a MessageID and NodeID and embed data.
func NewMessageWithNode(id MessageID, node string, data interface{}) Message {
return Message{id: id, node: node, payload: data}
diff --git a/pkg/bytes/buffer.go b/pkg/bytes/buffer.go
new file mode 100644
index 00000000..713632f3
--- /dev/null
+++ b/pkg/bytes/buffer.go
@@ -0,0 +1,88 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Package bytes provides utilities for operating bytes.
+package bytes
+
+import (
+ "fmt"
+ "io"
+ "sync"
+
+ "github.com/apache/skywalking-banyandb/pkg/fs"
+)
+
+var (
+ _ fs.Writer = (*Buffer)(nil)
+ _ fs.Reader = (*Buffer)(nil)
+)
+
+// Buffer is a in-memory buffer.
+type Buffer struct {
+ Buf []byte
+}
+
+// Close implements fs.Writer.
+func (*Buffer) Close() error {
+ return nil
+}
+
+// Path implements fs.Writer.
+func (b *Buffer) Path() string {
+ return fmt.Sprintf("mem/%p", b)
+}
+
+// Write implements fs.Writer.
+func (b *Buffer) Write(bb []byte) (int, error) {
+ b.Buf = append(b.Buf, bb...)
+ return len(bb), nil
+}
+
+// Read implements fs.Reader.
+func (b *Buffer) Read(offset int64, buffer []byte) (int, error) {
+ var err error
+ n := copy(buffer, b.Buf[offset:])
+ if n < len(buffer) {
+ err = io.EOF
+ }
+ return n, err
+}
+
+// Reset resets the buffer.
+func (b *Buffer) Reset() {
+ b.Buf = b.Buf[:0]
+}
+
+// BufferPool is a pool of Buffer.
+type BufferPool struct {
+ p sync.Pool
+}
+
+// Generate generates a Buffer.
+func (bp *BufferPool) Generate() *Buffer {
+ bbv := bp.p.Get()
+ if bbv == nil {
+ return &Buffer{}
+ }
+ return bbv.(*Buffer)
+}
+
+// Release releases a Buffer.
+func (bp *BufferPool) Release(b *Buffer) {
+ b.Reset()
+ bp.p.Put(b)
+}
diff --git a/api/proto/banyandb/cluster/v1/rpc.proto b/pkg/bytes/resize.go
similarity index 58%
copy from api/proto/banyandb/cluster/v1/rpc.proto
copy to pkg/bytes/resize.go
index 732bbe1f..69595b6d 100644
--- a/api/proto/banyandb/cluster/v1/rpc.proto
+++ b/pkg/bytes/resize.go
@@ -6,7 +6,7 @@
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
-// http://www.apache.org/licenses/LICENSE-2.0
+// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
@@ -15,26 +15,29 @@
// specific language governing permissions and limitations
// under the License.
-syntax = "proto3";
+package bytes
-package banyandb.cluster.v1;
+import "math/bits"
-import "google/protobuf/any.proto";
-
-option go_package = "github.com/apache/skywalking-banyandb/api/proto/banyandb/cluster/v1";
-
-message SendRequest {
- string topic = 1;
- uint64 message_id = 2;
- google.protobuf.Any body = 3;
+// ResizeOver resizes the byte slice to the nearest power of 2.
+func ResizeOver(b []byte, n int) []byte {
+ if n <= cap(b) {
+ return b[:n]
+ }
+ nNew := roundToNearestPow2(n)
+ bNew := make([]byte, nNew)
+ return bNew[:n]
}
-message SendResponse {
- uint64 message_id = 1;
- string error = 2;
- google.protobuf.Any body = 3;
+// ResizeExact resizes the byte slice to the exact size.
+func ResizeExact(b []byte, n int) []byte {
+ if n <= cap(b) {
+ return b[:n]
+ }
+ return make([]byte, n)
}
-service Service {
- rpc Send(stream SendRequest) returns (stream SendResponse);
+func roundToNearestPow2(n int) int {
+ pow2 := uint8(bits.Len(uint(n - 1)))
+ return 1 << pow2
}
diff --git a/pkg/compress/zstd/zstd.go b/pkg/compress/zstd/zstd.go
new file mode 100644
index 00000000..9eaab14e
--- /dev/null
+++ b/pkg/compress/zstd/zstd.go
@@ -0,0 +1,92 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Package zstd provides ZSTD compression and decompression.
+package zstd
+
+import (
+ "sync"
+ "sync/atomic"
+
+ "github.com/klauspost/compress/zstd"
+
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+)
+
+var (
+ decoder *zstd.Decoder
+
+ mu sync.Mutex
+
+ av atomic.Value
+)
+
+func init() {
+ r := make(map[int]*zstd.Encoder)
+ av.Store(r)
+
+ var err error
+ decoder, err = zstd.NewReader(nil)
+ if err != nil {
+ logger.Panicf("failed to create ZSTD reader: %v", err)
+ }
+}
+
+// Decompress decompresses the src into dst.
+func Decompress(dst, src []byte) ([]byte, error) {
+ return decoder.DecodeAll(src, dst)
+}
+
+// Compress compresses the src into dst.
+func Compress(dst, src []byte, compressionLevel int) []byte {
+ e := getEncoder(compressionLevel)
+ return e.EncodeAll(src, dst)
+}
+
+func getEncoder(compressionLevel int) *zstd.Encoder {
+ r := av.Load().(map[int]*zstd.Encoder)
+ e := r[compressionLevel]
+ if e != nil {
+ return e
+ }
+
+ mu.Lock()
+ r1 := av.Load().(map[int]*zstd.Encoder)
+ if e = r1[compressionLevel]; e == nil {
+ e = newEncoder(compressionLevel)
+ r2 := make(map[int]*zstd.Encoder)
+ for k, v := range r1 {
+ r2[k] = v
+ }
+ r2[compressionLevel] = e
+ av.Store(r2)
+ }
+ mu.Unlock()
+
+ return e
+}
+
+func newEncoder(compressionLevel int) *zstd.Encoder {
+ level := zstd.EncoderLevelFromZstd(compressionLevel)
+ e, err := zstd.NewWriter(nil,
+ zstd.WithEncoderCRC(false), // Disable CRC for performance reasons.
+ zstd.WithEncoderLevel(level))
+ if err != nil {
+ logger.Panicf("failed to create ZSTD writer: %v", err)
+ }
+ return e
+}
diff --git a/pkg/compress/zstd/zstd_test.go b/pkg/compress/zstd/zstd_test.go
new file mode 100644
index 00000000..b33d7251
--- /dev/null
+++ b/pkg/compress/zstd/zstd_test.go
@@ -0,0 +1,79 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package zstd_test
+
+import (
+ "crypto/rand"
+ "math/big"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/apache/skywalking-banyandb/pkg/compress/zstd"
+)
+
+func randString(n int) []byte {
+ letters := []byte("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
+
+ b := make([]byte, n)
+ for i := range b {
+ max := big.NewInt(int64(len(letters)))
+ randIndex, err := rand.Int(rand.Reader, max)
+ if err != nil {
+ panic(err)
+ }
+ b[i] = letters[randIndex.Int64()]
+ }
+
+ return b
+}
+
+func TestCompressAndDecompress(t *testing.T) {
+ testCases := []struct {
+ name string
+ data []byte
+ }{
+ {
+ name: "SingleByte",
+ data: randString(1),
+ },
+ {
+ name: "NormalSizeBytes",
+ data: randString(1000), // 1000 bytes
+ },
+ {
+ name: "SuperBigBytes",
+ data: randString(1e6), // 1 million bytes
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ compressionLevel := 3
+
+ // Test Compress
+ compressed := zstd.Compress(nil, tc.data, compressionLevel)
+ require.NotEmpty(t, compressed, "Compress should return non-empty result")
+
+ // Test Decompress
+ decompressed, err := zstd.Decompress(nil, compressed)
+ require.NoError(t, err, "Decompress should not return an error")
+ require.Equal(t, tc.data, decompressed, "Decompressed data should be equal to the original data")
+ })
+ }
+}
diff --git a/pkg/encoding/bytes.go b/pkg/encoding/bytes.go
new file mode 100644
index 00000000..6aab3f89
--- /dev/null
+++ b/pkg/encoding/bytes.go
@@ -0,0 +1,300 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package encoding
+
+import (
+ "fmt"
+
+ "github.com/apache/skywalking-banyandb/pkg/bytes"
+ "github.com/apache/skywalking-banyandb/pkg/compress/zstd"
+)
+
+// EncodeBytes encodes a string into dst.
+func EncodeBytes(dst, b []byte) []byte {
+ dst = VarUint64ToBytes(dst, uint64(len(b)))
+ dst = append(dst, b...)
+ return dst
+}
+
+// DecodeBytes decodes a string from src.
+func DecodeBytes(src []byte) ([]byte, []byte, error) {
+ tail, n, err := BytesToVarUint64(src)
+ if err != nil {
+ return nil, nil, fmt.Errorf("cannot decode string size: %w", err)
+ }
+ src = tail
+ if uint64(len(src)) < n {
+ return nil, nil, fmt.Errorf("src is too short for reading string with size %d; len(src)=%d", n, len(src))
+ }
+ return src[n:], src[:n], nil
+}
+
+// EncodeBytesBlock encodes a block of strings into dst.
+func EncodeBytesBlock(dst []byte, a [][]byte) []byte {
+ u64s := GenerateUint64List(len(a))
+ aLens := u64s.L[:0]
+ for _, s := range a {
+ aLens = append(aLens, uint64(len(s)))
+ }
+ u64s.L = aLens
+ dst = encodeUint64Block(dst, u64s.L)
+ ReleaseUint64List(u64s)
+
+ bb := bbPool.Generate()
+ b := bb.Buf
+ for _, s := range a {
+ b = append(b, s...)
+ }
+ bb.Buf = b
+ dst = compressBlock(dst, bb.Buf)
+ bbPool.Release(bb)
+
+ return dst
+}
+
+// BytesBlockDecoder decodes a block of strings from src.
+// It reuses the underlying buffer for storing the decoded strings.
+type BytesBlockDecoder struct {
+ data []byte
+}
+
+// Reset resets the decoder.
+func (bbd *BytesBlockDecoder) Reset() {
+ bbd.data = bbd.data[:0]
+}
+
+// Decode decodes a block of strings from src.
+func (bbd *BytesBlockDecoder) Decode(dst [][]byte, src []byte, itemsCount uint64) ([][]byte, error) {
+ u64List := GenerateUint64List(0)
+ defer ReleaseUint64List(u64List)
+
+ var tail []byte
+ var err error
+ u64List.L, tail, err = decodeUint64Block(u64List.L[:0], src, itemsCount)
+ if err != nil {
+ return dst, fmt.Errorf("cannot decode string lengths: %w", err)
+ }
+ aLens := u64List.L
+ src = tail
+
+ dataLen := len(bbd.data)
+ bbd.data, tail, err = decompressBlock(bbd.data, src)
+ if err != nil {
+ return dst, fmt.Errorf("cannot decode bytes block with strings: %w", err)
+ }
+ if len(tail) > 0 {
+ return dst, fmt.Errorf("unexpected non-empty tail after reading bytes block with strings; len(tail)=%d", len(tail))
+ }
+
+ data := bbd.data[dataLen:]
+ for _, sLen := range aLens {
+ if uint64(len(data)) < sLen {
+ return dst, fmt.Errorf("cannot decode a string with the length %d bytes from %d bytes", sLen, len(data))
+ }
+ if sLen == 0 {
+ dst = append(dst, nil)
+ continue
+ }
+ dst = append(dst, data[:sLen])
+ data = data[sLen:]
+ }
+
+ return dst, nil
+}
+
+func encodeUint64Block(dst []byte, a []uint64) []byte {
+ bb := bbPool.Generate()
+ bb.Buf = encodeUint64List(bb.Buf[:0], a)
+ dst = compressBlock(dst, bb.Buf)
+ bbPool.Release(bb)
+ return dst
+}
+
+func decodeUint64Block(dst []uint64, src []byte, itemsCount uint64) ([]uint64, []byte, error) {
+ bb := bbPool.Generate()
+ defer bbPool.Release(bb)
+
+ var err error
+ bb.Buf, src, err = decompressBlock(bb.Buf[:0], src)
+ if err != nil {
+ return dst, src, fmt.Errorf("cannot decode bytes block: %w", err)
+ }
+
+ dst, err = decodeUint64List(dst, bb.Buf, itemsCount)
+ if err != nil {
+ return dst, src, fmt.Errorf("cannot decode %d uint64 items from bytes block of length %d bytes: %w", itemsCount, len(bb.Buf), err)
+ }
+ return dst, src, nil
+}
+
+const (
+ uintBlockType8 = 0
+ uintBlockType16 = 1
+ uintBlockType32 = 2
+ uintBlockType64 = 3
+)
+
+func encodeUint64List(dst []byte, a []uint64) []byte {
+ nMax := uint64(0)
+ for _, n := range a {
+ if n > nMax {
+ nMax = n
+ }
+ }
+ switch {
+ case nMax < (1 << 8):
+ dst = append(dst, uintBlockType8)
+ for _, n := range a {
+ dst = append(dst, byte(n))
+ }
+ case nMax < (1 << 16):
+ dst = append(dst, uintBlockType16)
+ for _, n := range a {
+ dst = Uint16ToBytes(dst, uint16(n))
+ }
+ case nMax < (1 << 32):
+ dst = append(dst, uintBlockType32)
+ for _, n := range a {
+ dst = Uint32ToBytes(dst, uint32(n))
+ }
+ default:
+ dst = append(dst, uintBlockType64)
+ for _, n := range a {
+ dst = Uint64ToBytes(dst, n)
+ }
+ }
+ return dst
+}
+
+func decodeUint64List(dst []uint64, src []byte, itemsCount uint64) ([]uint64, error) {
+ if len(src) < 1 {
+ return dst, fmt.Errorf("cannot decode uint64 block type from empty src")
+ }
+ blockType := src[0]
+ src = src[1:]
+
+ switch blockType {
+ case uintBlockType8:
+ if uint64(len(src)) != itemsCount {
+ return dst, fmt.Errorf("unexpected block length for %d items; got %d bytes; want %d bytes", itemsCount, len(src), itemsCount)
+ }
+ for _, v := range src {
+ dst = append(dst, uint64(v))
+ }
+ case uintBlockType16:
+ if uint64(len(src)) != 2*itemsCount {
+ return dst, fmt.Errorf("unexpected block length for %d items; got %d bytes; want %d bytes", itemsCount, len(src), 2*itemsCount)
+ }
+ for len(src) > 0 {
+ v := BytesToUint16(src)
+ src = src[2:]
+ dst = append(dst, uint64(v))
+ }
+ case uintBlockType32:
+ if uint64(len(src)) != 4*itemsCount {
+ return dst, fmt.Errorf("unexpected block length for %d items; got %d bytes; want %d bytes", itemsCount, len(src), 4*itemsCount)
+ }
+ for len(src) > 0 {
+ v := BytesToUint32(src)
+ src = src[4:]
+ dst = append(dst, uint64(v))
+ }
+ case uintBlockType64:
+ if uint64(len(src)) != 8*itemsCount {
+ return dst, fmt.Errorf("unexpected block length for %d items; got %d bytes; want %d bytes", itemsCount, len(src), 8*itemsCount)
+ }
+ for len(src) > 0 {
+ v := BytesToUint64(src)
+ src = src[8:]
+ dst = append(dst, v)
+ }
+ default:
+ return dst, fmt.Errorf("unexpected uint64 block type: %d; want 0, 1, 2 or 3", blockType)
+ }
+ return dst, nil
+}
+
+const (
+ compressTypePlain = 0
+ compressTypeZSTD = 1
+)
+
+func compressBlock(dst, src []byte) []byte {
+ if len(src) < 128 {
+ dst = append(dst, compressTypePlain, byte(len(src)))
+ return append(dst, src...)
+ }
+
+ dst = append(dst, compressTypeZSTD)
+ bb := bbPool.Generate()
+ bb.Buf = zstd.Compress(bb.Buf[:0], src, 1)
+ dst = VarUint64ToBytes(dst, uint64(len(bb.Buf)))
+ dst = append(dst, bb.Buf...)
+ bbPool.Release(bb)
+ return dst
+}
+
+func decompressBlock(dst, src []byte) ([]byte, []byte, error) {
+ if len(src) < 1 {
+ return dst, src, fmt.Errorf("cannot decode block type from empty src")
+ }
+ blockType := src[0]
+ src = src[1:]
+ switch blockType {
+ case compressTypePlain:
+ if len(src) < 1 {
+ return dst, src, fmt.Errorf("cannot decode plain block size from empty src")
+ }
+ blockLen := int(src[0])
+ src = src[1:]
+ if len(src) < blockLen {
+ return dst, src, fmt.Errorf("cannot read plain block with the size %d bytes from %b bytes", blockLen, len(src))
+ }
+
+ dst = append(dst, src[:blockLen]...)
+ src = src[blockLen:]
+ return dst, src, nil
+ case compressTypeZSTD:
+ tail, blockLen, err := BytesToVarUint64(src)
+ if err != nil {
+ return dst, src, fmt.Errorf("cannot decode compressed block size: %w", err)
+ }
+ src = tail
+ if uint64(len(src)) < blockLen {
+ return dst, src, fmt.Errorf("cannot read compressed block with the size %d bytes from %d bytes", blockLen, len(src))
+ }
+ compressedBlock := src[:blockLen]
+ src = src[blockLen:]
+
+ // Decompress the block
+ bb := bbPool.Generate()
+ bb.Buf, err = zstd.Decompress(bb.Buf[:0], compressedBlock)
+ if err != nil {
+ return dst, src, fmt.Errorf("cannot decompress block: %w", err)
+ }
+
+ // Copy the decompressed block to dst.
+ dst = append(dst, bb.Buf...)
+ bbPool.Release(bb)
+ return dst, src, nil
+ default:
+ return dst, src, fmt.Errorf("unexpected block type: %d; supported types: 0, 1", blockType)
+ }
+}
+
+var bbPool bytes.BufferPool
diff --git a/pkg/encoding/bytes_test.go b/pkg/encoding/bytes_test.go
new file mode 100644
index 00000000..ec4eccd2
--- /dev/null
+++ b/pkg/encoding/bytes_test.go
@@ -0,0 +1,62 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package encoding_test
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "github.com/apache/skywalking-banyandb/pkg/encoding"
+)
+
+func TestDecodeBytes(t *testing.T) {
+ src := [][]byte{
+ []byte("Hello, "),
+ []byte("world!"),
+ }
+ encoded := make([]byte, 0)
+ for _, d := range src {
+ encoded = encoding.EncodeBytes(encoded, d)
+ }
+ var decoded []byte
+ var err error
+ for i := range src {
+ encoded, decoded, err = encoding.DecodeBytes(encoded)
+ require.Nil(t, err)
+ assert.Equal(t, src[i], decoded)
+ }
+}
+
+func TestEncodeBlockAndDecode(t *testing.T) {
+ slices := [][]byte{
+ []byte("Hello, "),
+ []byte("world!"),
+ []byte("Testing bytes"),
+ }
+
+ encoded := encoding.EncodeBytesBlock(nil, slices)
+ require.NotNil(t, encoded)
+ blockDecoder := &encoding.BytesBlockDecoder{}
+ decoded, err := blockDecoder.Decode(nil, encoded, uint64(len(slices)))
+ require.Nil(t, err)
+ for i, slice := range slices {
+ assert.Equal(t, slice, decoded[i])
+ }
+}
diff --git a/pkg/encoding/delta.go b/pkg/encoding/delta.go
new file mode 100644
index 00000000..4931aafe
--- /dev/null
+++ b/pkg/encoding/delta.go
@@ -0,0 +1,118 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package encoding
+
+import (
+ "fmt"
+
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+)
+
+func int64ListDeltaToBytes(dst []byte, src []int64) (result []byte, firstValue int64) {
+ if len(src) < 1 {
+ logger.Panicf("BUG: src must contain at least 1 item; got %d items", len(src))
+ }
+
+ firstValue = src[0]
+ v := src[0]
+ src = src[1:]
+ is := GenerateInt64List(len(src))
+ for i, next := range src {
+ d := next - v
+ v += d
+ is.L[i] = d
+ }
+ dst = VarInt64ListToBytes(dst, is.L)
+ ReleaseInt64List(is)
+ return dst, firstValue
+}
+
+func bytesDeltaToInt64List(dst []int64, src []byte, firstValue int64, itemsCount int) ([]int64, error) {
+ if itemsCount < 1 {
+ logger.Panicf("BUG: itemsCount must be greater than 0; got %d", itemsCount)
+ }
+
+ is := GenerateInt64List(itemsCount - 1)
+ defer ReleaseInt64List(is)
+
+ tail, err := BytesToVarInt64List(is.L, src)
+ if err != nil {
+ return nil, fmt.Errorf("cannot decode nearest delta from %d bytes; src=%X: %w", len(src), src, err)
+ }
+ if len(tail) > 0 {
+ return nil, fmt.Errorf("unexpected tail left after decodeing %d items from %d bytes; tail size=%d; src=%X; tail=%X", itemsCount, len(src), len(tail), src, tail)
+ }
+
+ v := firstValue
+ dst = append(dst, v)
+ for _, d := range is.L {
+ v += d
+ dst = append(dst, v)
+ }
+ return dst, nil
+}
+
+func int64sDeltaOfDeltaToBytes(dst []byte, src []int64) (result []byte, firstValue int64) {
+ if len(src) < 2 {
+ logger.Panicf("src must contain at least 2 items; got %d items", len(src))
+ }
+ firstValue = src[0]
+ d1 := src[1] - src[0]
+ dst = VarInt64ToBytes(dst, d1)
+ v := src[1]
+ src = src[2:]
+ is := GenerateInt64List(len(src))
+ for i, next := range src {
+ d2 := next - v - d1
+ d1 += d2
+ v += d1
+ is.L[i] = d2
+ }
+ dst = VarInt64ListToBytes(dst, is.L)
+ ReleaseInt64List(is)
+ return dst, firstValue
+}
+
+func bytesDeltaOfDeltaToInt64s(dst []int64, src []byte, firstValue int64, itemsCount int) ([]int64, error) {
+ if itemsCount < 2 {
+ logger.Panicf("itemsCount must be greater than 1; got %d", itemsCount)
+ }
+
+ is := GenerateInt64List(itemsCount - 1)
+ defer ReleaseInt64List(is)
+
+ tail, err := BytesToVarInt64List(is.L, src)
+ if err != nil {
+ return nil, fmt.Errorf("cannot decode nearest delta from %d bytes; src=%X: %w", len(src), src, err)
+ }
+ if len(tail) > 0 {
+ return nil, fmt.Errorf("unexpected tail left after decodeing %d items from %d bytes; tail size=%d; src=%X; tail=%X", itemsCount, len(src), len(tail), src, tail)
+ }
+
+ v := firstValue
+ d1 := is.L[0]
+ dst = append(dst, v)
+ v += d1
+ dst = append(dst, v)
+ for _, d2 := range is.L[1:] {
+ d1 += d2
+ v += d1
+ dst = append(dst, v)
+ }
+ return dst, nil
+}
diff --git a/pkg/encoding/encoding.go b/pkg/encoding/encoding.go
index 33ecdc79..0646cdbb 100644
--- a/pkg/encoding/encoding.go
+++ b/pkg/encoding/encoding.go
@@ -78,3 +78,16 @@ type BufferWriter interface {
WriteByte(b byte) error
Bytes() []byte
}
+
+// EncodeType indicates the encoding type of a series.
+type EncodeType byte
+
+// EncodeType constants.
+const (
+ EncodeTypeUnknown EncodeType = iota
+ EncodeTypeConst
+ EncodeTypeDeltaConst
+ EncodeTypeDelta
+ EncodeTypeDeltaOfDelta
+ EncodeTypeXOR
+)
diff --git a/pkg/encoding/int.go b/pkg/encoding/int.go
new file mode 100644
index 00000000..21b77482
--- /dev/null
+++ b/pkg/encoding/int.go
@@ -0,0 +1,276 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package encoding
+
+import (
+ "encoding/binary"
+ "fmt"
+ "sync"
+)
+
+// Uint16ToBytes appends the bytes of the given uint16 to the given byte slice.
+func Uint16ToBytes(dst []byte, u uint16) []byte {
+ return append(dst, byte(u>>8), byte(u))
+}
+
+// BytesToUint16 converts the first two bytes of the given byte slice to a uint16.
+func BytesToUint16(src []byte) uint16 {
+ return binary.BigEndian.Uint16(src[:2])
+}
+
+// Uint32ToBytes appends the bytes of the given uint32 to the given byte slice.
+func Uint32ToBytes(dst []byte, u uint32) []byte {
+ return append(dst, byte(u>>24), byte(u>>16), byte(u>>8), byte(u))
+}
+
+// BytesToUint32 converts the first four bytes of the given byte slice to a uint32.
+func BytesToUint32(src []byte) uint32 {
+ return binary.BigEndian.Uint32(src[:4])
+}
+
+// Uint64ToBytes appends the bytes of the given uint64 to the given byte slice.
+func Uint64ToBytes(dst []byte, u uint64) []byte {
+ return append(dst, byte(u>>56), byte(u>>48), byte(u>>40), byte(u>>32), byte(u>>24), byte(u>>16), byte(u>>8), byte(u))
+}
+
+// BytesToUint64 converts the first eight bytes of the given byte slice to a uint64.
+func BytesToUint64(src []byte) uint64 {
+ return binary.BigEndian.Uint64(src[:8])
+}
+
+// Int64ToBytes appends the bytes of the given int64 to the given byte slice.
+func Int64ToBytes(dst []byte, v int64) []byte {
+ v = (v << 1) ^ (v >> 63)
+ u := uint64(v)
+ return append(dst, byte(u>>56), byte(u>>48), byte(u>>40), byte(u>>32), byte(u>>24), byte(u>>16), byte(u>>8), byte(u))
+}
+
+// BytesToInt64 converts the first eight bytes of the given byte slice to an int64.
+func BytesToInt64(src []byte) int64 {
+ u := binary.BigEndian.Uint64(src[:8])
+ v := int64(u>>1) ^ (int64(u<<63) >> 63)
+ return v
+}
+
+// VarInt64ToBytes appends the bytes of the given int64 to the given byte slice.
+// It uses variable-length encoding.
+func VarInt64ToBytes(dst []byte, v int64) []byte {
+ var tmp [1]int64
+ tmp[0] = v
+ return VarInt64ListToBytes(dst, tmp[:])
+}
+
+// VarInt64ListToBytes appends the bytes of the given int64s to the given byte slice.
+// It uses variable-length encoding.
+func VarInt64ListToBytes(dst []byte, vs []int64) []byte {
+ for _, v := range vs {
+ if v < 0x40 && v > -0x40 {
+ c := int8(v)
+ v := (c << 1) ^ (c >> 7)
+ dst = append(dst, byte(v))
+ continue
+ }
+
+ v = (v << 1) ^ (v >> 63)
+ u := uint64(v)
+ for u > 0x7f {
+ dst = append(dst, 0x80|byte(u))
+ u >>= 7
+ }
+ dst = append(dst, byte(u))
+ }
+ return dst
+}
+
+// BytesToVarInt64 converts the first bytes of the given byte slice to an int64.
+// It uses variable-length encoding.
+func BytesToVarInt64(src []byte) ([]byte, int64, error) {
+ var tmp [1]int64
+ tail, err := BytesToVarInt64List(tmp[:], src)
+ return tail, tmp[0], err
+}
+
+// BytesToVarInt64List converts the first bytes of the given byte slice to an int64s.
+// It uses variable-length encoding.
+func BytesToVarInt64List(dst []int64, src []byte) ([]byte, error) {
+ idx := uint(0)
+ for i := range dst {
+ // Check if there's enough data to decode
+ if idx >= uint(len(src)) {
+ return nil, fmt.Errorf("cannot decode varint from empty data")
+ }
+ c := src[idx]
+ idx++
+ // If c < 128, the integer is represented by a single byte
+ if c < 0x80 {
+ v := int8(c>>1) ^ (int8(c<<7) >> 7)
+ dst[i] = int64(v)
+ continue
+ }
+
+ // If c >= 128, the integer is represented by multiple bytes
+ u := uint64(c & 0x7f)
+ startIdx := idx - 1
+ shift := uint8(0)
+ for c >= 0x80 {
+ if idx >= uint(len(src)) {
+ return nil, fmt.Errorf("unexpected end of encoded varint at byte %d; src=%x", idx-startIdx, src[startIdx:])
+ }
+ if idx-startIdx > 9 {
+ return src[idx:], fmt.Errorf("too long encoded varint; the maximum allowed length is 10 bytes; got %d bytes; src=%x",
+ (idx-startIdx)+1, src[startIdx:])
+ }
+ c = src[idx]
+ idx++
+ shift += 7
+ u |= uint64(c&0x7f) << shift
+ }
+ v := int64(u>>1) ^ (int64(u<<63) >> 63)
+ dst[i] = v
+ }
+ return src[idx:], nil
+}
+
+// VarUint64ToBytes appends the bytes of the given uint64 to the given byte slice.
+// It uses variable-length encoding.
+func VarUint64ToBytes(dst []byte, u uint64) []byte {
+ var tmp [1]uint64
+ tmp[0] = u
+ return VarUint64sToBytes(dst, tmp[:])
+}
+
+// VarUint64sToBytes appends the bytes of the given uint64s to the given byte slice.
+// It uses variable-length encoding.
+func VarUint64sToBytes(dst []byte, us []uint64) []byte {
+ for _, u := range us {
+ if u < 0x80 {
+ dst = append(dst, byte(u))
+ continue
+ }
+ for u > 0x7f {
+ dst = append(dst, 0x80|byte(u))
+ u >>= 7
+ }
+ dst = append(dst, byte(u))
+ }
+ return dst
+}
+
+// BytesToVarUint64 converts the first bytes of the given byte slice to a uint64.
+// It uses variable-length encoding.
+func BytesToVarUint64(src []byte) ([]byte, uint64, error) {
+ var tmp [1]uint64
+ tail, err := BytesToVarUint64s(tmp[:], src)
+ return tail, tmp[0], err
+}
+
+// BytesToVarUint64s converts the first bytes of the given byte slice to a uint64s.
+// It uses variable-length encoding.
+func BytesToVarUint64s(dst []uint64, src []byte) ([]byte, error) {
+ idx := uint(0)
+ for i := range dst {
+ if idx >= uint(len(src)) {
+ return nil, fmt.Errorf("cannot decode varuint from empty data")
+ }
+ c := src[idx]
+ idx++
+ if c < 0x80 {
+ dst[i] = uint64(c)
+ continue
+ }
+
+ u := uint64(c & 0x7f)
+ startIdx := idx - 1
+ shift := uint8(0)
+ for c >= 0x80 {
+ if idx >= uint(len(src)) {
+ return nil, fmt.Errorf("unexpected end of encoded varint at byte %d; src=%x", idx-startIdx, src[startIdx:])
+ }
+ if idx-startIdx > 9 {
+ return src[idx:], fmt.Errorf("too long encoded varint; the maximum allowed length is 10 bytes; got %d bytes; src=%x",
+ (idx-startIdx)+1, src[startIdx:])
+ }
+ c = src[idx]
+ idx++
+ shift += 7
+ u |= uint64(c&0x7f) << shift
+ }
+ dst[i] = u
+ }
+ return src[idx:], nil
+}
+
+// GenerateInt64List generates a list of int64 with the given size.
+// The returned list may be from a pool and should be released after use.
+func GenerateInt64List(size int) *Int64List {
+ v := int64ListPool.Get()
+ if v == nil {
+ return &Int64List{
+ L: make([]int64, size),
+ }
+ }
+ is := v.(*Int64List)
+ if n := size - cap(is.L); n > 0 {
+ is.L = append(is.L[:cap(is.L)], make([]int64, n)...)
+ }
+ is.L = is.L[:size]
+ return is
+}
+
+// ReleaseInt64List releases the given list of int64.
+// The list may be put into a pool for reuse.
+func ReleaseInt64List(is *Int64List) {
+ int64ListPool.Put(is)
+}
+
+// Int64List is a list of int64.
+type Int64List struct {
+ L []int64
+}
+
+var int64ListPool sync.Pool
+
+// GenerateUint64List generates a list of uint64 with the given size.
+// The returned list may be from a pool and should be released after use.
+func GenerateUint64List(size int) *Uint64List {
+ v := uint64ListPool.Get()
+ if v == nil {
+ return &Uint64List{
+ L: make([]uint64, size),
+ }
+ }
+ is := v.(*Uint64List)
+ if n := size - cap(is.L); n > 0 {
+ is.L = append(is.L[:cap(is.L)], make([]uint64, n)...)
+ }
+ is.L = is.L[:size]
+ return is
+}
+
+// ReleaseUint64List releases the given list of uint64.
+// The list may be put into a pool for reuse.
+func ReleaseUint64List(is *Uint64List) {
+ uint64ListPool.Put(is)
+}
+
+// Uint64List is a list of uint64.
+type Uint64List struct {
+ L []uint64
+}
+
+var uint64ListPool sync.Pool
diff --git a/pkg/encoding/int_list.go b/pkg/encoding/int_list.go
new file mode 100644
index 00000000..7ea736d9
--- /dev/null
+++ b/pkg/encoding/int_list.go
@@ -0,0 +1,178 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package encoding
+
+import (
+ "fmt"
+
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+)
+
+// Int64ListToBytes encodes a list of int64 into bytes.
+func Int64ListToBytes(dst []byte, a []int64) (result []byte, mt EncodeType, firstValue int64) {
+ if len(a) == 0 {
+ logger.Panicf("a must contain at least one item")
+ }
+ if isConst(a) {
+ firstValue = a[0]
+ return dst, EncodeTypeConst, firstValue
+ }
+ isDelta, isDeltaConst := isDelta(a)
+ if isDeltaConst {
+ firstValue = a[0]
+ dst = VarInt64ToBytes(dst, a[1]-a[0])
+ return dst, EncodeTypeDeltaConst, firstValue
+ }
+ if isDelta {
+ dst, firstValue = int64sDeltaOfDeltaToBytes(dst, a)
+ return dst, EncodeTypeDeltaOfDelta, firstValue
+ }
+
+ if isIncremental(a) {
+ mt = EncodeTypeDeltaOfDelta
+ dst, firstValue = int64sDeltaOfDeltaToBytes(dst, a)
+ return dst, mt, firstValue
+ }
+ mt = EncodeTypeDelta
+ dst, firstValue = int64ListDeltaToBytes(dst, a)
+ return dst, mt, firstValue
+}
+
+// BytesToInt64List decodes bytes into a list of int64.
+func BytesToInt64List(dst []int64, src []byte, mt EncodeType, firstValue int64, itemsCount int) ([]int64, error) {
+ dst = extendInt64ListCapacity(dst, itemsCount)
+
+ var err error
+ switch mt {
+ case EncodeTypeDelta:
+ dst, err = bytesDeltaToInt64List(dst, src, firstValue, itemsCount)
+ if err != nil {
+ return nil, fmt.Errorf("cannot decode nearest delta data: %w", err)
+ }
+ return dst, nil
+ case EncodeTypeDeltaOfDelta:
+ dst, err = bytesDeltaOfDeltaToInt64s(dst, src, firstValue, itemsCount)
+ if err != nil {
+ return nil, fmt.Errorf("cannot decode nearest delta2 data: %w", err)
+ }
+ return dst, nil
+ case EncodeTypeConst:
+ if len(src) > 0 {
+ return nil, fmt.Errorf("unexpected data left in const encoding: %d bytes", len(src))
+ }
+ for itemsCount > 0 {
+ dst = append(dst, firstValue)
+ itemsCount--
+ }
+ return dst, nil
+ case EncodeTypeDeltaConst:
+ v := firstValue
+ tail, d, err := BytesToVarInt64(src)
+ if err != nil {
+ return nil, fmt.Errorf("cannot decode delta value for delta const: %w", err)
+ }
+ if len(tail) > 0 {
+ return nil, fmt.Errorf("unexpected trailing data after delta const (d=%d): %d bytes", d, len(tail))
+ }
+ for itemsCount > 0 {
+ dst = append(dst, v)
+ itemsCount--
+ v += d
+ }
+ return dst, nil
+ default:
+ return nil, fmt.Errorf("unknown EncodeType=%d", mt)
+ }
+}
+
+func extendInt64ListCapacity(dst []int64, additionalItems int) []int64 {
+ dstLen := len(dst)
+ if n := dstLen + additionalItems - cap(dst); n > 0 {
+ dst = append(dst[:cap(dst)], make([]int64, n)...)
+ }
+ return dst[:dstLen]
+}
+
+func isConst(a []int64) bool {
+ if len(a) == 0 {
+ return false
+ }
+ v1 := a[0]
+ for _, v := range a {
+ if v != v1 {
+ return false
+ }
+ }
+ return true
+}
+
+func isDelta(a []int64) (bool, bool) {
+ if len(a) < 2 {
+ return false, false
+ }
+ ct := true
+ d1 := a[1] - a[0]
+ asc := getSignBit(d1)
+ prev := a[1]
+ for _, next := range a[2:] {
+ d := next - prev
+ if (getSignBit(d) ^ asc) == 1 {
+ return false, false
+ }
+ if ct && d != d1 {
+ ct = false
+ }
+ prev = next
+ }
+ return true, ct
+}
+
+func getSignBit(n int64) int64 {
+ return n >> 63 & 1
+}
+
+func isIncremental(a []int64) bool {
+ if len(a) < 2 {
+ return false
+ }
+
+ resets := 0
+ vPrev := a[0]
+ if vPrev < 0 {
+ return true
+ }
+ for _, v := range a[1:] {
+ if v < vPrev {
+ if v < 0 {
+ return false
+ }
+ if v > (vPrev >> 3) {
+ // Decremental data is found.
+ return false
+ }
+ resets++
+ }
+ vPrev = v
+ }
+ if resets <= 2 {
+ return true
+ }
+
+ // Assume an incremental list has less than len(a)/8 resets .
+ return resets < (len(a) >> 3)
+}
diff --git a/pkg/encoding/int_list_test.go b/pkg/encoding/int_list_test.go
new file mode 100644
index 00000000..dd7c75c4
--- /dev/null
+++ b/pkg/encoding/int_list_test.go
@@ -0,0 +1,77 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package encoding_test
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/apache/skywalking-banyandb/pkg/encoding"
+)
+
+func TestInt64ListToBytes(t *testing.T) {
+ testCases := []struct {
+ name string
+ values []int64
+ firstValue int64
+ expectedDstLen int
+ mt encoding.EncodeType
+ }{
+ {
+ name: "EncodeTypeDelta",
+ mt: encoding.EncodeTypeDelta,
+ firstValue: 0,
+ values: []int64{0, 2, 1, 3, 4},
+ },
+ {
+ name: "EncodeTypeDeltaOfDelta",
+ mt: encoding.EncodeTypeDeltaOfDelta,
+ firstValue: 0,
+ values: []int64{0, 1, 4, 6, 9},
+ },
+ {
+ name: "EncodeTypeConst",
+ mt: encoding.EncodeTypeConst,
+ firstValue: 0,
+ values: []int64{0, 0, 0, 0, 0},
+ },
+ {
+ name: "EncodeTypeDeltaConst",
+ mt: encoding.EncodeTypeDeltaConst,
+ firstValue: 0,
+ values: []int64{0, 1, 2, 3, 4},
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ dst := make([]byte, 0)
+ var encodeType encoding.EncodeType
+ var firstValue int64
+ dst, encodeType, firstValue = encoding.Int64ListToBytes(dst, tc.values)
+ require.Equal(t, tc.mt, encodeType, "EncodeType should be equal to the original EncodeType")
+ require.Equal(t, tc.firstValue, firstValue, "FirstValue should be equal to the original FirstValue")
+
+ decoded := make([]int64, 0, len(tc.values))
+ decoded, err := encoding.BytesToInt64List(decoded, dst, tc.mt, tc.firstValue, len(tc.values))
+ require.NoError(t, err, "BytesToInt64List should not return an error")
+ require.Equal(t, tc.values, decoded, "Decoded int64s should be equal to the original int64s")
+ })
+ }
+}
diff --git a/pkg/encoding/int_test.go b/pkg/encoding/int_test.go
new file mode 100644
index 00000000..0237d41c
--- /dev/null
+++ b/pkg/encoding/int_test.go
@@ -0,0 +1,248 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package encoding_test
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/apache/skywalking-banyandb/pkg/encoding"
+)
+
+func TestUint16ToBytes(t *testing.T) {
+ testCases := []struct {
+ name string
+ v uint16
+ }{
+ {
+ name: "Zero",
+ v: 0,
+ },
+ {
+ name: "PositiveSmall",
+ v: 12345,
+ },
+ {
+ name: "PositiveLarge",
+ v: 54321,
+ },
+ {
+ name: "MaxUint16",
+ v: 65535,
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ dst := make([]byte, 0)
+ encoded := encoding.Uint16ToBytes(dst, tc.v)
+
+ // Check that the encoded bytes can be decoded back to the original uint16
+ decoded := encoding.BytesToUint16(encoded)
+ require.Equal(t, tc.v, decoded, "Decoded uint16 should be equal to the original uint16")
+ })
+ }
+}
+
+func TestUint32ToBytes(t *testing.T) {
+ testCases := []struct {
+ name string
+ v uint32
+ }{
+ {
+ name: "Zero",
+ v: 0,
+ },
+ {
+ name: "PositiveSmall",
+ v: 123456789,
+ },
+ {
+ name: "PositiveLarge",
+ v: 987654321,
+ },
+ {
+ name: "MaxUint32",
+ v: 4294967295,
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ dst := make([]byte, 0)
+ encoded := encoding.Uint32ToBytes(dst, tc.v)
+
+ // Check that the encoded bytes can be decoded back to the original uint32
+ decoded := encoding.BytesToUint32(encoded)
+ require.Equal(t, tc.v, decoded, "Decoded uint32 should be equal to the original uint32")
+ })
+ }
+}
+
+func TestUint64ToBytes(t *testing.T) {
+ testCases := []struct {
+ name string
+ v uint64
+ }{
+ {
+ name: "Zero",
+ v: 0,
+ },
+ {
+ name: "PositiveSmall",
+ v: 1234567890123456789,
+ },
+ {
+ name: "PositiveLarge",
+ v: 9876543210987654321,
+ },
+ {
+ name: "MaxUint64",
+ v: 18446744073709551615,
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ dst := make([]byte, 0)
+ encoded := encoding.Uint64ToBytes(dst, tc.v)
+
+ // Check that the encoded bytes can be decoded back to the original uint64
+ decoded := encoding.BytesToUint64(encoded)
+ require.Equal(t, tc.v, decoded, "Decoded uint64 should be equal to the original uint64")
+ })
+ }
+}
+
+func TestInt64ToBytes(t *testing.T) {
+ testCases := []struct {
+ name string
+ v int64
+ }{
+ {
+ name: "Zero",
+ v: 0,
+ },
+ {
+ name: "Positive",
+ v: 1234567890,
+ },
+ {
+ name: "Negative",
+ v: -1234567890,
+ },
+ {
+ name: "MaxInt64",
+ v: 9223372036854775807,
+ },
+ {
+ name: "MinInt64",
+ v: -9223372036854775808,
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ dst := make([]byte, 0)
+ encoded := encoding.Int64ToBytes(dst, tc.v)
+
+ // Check that the encoded bytes can be decoded back to the original int64
+ decoded := encoding.BytesToInt64(encoded)
+ require.Equal(t, tc.v, decoded, "Decoded int64 should be equal to the original int64")
+ })
+ }
+}
+
+func TestBytesToVarInt64List(t *testing.T) {
+ testCases := []struct {
+ name string
+ v []int64
+ }{
+ {
+ name: "Zero",
+ v: []int64{0},
+ },
+ {
+ name: "Positive",
+ v: []int64{1234567890},
+ },
+ {
+ name: "Negative",
+ v: []int64{-1234567890},
+ },
+ {
+ name: "MultipleValues",
+ v: []int64{0, 1234567890, -1234567890, 9223372036854775807, -9223372036854775808},
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ dst := make([]byte, 0)
+ dst = encoding.VarInt64ListToBytes(dst, tc.v)
+
+ decoded := make([]int64, len(tc.v))
+ dst, err := encoding.BytesToVarInt64List(decoded, dst)
+ require.NoError(t, err, "BytesToVarInt64s should not return an error")
+ require.Len(t, dst, 0, "BytesToVarInt64s should consume all bytes")
+ require.Equal(t, tc.v, decoded, "Decoded int64s should be equal to the original int64s")
+ })
+ }
+}
+
+func TestBytesToVarUint64List(t *testing.T) {
+ testCases := []struct {
+ name string
+ v []uint64
+ }{
+ {
+ name: "Zero",
+ v: []uint64{0},
... 2931 lines suppressed ...