You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ha...@apache.org on 2023/12/17 21:49:28 UTC
(skywalking-banyandb) branch storage-column updated (0862fadd -> 7f157e80)
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a change to branch storage-column
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
from 0862fadd Add column test
new 835b36fe Finish test block
new 3fe134f3 Test part
new d91f5f26 Test tsTable
new 7cb0650c Test query result
new ad3f5c8e Pass the smoke test of the integration test.
new c651c164 Add projection
new 7f157e80 Pass integration test
The 7 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
banyand/internal/storage/index.go | 2 +-
banyand/internal/storage/tsdb.go | 39 +-
banyand/liaison/grpc/measure.go | 9 +-
banyand/measure/block.go | 260 ++++++++--
banyand/measure/block_metadata.go | 6 +-
banyand/measure/block_test.go | 265 ++++++++--
banyand/measure/block_writer.go | 35 +-
banyand/measure/column_metadata.go | 2 +-
banyand/measure/datapoints.go | 2 +-
banyand/measure/measure.go | 2 +
banyand/measure/measure_query.go | 572 ---------------------
banyand/measure/part.go | 9 +-
banyand/measure/part_iter.go | 166 +++---
banyand/measure/part_iter_test.go | 142 +++++
banyand/measure/part_metadata.go | 2 +
banyand/measure/part_test.go | 180 +++++++
banyand/measure/query.go | 430 ++++++++++++++++
banyand/measure/query_test.go | 443 ++++++++++++++++
banyand/measure/{measure_topn.go => topn.go} | 13 +-
banyand/measure/tstable.go | 223 +-------
banyand/measure/tstable_test.go | 318 ++++++++++++
banyand/measure/{measure_write.go => write.go} | 26 +-
banyand/queue/local.go | 21 +-
pkg/bytes/buffer.go | 8 +-
pkg/index/inverted/inverted_series.go | 11 +-
pkg/index/inverted/inverted_series_test.go | 1 +
pkg/pb/v1/metadata.go | 4 +-
pkg/pb/v1/value.go | 5 +
pkg/query/logical/index_filter.go | 2 +-
.../measure/measure_plan_indexscan_local.go | 30 +-
pkg/query/logical/measure/topn_plan_localscan.go | 8 +-
test/cases/measure/data/want/all_only_fields.yaml | 66 ---
test/cases/measure/measure.go | 4 +-
33 files changed, 2202 insertions(+), 1104 deletions(-)
delete mode 100644 banyand/measure/measure_query.go
create mode 100644 banyand/measure/part_iter_test.go
create mode 100644 banyand/measure/part_test.go
create mode 100644 banyand/measure/query.go
create mode 100644 banyand/measure/query_test.go
rename banyand/measure/{measure_topn.go => topn.go} (98%)
create mode 100644 banyand/measure/tstable_test.go
rename banyand/measure/{measure_write.go => write.go} (92%)
(skywalking-banyandb) 06/07: Add projection
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch storage-column
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit c651c164c190db03bb153fe2e6fa724d63664da8
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Sun Dec 17 12:26:35 2023 +0800
Add projection
Signed-off-by: Gao Hongtao <ha...@gmail.com>
---
banyand/measure/block.go | 89 ++++++++++++++++------
banyand/measure/block_metadata.go | 4 +
banyand/measure/block_test.go | 35 +++++++--
banyand/measure/query.go | 4 +-
banyand/measure/query_test.go | 10 ++-
banyand/measure/tstable.go | 2 +-
banyand/measure/tstable_test.go | 18 ++++-
pkg/pb/v1/metadata.go | 2 +-
.../measure/measure_plan_indexscan_local.go | 8 +-
pkg/query/logical/measure/topn_plan_localscan.go | 8 +-
test/cases/measure/data/want/all_only_fields.yaml | 66 ----------------
test/cases/measure/measure.go | 2 +-
12 files changed, 133 insertions(+), 115 deletions(-)
diff --git a/banyand/measure/block.go b/banyand/measure/block.go
index adf1a5e0..fd0661e1 100644
--- a/banyand/measure/block.go
+++ b/banyand/measure/block.go
@@ -195,7 +195,10 @@ func (b *block) marshalTagFamily(tf ColumnFamily, bm *blockMetadata, ww *writers
hw.MustWrite(bb.Buf)
}
-func (b *block) unmarshalTagFamily(decoder *encoding.BytesBlockDecoder, tfIndex int, name string, columnFamilyMetadataBlock *dataBlock, metaReader, valueReader fs.Reader) {
+func (b *block) unmarshalTagFamily(decoder *encoding.BytesBlockDecoder, tfIndex int, name string, columnFamilyMetadataBlock *dataBlock, tagProjection []string, metaReader, valueReader fs.Reader) {
+ if len(tagProjection) < 1 {
+ return
+ }
bb := bigValuePool.Generate()
bb.Buf = bytes.ResizeExact(bb.Buf, int(columnFamilyMetadataBlock.size))
fs.MustReadData(metaReader, int64(columnFamilyMetadataBlock.offset), bb.Buf)
@@ -207,9 +210,14 @@ func (b *block) unmarshalTagFamily(decoder *encoding.BytesBlockDecoder, tfIndex
}
bigValuePool.Release(bb)
b.tagFamilies[tfIndex].Name = name
- cc := b.tagFamilies[tfIndex].resizeColumns(len(cfm.columnMetadata))
- for i := range cc {
- cc[i].mustReadValues(decoder, valueReader, cfm.columnMetadata[i], uint64(b.Len()))
+ cc := b.tagFamilies[tfIndex].resizeColumns(len(tagProjection))
+
+ for j := range tagProjection {
+ for i := range cc {
+ if tagProjection[j] == cfm.columnMetadata[i].name {
+ cc[i].mustReadValues(decoder, valueReader, cfm.columnMetadata[i], uint64(b.Len()))
+ }
+ }
}
}
@@ -245,16 +253,19 @@ func (b *block) uncompressedSizeBytes() uint64 {
return n
}
-func (b *block) mustReadFrom(decoder *encoding.BytesBlockDecoder, p *part, bm *blockMetadata) {
+func (b *block) mustReadFrom(decoder *encoding.BytesBlockDecoder, p *part, bm blockMetadata) {
b.reset()
b.timestamps = mustReadTimestampsFrom(b.timestamps, &bm.timestamps, int(bm.count), p.timestamps)
- _ = b.resizeTagFamilies(len(bm.tagFamilies))
- var i int
- for name, block := range bm.tagFamilies {
- b.unmarshalTagFamily(decoder, i, name, block, p.tagFamilyMetadata[name], p.tagFamilies[name])
- i++
+ _ = b.resizeTagFamilies(len(bm.tagProjection))
+ for i := range bm.tagProjection {
+ name := bm.tagProjection[i].Family
+ block, ok := bm.tagFamilies[name]
+ if !ok {
+ continue
+ }
+ b.unmarshalTagFamily(decoder, i, name, block, bm.tagProjection[i].Names, p.tagFamilyMetadata[name], p.tagFamilies[name])
}
cc := b.field.resizeColumns(len(bm.field.columnMetadata))
for i := range cc {
@@ -326,6 +337,8 @@ type blockCursor struct {
bm blockMetadata
minTimestamp int64
maxTimestamp int64
+ tagProjection []pbv1.TagProjection
+ fieldProjection []string
}
func (bc *blockCursor) reset() {
@@ -334,6 +347,8 @@ func (bc *blockCursor) reset() {
bc.bm = blockMetadata{}
bc.minTimestamp = 0
bc.maxTimestamp = 0
+ bc.tagProjection = bc.tagProjection[:0]
+ bc.fieldProjection = bc.fieldProjection[:0]
bc.timestamps = bc.timestamps[:0]
@@ -345,12 +360,14 @@ func (bc *blockCursor) reset() {
bc.fields.reset()
}
-func (bc *blockCursor) init(p *part, bm blockMetadata, minTimestamp, maxTimestamp int64) {
+func (bc *blockCursor) init(p *part, bm blockMetadata, queryOpts QueryOptions) {
bc.reset()
bc.p = p
bc.bm = bm
- bc.minTimestamp = minTimestamp
- bc.maxTimestamp = maxTimestamp
+ bc.minTimestamp = queryOpts.minTimestamp
+ bc.maxTimestamp = queryOpts.maxTimestamp
+ bc.tagProjection = queryOpts.TagProjection
+ bc.fieldProjection = queryOpts.FieldProjection
}
func (bc *blockCursor) copyAllTo(r *pbv1.Result) {
@@ -385,30 +402,36 @@ func (bc *blockCursor) copyAllTo(r *pbv1.Result) {
func (bc *blockCursor) copyTo(r *pbv1.Result) {
r.SID = bc.bm.seriesID
r.Timestamps = append(r.Timestamps, bc.timestamps[bc.idx])
- if len(r.TagFamilies) != len(bc.tagFamilies) {
- for _, cf := range bc.tagFamilies {
+ if len(r.TagFamilies) != len(bc.tagProjection) {
+ for _, tp := range bc.tagProjection {
tf := pbv1.TagFamily{
- Name: cf.Name,
+ Name: tp.Family,
}
- for _, c := range cf.Columns {
+ for _, n := range tp.Names {
t := pbv1.Tag{
- Name: c.Name,
+ Name: n,
}
tf.Tags = append(tf.Tags, t)
}
r.TagFamilies = append(r.TagFamilies, tf)
}
}
+ if len(bc.tagFamilies) != len(r.TagFamilies) {
+ logger.Panicf("unexpected number of tag families: got %d; want %d", len(bc.tagFamilies), len(r.TagFamilies))
+ }
for i, cf := range bc.tagFamilies {
+ if len(r.TagFamilies[i].Tags) != len(cf.Columns) {
+ logger.Panicf("unexpected number of tags: got %d; want %d", len(r.TagFamilies[i].Tags), len(bc.tagProjection[i].Names))
+ }
for i2, c := range cf.Columns {
r.TagFamilies[i].Tags[i2].Values = append(r.TagFamilies[i].Tags[i2].Values, mustDecodeTagValue(c.ValueType, c.Values[bc.idx]))
}
}
- if len(r.Fields) != len(bc.fields.Columns) {
- for _, c := range bc.fields.Columns {
+ if len(r.Fields) != len(bc.fieldProjection) {
+ for _, n := range bc.fieldProjection {
f := pbv1.Field{
- Name: c.Name,
+ Name: n,
}
r.Fields = append(r.Fields, f)
}
@@ -420,7 +443,26 @@ func (bc *blockCursor) copyTo(r *pbv1.Result) {
func (bc *blockCursor) loadData(tmpBlock *block) bool {
tmpBlock.reset()
- tmpBlock.mustReadFrom(&bc.columnValuesDecoder, bc.p, &bc.bm)
+ cfm := make([]columnMetadata, 0, len(bc.fieldProjection))
+ for j := range bc.fieldProjection {
+ for i := range bc.bm.field.columnMetadata {
+ if bc.bm.field.columnMetadata[i].name == bc.fieldProjection[j] {
+ cfm = append(cfm, bc.bm.field.columnMetadata[i])
+ }
+ }
+ }
+ bc.bm.field.columnMetadata = cfm
+ bc.bm.tagProjection = bc.tagProjection
+ tf := make(map[string]*dataBlock, len(bc.tagProjection))
+ for i := range bc.tagProjection {
+ for tfName, block := range bc.bm.tagFamilies {
+ if bc.tagProjection[i].Family == tfName {
+ tf[tfName] = block
+ }
+ }
+ }
+ bc.bm.tagFamilies = tf
+ tmpBlock.mustReadFrom(&bc.columnValuesDecoder, bc.p, bc.bm)
start, end, ok := findRange(tmpBlock.timestamps, bc.minTimestamp, bc.maxTimestamp)
if !ok {
@@ -428,9 +470,6 @@ func (bc *blockCursor) loadData(tmpBlock *block) bool {
}
bc.timestamps = append(bc.timestamps, tmpBlock.timestamps[start:end]...)
- sort.Slice(tmpBlock.tagFamilies, func(i, j int) bool {
- return tmpBlock.tagFamilies[i].Name < tmpBlock.tagFamilies[j].Name
- })
for _, cf := range tmpBlock.tagFamilies {
tf := ColumnFamily{
Name: cf.Name,
diff --git a/banyand/measure/block_metadata.go b/banyand/measure/block_metadata.go
index b77f0b03..77afe66d 100644
--- a/banyand/measure/block_metadata.go
+++ b/banyand/measure/block_metadata.go
@@ -25,6 +25,7 @@ import (
"github.com/apache/skywalking-banyandb/api/common"
"github.com/apache/skywalking-banyandb/pkg/convert"
"github.com/apache/skywalking-banyandb/pkg/encoding"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
)
type dataBlock struct {
@@ -73,6 +74,8 @@ type blockMetadata struct {
timestamps timestampsMetadata
field columnFamilyMetadata
tagFamilies map[string]*dataBlock
+
+ tagProjection []pbv1.TagProjection
}
func (bh *blockMetadata) getTagFamilyMetadata(name string) *dataBlock {
@@ -97,6 +100,7 @@ func (bh *blockMetadata) reset() {
bh.tagFamilies[k].reset()
delete(bh.tagFamilies, k)
}
+ bh.tagProjection = bh.tagProjection[:0]
}
func (bh *blockMetadata) copyFrom(src *blockMetadata) {
diff --git a/banyand/measure/block_test.go b/banyand/measure/block_test.go
index a393d7b7..c7768618 100644
--- a/banyand/measure/block_test.go
+++ b/banyand/measure/block_test.go
@@ -23,6 +23,8 @@ import (
"testing"
"time"
+ "github.com/google/go-cmp/cmp"
+
"github.com/apache/skywalking-banyandb/api/common"
"github.com/apache/skywalking-banyandb/pkg/bytes"
"github.com/apache/skywalking-banyandb/pkg/convert"
@@ -71,6 +73,18 @@ func Test_block_reset(t *testing.T) {
}
}
+func toTagProjection(b block) map[string][]string {
+ result := make(map[string][]string, len(b.tagFamilies))
+ for i := range b.tagFamilies {
+ names := make([]string, len(b.tagFamilies[i].Columns))
+ for i2 := range b.tagFamilies[i].Columns {
+ names[i2] = b.tagFamilies[i].Columns[i2].Name
+ }
+ result[b.tagFamilies[i].Name] = names
+ }
+ return result
+}
+
var conventionalBlock = block{
timestamps: []int64{1, 2},
tagFamilies: []ColumnFamily{
@@ -250,6 +264,7 @@ func Test_marshalAndUnmarshalTagFamily(t *testing.T) {
tagFamilyWriters: make(map[string]*writer),
}
b := &conventionalBlock
+ tagProjection := toTagProjection(*b)
tfIndex := 0
name := "arrTag"
decoder := &encoding.BytesBlockDecoder{}
@@ -273,10 +288,10 @@ func Test_marshalAndUnmarshalTagFamily(t *testing.T) {
unmarshaled.timestamps = make([]int64, len(b.timestamps))
unmarshaled.resizeTagFamilies(1)
- unmarshaled.unmarshalTagFamily(decoder, tfIndex, name, bm.getTagFamilyMetadata(name), metaBuffer, dataBuffer)
+ unmarshaled.unmarshalTagFamily(decoder, tfIndex, name, bm.getTagFamilyMetadata(name), tagProjection[name], metaBuffer, dataBuffer)
- if !reflect.DeepEqual(b.tagFamilies[0], unmarshaled.tagFamilies[0]) {
- t.Errorf("block.unmarshalTagFamily() = %+v, want %+v", unmarshaled.tagFamilies, b.tagFamilies)
+ if diff := cmp.Diff(unmarshaled.tagFamilies[0], b.tagFamilies[0]); diff != "" {
+ t.Errorf("block.unmarshalTagFamily() (-got +want):\n%s", diff)
}
}
@@ -296,11 +311,12 @@ func Test_marshalAndUnmarshalBlock(t *testing.T) {
fieldValues: fieldBuffer,
}
b := &conventionalBlock
+ tagProjection := toTagProjection(*b)
decoder := &encoding.BytesBlockDecoder{}
sid := common.SeriesID(1)
- bm := &blockMetadata{}
+ bm := blockMetadata{}
- b.mustWriteTo(sid, bm, ww)
+ b.mustWriteTo(sid, &bm, ww)
tagFamilyMetadataReaders := make(map[string]fs.Reader)
tagFamilyReaders := make(map[string]fs.Reader)
@@ -314,6 +330,15 @@ func Test_marshalAndUnmarshalBlock(t *testing.T) {
unmarshaled := generateBlock()
defer releaseBlock(unmarshaled)
+
+ var tp []pbv1.TagProjection
+ for family, names := range tagProjection {
+ tp = append(tp, pbv1.TagProjection{
+ Family: family,
+ Names: names,
+ })
+ }
+ bm.tagProjection = tp
unmarshaled.mustReadFrom(decoder, p, bm)
// blockMetadata is using a map, so the order of tag families is not guaranteed
unmarshaled.sortTagFamilies()
diff --git a/banyand/measure/query.go b/banyand/measure/query.go
index 94984dbf..15a9193f 100644
--- a/banyand/measure/query.go
+++ b/banyand/measure/query.go
@@ -87,7 +87,7 @@ func (s *measure) Query(ctx context.Context, mqo pbv1.MeasureQueryOptions) (pbv1
}
var pws []*partWrapper
var parts []*part
- qo := &QueryOptions{
+ qo := QueryOptions{
MeasureQueryOptions: mqo,
minTimestamp: mqo.TimeRange.Start.UnixNano(),
maxTimestamp: mqo.TimeRange.End.UnixNano(),
@@ -107,7 +107,7 @@ func (s *measure) Query(ctx context.Context, mqo pbv1.MeasureQueryOptions) (pbv1
for tstIter.nextBlock() {
bc := generateBlockCursor()
p := tstIter.piHeap[0]
- bc.init(p.p, p.curBlock, qo.minTimestamp, qo.maxTimestamp)
+ bc.init(p.p, p.curBlock, qo)
result.data = append(result.data, bc)
}
if tstIter.Error() != nil {
diff --git a/banyand/measure/query_test.go b/banyand/measure/query_test.go
index 85a92288..0f0f40dc 100644
--- a/banyand/measure/query_test.go
+++ b/banyand/measure/query_test.go
@@ -370,10 +370,11 @@ func TestQueryResult(t *testing.T) {
tst.mustAddDataPoints(dps)
time.Sleep(100 * time.Millisecond)
}
- pws, pp := tst.getParts(nil, nil, &QueryOptions{
+ queryOpts := QueryOptions{
minTimestamp: tt.minTimestamp,
maxTimestamp: tt.maxTimestamp,
- })
+ }
+ pws, pp := tst.getParts(nil, nil, queryOpts)
defer func() {
for _, pw := range pws {
pw.decRef()
@@ -391,7 +392,10 @@ func TestQueryResult(t *testing.T) {
for ti.nextBlock() {
bc := generateBlockCursor()
p := ti.piHeap[0]
- bc.init(p.p, p.curBlock, tt.minTimestamp, tt.maxTimestamp)
+ opts := queryOpts
+ opts.TagProjection = tagProjections[int(p.curBlock.seriesID)]
+ opts.FieldProjection = fieldProjections[int(p.curBlock.seriesID)]
+ bc.init(p.p, p.curBlock, opts)
result.data = append(result.data, bc)
}
defer result.Release()
diff --git a/banyand/measure/tstable.go b/banyand/measure/tstable.go
index 3d900bfc..838e37c1 100644
--- a/banyand/measure/tstable.go
+++ b/banyand/measure/tstable.go
@@ -62,7 +62,7 @@ func (tst *tsTable) mustAddDataPoints(dps *dataPoints) {
tst.memParts = append(tst.memParts, pw)
}
-func (tst *tsTable) getParts(dst []*partWrapper, dstPart []*part, opts *QueryOptions) ([]*partWrapper, []*part) {
+func (tst *tsTable) getParts(dst []*partWrapper, dstPart []*part, opts QueryOptions) ([]*partWrapper, []*part) {
tst.RLock()
defer tst.RUnlock()
for _, p := range tst.memParts {
diff --git a/banyand/measure/tstable_test.go b/banyand/measure/tstable_test.go
index 6787a424..a56846ed 100644
--- a/banyand/measure/tstable_test.go
+++ b/banyand/measure/tstable_test.go
@@ -168,7 +168,7 @@ func Test_tstIter(t *testing.T) {
tst.mustAddDataPoints(dps)
time.Sleep(100 * time.Millisecond)
}
- pws, pp := tst.getParts(nil, nil, &QueryOptions{
+ pws, pp := tst.getParts(nil, nil, QueryOptions{
minTimestamp: tt.minTimestamp,
maxTimestamp: tt.maxTimestamp,
})
@@ -203,6 +203,22 @@ func Test_tstIter(t *testing.T) {
}
}
+var tagProjections = map[int][]pbv1.TagProjection{
+ 1: {
+ {Family: "arrTag", Names: []string{"strArrTag", "intArrTag"}},
+ {Family: "binaryTag", Names: []string{"binaryTag"}},
+ {Family: "singleTag", Names: []string{"strTag", "intTag"}},
+ },
+ 2: {
+ {Family: "singleTag", Names: []string{"strTag1", "strTag2"}},
+ },
+}
+
+var fieldProjections = map[int][]string{
+ 1: {"strField", "intField", "floatField", "binaryField"},
+ 3: {"intField"},
+}
+
var dps_ts1 = &dataPoints{
seriesIDs: []common.SeriesID{1, 2, 3},
timestamps: []int64{1, 1, 1},
diff --git a/pkg/pb/v1/metadata.go b/pkg/pb/v1/metadata.go
index 87dda12f..451304e6 100644
--- a/pkg/pb/v1/metadata.go
+++ b/pkg/pb/v1/metadata.go
@@ -106,7 +106,7 @@ type Result struct {
type TagProjection struct {
Family string
- Name string
+ Names []string
}
type MeasureQueryOptions struct {
diff --git a/pkg/query/logical/measure/measure_plan_indexscan_local.go b/pkg/query/logical/measure/measure_plan_indexscan_local.go
index aa29dc0e..620a17bc 100644
--- a/pkg/query/logical/measure/measure_plan_indexscan_local.go
+++ b/pkg/query/logical/measure/measure_plan_indexscan_local.go
@@ -48,15 +48,13 @@ type unresolvedIndexScan struct {
}
func (uis *unresolvedIndexScan) Analyze(s logical.Schema) (logical.Plan, error) {
- var projTags []pbv1.TagProjection
+ projTags := make([]pbv1.TagProjection, len(uis.projectionTags))
var projTagsRefs [][]*logical.TagRef
if len(uis.projectionTags) > 0 {
for i := range uis.projectionTags {
for _, tag := range uis.projectionTags[i] {
- projTags = append(projTags, pbv1.TagProjection{
- Family: tag.GetFamilyName(),
- Name: tag.GetTagName(),
- })
+ projTags[i].Family = tag.GetFamilyName()
+ projTags[i].Names = append(projTags[i].Names, tag.GetTagName())
}
}
var err error
diff --git a/pkg/query/logical/measure/topn_plan_localscan.go b/pkg/query/logical/measure/topn_plan_localscan.go
index 10c3c867..41d901d3 100644
--- a/pkg/query/logical/measure/topn_plan_localscan.go
+++ b/pkg/query/logical/measure/topn_plan_localscan.go
@@ -48,14 +48,12 @@ type unresolvedLocalScan struct {
func (uls *unresolvedLocalScan) Analyze(s logical.Schema) (logical.Plan, error) {
var projTagsRefs [][]*logical.TagRef
- var projTags []pbv1.TagProjection
+ projTags := make([]pbv1.TagProjection, len(uls.projectionTags))
if len(uls.projectionTags) > 0 {
for i := range uls.projectionTags {
for _, tag := range uls.projectionTags[i] {
- projTags = append(projTags, pbv1.TagProjection{
- Family: tag.GetFamilyName(),
- Name: tag.GetTagName(),
- })
+ projTags[i].Family = tag.GetFamilyName()
+ projTags[i].Names = append(projTags[i].Names, tag.GetTagName())
}
}
var err error
diff --git a/test/cases/measure/data/want/all_only_fields.yaml b/test/cases/measure/data/want/all_only_fields.yaml
index 4c18f1ce..e6c80586 100644
--- a/test/cases/measure/data/want/all_only_fields.yaml
+++ b/test/cases/measure/data/want/all_only_fields.yaml
@@ -25,15 +25,6 @@ dataPoints:
value:
int:
value: "1"
-- fields:
- - name: total
- value:
- int:
- value: "100"
- - name: value
- value:
- int:
- value: "10"
- fields:
- name: total
value:
@@ -43,15 +34,6 @@ dataPoints:
value:
int:
value: "2"
-- fields:
- - name: total
- value:
- int:
- value: "100"
- - name: value
- value:
- int:
- value: "9"
- fields:
- name: total
value:
@@ -61,15 +43,6 @@ dataPoints:
value:
int:
value: "3"
-- fields:
- - name: total
- value:
- int:
- value: "100"
- - name: value
- value:
- int:
- value: "8"
- fields:
- name: total
value:
@@ -79,15 +52,6 @@ dataPoints:
value:
int:
value: "5"
-- fields:
- - name: total
- value:
- int:
- value: "100"
- - name: value
- value:
- int:
- value: "11"
- fields:
- name: total
value:
@@ -97,27 +61,6 @@ dataPoints:
value:
int:
value: "4"
- tagFamilies:
- - name: default
- tags:
- - key: id
- value:
- str:
- value: "2"
- - key: entity_id
- value:
- str:
- value: entity_1
- timestamp: "2022-10-17T12:53:45.912Z"
-- fields:
- - name: total
- value:
- int:
- value: "50"
- - name: value
- value:
- int:
- value: "12"
- fields:
- name: total
value:
@@ -127,12 +70,3 @@ dataPoints:
value:
int:
value: "6"
-- fields:
- - name: total
- value:
- int:
- value: "300"
- - name: value
- value:
- int:
- value: "7"
diff --git a/test/cases/measure/measure.go b/test/cases/measure/measure.go
index c2a43223..bfcb6642 100644
--- a/test/cases/measure/measure.go
+++ b/test/cases/measure/measure.go
@@ -41,7 +41,7 @@ var (
var _ = g.DescribeTable("Scanning Measures", verify,
g.Entry("all", helpers.Args{Input: "all", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
- g.Entry("all_only_fields", helpers.Args{Input: "all", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
+ g.FEntry("all only fields", helpers.Args{Input: "all_only_fields", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
g.Entry("filter by tag", helpers.Args{Input: "tag_filter", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
g.Entry("filter by a integer tag", helpers.Args{Input: "tag_filter_int", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
g.Entry("filter by an unknown tag", helpers.Args{Input: "tag_filter_unknown", Duration: 25 * time.Minute, Offset: -20 * time.Minute, WantEmpty: true}),
(skywalking-banyandb) 03/07: Test tsTable
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch storage-column
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit d91f5f26a0a1a2f07cdce0cac5a0545682146568
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Thu Dec 14 08:13:40 2023 +0800
Test tsTable
Signed-off-by: Gao Hongtao <ha...@gmail.com>
---
banyand/measure/block_writer.go | 2 +
banyand/measure/measure_query.go | 66 ++-----
banyand/measure/measure_write.go | 5 +-
banyand/measure/part_iter.go | 6 +-
banyand/measure/part_iter_test.go | 2 +-
banyand/measure/part_metadata.go | 2 +
banyand/measure/tstable.go | 20 +-
banyand/measure/tstable_test.go | 386 ++++++++++++++++++++++++++++++++++++++
8 files changed, 424 insertions(+), 65 deletions(-)
diff --git a/banyand/measure/block_writer.go b/banyand/measure/block_writer.go
index 4fd50190..546a8d32 100644
--- a/banyand/measure/block_writer.go
+++ b/banyand/measure/block_writer.go
@@ -19,6 +19,7 @@ package measure
import (
"sync"
+ "time"
"github.com/apache/skywalking-banyandb/api/common"
"github.com/apache/skywalking-banyandb/pkg/compress/zstd"
@@ -255,6 +256,7 @@ func (bsw *blockWriter) Flush(ph *partMetadata) {
ph.BlocksCount = bsw.totalBlocksCount
ph.MinTimestamp = bsw.totalMinTimestamp
ph.MaxTimestamp = bsw.totalMaxTimestamp
+ ph.Version = time.Now().UnixNano() // TODO: use a global version
bsw.mustFlushPrimaryBlock(bsw.primaryBlockData)
diff --git a/banyand/measure/measure_query.go b/banyand/measure/measure_query.go
index 4b2960c9..5b13923a 100644
--- a/banyand/measure/measure_query.go
+++ b/banyand/measure/measure_query.go
@@ -23,7 +23,6 @@ import (
"fmt"
"io"
"sort"
- "sync"
"time"
"github.com/pkg/errors"
@@ -54,7 +53,6 @@ type Query interface {
type Measure interface {
io.Closer
Query(ctx context.Context, opts pbv1.MeasureQueryOptions) (pbv1.MeasureQueryResult, error)
- Shards(entity tsdb.Entity) ([]tsdb.Shard, error)
CompanionShards(metadata *commonv1.Metadata) ([]tsdb.Shard, error)
Shard(id common.ShardID) (tsdb.Shard, error)
ParseTagFamily(family string, item tsdb.Item) (*modelv1.TagFamily, error)
@@ -89,8 +87,6 @@ type QueryResult struct {
asc bool
}
-const round = time.Millisecond
-
func (qr *QueryResult) Pull() *pbv1.Result {
if len(qr.data) == 0 {
return nil
@@ -111,7 +107,7 @@ func (qr *QueryResult) Pull() *pbv1.Result {
bc.copyAllTo(r)
return r
}
- return qr.merge(round)
+ return qr.merge()
}
originalData := qr.data
var lastSeriesID common.SeriesID
@@ -121,13 +117,13 @@ func (qr *QueryResult) Pull() *pbv1.Result {
qr.data = qr.data[:0]
qr.data = append(qr.data, originalData[qr.lastIndex:i]...)
qr.lastIndex = i
- return qr.merge(round)
+ return qr.merge()
}
lastSeriesID = bc.bm.seriesID
}
if qr.lastIndex < len(originalData)-1 {
qr.data = originalData[qr.lastIndex:]
- return qr.merge(round)
+ return qr.merge()
}
return nil
}
@@ -171,23 +167,20 @@ func (qr *QueryResult) Pop() interface{} {
return x
}
-func (qr *QueryResult) merge(round time.Duration) *pbv1.Result {
+func (qr *QueryResult) merge() *pbv1.Result {
result := &pbv1.Result{}
- var lastOriginalTimestamp int64
- r := int64(round)
+ var lastPartVersion int64
for qr.Len() > 0 {
bc := heap.Pop(qr).(*blockCursor)
- roundedTimestamp := (bc.timestamps[bc.idx] / r) * r
-
- if len(result.Timestamps) > 0 && roundedTimestamp == result.Timestamps[len(result.Timestamps)-1] {
- if bc.timestamps[bc.idx] > lastOriginalTimestamp {
- lastOriginalTimestamp = bc.timestamps[bc.idx]
+ if len(result.Timestamps) > 0 && bc.timestamps[bc.idx] == result.Timestamps[len(result.Timestamps)-1] {
+ if bc.p.partMetadata.Version > lastPartVersion {
+ lastPartVersion = bc.p.partMetadata.Version
}
} else {
bc.copyTo(result)
- lastOriginalTimestamp = bc.timestamps[bc.idx]
+ lastPartVersion = bc.p.partMetadata.Version
}
bc.idx++
@@ -235,12 +228,12 @@ func (s *measure) Query(ctx context.Context, mqo pbv1.MeasureQueryOptions) (pbv1
originalSids := make([]common.SeriesID, len(sids))
copy(originalSids, sids)
sort.Slice(sids, func(i, j int) bool { return sids[i] < sids[j] })
- tstIter.init(parts, sids, qo)
+ tstIter.init(parts, sids, qo.minTimestamp, qo.maxTimestamp)
if tstIter.err != nil {
return nil, fmt.Errorf("cannot init tstIter: %w", tstIter.err)
}
var result QueryResult
- for tstIter.NextBlock() {
+ for tstIter.nextBlock() {
bc := generateBlockCursor()
p := tstIter.piHeap[0]
bc.init(p.p, p.curBlock, qo)
@@ -263,37 +256,6 @@ func (s *measure) Query(ctx context.Context, mqo pbv1.MeasureQueryOptions) (pbv1
return &result, nil
}
-type tableIterator struct {
- sids []common.SeriesID
- tsTables []*tsTable
-}
-
-func (si *tableIterator) reset() {
-}
-
-// func (si *tableIterator) init(seriesTablesList []*seriesTables) {
-
-// }
-
-func (si *tableIterator) nextBlock() bool {
- return false
-}
-
-func generateTableIterator() *tableIterator {
- v := tiPool.Get()
- if v == nil {
- return &tableIterator{}
- }
- return v.(*tableIterator)
-}
-
-func releaseTableIterator(ti *tableIterator) {
- ti.reset()
- tiPool.Put(ti)
-}
-
-var tiPool sync.Pool
-
func (bc *blockCursor) copyAllTo(r *pbv1.Result) {
r.Timestamps = bc.timestamps
for _, cf := range bc.tagFamilies {
@@ -358,6 +320,9 @@ func (bc *blockCursor) copyTo(r *pbv1.Result) {
}
func mustDecodeTagValue(valueType pbv1.ValueType, value []byte) *modelv1.TagValue {
+ if value == nil {
+ return pbv1.NullTagValue
+ }
switch valueType {
case pbv1.ValueTypeInt64:
return &modelv1.TagValue{
@@ -420,6 +385,9 @@ func mustDecodeTagValue(valueType pbv1.ValueType, value []byte) *modelv1.TagValu
}
func mustDecodeFieldValue(valueType pbv1.ValueType, value []byte) *modelv1.FieldValue {
+ if value == nil {
+ return pbv1.NullFieldValue
+ }
switch valueType {
case pbv1.ValueTypeInt64:
return &modelv1.FieldValue{
diff --git a/banyand/measure/measure_write.go b/banyand/measure/measure_write.go
index cdee876c..12c136f5 100644
--- a/banyand/measure/measure_write.go
+++ b/banyand/measure/measure_write.go
@@ -50,11 +50,10 @@ func setUpWriteCallback(l *logger.Logger, schemaRepo *schemaRepo) bus.MessageLis
func (w *writeCallback) handle(dst map[string]*dataPointsInGroup, writeEvent *measurev1.InternalWriteRequest) (map[string]*dataPointsInGroup, error) {
req := writeEvent.Request
- tp := req.DataPoint.Timestamp.AsTime().Local()
- if err := timestamp.Check(tp); err != nil {
+ t := req.DataPoint.Timestamp.AsTime().Local()
+ if err := timestamp.Check(t); err != nil {
return nil, fmt.Errorf("invalid timestamp: %s", err)
}
- t := timestamp.MToN(tp)
ts := uint64(t.UnixNano())
gn := req.Metadata.Group
diff --git a/banyand/measure/part_iter.go b/banyand/measure/part_iter.go
index 84a93d0a..87cc007e 100644
--- a/banyand/measure/part_iter.go
+++ b/banyand/measure/part_iter.go
@@ -63,13 +63,13 @@ func (pi *partIter) reset() {
pi.err = nil
}
-func (pi *partIter) init(p *part, sids []common.SeriesID, opt *QueryOptions) {
+func (pi *partIter) init(p *part, sids []common.SeriesID, minTimestamp, maxTimestamp int64) {
pi.reset()
pi.p = p
pi.sids = sids
- pi.minTimestamp = opt.minTimestamp
- pi.maxTimestamp = opt.maxTimestamp
+ pi.minTimestamp = minTimestamp
+ pi.maxTimestamp = maxTimestamp
pi.primaryBlockMetadata = p.primaryBlockMetadata
diff --git a/banyand/measure/part_iter_test.go b/banyand/measure/part_iter_test.go
index 58e8c7af..5ecc854d 100644
--- a/banyand/measure/part_iter_test.go
+++ b/banyand/measure/part_iter_test.go
@@ -114,7 +114,7 @@ func Test_partIter_nextBlock(t *testing.T) {
p := openMemPart(mp)
pi := partIter{}
- pi.init(p, tt.sids, tt.opt)
+ pi.init(p, tt.sids, tt.opt.minTimestamp, tt.opt.maxTimestamp)
var got []blockMetadata
for pi.nextBlock() {
diff --git a/banyand/measure/part_metadata.go b/banyand/measure/part_metadata.go
index 9a79040d..6c6f6e17 100644
--- a/banyand/measure/part_metadata.go
+++ b/banyand/measure/part_metadata.go
@@ -24,6 +24,7 @@ type partMetadata struct {
BlocksCount uint64
MinTimestamp int64
MaxTimestamp int64
+ Version int64
}
func (ph *partMetadata) reset() {
@@ -33,4 +34,5 @@ func (ph *partMetadata) reset() {
ph.BlocksCount = 0
ph.MinTimestamp = 0
ph.MaxTimestamp = 0
+ ph.Version = 0
}
diff --git a/banyand/measure/tstable.go b/banyand/measure/tstable.go
index 7f758c02..3d900bfc 100644
--- a/banyand/measure/tstable.go
+++ b/banyand/measure/tstable.go
@@ -38,6 +38,11 @@ type tsTable struct {
}
func (tst *tsTable) Close() error {
+ tst.Lock()
+ defer tst.Unlock()
+ for _, p := range tst.memParts {
+ p.decRef()
+ }
return nil
}
@@ -73,8 +78,6 @@ func (tst *tsTable) getParts(dst []*partWrapper, dstPart []*part, opts *QueryOpt
}
type tstIter struct {
- queryOpts *QueryOptions
-
parts []*part
piPool []partIter
@@ -105,17 +108,16 @@ func (ti *tstIter) reset() {
ti.nextBlockNoop = false
}
-func (ti *tstIter) init(parts []*part, sids []common.SeriesID, queryOpts *QueryOptions) {
+func (ti *tstIter) init(parts []*part, sids []common.SeriesID, minTimestamp, maxTimestamp int64) {
ti.reset()
ti.parts = parts
- ti.queryOpts = queryOpts
if n := len(ti.parts) - cap(ti.piPool); n > 0 {
ti.piPool = append(ti.piPool[:cap(ti.piPool)], make([]partIter, n)...)
}
ti.piPool = ti.piPool[:len(ti.parts)]
for i, p := range ti.parts {
- ti.piPool[i].init(p, sids, queryOpts)
+ ti.piPool[i].init(p, sids, minTimestamp, maxTimestamp)
}
ti.piHeap = ti.piHeap[:0]
@@ -123,7 +125,7 @@ func (ti *tstIter) init(parts []*part, sids []common.SeriesID, queryOpts *QueryO
ps := &ti.piPool[i]
if !ps.nextBlock() {
if err := ps.Error(); err != nil {
- ti.err = fmt.Errorf("cannot initialize tstable iteration: %w", err)
+ ti.err = fmt.Errorf("cannot initialize tsTable iteration: %w", err)
return
}
continue
@@ -138,7 +140,7 @@ func (ti *tstIter) init(parts []*part, sids []common.SeriesID, queryOpts *QueryO
ti.nextBlockNoop = true
}
-func (ti *tstIter) NextBlock() bool {
+func (ti *tstIter) nextBlock() bool {
if ti.err != nil {
return false
}
@@ -147,7 +149,7 @@ func (ti *tstIter) NextBlock() bool {
return true
}
- ti.err = ti.nextBlock()
+ ti.err = ti.next()
if ti.err != nil {
if ti.err != io.EOF {
ti.err = fmt.Errorf("cannot obtain the next block to search in the partition: %w", ti.err)
@@ -157,7 +159,7 @@ func (ti *tstIter) NextBlock() bool {
return true
}
-func (ti *tstIter) nextBlock() error {
+func (ti *tstIter) next() error {
psMin := ti.piHeap[0]
if psMin.nextBlock() {
heap.Fix(&ti.piHeap, 0)
diff --git a/banyand/measure/tstable_test.go b/banyand/measure/tstable_test.go
new file mode 100644
index 00000000..0f7ee9c4
--- /dev/null
+++ b/banyand/measure/tstable_test.go
@@ -0,0 +1,386 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package measure
+
+import (
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "github.com/apache/skywalking-banyandb/api/common"
+ "github.com/apache/skywalking-banyandb/pkg/convert"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
+)
+
+func Test_tsTable_mustAddDataPoints(t *testing.T) {
+ tests := []struct {
+ name string
+ dpsList []*dataPoints
+ want int
+ }{
+ {
+ name: "Test with empty dataPoints",
+ dpsList: []*dataPoints{
+ {
+ timestamps: []int64{},
+ seriesIDs: []common.SeriesID{},
+ tagFamilies: make([][]nameValues, 0),
+ fields: make([]nameValues, 0),
+ },
+ },
+ want: 0,
+ },
+ {
+ name: "Test with one item in dataPoints",
+ dpsList: []*dataPoints{
+ {
+ timestamps: []int64{1},
+ seriesIDs: []common.SeriesID{1},
+ tagFamilies: [][]nameValues{{{
+ "arrTag", []*nameValue{
+ {"strArrTag", pbv1.ValueTypeStrArr, nil, [][]byte{[]byte("value1"), []byte("value2")}},
+ {"intArrTag", pbv1.ValueTypeInt64Arr, nil, [][]byte{convert.Int64ToBytes(25), convert.Int64ToBytes(30)}},
+ },
+ }}},
+ fields: []nameValues{{
+ "skipped", []*nameValue{
+ {"intField", pbv1.ValueTypeInt64, convert.Int64ToBytes(1110), nil},
+ },
+ }},
+ },
+ },
+ want: 1,
+ },
+ {
+ name: "Test with multiple calls to mustAddDataPoints",
+ dpsList: []*dataPoints{
+ dps_ts1,
+ dps_ts2,
+ },
+ want: 2,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ tst := &tsTable{}
+ defer tst.Close()
+ for _, dps := range tt.dpsList {
+ tst.mustAddDataPoints(dps)
+ time.Sleep(100 * time.Millisecond)
+ }
+ assert.Equal(t, tt.want, len(tst.memParts))
+ var lastVersion int64
+ for _, pw := range tst.memParts {
+ if lastVersion == 0 {
+ lastVersion = pw.p.partMetadata.Version
+ } else {
+ require.Less(t, lastVersion, pw.p.partMetadata.Version)
+ }
+ }
+ })
+ }
+}
+
+func Test_tstIter_init(t *testing.T) {
+ tests := []struct {
+ name string
+ dpsList []*dataPoints
+ sids []common.SeriesID
+ minTimestamp int64
+ maxTimestamp int64
+ want []blockMetadata
+ wantErr error
+ }{
+ {
+ name: "Test with no data points",
+ dpsList: []*dataPoints{},
+ sids: []common.SeriesID{1, 2, 3},
+ minTimestamp: 1,
+ maxTimestamp: 1,
+ },
+ {
+ name: "Test with single part",
+ dpsList: []*dataPoints{dps_ts1},
+ sids: []common.SeriesID{1, 2, 3},
+ minTimestamp: 1,
+ maxTimestamp: 1,
+ want: []blockMetadata{
+ {seriesID: 1, count: 2, uncompressedSizeBytes: 3426},
+ {seriesID: 2, count: 2, uncompressedSizeBytes: 106},
+ {seriesID: 3, count: 2, uncompressedSizeBytes: 48},
+ },
+ },
+ {
+ name: "Test with multiple parts with different ts",
+ dpsList: []*dataPoints{dps_ts1, dps_ts2},
+ sids: []common.SeriesID{1, 2, 3},
+ minTimestamp: 1,
+ maxTimestamp: 2,
+ want: []blockMetadata{
+ {seriesID: 1, count: 2, uncompressedSizeBytes: 3426},
+ {seriesID: 1, count: 2, uncompressedSizeBytes: 3426},
+ {seriesID: 2, count: 2, uncompressedSizeBytes: 106},
+ {seriesID: 2, count: 2, uncompressedSizeBytes: 106},
+ {seriesID: 3, count: 2, uncompressedSizeBytes: 48},
+ {seriesID: 3, count: 2, uncompressedSizeBytes: 48},
+ },
+ },
+ {
+ name: "Test with multiple parts with same ts",
+ dpsList: []*dataPoints{dps_ts1, dps_ts1},
+ sids: []common.SeriesID{1, 2, 3},
+ minTimestamp: 1,
+ maxTimestamp: 2,
+ want: []blockMetadata{
+ {seriesID: 1, count: 2, uncompressedSizeBytes: 3426},
+ {seriesID: 1, count: 2, uncompressedSizeBytes: 3426},
+ {seriesID: 2, count: 2, uncompressedSizeBytes: 106},
+ {seriesID: 2, count: 2, uncompressedSizeBytes: 106},
+ {seriesID: 3, count: 2, uncompressedSizeBytes: 48},
+ {seriesID: 3, count: 2, uncompressedSizeBytes: 48},
+ },
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ tst := &tsTable{}
+ defer tst.Close()
+ for _, dps := range tt.dpsList {
+ tst.mustAddDataPoints(dps)
+ time.Sleep(100 * time.Millisecond)
+ }
+ pws, pp := tst.getParts(nil, nil, &QueryOptions{
+ minTimestamp: tt.minTimestamp,
+ maxTimestamp: tt.maxTimestamp,
+ })
+ defer func() {
+ for _, pw := range pws {
+ pw.decRef()
+ }
+ }()
+ ti := &tstIter{}
+ ti.init(pp, tt.sids, tt.minTimestamp, tt.maxTimestamp)
+ var got []blockMetadata
+ for ti.nextBlock() {
+ if ti.piHeap[0].curBlock.seriesID == 0 {
+ t.Errorf("Expected curBlock to be initialized, but it was nil")
+ }
+ got = append(got, ti.piHeap[0].curBlock)
+ }
+
+ if ti.Error() != tt.wantErr {
+ t.Errorf("Unexpected error: got %v, want %v", ti.err, tt.wantErr)
+ }
+
+ if diff := cmp.Diff(got, tt.want,
+ cmpopts.IgnoreFields(blockMetadata{}, "timestamps"),
+ cmpopts.IgnoreFields(blockMetadata{}, "field"),
+ cmpopts.IgnoreFields(blockMetadata{}, "tagFamilies"),
+ cmp.AllowUnexported(blockMetadata{}),
+ ); diff != "" {
+ t.Errorf("Unexpected blockMetadata (-got +want):\n%s", diff)
+ }
+ })
+ }
+}
+
+var dps_ts1 = &dataPoints{
+ seriesIDs: []common.SeriesID{1, 1, 2, 2, 3, 3},
+ timestamps: []int64{1, 1, 1, 1, 1, 1},
+ tagFamilies: [][]nameValues{
+ {
+ {
+ "arrTag", []*nameValue{
+ {"strArrTag", pbv1.ValueTypeStrArr, nil, [][]byte{[]byte("value1"), []byte("value2")}},
+ {"intArrTag", pbv1.ValueTypeInt64Arr, nil, [][]byte{convert.Int64ToBytes(25), convert.Int64ToBytes(30)}},
+ },
+ }, {
+ "binaryTag", []*nameValue{
+ {"binaryTag", pbv1.ValueTypeBinaryData, []byte(longText), nil},
+ },
+ }, {
+ "singleTag", []*nameValue{
+ {"strTag", pbv1.ValueTypeStr, []byte("value1"), nil},
+ {"intTag", pbv1.ValueTypeInt64, convert.Int64ToBytes(10), nil},
+ {"floatTag", pbv1.ValueTypeFloat64, convert.Float64ToBytes(12233.343), nil},
+ },
+ },
+ },
+ {
+ {
+ "arrTag", []*nameValue{
+ {"strArrTag", pbv1.ValueTypeStrArr, nil, [][]byte{[]byte("value3"), []byte("value4")}},
+ {"intArrTag", pbv1.ValueTypeInt64Arr, nil, [][]byte{convert.Int64ToBytes(50), convert.Int64ToBytes(60)}},
+ },
+ }, {
+ "binaryTag", []*nameValue{
+ {"binaryTag", pbv1.ValueTypeBinaryData, []byte(longText), nil},
+ },
+ }, {
+ "singleTag", []*nameValue{
+ {"strTag", pbv1.ValueTypeStr, []byte("value2"), nil},
+ {"intTag", pbv1.ValueTypeInt64, convert.Int64ToBytes(20), nil},
+ {"floatTag", pbv1.ValueTypeFloat64, convert.Float64ToBytes(24466.686), nil},
+ },
+ },
+ },
+ {
+ {
+ "singleTag", []*nameValue{
+ {"strTag", pbv1.ValueTypeStr, []byte("tag1"), nil},
+ {"strTag", pbv1.ValueTypeInt64, []byte("tag2"), nil},
+ },
+ },
+ },
+ {
+ {
+ "singleTag", []*nameValue{
+ {"strTag", pbv1.ValueTypeStr, []byte("tag11"), nil},
+ {"strTag", pbv1.ValueTypeInt64, []byte("tag22"), nil},
+ },
+ },
+ },
+ {},
+ {}, // empty tagFamilies for seriesID 3
+ },
+ fields: []nameValues{
+ {
+ "skipped", []*nameValue{
+ {"strField", pbv1.ValueTypeStr, []byte("field1"), nil},
+ {"intField", pbv1.ValueTypeInt64, convert.Int64ToBytes(1110), nil},
+ {"floatField", pbv1.ValueTypeFloat64, convert.Float64ToBytes(1221233.343), nil},
+ {"binaryField", pbv1.ValueTypeBinaryData, []byte(longText), nil},
+ },
+ },
+ {
+ "skipped", []*nameValue{
+ {"strField", pbv1.ValueTypeStr, []byte("field2"), nil},
+ {"intField", pbv1.ValueTypeInt64, convert.Int64ToBytes(2220), nil},
+ {"floatField", pbv1.ValueTypeFloat64, convert.Float64ToBytes(2442466.686), nil},
+ {"binaryField", pbv1.ValueTypeBinaryData, []byte(longText), nil},
+ },
+ },
+ {},
+ {}, // empty fields for seriesID 2
+ {
+ "onlyFields", []*nameValue{
+ {"intField", pbv1.ValueTypeInt64, convert.Int64ToBytes(1110), nil},
+ },
+ },
+ {
+ "onlyFields", []*nameValue{
+ {"intField", pbv1.ValueTypeInt64, convert.Int64ToBytes(2220), nil},
+ },
+ },
+ },
+}
+
+var dps_ts2 = &dataPoints{
+ seriesIDs: []common.SeriesID{1, 1, 2, 2, 3, 3},
+ timestamps: []int64{2, 2, 2, 2, 2, 2},
+ tagFamilies: [][]nameValues{
+ {
+ {
+ "arrTag", []*nameValue{
+ {"strArrTag", pbv1.ValueTypeStrArr, nil, [][]byte{[]byte("value5"), []byte("value6")}},
+ {"intArrTag", pbv1.ValueTypeInt64Arr, nil, [][]byte{convert.Int64ToBytes(35), convert.Int64ToBytes(40)}},
+ },
+ }, {
+ "binaryTag", []*nameValue{
+ {"binaryTag", pbv1.ValueTypeBinaryData, []byte(longText), nil},
+ },
+ }, {
+ "singleTag", []*nameValue{
+ {"strTag", pbv1.ValueTypeStr, []byte("value3"), nil},
+ {"intTag", pbv1.ValueTypeInt64, convert.Int64ToBytes(30), nil},
+ {"floatTag", pbv1.ValueTypeFloat64, convert.Float64ToBytes(36699.029), nil},
+ },
+ },
+ },
+ {
+ {
+ "arrTag", []*nameValue{
+ {"strArrTag", pbv1.ValueTypeStrArr, nil, [][]byte{[]byte("value7"), []byte("value8")}},
+ {"intArrTag", pbv1.ValueTypeInt64Arr, nil, [][]byte{convert.Int64ToBytes(70), convert.Int64ToBytes(80)}},
+ },
+ }, {
+ "binaryTag", []*nameValue{
+ {"binaryTag", pbv1.ValueTypeBinaryData, []byte(longText), nil},
+ },
+ }, {
+ "singleTag", []*nameValue{
+ {"strTag", pbv1.ValueTypeStr, []byte("value4"), nil},
+ {"intTag", pbv1.ValueTypeInt64, convert.Int64ToBytes(40), nil},
+ {"floatTag", pbv1.ValueTypeFloat64, convert.Float64ToBytes(48999.372), nil},
+ },
+ },
+ },
+ {
+ {
+ "singleTag", []*nameValue{
+ {"strTag", pbv1.ValueTypeStr, []byte("tag3"), nil},
+ {"strTag", pbv1.ValueTypeInt64, []byte("tag4"), nil},
+ },
+ },
+ },
+ {
+ {
+ "singleTag", []*nameValue{
+ {"strTag", pbv1.ValueTypeStr, []byte("tag33"), nil},
+ {"strTag", pbv1.ValueTypeInt64, []byte("tag44"), nil},
+ },
+ },
+ },
+ {},
+ {}, // empty tagFamilies for seriesID 6
+ },
+ fields: []nameValues{
+ {
+ "skipped", []*nameValue{
+ {"strField", pbv1.ValueTypeStr, []byte("field3"), nil},
+ {"intField", pbv1.ValueTypeInt64, convert.Int64ToBytes(3330), nil},
+ {"floatField", pbv1.ValueTypeFloat64, convert.Float64ToBytes(3663699.029), nil},
+ {"binaryField", pbv1.ValueTypeBinaryData, []byte(longText), nil},
+ },
+ },
+ {
+ "skipped", []*nameValue{
+ {"strField", pbv1.ValueTypeStr, []byte("field4"), nil},
+ {"intField", pbv1.ValueTypeInt64, convert.Int64ToBytes(4440), nil},
+ {"floatField", pbv1.ValueTypeFloat64, convert.Float64ToBytes(4899899.372), nil},
+ {"binaryField", pbv1.ValueTypeBinaryData, []byte(longText), nil},
+ },
+ },
+ {},
+ {}, // empty fields for seriesID 5
+ {
+ "onlyFields", []*nameValue{
+ {"intField", pbv1.ValueTypeInt64, convert.Int64ToBytes(3330), nil},
+ },
+ },
+ {
+ "onlyFields", []*nameValue{
+ {"intField", pbv1.ValueTypeInt64, convert.Int64ToBytes(4440), nil},
+ },
+ },
+ },
+}
(skywalking-banyandb) 02/07: Test part
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch storage-column
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit 3fe134f3170a679ae3d8c408f599ea5a047aafd4
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Thu Dec 14 05:53:25 2023 +0800
Test part
Signed-off-by: Gao Hongtao <ha...@gmail.com>
---
banyand/measure/block.go | 9 +-
banyand/measure/block_metadata.go | 2 +-
banyand/measure/measure.go | 2 +
banyand/measure/measure_query.go | 2 +-
banyand/measure/part.go | 9 +-
banyand/measure/part_iter.go | 166 +++++++++++++++----------------
banyand/measure/part_iter_test.go | 142 +++++++++++++++++++++++++++
banyand/measure/part_test.go | 182 ++++++++++++++++++++++++++++++++++
banyand/measure/tstable.go | 201 ++------------------------------------
pkg/bytes/buffer.go | 8 +-
10 files changed, 434 insertions(+), 289 deletions(-)
diff --git a/banyand/measure/block.go b/banyand/measure/block.go
index 2c0407ce..8fd64781 100644
--- a/banyand/measure/block.go
+++ b/banyand/measure/block.go
@@ -322,14 +322,14 @@ type blockCursor struct {
columnValuesDecoder encoding.BytesBlockDecoder
p *part
- bm *blockMetadata
+ bm blockMetadata
opts *QueryOptions
}
func (bc *blockCursor) reset() {
bc.idx = 0
bc.p = nil
- bc.bm = nil
+ bc.bm = blockMetadata{}
bc.opts = nil
bc.timestamps = bc.timestamps[:0]
@@ -342,7 +342,7 @@ func (bc *blockCursor) reset() {
bc.fields.reset()
}
-func (bc *blockCursor) init(p *part, bm *blockMetadata, opts *QueryOptions) {
+func (bc *blockCursor) init(p *part, bm blockMetadata, opts *QueryOptions) {
bc.reset()
bc.p = p
bc.bm = bm
@@ -350,9 +350,8 @@ func (bc *blockCursor) init(p *part, bm *blockMetadata, opts *QueryOptions) {
}
func (bc *blockCursor) loadData(tmpBlock *block) bool {
- bc.reset()
tmpBlock.reset()
- tmpBlock.mustReadFrom(&bc.columnValuesDecoder, bc.p, bc.bm)
+ tmpBlock.mustReadFrom(&bc.columnValuesDecoder, bc.p, &bc.bm)
start, end, ok := findRange(tmpBlock.timestamps, bc.opts.minTimestamp, bc.opts.maxTimestamp)
if !ok {
diff --git a/banyand/measure/block_metadata.go b/banyand/measure/block_metadata.go
index 857f9125..b77f0b03 100644
--- a/banyand/measure/block_metadata.go
+++ b/banyand/measure/block_metadata.go
@@ -175,7 +175,7 @@ func (bh *blockMetadata) unmarshal(src []byte) ([]byte, error) {
return src, nil
}
-func (bh *blockMetadata) less(other *blockMetadata) bool {
+func (bh blockMetadata) less(other blockMetadata) bool {
if bh.seriesID == other.seriesID {
return bh.timestamps.min < other.timestamps.min
}
diff --git a/banyand/measure/measure.go b/banyand/measure/measure.go
index 739698b8..bfe7d7b2 100644
--- a/banyand/measure/measure.go
+++ b/banyand/measure/measure.go
@@ -44,6 +44,8 @@ const (
maxTagFamiliesMetadataSize = 8 * 1024 * 1024
maxUncompressedBlockSize = 2 * 1024 * 1024
maxUncompressedPrimaryBlockSize = 128 * 1024
+
+ maxBlockLength = 8 * 1024
)
type measure struct {
diff --git a/banyand/measure/measure_query.go b/banyand/measure/measure_query.go
index 2175459c..4b2960c9 100644
--- a/banyand/measure/measure_query.go
+++ b/banyand/measure/measure_query.go
@@ -243,7 +243,7 @@ func (s *measure) Query(ctx context.Context, mqo pbv1.MeasureQueryOptions) (pbv1
for tstIter.NextBlock() {
bc := generateBlockCursor()
p := tstIter.piHeap[0]
- bc.init(p.p, p.bm, qo)
+ bc.init(p.p, p.curBlock, qo)
result.data = append(result.data, bc)
}
if mqo.Order != nil {
diff --git a/banyand/measure/part.go b/banyand/measure/part.go
index dd211449..d24b70d4 100644
--- a/banyand/measure/part.go
+++ b/banyand/measure/part.go
@@ -129,7 +129,8 @@ func (mp *memPart) mustInitFromDataPoints(dps *dataPoints) {
sidPrev = sid
}
- if uncompressedBlockSizeBytes >= maxUncompressedBlockSize || sid != sidPrev {
+ if uncompressedBlockSizeBytes >= maxUncompressedBlockSize ||
+ (i-indexPrev) > maxBlockLength || sid != sidPrev {
bsw.MustWriteDataPoints(sidPrev, dps.timestamps[indexPrev:i], dps.tagFamilies[indexPrev:i], dps.fields[indexPrev:i])
sidPrev = sid
indexPrev = i
@@ -157,7 +158,7 @@ func uncompressedDataPointSizeBytes(index int, dps *dataPoints) uint64 {
return n
}
-func getMemPart() *memPart {
+func generateMemPart() *memPart {
v := memPartPool.Get()
if v == nil {
return &memPart{}
@@ -165,7 +166,7 @@ func getMemPart() *memPart {
return v.(*memPart)
}
-func putMemPart(mp *memPart) {
+func releaseMemPart(mp *memPart) {
mp.reset()
memPartPool.Put(mp)
}
@@ -193,7 +194,7 @@ func (pw *partWrapper) decRef() {
return
}
if pw.mp != nil {
- putMemPart(pw.mp)
+ releaseMemPart(pw.mp)
pw.mp = nil
}
}
diff --git a/banyand/measure/part_iter.go b/banyand/measure/part_iter.go
index 87628fa4..84a93d0a 100644
--- a/banyand/measure/part_iter.go
+++ b/banyand/measure/part_iter.go
@@ -20,9 +20,7 @@ package measure
import (
"fmt"
"io"
- "os"
"sort"
- "strings"
"github.com/apache/skywalking-banyandb/api/common"
"github.com/apache/skywalking-banyandb/pkg/bytes"
@@ -32,7 +30,7 @@ import (
)
type partIter struct {
- bm *blockMetadata
+ curBlock blockMetadata
p *part
@@ -48,122 +46,126 @@ type partIter struct {
bms []blockMetadata
compressedPrimaryBuf []byte
- indexBuf []byte
+ primaryBuf []byte
err error
}
-func (ps *partIter) reset() {
- ps.bm = nil
- ps.p = nil
- ps.sids = nil
- ps.sidIdx = 0
- ps.primaryBlockMetadata = nil
- ps.bms = nil
- ps.compressedPrimaryBuf = ps.compressedPrimaryBuf[:0]
- ps.indexBuf = ps.indexBuf[:0]
- ps.err = nil
+func (pi *partIter) reset() {
+ pi.curBlock = blockMetadata{}
+ pi.p = nil
+ pi.sids = nil
+ pi.sidIdx = 0
+ pi.primaryBlockMetadata = nil
+ pi.bms = nil
+ pi.compressedPrimaryBuf = pi.compressedPrimaryBuf[:0]
+ pi.primaryBuf = pi.primaryBuf[:0]
+ pi.err = nil
}
-var isInTest = func() bool {
- return strings.HasSuffix(os.Args[0], ".test")
-}()
+func (pi *partIter) init(p *part, sids []common.SeriesID, opt *QueryOptions) {
+ pi.reset()
+ pi.p = p
-func (ps *partIter) init(p *part, sids []common.SeriesID, opt *QueryOptions) {
- ps.reset()
- ps.p = p
+ pi.sids = sids
+ pi.minTimestamp = opt.minTimestamp
+ pi.maxTimestamp = opt.maxTimestamp
- ps.sids = sids
- ps.minTimestamp = opt.minTimestamp
- ps.maxTimestamp = opt.maxTimestamp
+ pi.primaryBlockMetadata = p.primaryBlockMetadata
- ps.primaryBlockMetadata = p.primaryBlockMetadata
-
- ps.nextSeriesID()
+ pi.nextSeriesID()
}
-func (ps *partIter) NextBlock() bool {
+func (pi *partIter) nextBlock() bool {
for {
- if ps.err != nil {
+ if pi.err != nil {
return false
}
- if len(ps.bms) == 0 {
- if !ps.nextBlockMetadata() {
+ if len(pi.bms) == 0 {
+ if !pi.loadNextBlockMetadata() {
return false
}
}
- if ps.searchBlockMetadata() {
+ if pi.findBlock() {
return true
}
}
}
-func (ps *partIter) Error() error {
- if ps.err == io.EOF {
+func (pi *partIter) Error() error {
+ if pi.err == io.EOF {
return nil
}
- return ps.err
+ return pi.err
}
-func (ps *partIter) nextSeriesID() bool {
- if ps.sidIdx >= len(ps.sids) {
- ps.err = io.EOF
+func (pi *partIter) nextSeriesID() bool {
+ if pi.sidIdx >= len(pi.sids) {
+ pi.err = io.EOF
return false
}
- ps.bm.seriesID = ps.sids[ps.sidIdx]
- ps.sidIdx++
+ pi.curBlock.seriesID = pi.sids[pi.sidIdx]
+ pi.sidIdx++
return true
}
-func (ps *partIter) skipSeriesIDsSmallerThan(sid common.SeriesID) bool {
- sids := ps.sids[ps.sidIdx:]
- ps.sidIdx += sort.Search(len(sids), func(i int) bool {
+func (pi *partIter) searchTargetSeriesID(sid common.SeriesID) bool {
+ if pi.curBlock.seriesID >= sid {
+ return true
+ }
+ if !pi.nextSeriesID() {
+ return false
+ }
+ if pi.curBlock.seriesID >= sid {
+ return true
+ }
+ sids := pi.sids[pi.sidIdx:]
+ pi.sidIdx += sort.Search(len(sids), func(i int) bool {
return sid < sids[i]
})
- if ps.sidIdx >= len(ps.sids) {
- ps.sidIdx = len(ps.sids)
- ps.err = io.EOF
+ if pi.sidIdx >= len(pi.sids) {
+ pi.sidIdx = len(pi.sids)
+ pi.err = io.EOF
return false
}
- ps.bm.seriesID = ps.sids[ps.sidIdx]
- ps.sidIdx++
+ pi.curBlock.seriesID = pi.sids[pi.sidIdx]
+ pi.sidIdx++
return true
}
-func (ps *partIter) nextBlockMetadata() bool {
- for len(ps.primaryBlockMetadata) > 0 {
- if !ps.skipSeriesIDsSmallerThan(ps.primaryBlockMetadata[0].seriesID) {
+func (pi *partIter) loadNextBlockMetadata() bool {
+ for len(pi.primaryBlockMetadata) > 0 {
+ if !pi.searchTargetSeriesID(pi.primaryBlockMetadata[0].seriesID) {
return false
}
- ps.primaryBlockMetadata = skipSmallMetaindexRows(ps.primaryBlockMetadata, ps.bm.seriesID)
+ pi.primaryBlockMetadata = searchPBM(pi.primaryBlockMetadata, pi.curBlock.seriesID)
- mr := &ps.primaryBlockMetadata[0]
- ps.primaryBlockMetadata = ps.primaryBlockMetadata[1:]
- if ps.bm.seriesID < mr.seriesID {
- logger.Panicf("BUG: invariant violation: ps.BlockRef.bh.TSID cannot be smaller than mr.TSID; got %+v vs %+v", &ps.bm.seriesID, &mr.seriesID)
+ pbm := &pi.primaryBlockMetadata[0]
+ pi.primaryBlockMetadata = pi.primaryBlockMetadata[1:]
+ if pi.curBlock.seriesID < pbm.seriesID {
+ logger.Panicf("invariant violation: pi.curBlock.seriesID cannot be smaller than pbm.seriesID; got %+v vs %+v", &pi.curBlock.seriesID, &pbm.seriesID)
}
- if mr.maxTimestamp < ps.minTimestamp || mr.minTimestamp > ps.maxTimestamp {
+ if pbm.maxTimestamp < pi.minTimestamp || pbm.minTimestamp > pi.maxTimestamp {
continue
}
- bm, err := ps.readPrimaryBlock(mr)
+ bm, err := pi.readPrimaryBlock(pbm)
if err != nil {
- ps.err = fmt.Errorf("cannot read index block for part %q at offset %d with size %d: %w",
- &ps.p.partMetadata, mr.offset, mr.size, err)
+ pi.err = fmt.Errorf("cannot read primary block for part %q at offset %d with size %d: %w",
+ &pi.p.partMetadata, pbm.offset, pbm.size, err)
return false
}
- ps.bms = bm
+ pi.bms = bm
return true
}
-
- ps.err = io.EOF
+ pi.err = io.EOF
return false
}
-func skipSmallMetaindexRows(pbmIndex []primaryBlockMetadata, sid common.SeriesID) []primaryBlockMetadata {
+func searchPBM(pbmIndex []primaryBlockMetadata, sid common.SeriesID) []primaryBlockMetadata {
if sid < pbmIndex[0].seriesID {
- logger.Panicf("BUG: invariant violation: tsid cannot be smaller than metaindex[0]; got %d vs %d", sid, &pbmIndex[0].seriesID)
+ logger.Panicf("invariant violation: sid cannot be smaller than pbmIndex[0]; got %d vs %d", sid, &pbmIndex[0].seriesID)
}
if sid == pbmIndex[0].seriesID {
@@ -171,39 +173,39 @@ func skipSmallMetaindexRows(pbmIndex []primaryBlockMetadata, sid common.SeriesID
}
n := sort.Search(len(pbmIndex), func(i int) bool {
- return sid > pbmIndex[i].seriesID
+ return sid <= pbmIndex[i].seriesID
})
if n == 0 {
- logger.Panicf("BUG: invariant violation: sort.Search returned 0 for tsid > metaindex[0].TSID; tsid=%+v; metaindex[0].TSID=%+v",
+ logger.Panicf("invariant violation: sort.Search returned 0 for sid > pbmIndex[0].seriesID; sid=%+v; pbmIndex[0].seriesID=%+v",
sid, &pbmIndex[0].seriesID)
}
return pbmIndex[n-1:]
}
-func (ps *partIter) readPrimaryBlock(mr *primaryBlockMetadata) ([]blockMetadata, error) {
- ps.compressedPrimaryBuf = bytes.ResizeOver(ps.compressedPrimaryBuf, int(mr.size))
- fs.MustReadData(ps.p.primary, int64(mr.offset), ps.compressedPrimaryBuf)
+func (pi *partIter) readPrimaryBlock(mr *primaryBlockMetadata) ([]blockMetadata, error) {
+ pi.compressedPrimaryBuf = bytes.ResizeOver(pi.compressedPrimaryBuf, int(mr.size))
+ fs.MustReadData(pi.p.primary, int64(mr.offset), pi.compressedPrimaryBuf)
var err error
- ps.indexBuf, err = zstd.Decompress(ps.indexBuf[:0], ps.compressedPrimaryBuf)
+ pi.primaryBuf, err = zstd.Decompress(pi.primaryBuf[:0], pi.compressedPrimaryBuf)
if err != nil {
return nil, fmt.Errorf("cannot decompress index block: %w", err)
}
bm := make([]blockMetadata, 0)
- bm, err = unmarshalBlockMetadata(bm, ps.indexBuf)
+ bm, err = unmarshalBlockMetadata(bm, pi.primaryBuf)
if err != nil {
return nil, fmt.Errorf("cannot unmarshal index block: %w", err)
}
return bm, nil
}
-func (ps *partIter) searchBlockMetadata() bool {
- bhs := ps.bms
+func (pi *partIter) findBlock() bool {
+ bhs := pi.bms
for len(bhs) > 0 {
- sid := ps.bm.seriesID
+ sid := pi.curBlock.seriesID
if bhs[0].seriesID < sid {
n := sort.Search(len(bhs), func(i int) bool {
- return sid > bhs[i].seriesID
+ return sid <= bhs[i].seriesID
})
if n == len(bhs) {
break
@@ -213,28 +215,28 @@ func (ps *partIter) searchBlockMetadata() bool {
bm := &bhs[0]
if bm.seriesID != sid {
- if !ps.skipSeriesIDsSmallerThan(bm.seriesID) {
+ if !pi.searchTargetSeriesID(bm.seriesID) {
return false
}
continue
}
- if bm.timestamps.max < ps.minTimestamp {
+ if bm.timestamps.max < pi.minTimestamp {
bhs = bhs[1:]
continue
}
- if bm.timestamps.min > ps.maxTimestamp {
- if !ps.nextSeriesID() {
+ if bm.timestamps.min > pi.maxTimestamp {
+ if !pi.nextSeriesID() {
return false
}
continue
}
- ps.bm = bm
+ pi.curBlock = *bm
- ps.bms = bhs[1:]
+ pi.bms = bhs[1:]
return true
}
- ps.bms = nil
+ pi.bms = nil
return false
}
diff --git a/banyand/measure/part_iter_test.go b/banyand/measure/part_iter_test.go
new file mode 100644
index 00000000..58e8c7af
--- /dev/null
+++ b/banyand/measure/part_iter_test.go
@@ -0,0 +1,142 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package measure
+
+import (
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+
+ "github.com/apache/skywalking-banyandb/api/common"
+)
+
+// TODO: test more scenarios
+func Test_partIter_nextBlock(t *testing.T) {
+ tests := []struct {
+ name string
+ sids []common.SeriesID
+ opt *QueryOptions
+ want []blockMetadata
+ wantErr error
+ }{
+ {
+ name: "Test with all seriesIDs",
+ sids: []common.SeriesID{1, 2, 3},
+ opt: &QueryOptions{
+ minTimestamp: 1,
+ maxTimestamp: 220,
+ },
+ want: []blockMetadata{
+ {seriesID: 1, count: 2}, {seriesID: 2, count: 2}, {seriesID: 3, count: 2},
+ },
+ },
+ {
+ name: "Test with no seriesIDs",
+ sids: []common.SeriesID{},
+ opt: &QueryOptions{
+ minTimestamp: 1,
+ maxTimestamp: 220,
+ },
+ want: nil,
+ },
+ {
+ name: "Test with a single seriesID",
+ sids: []common.SeriesID{1},
+ opt: &QueryOptions{
+ minTimestamp: 1,
+ maxTimestamp: 220,
+ },
+ want: []blockMetadata{{seriesID: 1, count: 2}},
+ },
+ {
+ name: "Test with non-sequential seriesIDs",
+ sids: []common.SeriesID{1, 3},
+ opt: &QueryOptions{
+ minTimestamp: 1,
+ maxTimestamp: 220,
+ },
+ want: []blockMetadata{{seriesID: 1, count: 2}, {seriesID: 3, count: 2}},
+ },
+ {
+ name: "Test with seriesID not in data",
+ sids: []common.SeriesID{4},
+ opt: &QueryOptions{
+ minTimestamp: 1,
+ maxTimestamp: 220,
+ },
+ want: nil,
+ wantErr: nil,
+ },
+ {
+ name: "Test with multiple seriesIDs not in data",
+ sids: []common.SeriesID{4, 5, 6},
+ opt: &QueryOptions{
+ minTimestamp: 1,
+ maxTimestamp: 220,
+ },
+ want: nil,
+ wantErr: nil,
+ },
+ {
+ name: "Test with some seriesIDs in data and some not",
+ sids: []common.SeriesID{1, 4},
+ opt: &QueryOptions{
+ minTimestamp: 1,
+ maxTimestamp: 220,
+ },
+ want: []blockMetadata{{seriesID: 1, count: 2}},
+ wantErr: nil,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ mp := generateMemPart()
+ releaseMemPart(mp)
+ mp.mustInitFromDataPoints(dps)
+
+ p := openMemPart(mp)
+
+ pi := partIter{}
+ pi.init(p, tt.sids, tt.opt)
+
+ var got []blockMetadata
+ for pi.nextBlock() {
+ if pi.curBlock.seriesID == 0 {
+ t.Errorf("Expected currBlock to be initialized, but it was nil")
+ }
+ got = append(got, pi.curBlock)
+ }
+
+ if pi.Error() != tt.wantErr {
+ t.Errorf("Unexpected error: got %v, want %v", pi.err, tt.wantErr)
+ }
+
+ if diff := cmp.Diff(got, tt.want,
+ cmpopts.IgnoreFields(blockMetadata{}, "uncompressedSizeBytes"),
+ cmpopts.IgnoreFields(blockMetadata{}, "timestamps"),
+ cmpopts.IgnoreFields(blockMetadata{}, "field"),
+ cmpopts.IgnoreFields(blockMetadata{}, "tagFamilies"),
+ cmp.AllowUnexported(blockMetadata{}),
+ ); diff != "" {
+ t.Errorf("Unexpected blockMetadata (-got +want):\n%s", diff)
+ }
+ })
+ }
+}
diff --git a/banyand/measure/part_test.go b/banyand/measure/part_test.go
new file mode 100644
index 00000000..30b2febf
--- /dev/null
+++ b/banyand/measure/part_test.go
@@ -0,0 +1,182 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package measure
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+
+ "github.com/apache/skywalking-banyandb/api/common"
+ "github.com/apache/skywalking-banyandb/pkg/convert"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
+)
+
+func Test_memPart_mustInitFromDataPoints(t *testing.T) {
+ tests := []struct {
+ name string
+ dps *dataPoints
+ want partMetadata
+ }{
+ {
+ name: "Test with empty dataPoints",
+ dps: &dataPoints{
+ timestamps: []int64{},
+ seriesIDs: []common.SeriesID{},
+ tagFamilies: make([][]nameValues, 0),
+ fields: make([]nameValues, 0),
+ },
+ want: partMetadata{},
+ },
+ {
+ name: "Test with one item in dataPoints",
+ dps: &dataPoints{
+ timestamps: []int64{1},
+ seriesIDs: []common.SeriesID{1},
+ tagFamilies: [][]nameValues{{{
+ "arrTag", []*nameValue{
+ {"strArrTag", pbv1.ValueTypeStrArr, nil, [][]byte{[]byte("value1"), []byte("value2")}},
+ {"intArrTag", pbv1.ValueTypeInt64Arr, nil, [][]byte{convert.Int64ToBytes(25), convert.Int64ToBytes(30)}},
+ },
+ }}},
+ fields: []nameValues{{
+ "skipped", []*nameValue{
+ {"intField", pbv1.ValueTypeInt64, convert.Int64ToBytes(1110), nil},
+ },
+ }},
+ },
+ want: partMetadata{
+ BlocksCount: 1,
+ MinTimestamp: 1,
+ MaxTimestamp: 1,
+ TotalCount: 1,
+ },
+ },
+ {
+ name: "Test with multiple items in dataPoints",
+ dps: dps,
+ want: partMetadata{
+ BlocksCount: 3,
+ MinTimestamp: 1,
+ MaxTimestamp: 220,
+ TotalCount: 6,
+ },
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ mp := &memPart{}
+ mp.mustInitFromDataPoints(tt.dps)
+ assert.Equal(t, tt.want.BlocksCount, mp.partMetadata.BlocksCount)
+ assert.Equal(t, tt.want.MinTimestamp, mp.partMetadata.MinTimestamp)
+ assert.Equal(t, tt.want.MaxTimestamp, mp.partMetadata.MaxTimestamp)
+ assert.Equal(t, tt.want.TotalCount, mp.partMetadata.TotalCount)
+ })
+ }
+}
+
+var dps = &dataPoints{
+ seriesIDs: []common.SeriesID{1, 1, 2, 2, 3, 3},
+ timestamps: []int64{1, 2, 8, 10, 100, 220},
+ tagFamilies: [][]nameValues{
+ {
+ {
+ "arrTag", []*nameValue{
+ {"strArrTag", pbv1.ValueTypeStrArr, nil, [][]byte{[]byte("value1"), []byte("value2")}},
+ {"intArrTag", pbv1.ValueTypeInt64Arr, nil, [][]byte{convert.Int64ToBytes(25), convert.Int64ToBytes(30)}},
+ },
+ }, {
+ "binaryTag", []*nameValue{
+ {"binaryTag", pbv1.ValueTypeBinaryData, []byte(longText), nil},
+ },
+ }, {
+ "singleTag", []*nameValue{
+ {"strTag", pbv1.ValueTypeStr, []byte("value1"), nil},
+ {"intTag", pbv1.ValueTypeInt64, convert.Int64ToBytes(10), nil},
+ {"floatTag", pbv1.ValueTypeFloat64, convert.Float64ToBytes(12233.343), nil},
+ },
+ },
+ },
+ {
+ {
+ "arrTag", []*nameValue{
+ {"strArrTag", pbv1.ValueTypeStrArr, nil, [][]byte{[]byte("value3"), []byte("value4")}},
+ {"intArrTag", pbv1.ValueTypeInt64Arr, nil, [][]byte{convert.Int64ToBytes(50), convert.Int64ToBytes(60)}},
+ },
+ }, {
+ "binaryTag", []*nameValue{
+ {"binaryTag", pbv1.ValueTypeBinaryData, []byte(longText), nil},
+ },
+ }, {
+ "singleTag", []*nameValue{
+ {"strTag", pbv1.ValueTypeStr, []byte("value2"), nil},
+ {"intTag", pbv1.ValueTypeInt64, convert.Int64ToBytes(20), nil},
+ {"floatTag", pbv1.ValueTypeFloat64, convert.Float64ToBytes(24466.686), nil},
+ },
+ },
+ },
+ {
+ {
+ "singleTag", []*nameValue{
+ {"strTag", pbv1.ValueTypeStr, []byte("tag1"), nil},
+ {"strTag", pbv1.ValueTypeInt64, []byte("tag2"), nil},
+ },
+ },
+ },
+ {
+ {
+ "singleTag", []*nameValue{
+ {"strTag", pbv1.ValueTypeStr, []byte("tag11"), nil},
+ {"strTag", pbv1.ValueTypeInt64, []byte("tag22"), nil},
+ },
+ },
+ },
+ {},
+ {}, // empty tagFamilies for seriesID 3
+ },
+ fields: []nameValues{
+ {
+ "skipped", []*nameValue{
+ {"strField", pbv1.ValueTypeStr, []byte("field1"), nil},
+ {"intField", pbv1.ValueTypeInt64, convert.Int64ToBytes(1110), nil},
+ {"floatField", pbv1.ValueTypeFloat64, convert.Float64ToBytes(1221233.343), nil},
+ {"binaryField", pbv1.ValueTypeBinaryData, []byte(longText), nil},
+ },
+ },
+ {
+ "skipped", []*nameValue{
+ {"strField", pbv1.ValueTypeStr, []byte("field2"), nil},
+ {"intField", pbv1.ValueTypeInt64, convert.Int64ToBytes(2220), nil},
+ {"floatField", pbv1.ValueTypeFloat64, convert.Float64ToBytes(2442466.686), nil},
+ {"binaryField", pbv1.ValueTypeBinaryData, []byte(longText), nil},
+ },
+ },
+ {},
+ {}, // empty fields for seriesID 2
+ {
+ "onlyFields", []*nameValue{
+ {"intField", pbv1.ValueTypeInt64, convert.Int64ToBytes(1110), nil},
+ },
+ },
+ {
+ "onlyFields", []*nameValue{
+ {"intField", pbv1.ValueTypeInt64, convert.Int64ToBytes(2220), nil},
+ },
+ },
+ },
+}
diff --git a/banyand/measure/tstable.go b/banyand/measure/tstable.go
index 35b96d02..7f758c02 100644
--- a/banyand/measure/tstable.go
+++ b/banyand/measure/tstable.go
@@ -21,204 +21,15 @@ import (
"container/heap"
"fmt"
"io"
- "path"
- "strings"
"sync"
- "sync/atomic"
- "time"
-
- "github.com/dgraph-io/badger/v3"
- "github.com/dgraph-io/badger/v3/skl"
- "go.uber.org/multierr"
"github.com/apache/skywalking-banyandb/api/common"
- databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
- "github.com/apache/skywalking-banyandb/banyand/kv"
- "github.com/apache/skywalking-banyandb/banyand/tsdb"
- "github.com/apache/skywalking-banyandb/pkg/encoding"
"github.com/apache/skywalking-banyandb/pkg/logger"
- pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
"github.com/apache/skywalking-banyandb/pkg/timestamp"
)
-const (
- defaultNumBufferShards = 2
- defaultWriteConcurrency = 1000
- defaultWriteWal = true
- plain = "tst"
- encoded = "encoded"
-)
-
-var _ tsdb.TSTable = (*tsTableDeprecated)(nil)
-
-type tsTableDeprecated struct {
- encoderSST kv.TimeSeriesStore
- sst kv.TimeSeriesStore
- bufferSupplier *tsdb.BufferSupplier
- l *logger.Logger
- encoderBuffer *tsdb.Buffer
- buffer *tsdb.Buffer
- position common.Position
- id string
- bufferSize int64
- encoderBufferSize int64
- lock sync.RWMutex
-}
-
-func (t *tsTableDeprecated) SizeOnDisk() int64 {
- return t.encoderSST.SizeOnDisk()
-}
-
-func (t *tsTableDeprecated) openBuffer() (err error) {
- t.lock.Lock()
- defer t.lock.Unlock()
- if t.encoderBuffer != nil {
- return nil
- }
- bufferSize := int(t.encoderBufferSize / defaultNumBufferShards)
- if t.encoderBuffer, err = t.bufferSupplier.Borrow(encoded, t.id, bufferSize, t.encoderFlush); err != nil {
- return fmt.Errorf("failed to create encoder buffer: %w", err)
- }
- bufferSize = int(t.bufferSize / defaultNumBufferShards)
- if t.buffer, err = t.bufferSupplier.Borrow(plain, t.id, bufferSize, t.flush); err != nil {
- return fmt.Errorf("failed to create buffer: %w", err)
- }
- return nil
-}
-
-func (t *tsTableDeprecated) Close() (err error) {
- t.lock.Lock()
- defer t.lock.Unlock()
- for _, b := range []io.Closer{t.sst, t.encoderSST} {
- if b != nil {
- err = multierr.Append(err, b.Close())
- }
- }
- for _, b := range []string{encoded, plain} {
- t.bufferSupplier.Return(b, t.id)
- }
- return err
-}
-
-func (t *tsTableDeprecated) CollectStats() *badger.Statistics {
- mergedMap := &sync.Map{}
- for _, s := range []*badger.Statistics{t.encoderSST.CollectStats(), t.sst.CollectStats()} {
- if s != nil && s.TableBuilderSize != nil {
- s.TableBuilderSize.Range(func(key, value interface{}) bool {
- val := value.(*atomic.Int64)
- if val.Load() > 0 {
- mergedMap.Store(key, val)
- }
- return true
- })
- }
- }
- return &badger.Statistics{
- TableBuilderSize: mergedMap,
- }
-}
-
-func (t *tsTableDeprecated) Get(key []byte, ts time.Time) ([]byte, error) {
- if t.toEncode(key) {
- if v, ok := t.encoderBuffer.Read(key, ts); ok {
- return v, nil
- }
- return t.encoderSST.Get(key, uint64(ts.UnixNano()))
- }
- if v, ok := t.buffer.Read(key, ts); ok {
- return v, nil
- }
- return t.sst.Get(key, uint64(ts.UnixNano()))
-}
-
-func (t *tsTableDeprecated) Put(key []byte, val []byte, ts time.Time) error {
- t.lock.RLock()
- if t.encoderBuffer != nil {
- defer t.lock.RUnlock()
- return t.writeToBuffer(key, val, ts)
- }
- t.lock.RUnlock()
- if err := t.openBuffer(); err != nil {
- return err
- }
- return t.writeToBuffer(key, val, ts)
-}
-
-func (t *tsTableDeprecated) writeToBuffer(key []byte, val []byte, ts time.Time) error {
- if t.toEncode(key) {
- return t.encoderBuffer.Write(key, val, ts)
- }
- return t.buffer.Write(key, val, ts)
-}
-
-func (t *tsTableDeprecated) encoderFlush(shardIndex int, skl *skl.Skiplist) error {
- t.l.Info().Int("shard", shardIndex).Msg("flushing encoder buffer")
- return t.encoderSST.Handover(skl)
-}
-
-func (t *tsTableDeprecated) flush(shardIndex int, skl *skl.Skiplist) error {
- t.l.Info().Int("shard", shardIndex).Msg("flushing buffer")
- return t.sst.Handover(skl)
-}
-
-func (t *tsTableDeprecated) toEncode(key []byte) bool {
- fieldSpec, _, err := pbv1.DecodeFieldFlag(key)
- if err != nil {
- t.l.Err(err).Msg("failed to decode field flag")
- }
- return fieldSpec.EncodingMethod == databasev1.EncodingMethod_ENCODING_METHOD_GORILLA
-}
-
-var _ tsdb.TSTableFactory = (*tsTableFactory)(nil)
-
-type tsTableFactory struct {
- encoderPool encoding.SeriesEncoderPool
- decoderPool encoding.SeriesDecoderPool
- bufferSize int64
- encoderBufferSize int64
- plainChunkSize int64
- encodingChunkSize int
- compressionMethod databasev1.CompressionMethod
-}
-
-func (ttf *tsTableFactory) NewTSTable(bufferSupplier *tsdb.BufferSupplier, root string,
- position common.Position, l *logger.Logger, timeRange timestamp.TimeRange,
-) (tsdb.TSTable, error) {
- encoderSST, err := kv.OpenTimeSeriesStore(
- path.Join(root, encoded),
- timeRange,
- kv.TSSWithMemTableSize(ttf.bufferSize),
- kv.TSSWithLogger(l.Named(encoded)),
- kv.TSSWithEncoding(ttf.encoderPool, ttf.decoderPool, ttf.encodingChunkSize),
- )
- if err != nil {
- return nil, fmt.Errorf("failed to create time series table: %w", err)
- }
- sst, err := kv.OpenTimeSeriesStore(
- path.Join(root, plain),
- timeRange,
- kv.TSSWithMemTableSize(ttf.bufferSize),
- kv.TSSWithLogger(l.Named(plain)),
- kv.TSSWithZSTDCompression(int(ttf.plainChunkSize)),
- )
- if err != nil {
- return nil, fmt.Errorf("failed to create time series table: %w", err)
- }
- table := &tsTableDeprecated{
- bufferSize: ttf.bufferSize,
- encoderBufferSize: ttf.encoderBufferSize,
- l: l,
- position: position,
- encoderSST: encoderSST,
- sst: sst,
- id: strings.Join([]string{position.Segment, position.Block}, "-"),
- bufferSupplier: bufferSupplier,
- }
- return table, nil
-}
-
func newTSTable(root string, position common.Position, l *logger.Logger, timeRange timestamp.TimeRange) (*tsTable, error) {
- return nil, nil
+ return &tsTable{}, nil
}
type tsTable struct {
@@ -227,7 +38,7 @@ type tsTable struct {
}
func (tst *tsTable) Close() error {
- panic("implement me")
+ return nil
}
func (tst *tsTable) mustAddDataPoints(dps *dataPoints) {
@@ -235,7 +46,7 @@ func (tst *tsTable) mustAddDataPoints(dps *dataPoints) {
return
}
- mp := getMemPart()
+ mp := generateMemPart()
mp.mustInitFromDataPoints(dps)
p := openMemPart(mp)
@@ -310,7 +121,7 @@ func (ti *tstIter) init(parts []*part, sids []common.SeriesID, queryOpts *QueryO
ti.piHeap = ti.piHeap[:0]
for i := range ti.piPool {
ps := &ti.piPool[i]
- if !ps.NextBlock() {
+ if !ps.nextBlock() {
if err := ps.Error(); err != nil {
ti.err = fmt.Errorf("cannot initialize tstable iteration: %w", err)
return
@@ -348,7 +159,7 @@ func (ti *tstIter) NextBlock() bool {
func (ti *tstIter) nextBlock() error {
psMin := ti.piHeap[0]
- if psMin.NextBlock() {
+ if psMin.nextBlock() {
heap.Fix(&ti.piHeap, 0)
return nil
}
@@ -380,7 +191,7 @@ func (pih *partIterHeap) Len() int {
func (pih *partIterHeap) Less(i, j int) bool {
x := *pih
- return x[i].bm.less(x[j].bm)
+ return x[i].curBlock.less(x[j].curBlock)
}
func (pih *partIterHeap) Swap(i, j int) {
diff --git a/pkg/bytes/buffer.go b/pkg/bytes/buffer.go
index 1c1489f5..3c7d9cbf 100644
--- a/pkg/bytes/buffer.go
+++ b/pkg/bytes/buffer.go
@@ -19,6 +19,7 @@ package bytes
import (
"fmt"
+ "io"
"sync"
"github.com/apache/skywalking-banyandb/pkg/fs"
@@ -51,7 +52,12 @@ func (b *Buffer) Write(bb []byte) (int, error) {
// Read implements fs.Reader.
func (b *Buffer) Read(offset int64, buffer []byte) (int, error) {
- return copy(buffer, b.Buf[offset:]), nil
+ var err error
+ n := copy(buffer, b.Buf[offset:])
+ if n < len(buffer) {
+ err = io.EOF
+ }
+ return n, err
}
func (b *Buffer) Reset() {
(skywalking-banyandb) 05/07: Pass the smoke test of the integration test.
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch storage-column
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit ad3f5c8e7a0efd6de4281085d542144164870325
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Sun Dec 17 08:30:40 2023 +0800
Pass the smoke test of the integration test.
Signed-off-by: Gao Hongtao <ha...@gmail.com>
---
banyand/internal/storage/index.go | 2 +-
banyand/internal/storage/tsdb.go | 39 +++++++++++++---------
banyand/liaison/grpc/measure.go | 9 ++---
banyand/measure/datapoints.go | 2 +-
banyand/measure/query.go | 13 ++++++--
banyand/measure/topn.go | 13 +++++---
banyand/measure/write.go | 21 ++++++------
banyand/queue/local.go | 21 ++++++++++--
pkg/pb/v1/value.go | 5 +++
.../measure/measure_plan_indexscan_local.go | 2 ++
10 files changed, 84 insertions(+), 43 deletions(-)
diff --git a/banyand/internal/storage/index.go b/banyand/internal/storage/index.go
index 780a2fea..ecca0736 100644
--- a/banyand/internal/storage/index.go
+++ b/banyand/internal/storage/index.go
@@ -184,7 +184,7 @@ func (s *seriesIndex) Search(ctx context.Context, series *pbv1.Series, filter in
}
}
- if order == nil && order.Index == nil {
+ if order == nil || order.Index == nil {
return filterSeriesList(seriesList, pl), nil
}
diff --git a/banyand/internal/storage/tsdb.go b/banyand/internal/storage/tsdb.go
index d6fdfcad..bb3e1169 100644
--- a/banyand/internal/storage/tsdb.go
+++ b/banyand/internal/storage/tsdb.go
@@ -83,7 +83,7 @@ func (d *database[T]) Close() error {
for _, s := range d.sLst {
s.closer()
}
- return nil
+ return d.index.Close()
}
// OpenTSDB returns a new tsdb runtime. This constructor will create a new database if it's absent,
@@ -120,29 +120,38 @@ func (d *database[T]) Register(shardID common.ShardID, series *pbv1.Series) (*pb
if series, err = d.index.createPrimary(series); err != nil {
return nil, err
}
- id := int(shardID)
- if id < int(atomic.LoadUint32(&d.sLen)) {
- return series, nil
- }
- d.Lock()
- defer d.Unlock()
- if id < len(d.sLst) {
- return series, nil
- }
- d.logger.Info().Int("shard_id", id).Msg("creating a shard")
- if err = d.registerShard(id); err != nil {
- return nil, err
- }
return series, nil
}
func (d *database[T]) CreateTSTableIfNotExist(shardID common.ShardID, ts time.Time) (TSTableWrapper[T], error) {
+ id := uint32(shardID)
+ if id >= atomic.LoadUint32(&d.sLen) {
+ return func() (TSTableWrapper[T], error) {
+ d.Lock()
+ defer d.Unlock()
+ if int(id) >= len(d.sLst) {
+ for i := len(d.sLst); i <= int(id); i++ {
+ d.logger.Info().Int("shard_id", i).Msg("creating a shard")
+ if err := d.registerShard(i); err != nil {
+ return nil, err
+ }
+ }
+ }
+ return d.createTSTTable(shardID, ts)
+ }()
+ }
+ d.RLock()
+ defer d.RUnlock()
+ return d.createTSTTable(shardID, ts)
+}
+
+func (d *database[T]) createTSTTable(shardID common.ShardID, ts time.Time) (TSTableWrapper[T], error) {
timeRange := timestamp.NewInclusiveTimeRange(ts, ts)
ss := d.sLst[shardID].segmentController.selectTSTables(timeRange)
if len(ss) > 0 {
return ss[0], nil
}
- return d.sLst[shardID].segmentController.createTSTable(timeRange.Start)
+ return d.sLst[shardID].segmentController.createTSTable(ts)
}
func (d *database[T]) SelectTSTables(timeRange timestamp.TimeRange) []TSTableWrapper[T] {
diff --git a/banyand/liaison/grpc/measure.go b/banyand/liaison/grpc/measure.go
index ebb5c58f..afc07fab 100644
--- a/banyand/liaison/grpc/measure.go
+++ b/banyand/liaison/grpc/measure.go
@@ -113,10 +113,11 @@ func (ms *measureService) Write(measure measurev1.MeasureService_WriteServer) er
}
}
iwr := &measurev1.InternalWriteRequest{
- Request: writeRequest,
- ShardId: uint32(shardID),
- SeriesHash: tsdb.HashEntity(entity),
- EntityValues: tagValues.Encode(),
+ Request: writeRequest,
+ ShardId: uint32(shardID),
+ SeriesHash: tsdb.HashEntity(entity),
+ // TODO: remove the first value (measure name) of tagValues
+ EntityValues: tagValues[1:].Encode(),
}
nodeID, errPickNode := ms.nodeRegistry.Locate(writeRequest.GetMetadata().GetGroup(), writeRequest.GetMetadata().GetName(), uint32(shardID))
if errPickNode != nil {
diff --git a/banyand/measure/datapoints.go b/banyand/measure/datapoints.go
index bc60ab71..6884ae16 100644
--- a/banyand/measure/datapoints.go
+++ b/banyand/measure/datapoints.go
@@ -143,7 +143,7 @@ type dataPointsInTable struct {
timeRange timestamp.TimeRange
tsTable storage.TSTableWrapper[*tsTable]
- dataPoints *dataPoints
+ dataPoints dataPoints
}
type dataPointsInGroup struct {
diff --git a/banyand/measure/query.go b/banyand/measure/query.go
index 0028b591..94984dbf 100644
--- a/banyand/measure/query.go
+++ b/banyand/measure/query.go
@@ -76,6 +76,11 @@ func (s *measure) Query(ctx context.Context, mqo pbv1.MeasureQueryOptions) (pbv1
if err != nil {
return nil, err
}
+
+ var result QueryResult
+ if len(sl) < 1 {
+ return &result, nil
+ }
var sids []common.SeriesID
for i := range sl {
sids = append(sids, sl[i].ID)
@@ -96,16 +101,18 @@ func (s *measure) Query(ctx context.Context, mqo pbv1.MeasureQueryOptions) (pbv1
copy(originalSids, sids)
sort.Slice(sids, func(i, j int) bool { return sids[i] < sids[j] })
tstIter.init(parts, sids, qo.minTimestamp, qo.maxTimestamp)
- if tstIter.err != nil {
- return nil, fmt.Errorf("cannot init tstIter: %w", tstIter.err)
+ if tstIter.Error() != nil {
+ return nil, fmt.Errorf("cannot init tstIter: %w", tstIter.Error())
}
- var result QueryResult
for tstIter.nextBlock() {
bc := generateBlockCursor()
p := tstIter.piHeap[0]
bc.init(p.p, p.curBlock, qo.minTimestamp, qo.maxTimestamp)
result.data = append(result.data, bc)
}
+ if tstIter.Error() != nil {
+ return nil, fmt.Errorf("cannot iterate tstIter: %w", tstIter.Error())
+ }
if mqo.Order == nil {
result.orderByTS = true
result.ascTS = true
diff --git a/banyand/measure/topn.go b/banyand/measure/topn.go
index 79263a1c..3f41c4e2 100644
--- a/banyand/measure/topn.go
+++ b/banyand/measure/topn.go
@@ -171,6 +171,11 @@ func (t *topNStreamingProcessor) writeData(publisher queue.BatchPublisher, event
return errors.New("fail to extract tag values from topN result")
}
}
+ for _, tv := range tagValues {
+ if tv.Value == nil {
+ t.l.Warn().Msg("tag value is nil")
+ }
+ }
series, shardID, err := t.locate(tagValues, rankNum)
if err != nil {
return err
@@ -212,10 +217,8 @@ func (t *topNStreamingProcessor) writeData(publisher queue.BatchPublisher, event
},
},
},
- ShardId: uint32(shardID),
- }
- if t.l.Debug().Enabled() {
- iwr.EntityValues = series.EntityValues
+ EntityValues: series.EntityValues,
+ ShardId: uint32(shardID),
}
message := bus.NewMessageWithNode(bus.MessageID(time.Now().UnixNano()), "local", iwr)
_, errWritePub := publisher.Publish(apiData.TopicMeasureWrite, message)
@@ -259,7 +262,7 @@ func (t *topNStreamingProcessor) locate(tagValues []*modelv1.TagValue, rankNum i
}
// measureID as sharding key
for idx, tagVal := range tagValues {
- series.EntityValues[idx+3] = tagVal
+ series.EntityValues[idx+2] = tagVal
}
if err := series.Marshal(); err != nil {
return nil, 0, fmt.Errorf("fail to marshal series: %w", err)
diff --git a/banyand/measure/write.go b/banyand/measure/write.go
index 12c136f5..b0d6b241 100644
--- a/banyand/measure/write.go
+++ b/banyand/measure/write.go
@@ -89,8 +89,7 @@ func (w *writeCallback) handle(dst map[string]*dataPointsInGroup, writeEvent *me
}
dpg.tables = append(dpg.tables, dpt)
}
- dps := dpt.dataPoints
- dps.timestamps = append(dps.timestamps, int64(ts))
+ dpt.dataPoints.timestamps = append(dpt.dataPoints.timestamps, int64(ts))
stm, ok := w.schemaRepo.loadMeasure(writeEvent.GetRequest().GetMetadata())
if !ok {
return nil, fmt.Errorf("cannot find measure definition: %s", writeEvent.GetRequest().GetMetadata())
@@ -104,12 +103,12 @@ func (w *writeCallback) handle(dst map[string]*dataPointsInGroup, writeEvent *me
}
series, err := tsdb.Register(shardID, &pbv1.Series{
Subject: req.Metadata.Name,
- EntityValues: writeEvent.EntityValues[1:],
+ EntityValues: writeEvent.EntityValues,
})
if err != nil {
return nil, fmt.Errorf("cannot register series: %w", err)
}
- dps.seriesIDs = append(dps.seriesIDs, series.ID)
+ dpt.dataPoints.seriesIDs = append(dpt.dataPoints.seriesIDs, series.ID)
field := nameValues{}
for i := range stm.GetSchema().GetFields() {
var v *modelv1.FieldValue
@@ -124,8 +123,9 @@ func (w *writeCallback) handle(dst map[string]*dataPointsInGroup, writeEvent *me
v,
))
}
- dps.fields = append(dps.fields, field)
- tagFamilies := make([]nameValues, len(req.DataPoint.TagFamilies))
+ dpt.dataPoints.fields = append(dpt.dataPoints.fields, field)
+ tagFamilies := make([]nameValues, len(stm.schema.TagFamilies))
+ dpt.dataPoints.tagFamilies = append(dpt.dataPoints.tagFamilies, tagFamilies)
for i := range stm.GetSchema().GetTagFamilies() {
var tagFamily *modelv1.TagFamilyForWrite
if len(req.DataPoint.TagFamilies) <= i {
@@ -133,8 +133,8 @@ func (w *writeCallback) handle(dst map[string]*dataPointsInGroup, writeEvent *me
} else {
tagFamily = req.DataPoint.TagFamilies[i]
}
- tf := nameValues{}
tagFamilySpec := stm.GetSchema().GetTagFamilies()[i]
+ tagFamilies[i].name = tagFamilySpec.Name
for j := range tagFamilySpec.Tags {
var tagValue *modelv1.TagValue
if tagFamily == pbv1.NullTagFamily || len(tagFamily.Tags) <= j {
@@ -142,14 +142,13 @@ func (w *writeCallback) handle(dst map[string]*dataPointsInGroup, writeEvent *me
} else {
tagValue = tagFamily.Tags[j]
}
- tf.values = append(tf.values, encodeTagValue(
+ tagFamilies[i].values = append(tagFamilies[i].values, encodeTagValue(
tagFamilySpec.Tags[j].Name,
tagFamilySpec.Tags[j].Type,
tagValue,
))
}
}
- dps.tagFamilies = append(dps.tagFamilies, tagFamilies)
if stm.processorManager != nil {
stm.processorManager.onMeasureWrite(&measurev1.InternalWriteRequest{
@@ -158,7 +157,7 @@ func (w *writeCallback) handle(dst map[string]*dataPointsInGroup, writeEvent *me
DataPoint: req.DataPoint,
MessageId: uint64(time.Now().UnixNano()),
},
- EntityValues: writeEvent.EntityValues[1:],
+ EntityValues: writeEvent.EntityValues,
})
}
var fields []index.Field
@@ -219,7 +218,7 @@ func (w *writeCallback) Rev(message bus.Message) (resp bus.Message) {
g := groups[i]
for j := range g.tables {
dps := g.tables[j]
- dps.tsTable.Table().mustAddDataPoints(dps.dataPoints)
+ dps.tsTable.Table().mustAddDataPoints(&dps.dataPoints)
dps.tsTable.DecRef()
}
g.tsdb.IndexDB().Write(g.docs)
diff --git a/banyand/queue/local.go b/banyand/queue/local.go
index a6872881..81607e3d 100644
--- a/banyand/queue/local.go
+++ b/banyand/queue/local.go
@@ -89,13 +89,28 @@ func (*local) Register(schema.EventHandler) {
}
type localBatchPublisher struct {
- local *bus.Bus
+ local *bus.Bus
+ messages []any
+ topic *bus.Topic
}
-func (l *localBatchPublisher) Publish(topic bus.Topic, message ...bus.Message) (bus.Future, error) {
- return l.local.Publish(topic, message...)
+func (l *localBatchPublisher) Publish(topic bus.Topic, messages ...bus.Message) (bus.Future, error) {
+ if l.topic == nil {
+ l.topic = &topic
+ }
+ for i := range messages {
+ l.messages = append(l.messages, messages[i].Data())
+ }
+ return nil, nil
}
func (l *localBatchPublisher) Close() error {
+ newMessage := bus.NewMessage(1, l.messages)
+ _, err := l.local.Publish(*l.topic, newMessage)
+ if err != nil {
+ return err
+ }
+ l.messages = nil
+ l.topic = nil
return nil
}
diff --git a/pkg/pb/v1/value.go b/pkg/pb/v1/value.go
index 4ae6497a..0046f617 100644
--- a/pkg/pb/v1/value.go
+++ b/pkg/pb/v1/value.go
@@ -42,6 +42,8 @@ const (
func MustTagValueToValueType(tag *modelv1.TagValue) ValueType {
switch tag.Value.(type) {
+ case *modelv1.TagValue_Null:
+ return ValueTypeUnknown
case *modelv1.TagValue_Str:
return ValueTypeStr
case *modelv1.TagValue_Int:
@@ -64,6 +66,7 @@ func marshalTagValue(dest []byte, tv *modelv1.TagValue) ([]byte, error) {
}
dest = append(dest, byte(MustTagValueToValueType(tv)))
switch tv.Value.(type) {
+ case *modelv1.TagValue_Null:
case *modelv1.TagValue_Str:
dest = marshalEntityValue(dest, convert.StringToBytes(tv.GetStr().Value))
case *modelv1.TagValue_Int:
@@ -84,6 +87,8 @@ func unmarshalTagValue(dest []byte, src []byte) ([]byte, []byte, *modelv1.TagVal
var err error
vt := ValueType(src[0])
switch vt {
+ case ValueTypeUnknown:
+ return dest, src[1:], NullTagValue, nil
case ValueTypeStr:
if dest, src, err = unmarshalEntityValue(dest, src[1:]); err != nil {
return nil, nil, nil, errors.WithMessage(err, "unmarshal string tag value")
diff --git a/pkg/query/logical/measure/measure_plan_indexscan_local.go b/pkg/query/logical/measure/measure_plan_indexscan_local.go
index d7866d80..aa29dc0e 100644
--- a/pkg/query/logical/measure/measure_plan_indexscan_local.go
+++ b/pkg/query/logical/measure/measure_plan_indexscan_local.go
@@ -222,6 +222,7 @@ func (ei *resultMIterator) Next() bool {
ei.index++
return ei.Next()
}
+ ei.current = ei.current[:0]
for i := range r.Timestamps {
dp := &measurev1.DataPoint{
Timestamp: timestamppb.New(time.Unix(0, int64(r.Timestamps[i]))),
@@ -245,6 +246,7 @@ func (ei *resultMIterator) Next() bool {
Value: f.Values[i],
})
}
+ ei.current = append(ei.current, dp)
}
return true
(skywalking-banyandb) 04/07: Test query result
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch storage-column
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit 7cb0650cd0837e19c88c6042b30bd3254abc5261
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Sat Dec 16 09:46:06 2023 +0800
Test query result
Signed-off-by: Gao Hongtao <ha...@gmail.com>
---
banyand/measure/block.go | 120 +++++-
banyand/measure/block_test.go | 88 +++-
banyand/measure/measure_query.go | 540 -------------------------
banyand/measure/part_test.go | 2 -
banyand/measure/query.go | 419 +++++++++++++++++++
banyand/measure/query_test.go | 439 ++++++++++++++++++++
banyand/measure/{measure_topn.go => topn.go} | 0
banyand/measure/tstable_test.go | 102 +----
banyand/measure/{measure_write.go => write.go} | 0
pkg/pb/v1/metadata.go | 2 +
10 files changed, 1061 insertions(+), 651 deletions(-)
diff --git a/banyand/measure/block.go b/banyand/measure/block.go
index 8fd64781..adf1a5e0 100644
--- a/banyand/measure/block.go
+++ b/banyand/measure/block.go
@@ -26,6 +26,7 @@ import (
"github.com/apache/skywalking-banyandb/pkg/encoding"
"github.com/apache/skywalking-banyandb/pkg/fs"
"github.com/apache/skywalking-banyandb/pkg/logger"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
)
type block struct {
@@ -323,14 +324,16 @@ type blockCursor struct {
columnValuesDecoder encoding.BytesBlockDecoder
p *part
bm blockMetadata
- opts *QueryOptions
+ minTimestamp int64
+ maxTimestamp int64
}
func (bc *blockCursor) reset() {
bc.idx = 0
bc.p = nil
bc.bm = blockMetadata{}
- bc.opts = nil
+ bc.minTimestamp = 0
+ bc.maxTimestamp = 0
bc.timestamps = bc.timestamps[:0]
@@ -342,47 +345,138 @@ func (bc *blockCursor) reset() {
bc.fields.reset()
}
-func (bc *blockCursor) init(p *part, bm blockMetadata, opts *QueryOptions) {
+func (bc *blockCursor) init(p *part, bm blockMetadata, minTimestamp, maxTimestamp int64) {
bc.reset()
bc.p = p
bc.bm = bm
- bc.opts = opts
+ bc.minTimestamp = minTimestamp
+ bc.maxTimestamp = maxTimestamp
+}
+
+func (bc *blockCursor) copyAllTo(r *pbv1.Result) {
+ r.SID = bc.bm.seriesID
+ r.Timestamps = bc.timestamps
+ for _, cf := range bc.tagFamilies {
+ tf := pbv1.TagFamily{
+ Name: cf.Name,
+ }
+ for _, c := range cf.Columns {
+ t := pbv1.Tag{
+ Name: c.Name,
+ }
+ for _, v := range c.Values {
+ t.Values = append(t.Values, mustDecodeTagValue(c.ValueType, v))
+ }
+ tf.Tags = append(tf.Tags, t)
+ }
+ r.TagFamilies = append(r.TagFamilies, tf)
+ }
+ for _, c := range bc.fields.Columns {
+ f := pbv1.Field{
+ Name: c.Name,
+ }
+ for _, v := range c.Values {
+ f.Values = append(f.Values, mustDecodeFieldValue(c.ValueType, v))
+ }
+ r.Fields = append(r.Fields, f)
+ }
+}
+
+func (bc *blockCursor) copyTo(r *pbv1.Result) {
+ r.SID = bc.bm.seriesID
+ r.Timestamps = append(r.Timestamps, bc.timestamps[bc.idx])
+ if len(r.TagFamilies) != len(bc.tagFamilies) {
+ for _, cf := range bc.tagFamilies {
+ tf := pbv1.TagFamily{
+ Name: cf.Name,
+ }
+ for _, c := range cf.Columns {
+ t := pbv1.Tag{
+ Name: c.Name,
+ }
+ tf.Tags = append(tf.Tags, t)
+ }
+ r.TagFamilies = append(r.TagFamilies, tf)
+ }
+ }
+ for i, cf := range bc.tagFamilies {
+ for i2, c := range cf.Columns {
+ r.TagFamilies[i].Tags[i2].Values = append(r.TagFamilies[i].Tags[i2].Values, mustDecodeTagValue(c.ValueType, c.Values[bc.idx]))
+ }
+ }
+
+ if len(r.Fields) != len(bc.fields.Columns) {
+ for _, c := range bc.fields.Columns {
+ f := pbv1.Field{
+ Name: c.Name,
+ }
+ r.Fields = append(r.Fields, f)
+ }
+ }
+ for i, c := range bc.fields.Columns {
+ r.Fields[i].Values = append(r.Fields[i].Values, mustDecodeFieldValue(c.ValueType, c.Values[bc.idx]))
+ }
}
func (bc *blockCursor) loadData(tmpBlock *block) bool {
tmpBlock.reset()
tmpBlock.mustReadFrom(&bc.columnValuesDecoder, bc.p, &bc.bm)
- start, end, ok := findRange(tmpBlock.timestamps, bc.opts.minTimestamp, bc.opts.maxTimestamp)
+ start, end, ok := findRange(tmpBlock.timestamps, bc.minTimestamp, bc.maxTimestamp)
if !ok {
return false
}
bc.timestamps = append(bc.timestamps, tmpBlock.timestamps[start:end]...)
- bc.fields.Name = tmpBlock.field.Name
- bc.fields.Columns = append(bc.fields.Columns, tmpBlock.field.Columns[start:end]...)
+
+ sort.Slice(tmpBlock.tagFamilies, func(i, j int) bool {
+ return tmpBlock.tagFamilies[i].Name < tmpBlock.tagFamilies[j].Name
+ })
for _, cf := range tmpBlock.tagFamilies {
tf := ColumnFamily{
Name: cf.Name,
}
- tf.Columns = append(tf.Columns, cf.Columns[start:end]...)
+ for i := range cf.Columns {
+ column := Column{
+ Name: cf.Columns[i].Name,
+ ValueType: cf.Columns[i].ValueType,
+ }
+ column.Values = append(column.Values, cf.Columns[i].Values[start:end]...)
+ tf.Columns = append(tf.Columns, column)
+ }
bc.tagFamilies = append(bc.tagFamilies, tf)
}
+ bc.fields.Name = tmpBlock.field.Name
+ for i := range tmpBlock.field.Columns {
+ column := Column{
+ Name: tmpBlock.field.Columns[i].Name,
+ ValueType: tmpBlock.field.Columns[i].ValueType,
+ }
+ column.Values = append(column.Values, tmpBlock.field.Columns[i].Values[start:end]...)
+ bc.fields.Columns = append(bc.fields.Columns, column)
+ }
return true
}
func findRange(timestamps []int64, min int64, max int64) (int, int, bool) {
+ l := len(timestamps)
start, end := -1, -1
- for i, t := range timestamps {
- if t >= min && start == -1 {
+ for i := 0; i < l; i++ {
+ if timestamps[i] > max || timestamps[l-i-1] < min {
+ break
+ }
+ if timestamps[i] >= min && start == -1 {
start = i
}
- if t <= max {
- end = i
+ if timestamps[l-i-1] <= max && end == -1 {
+ end = l - i
+ }
+ if start != -1 && end != -1 {
+ break
}
}
- if start == -1 || end == -1 {
+ if start == -1 || end == -1 || start >= end {
return 0, 0, false
}
diff --git a/banyand/measure/block_test.go b/banyand/measure/block_test.go
index 066a9020..a393d7b7 100644
--- a/banyand/measure/block_test.go
+++ b/banyand/measure/block_test.go
@@ -84,7 +84,6 @@ var conventionalBlock = block{
{"singleTag", []Column{
{"strTag", pbv1.ValueTypeStr, [][]byte{[]byte("value1"), []byte("value2")}},
{"intTag", pbv1.ValueTypeInt64, [][]byte{convert.Int64ToBytes(10), convert.Int64ToBytes(20)}},
- {"floatTag", pbv1.ValueTypeFloat64, [][]byte{convert.Float64ToBytes(12233.343), convert.Float64ToBytes(24466.686)}},
}},
},
field: ColumnFamily{Columns: []Column{
@@ -124,7 +123,6 @@ func Test_block_mustInitFromDataPoints(t *testing.T) {
"singleTag", []*nameValue{
{"strTag", pbv1.ValueTypeStr, []byte("value1"), nil},
{"intTag", pbv1.ValueTypeInt64, convert.Int64ToBytes(10), nil},
- {"floatTag", pbv1.ValueTypeFloat64, convert.Float64ToBytes(12233.343), nil},
},
},
}, {
@@ -141,7 +139,6 @@ func Test_block_mustInitFromDataPoints(t *testing.T) {
"singleTag", []*nameValue{
{"strTag", pbv1.ValueTypeStr, []byte("value2"), nil},
{"intTag", pbv1.ValueTypeInt64, convert.Int64ToBytes(20), nil},
- {"floatTag", pbv1.ValueTypeFloat64, convert.Float64ToBytes(24466.686), nil},
},
},
}},
@@ -325,3 +322,88 @@ func Test_marshalAndUnmarshalBlock(t *testing.T) {
t.Errorf("block.mustReadFrom() = %+v, want %+v", unmarshaled, b)
}
}
+
+func Test_findRange(t *testing.T) {
+ type args struct {
+ timestamps []int64
+ min int64
+ max int64
+ }
+ tests := []struct {
+ name string
+ args args
+ wantStart int
+ wantEnd int
+ wantExist bool
+ }{
+ {
+ name: "Test with empty timestamps",
+ args: args{
+ timestamps: []int64{},
+ min: 1,
+ max: 10,
+ },
+ wantStart: 0,
+ wantEnd: 0,
+ wantExist: false,
+ },
+ {
+ name: "Test with single timestamp",
+ args: args{
+ timestamps: []int64{1},
+ min: 1,
+ max: 1,
+ },
+ wantStart: 0,
+ wantEnd: 1,
+ wantExist: true,
+ },
+ {
+ name: "Test with range not in timestamps",
+ args: args{
+ timestamps: []int64{1, 2, 3, 4, 5},
+ min: 6,
+ max: 10,
+ },
+ wantStart: 0,
+ wantEnd: 0,
+ wantExist: false,
+ },
+ {
+ name: "Test with range in timestamps",
+ args: args{
+ timestamps: []int64{1, 2, 3, 4, 5},
+ min: 2,
+ max: 4,
+ },
+ wantStart: 1,
+ wantEnd: 4,
+ wantExist: true,
+ },
+ {
+ name: "Test with range as timestamps",
+ args: args{
+ timestamps: []int64{1, 2, 3, 4, 5},
+ min: 1,
+ max: 5,
+ },
+ wantStart: 0,
+ wantEnd: 5,
+ wantExist: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ gotStart, gotEnd, gotExist := findRange(tt.args.timestamps, tt.args.min, tt.args.max)
+ if gotStart != tt.wantStart {
+ t.Errorf("findRange() gotStart = %v, want %v", gotStart, tt.wantStart)
+ }
+ if gotEnd != tt.wantEnd {
+ t.Errorf("findRange() gotEnd = %v, want %v", gotEnd, tt.wantEnd)
+ }
+ if gotExist != tt.wantExist {
+ t.Errorf("findRange() gotExist = %v, want %v", gotExist, tt.wantExist)
+ }
+ })
+ }
+}
diff --git a/banyand/measure/measure_query.go b/banyand/measure/measure_query.go
deleted file mode 100644
index 5b13923a..00000000
--- a/banyand/measure/measure_query.go
+++ /dev/null
@@ -1,540 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package measure
-
-import (
- "container/heap"
- "context"
- "fmt"
- "io"
- "sort"
- "time"
-
- "github.com/pkg/errors"
- "google.golang.org/protobuf/proto"
-
- "github.com/apache/skywalking-banyandb/api/common"
- commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
- databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
- measurev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/measure/v1"
- modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
- "github.com/apache/skywalking-banyandb/banyand/internal/storage"
- "github.com/apache/skywalking-banyandb/banyand/tsdb"
- "github.com/apache/skywalking-banyandb/pkg/convert"
- "github.com/apache/skywalking-banyandb/pkg/logger"
- pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
- resourceSchema "github.com/apache/skywalking-banyandb/pkg/schema"
-)
-
-var errTagFamilyNotExist = errors.New("tag family doesn't exist")
-
-// Query allow to retrieve measure data points.
-type Query interface {
- LoadGroup(name string) (resourceSchema.Group, bool)
- Measure(measure *commonv1.Metadata) (Measure, error)
-}
-
-// Measure allows inspecting measure data points' details.
-type Measure interface {
- io.Closer
- Query(ctx context.Context, opts pbv1.MeasureQueryOptions) (pbv1.MeasureQueryResult, error)
- CompanionShards(metadata *commonv1.Metadata) ([]tsdb.Shard, error)
- Shard(id common.ShardID) (tsdb.Shard, error)
- ParseTagFamily(family string, item tsdb.Item) (*modelv1.TagFamily, error)
- ParseField(name string, item tsdb.Item) (*measurev1.DataPoint_Field, error)
- GetSchema() *databasev1.Measure
- GetIndexRules() []*databasev1.IndexRule
- GetInterval() time.Duration
- GetShardNum() uint32
- SetSchema(schema *databasev1.Measure)
-}
-
-var _ Measure = (*measure)(nil)
-
-func (s *measure) SetSchema(schema *databasev1.Measure) {
- s.schema = schema
-}
-
-func (s *measure) GetInterval() time.Duration {
- return s.interval
-}
-
-func (s *measure) GetShardNum() uint32 {
- return s.shardNum
-}
-
-type QueryResult struct {
- data []*blockCursor
- originalData []*blockCursor
- lastIndex int
- pws []*partWrapper
- shouldMergeAll bool
- asc bool
-}
-
-func (qr *QueryResult) Pull() *pbv1.Result {
- if len(qr.data) == 0 {
- return nil
- }
- // TODO:// Parallel load
- tmpBlock := generateBlock()
- defer releaseBlock(tmpBlock)
- for i := 0; i < len(qr.data); i++ {
- if !qr.data[i].loadData(tmpBlock) {
- qr.data = append(qr.data[:i], qr.data[i+1:]...)
- i--
- }
- }
- if qr.shouldMergeAll {
- if len(qr.data) == 1 {
- r := &pbv1.Result{}
- bc := qr.data[0]
- bc.copyAllTo(r)
- return r
- }
- return qr.merge()
- }
- originalData := qr.data
- var lastSeriesID common.SeriesID
- for i := qr.lastIndex; i < len(originalData); i++ {
- bc := originalData[i]
- if lastSeriesID != 0 && lastSeriesID != bc.bm.seriesID {
- qr.data = qr.data[:0]
- qr.data = append(qr.data, originalData[qr.lastIndex:i]...)
- qr.lastIndex = i
- return qr.merge()
- }
- lastSeriesID = bc.bm.seriesID
- }
- if qr.lastIndex < len(originalData)-1 {
- qr.data = originalData[qr.lastIndex:]
- return qr.merge()
- }
- return nil
-}
-
-func (qr *QueryResult) Release() {
- for i, v := range qr.data {
- releaseBlockCursor(v)
- qr.data[i] = nil
- }
- qr.data = qr.data[:0]
- for i := range qr.pws {
- qr.pws[i].decRef()
- }
- qr.pws = qr.pws[:0]
-}
-
-func (qr QueryResult) Len() int {
- return len(qr.data)
-}
-
-func (qr QueryResult) Less(i, j int) bool {
- if qr.asc {
- return qr.data[i].timestamps[qr.data[i].idx] < qr.data[j].timestamps[qr.data[j].idx]
- }
- return qr.data[i].timestamps[qr.data[i].idx] > qr.data[j].timestamps[qr.data[j].idx]
-}
-
-func (qr QueryResult) Swap(i, j int) {
- qr.data[i], qr.data[j] = qr.data[j], qr.data[i]
-}
-
-func (qr *QueryResult) Push(x interface{}) {
- qr.data = append(qr.data, x.(*blockCursor))
-}
-
-func (qr *QueryResult) Pop() interface{} {
- old := qr.data
- n := len(old)
- x := old[n-1]
- qr.data = old[0 : n-1]
- return x
-}
-
-func (qr *QueryResult) merge() *pbv1.Result {
- result := &pbv1.Result{}
- var lastPartVersion int64
-
- for qr.Len() > 0 {
- bc := heap.Pop(qr).(*blockCursor)
-
- if len(result.Timestamps) > 0 && bc.timestamps[bc.idx] == result.Timestamps[len(result.Timestamps)-1] {
- if bc.p.partMetadata.Version > lastPartVersion {
- lastPartVersion = bc.p.partMetadata.Version
- }
- } else {
- bc.copyTo(result)
- lastPartVersion = bc.p.partMetadata.Version
- }
-
- bc.idx++
-
- if bc.idx < len(bc.timestamps) {
- heap.Push(qr, bc)
- }
- }
-
- return result
-}
-
-type QueryOptions struct {
- pbv1.MeasureQueryOptions
- minTimestamp int64
- maxTimestamp int64
-}
-
-func (s *measure) Query(ctx context.Context, mqo pbv1.MeasureQueryOptions) (pbv1.MeasureQueryResult, error) {
- if mqo.TimeRange == nil || mqo.Entity == nil {
- return nil, errors.New("invalid query options: timeRange and series are required")
- }
- tsdb := s.databaseSupplier.SupplyTSDB().(storage.TSDB[*tsTable])
- tabWrappers := tsdb.SelectTSTables(*mqo.TimeRange)
- sl, err := tsdb.IndexDB().Search(ctx, &pbv1.Series{Subject: mqo.Name, EntityValues: mqo.Entity}, mqo.Filter, mqo.Order)
- if err != nil {
- return nil, err
- }
- var sids []common.SeriesID
- for i := range sl {
- sids = append(sids, sl[i].ID)
- }
- var pws []*partWrapper
- var parts []*part
- qo := &QueryOptions{
- MeasureQueryOptions: mqo,
- minTimestamp: mqo.TimeRange.Start.UnixNano(),
- maxTimestamp: mqo.TimeRange.End.UnixNano(),
- }
- for _, tw := range tabWrappers {
- pws, parts = tw.Table().getParts(pws, parts, qo)
- }
- // TODO: cache tstIter
- var tstIter tstIter
- originalSids := make([]common.SeriesID, len(sids))
- copy(originalSids, sids)
- sort.Slice(sids, func(i, j int) bool { return sids[i] < sids[j] })
- tstIter.init(parts, sids, qo.minTimestamp, qo.maxTimestamp)
- if tstIter.err != nil {
- return nil, fmt.Errorf("cannot init tstIter: %w", tstIter.err)
- }
- var result QueryResult
- for tstIter.nextBlock() {
- bc := generateBlockCursor()
- p := tstIter.piHeap[0]
- bc.init(p.p, p.curBlock, qo)
- result.data = append(result.data, bc)
- }
- if mqo.Order != nil {
- if mqo.Order.Sort == modelv1.Sort_SORT_ASC || mqo.Order.Sort == modelv1.Sort_SORT_UNSPECIFIED {
- result.asc = true
- }
- sidToIndex := make(map[common.SeriesID]int)
- for i, si := range originalSids {
- sidToIndex[si] = i
- }
- sort.Slice(result.data, func(i, j int) bool {
- return sidToIndex[result.data[i].bm.seriesID] < sidToIndex[result.data[j].bm.seriesID]
- })
- } else {
- result.asc = true
- }
- return &result, nil
-}
-
-func (bc *blockCursor) copyAllTo(r *pbv1.Result) {
- r.Timestamps = bc.timestamps
- for _, cf := range bc.tagFamilies {
- tf := pbv1.TagFamily{
- Name: cf.Name,
- }
- for _, c := range cf.Columns {
- t := pbv1.Tag{
- Name: c.Name,
- }
- for _, v := range c.Values {
- t.Values = append(t.Values, mustDecodeTagValue(c.ValueType, v))
- }
- tf.Tags = append(tf.Tags, t)
- }
- r.TagFamilies = append(r.TagFamilies, tf)
- }
- for _, c := range bc.fields.Columns {
- f := pbv1.Field{
- Name: c.Name,
- }
- for _, v := range c.Values {
- f.Values = append(f.Values, mustDecodeFieldValue(c.ValueType, v))
- }
- r.Fields = append(r.Fields, f)
- }
-}
-
-func (bc *blockCursor) copyTo(r *pbv1.Result) {
- r.Timestamps = append(r.Timestamps, bc.timestamps[bc.idx])
- if len(r.TagFamilies) != len(bc.tagFamilies) {
- for _, cf := range bc.tagFamilies {
- tf := pbv1.TagFamily{
- Name: cf.Name,
- }
- for _, c := range cf.Columns {
- t := pbv1.Tag{
- Name: c.Name,
- }
- tf.Tags = append(tf.Tags, t)
- }
- r.TagFamilies = append(r.TagFamilies, tf)
- }
- }
- for i, cf := range bc.tagFamilies {
- for i2, c := range cf.Columns {
- r.TagFamilies[i].Tags[i2].Values = append(r.TagFamilies[i].Tags[i2].Values, mustDecodeTagValue(c.ValueType, c.Values[bc.idx]))
- }
- }
-
- if len(r.Fields) != len(bc.fields.Columns) {
- for _, c := range bc.fields.Columns {
- f := pbv1.Field{
- Name: c.Name,
- }
- r.Fields = append(r.Fields, f)
- }
- }
- for i, c := range bc.fields.Columns {
- r.Fields[i].Values = append(r.Fields[i].Values, mustDecodeFieldValue(c.ValueType, c.Values[bc.idx]))
- }
-}
-
-func mustDecodeTagValue(valueType pbv1.ValueType, value []byte) *modelv1.TagValue {
- if value == nil {
- return pbv1.NullTagValue
- }
- switch valueType {
- case pbv1.ValueTypeInt64:
- return &modelv1.TagValue{
- Value: &modelv1.TagValue_Int{
- Int: &modelv1.Int{
- Value: convert.BytesToInt64(value),
- },
- },
- }
- case pbv1.ValueTypeStr:
- return &modelv1.TagValue{
- Value: &modelv1.TagValue_Str{
- Str: &modelv1.Str{
- Value: string(value),
- },
- },
- }
- case pbv1.ValueTypeBinaryData:
- data := make([]byte, len(value))
- copy(data, value)
- return &modelv1.TagValue{
- Value: &modelv1.TagValue_BinaryData{
- BinaryData: data,
- },
- }
- case pbv1.ValueTypeInt64Arr:
- var values []int64
- for i := 0; i < len(value); i += 8 {
- values = append(values, convert.BytesToInt64(value[i:i+8]))
- }
- return &modelv1.TagValue{
- Value: &modelv1.TagValue_IntArray{
- IntArray: &modelv1.IntArray{
- Value: values,
- },
- },
- }
- case pbv1.ValueTypeStrArr:
- var values []string
- bb := bigValuePool.Generate()
- var err error
- for len(value) > 0 {
- bb.Buf, value, err = unmarshalVarArray(bb.Buf[:0], value)
- if err != nil {
- logger.Panicf("unmarshalVarArray failed: %v", err)
- }
- values = append(values, string(bb.Buf))
- }
- return &modelv1.TagValue{
- Value: &modelv1.TagValue_StrArray{
- StrArray: &modelv1.StrArray{
- Value: values,
- },
- },
- }
- default:
- logger.Panicf("unsupported value type: %v", valueType)
- return nil
- }
-}
-
-func mustDecodeFieldValue(valueType pbv1.ValueType, value []byte) *modelv1.FieldValue {
- if value == nil {
- return pbv1.NullFieldValue
- }
- switch valueType {
- case pbv1.ValueTypeInt64:
- return &modelv1.FieldValue{
- Value: &modelv1.FieldValue_Int{
- Int: &modelv1.Int{
- Value: convert.BytesToInt64(value),
- },
- },
- }
- case pbv1.ValueTypeFloat64:
- return &modelv1.FieldValue{
- Value: &modelv1.FieldValue_Float{
- Float: &modelv1.Float{
- Value: convert.BytesToFloat64(value),
- },
- },
- }
- case pbv1.ValueTypeStr:
- return &modelv1.FieldValue{
- Value: &modelv1.FieldValue_Str{
- Str: &modelv1.Str{
- Value: string(value),
- },
- },
- }
- case pbv1.ValueTypeBinaryData:
- data := make([]byte, len(value))
- copy(data, value)
- return &modelv1.FieldValue{
- Value: &modelv1.FieldValue_BinaryData{
- BinaryData: data,
- },
- }
- default:
- logger.Panicf("unsupported value type: %v", valueType)
- return nil
- }
-}
-
-func (s *measure) Shards(entity tsdb.Entity) ([]tsdb.Shard, error) {
- panic("implement me")
- // wrap := func(shards []tsdb.Shard) []tsdb.Shard {
- // result := make([]tsdb.Shard, len(shards))
- // for i := 0; i < len(shards); i++ {
- // result[i] = tsdb.NewScopedShard(tsdb.Entry(s.name), shards[i])
- // }
- // return result
- // }
- // db := s.databaseSupplier.SupplyTSDB()
- // if len(entity) < 1 {
- // return wrap(db.Shards()), nil
- // }
- // for _, e := range entity {
- // if e == nil {
- // return wrap(db.Shards()), nil
- // }
- // }
- // shardID, err := partition.ShardID(entity.Prepend(tsdb.Entry(s.name)).Marshal(), s.shardNum)
- // if err != nil {
- // return nil, err
- // }
- // shard, err := s.databaseSupplier.SupplyTSDB().Shard(common.ShardID(shardID))
- // if err != nil {
- // if errors.Is(err, tsdb.ErrUnknownShard) {
- // return []tsdb.Shard{}, nil
- // }
- // return nil, err
- // }
- // return []tsdb.Shard{tsdb.NewScopedShard(tsdb.Entry(s.name), shard)}, nil
-}
-
-func (s *measure) CompanionShards(metadata *commonv1.Metadata) ([]tsdb.Shard, error) {
- panic("implement me")
- // wrap := func(shards []tsdb.Shard) []tsdb.Shard {
- // result := make([]tsdb.Shard, len(shards))
- // for i := 0; i < len(shards); i++ {
- // result[i] = tsdb.NewScopedShard(tsdb.Entry(formatMeasureCompanionPrefix(s.name, metadata.GetName())), shards[i])
- // }
- // return result
- // }
- // db := s.databaseSupplier.SupplyTSDB()
- // return wrap(db.Shards()), nil
-}
-
-func (s *measure) Shard(id common.ShardID) (tsdb.Shard, error) {
- panic("implement me")
- // shard, err := s.databaseSupplier.SupplyTSDB().Shard(id)
- // if err != nil {
- // return nil, err
- // }
- // return tsdb.NewScopedShard(tsdb.Entry(s.name), shard), nil
-}
-
-func (s *measure) ParseTagFamily(family string, item tsdb.Item) (*modelv1.TagFamily, error) {
- fid := familyIdentity(family, pbv1.TagFlag)
- familyRawBytes, err := item.Family(fid)
- if err != nil {
- return nil, errors.Wrapf(err, "measure %s.%s parse family %s", s.name, s.group, family)
- }
- tagFamily := &modelv1.TagFamilyForWrite{}
- err = proto.Unmarshal(familyRawBytes, tagFamily)
- if err != nil {
- return nil, err
- }
- tags := make([]*modelv1.Tag, len(tagFamily.GetTags()))
- var tagSpec []*databasev1.TagSpec
- for _, tf := range s.schema.GetTagFamilies() {
- if tf.GetName() == family {
- tagSpec = tf.GetTags()
- }
- }
- if tagSpec == nil {
- return nil, errTagFamilyNotExist
- }
- for i, tag := range tagFamily.GetTags() {
- tags[i] = &modelv1.Tag{
- Key: tagSpec[i].GetName(),
- Value: &modelv1.TagValue{
- Value: tag.GetValue(),
- },
- }
- }
- return &modelv1.TagFamily{
- Name: family,
- Tags: tags,
- }, err
-}
-
-func (s *measure) ParseField(name string, item tsdb.Item) (*measurev1.DataPoint_Field, error) {
- var fieldSpec *databasev1.FieldSpec
- for _, spec := range s.schema.GetFields() {
- if spec.GetName() == name {
- fieldSpec = spec
- break
- }
- }
- fid := familyIdentity(name, pbv1.EncoderFieldFlag(fieldSpec, s.interval))
- bytes, err := item.Family(fid)
- if err != nil {
- return nil, errors.Wrapf(err, "measure %s.%s parse field %s", s.name, s.group, name)
- }
- fieldValue, err := pbv1.DecodeFieldValue(bytes, fieldSpec)
- if err != nil {
- return nil, err
- }
- return &measurev1.DataPoint_Field{
- Name: name,
- Value: fieldValue,
- }, err
-}
diff --git a/banyand/measure/part_test.go b/banyand/measure/part_test.go
index 30b2febf..7e858197 100644
--- a/banyand/measure/part_test.go
+++ b/banyand/measure/part_test.go
@@ -108,7 +108,6 @@ var dps = &dataPoints{
"singleTag", []*nameValue{
{"strTag", pbv1.ValueTypeStr, []byte("value1"), nil},
{"intTag", pbv1.ValueTypeInt64, convert.Int64ToBytes(10), nil},
- {"floatTag", pbv1.ValueTypeFloat64, convert.Float64ToBytes(12233.343), nil},
},
},
},
@@ -126,7 +125,6 @@ var dps = &dataPoints{
"singleTag", []*nameValue{
{"strTag", pbv1.ValueTypeStr, []byte("value2"), nil},
{"intTag", pbv1.ValueTypeInt64, convert.Int64ToBytes(20), nil},
- {"floatTag", pbv1.ValueTypeFloat64, convert.Float64ToBytes(24466.686), nil},
},
},
},
diff --git a/banyand/measure/query.go b/banyand/measure/query.go
new file mode 100644
index 00000000..0028b591
--- /dev/null
+++ b/banyand/measure/query.go
@@ -0,0 +1,419 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package measure
+
+import (
+ "container/heap"
+ "context"
+ "fmt"
+ "io"
+ "sort"
+
+ "github.com/pkg/errors"
+
+ "github.com/apache/skywalking-banyandb/api/common"
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+ "github.com/apache/skywalking-banyandb/banyand/internal/storage"
+ "github.com/apache/skywalking-banyandb/pkg/convert"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
+ resourceSchema "github.com/apache/skywalking-banyandb/pkg/schema"
+)
+
+var errTagFamilyNotExist = errors.New("tag family doesn't exist")
+
+// Query allow to retrieve measure data points.
+type Query interface {
+ LoadGroup(name string) (resourceSchema.Group, bool)
+ Measure(measure *commonv1.Metadata) (Measure, error)
+}
+
+// Measure allows inspecting measure data points' details.
+type Measure interface {
+ io.Closer
+ Query(ctx context.Context, opts pbv1.MeasureQueryOptions) (pbv1.MeasureQueryResult, error)
+ GetSchema() *databasev1.Measure
+ GetIndexRules() []*databasev1.IndexRule
+ SetSchema(schema *databasev1.Measure)
+}
+
+var _ Measure = (*measure)(nil)
+
+func (s *measure) SetSchema(schema *databasev1.Measure) {
+ s.schema = schema
+}
+
+type QueryOptions struct {
+ pbv1.MeasureQueryOptions
+ minTimestamp int64
+ maxTimestamp int64
+}
+
+func (s *measure) Query(ctx context.Context, mqo pbv1.MeasureQueryOptions) (pbv1.MeasureQueryResult, error) {
+ if mqo.TimeRange == nil || mqo.Entity == nil {
+ return nil, errors.New("invalid query options: timeRange and series are required")
+ }
+ tsdb := s.databaseSupplier.SupplyTSDB().(storage.TSDB[*tsTable])
+ tabWrappers := tsdb.SelectTSTables(*mqo.TimeRange)
+ sl, err := tsdb.IndexDB().Search(ctx, &pbv1.Series{Subject: mqo.Name, EntityValues: mqo.Entity}, mqo.Filter, mqo.Order)
+ if err != nil {
+ return nil, err
+ }
+ var sids []common.SeriesID
+ for i := range sl {
+ sids = append(sids, sl[i].ID)
+ }
+ var pws []*partWrapper
+ var parts []*part
+ qo := &QueryOptions{
+ MeasureQueryOptions: mqo,
+ minTimestamp: mqo.TimeRange.Start.UnixNano(),
+ maxTimestamp: mqo.TimeRange.End.UnixNano(),
+ }
+ for _, tw := range tabWrappers {
+ pws, parts = tw.Table().getParts(pws, parts, qo)
+ }
+ // TODO: cache tstIter
+ var tstIter tstIter
+ originalSids := make([]common.SeriesID, len(sids))
+ copy(originalSids, sids)
+ sort.Slice(sids, func(i, j int) bool { return sids[i] < sids[j] })
+ tstIter.init(parts, sids, qo.minTimestamp, qo.maxTimestamp)
+ if tstIter.err != nil {
+ return nil, fmt.Errorf("cannot init tstIter: %w", tstIter.err)
+ }
+ var result QueryResult
+ for tstIter.nextBlock() {
+ bc := generateBlockCursor()
+ p := tstIter.piHeap[0]
+ bc.init(p.p, p.curBlock, qo.minTimestamp, qo.maxTimestamp)
+ result.data = append(result.data, bc)
+ }
+ if mqo.Order == nil {
+ result.orderByTS = true
+ result.ascTS = true
+ return &result, nil
+ }
+ if mqo.Order.Index == nil {
+ result.orderByTS = true
+ if mqo.Order.Sort == modelv1.Sort_SORT_ASC || mqo.Order.Sort == modelv1.Sort_SORT_UNSPECIFIED {
+ result.ascTS = true
+ }
+ }
+
+ result.sidToIndex = make(map[common.SeriesID]int)
+ for i, si := range originalSids {
+ result.sidToIndex[si] = i
+ }
+ return &result, nil
+}
+
+func mustDecodeTagValue(valueType pbv1.ValueType, value []byte) *modelv1.TagValue {
+ if value == nil {
+ return pbv1.NullTagValue
+ }
+ switch valueType {
+ case pbv1.ValueTypeInt64:
+ return int64TagValue(convert.BytesToInt64(value))
+ case pbv1.ValueTypeStr:
+ return strTagValue(string(value))
+ case pbv1.ValueTypeBinaryData:
+ return binaryDataTagValue(value)
+ case pbv1.ValueTypeInt64Arr:
+ var values []int64
+ for i := 0; i < len(value); i += 8 {
+ values = append(values, convert.BytesToInt64(value[i:i+8]))
+ }
+ return int64ArrTagValue(values)
+ case pbv1.ValueTypeStrArr:
+ var values []string
+ bb := bigValuePool.Generate()
+ var err error
+ for len(value) > 0 {
+ bb.Buf, value, err = unmarshalVarArray(bb.Buf[:0], value)
+ if err != nil {
+ logger.Panicf("unmarshalVarArray failed: %v", err)
+ }
+ values = append(values, string(bb.Buf))
+ }
+ return strArrTagValue(values)
+ default:
+ logger.Panicf("unsupported value type: %v", valueType)
+ return nil
+ }
+}
+
+func int64TagValue(value int64) *modelv1.TagValue {
+ return &modelv1.TagValue{
+ Value: &modelv1.TagValue_Int{
+ Int: &modelv1.Int{
+ Value: value,
+ },
+ },
+ }
+}
+
+func strTagValue(value string) *modelv1.TagValue {
+ return &modelv1.TagValue{
+ Value: &modelv1.TagValue_Str{
+ Str: &modelv1.Str{
+ Value: value,
+ },
+ },
+ }
+}
+
+func binaryDataTagValue(value []byte) *modelv1.TagValue {
+ data := make([]byte, len(value))
+ copy(data, value)
+ return &modelv1.TagValue{
+ Value: &modelv1.TagValue_BinaryData{
+ BinaryData: data,
+ },
+ }
+}
+
+func int64ArrTagValue(values []int64) *modelv1.TagValue {
+ return &modelv1.TagValue{
+ Value: &modelv1.TagValue_IntArray{
+ IntArray: &modelv1.IntArray{
+ Value: values,
+ },
+ },
+ }
+}
+
+func strArrTagValue(values []string) *modelv1.TagValue {
+ return &modelv1.TagValue{
+ Value: &modelv1.TagValue_StrArray{
+ StrArray: &modelv1.StrArray{
+ Value: values,
+ },
+ },
+ }
+}
+
+func mustDecodeFieldValue(valueType pbv1.ValueType, value []byte) *modelv1.FieldValue {
+ if value == nil {
+ return pbv1.NullFieldValue
+ }
+ switch valueType {
+ case pbv1.ValueTypeInt64:
+ return int64FieldValue(convert.BytesToInt64(value))
+ case pbv1.ValueTypeFloat64:
+ return float64FieldValue(convert.BytesToFloat64(value))
+ case pbv1.ValueTypeStr:
+ return strFieldValue(string(value))
+ case pbv1.ValueTypeBinaryData:
+ return binaryDataFieldValue(value)
+ default:
+ logger.Panicf("unsupported value type: %v", valueType)
+ return nil
+ }
+}
+
+func int64FieldValue(value int64) *modelv1.FieldValue {
+ return &modelv1.FieldValue{
+ Value: &modelv1.FieldValue_Int{
+ Int: &modelv1.Int{
+ Value: value,
+ },
+ },
+ }
+}
+
+func float64FieldValue(value float64) *modelv1.FieldValue {
+ return &modelv1.FieldValue{
+ Value: &modelv1.FieldValue_Float{
+ Float: &modelv1.Float{
+ Value: value,
+ },
+ },
+ }
+}
+
+func strFieldValue(value string) *modelv1.FieldValue {
+ return &modelv1.FieldValue{
+ Value: &modelv1.FieldValue_Str{
+ Str: &modelv1.Str{
+ Value: value,
+ },
+ },
+ }
+}
+
+func binaryDataFieldValue(value []byte) *modelv1.FieldValue {
+ data := make([]byte, len(value))
+ copy(data, value)
+ return &modelv1.FieldValue{
+ Value: &modelv1.FieldValue_BinaryData{
+ BinaryData: data,
+ },
+ }
+}
+
+type QueryResult struct {
+ data []*blockCursor
+ loaded bool
+ sidToIndex map[common.SeriesID]int
+ orderByTS bool
+ ascTS bool
+
+ lastIndex int
+
+ pws []*partWrapper
+}
+
+func (qr *QueryResult) Pull() *pbv1.Result {
+ if !qr.loaded {
+ if len(qr.data) == 0 {
+ return nil
+ }
+ // TODO:// Parallel load
+ tmpBlock := generateBlock()
+ defer releaseBlock(tmpBlock)
+ for i := 0; i < len(qr.data); i++ {
+ if !qr.data[i].loadData(tmpBlock) {
+ qr.data = append(qr.data[:i], qr.data[i+1:]...)
+ i--
+ }
+ if qr.orderByTimestampDesc() {
+ qr.data[i].idx = len(qr.data[i].timestamps) - 1
+ }
+ }
+ qr.loaded = true
+ heap.Init(qr)
+ }
+ if len(qr.data) == 0 {
+ return nil
+ }
+ if len(qr.data) == 1 {
+ r := &pbv1.Result{}
+ bc := qr.data[0]
+ bc.copyAllTo(r)
+ qr.data = qr.data[:0]
+ return r
+ }
+ return qr.merge()
+}
+
+func (qr *QueryResult) Release() {
+ for i, v := range qr.data {
+ releaseBlockCursor(v)
+ qr.data[i] = nil
+ }
+ qr.data = qr.data[:0]
+ for i := range qr.pws {
+ qr.pws[i].decRef()
+ }
+ qr.pws = qr.pws[:0]
+}
+
+func (qr QueryResult) Len() int {
+ return len(qr.data)
+}
+
+func (qr QueryResult) Less(i, j int) bool {
+ if qr.orderByTS {
+ leftTS := qr.data[i].timestamps[qr.data[i].idx]
+ rightTS := qr.data[j].timestamps[qr.data[j].idx]
+ if leftTS == rightTS {
+ // sort seriesID in ascending order if timestamps are equal
+ return qr.data[i].bm.seriesID < qr.data[j].bm.seriesID
+ }
+ if qr.ascTS {
+ return leftTS < rightTS
+ }
+ return leftTS > rightTS
+ }
+ leftSIDIndex := qr.sidToIndex[qr.data[i].bm.seriesID]
+ rightSIDIndex := qr.sidToIndex[qr.data[j].bm.seriesID]
+ if leftSIDIndex == rightSIDIndex {
+ // sort timestamps in ascending order if seriesID are equal
+ return qr.data[i].timestamps[qr.data[i].idx] < qr.data[j].timestamps[qr.data[j].idx]
+ }
+ return leftSIDIndex < rightSIDIndex
+}
+
+func (qr QueryResult) Swap(i, j int) {
+ qr.data[i], qr.data[j] = qr.data[j], qr.data[i]
+}
+
+func (qr *QueryResult) Push(x interface{}) {
+ qr.data = append(qr.data, x.(*blockCursor))
+}
+
+func (qr *QueryResult) Pop() interface{} {
+ old := qr.data
+ n := len(old)
+ x := old[n-1]
+ qr.data = old[0 : n-1]
+ return x
+}
+
+func (qr *QueryResult) orderByTimestampDesc() bool {
+ return qr.orderByTS && !qr.ascTS
+}
+
+func (qr *QueryResult) merge() *pbv1.Result {
+ step := 1
+ if qr.orderByTimestampDesc() {
+ step = -1
+ }
+ result := &pbv1.Result{}
+ var lastPartVersion int64
+ var lastSid common.SeriesID
+
+ for qr.Len() > 0 {
+ topBC := qr.data[0]
+ if lastSid != 0 && topBC.bm.seriesID != lastSid {
+ return result
+ }
+ lastSid = topBC.bm.seriesID
+
+ if len(result.Timestamps) > 0 &&
+ topBC.timestamps[topBC.idx] == result.Timestamps[len(result.Timestamps)-1] {
+ if topBC.p.partMetadata.Version > lastPartVersion {
+ lastPartVersion = topBC.p.partMetadata.Version
+ }
+ } else {
+ topBC.copyTo(result)
+ lastPartVersion = topBC.p.partMetadata.Version
+ }
+
+ topBC.idx += step
+
+ if qr.orderByTimestampDesc() {
+ if topBC.idx < 0 {
+ heap.Pop(qr)
+ } else {
+ heap.Fix(qr, 0)
+ }
+ } else {
+ if topBC.idx >= len(topBC.timestamps) {
+ heap.Pop(qr)
+ } else {
+ heap.Fix(qr, 0)
+ }
+ }
+ }
+
+ return result
+}
diff --git a/banyand/measure/query_test.go b/banyand/measure/query_test.go
new file mode 100644
index 00000000..85a92288
--- /dev/null
+++ b/banyand/measure/query_test.go
@@ -0,0 +1,439 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package measure
+
+import (
+ "sort"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "google.golang.org/protobuf/testing/protocmp"
+
+ "github.com/apache/skywalking-banyandb/api/common"
+ modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
+)
+
+func TestQueryResult(t *testing.T) {
+ tests := []struct {
+ name string
+ dpsList []*dataPoints
+ sids []common.SeriesID
+ minTimestamp int64
+ maxTimestamp int64
+ orderBySeries bool
+ ascTS bool
+
+ want []pbv1.Result
+ wantErr error
+ }{
+ {
+ name: "Test with multiple parts with duplicated data order by TS",
+ dpsList: []*dataPoints{dps_ts1, dps_ts1},
+ sids: []common.SeriesID{1, 2, 3},
+ minTimestamp: 1,
+ maxTimestamp: 1,
+ want: []pbv1.Result{{
+ SID: 1,
+ Timestamps: []int64{1},
+ TagFamilies: []pbv1.TagFamily{
+ {Name: "arrTag", Tags: []pbv1.Tag{
+ {Name: "strArrTag", Values: []*modelv1.TagValue{strArrTagValue([]string{"value1", "value2"})}},
+ {Name: "intArrTag", Values: []*modelv1.TagValue{int64ArrTagValue([]int64{25, 30})}},
+ }},
+ {Name: "binaryTag", Tags: []pbv1.Tag{
+ {Name: "binaryTag", Values: []*modelv1.TagValue{binaryDataTagValue(longText)}},
+ }},
+ {Name: "singleTag", Tags: []pbv1.Tag{
+ {Name: "strTag", Values: []*modelv1.TagValue{strTagValue("value1")}},
+ {Name: "intTag", Values: []*modelv1.TagValue{int64TagValue(10)}},
+ }},
+ },
+ Fields: []pbv1.Field{
+ {Name: "strField", Values: []*modelv1.FieldValue{strFieldValue("field1")}},
+ {Name: "intField", Values: []*modelv1.FieldValue{int64FieldValue(1110)}},
+ {Name: "floatField", Values: []*modelv1.FieldValue{float64FieldValue(1.221233343e+06)}},
+ {Name: "binaryField", Values: []*modelv1.FieldValue{binaryDataFieldValue(longText)}},
+ },
+ }, {
+ SID: 2,
+ Timestamps: []int64{1},
+ TagFamilies: []pbv1.TagFamily{
+ {Name: "singleTag", Tags: []pbv1.Tag{
+ {Name: "strTag1", Values: []*modelv1.TagValue{strTagValue("tag1")}},
+ {Name: "strTag2", Values: []*modelv1.TagValue{strTagValue("tag2")}},
+ }},
+ },
+ Fields: nil,
+ }, {
+ SID: 3,
+ Timestamps: []int64{1},
+ TagFamilies: nil,
+ Fields: []pbv1.Field{
+ {Name: "intField", Values: []*modelv1.FieldValue{int64FieldValue(1110)}},
+ },
+ }},
+ },
+ {
+ name: "Test with multiple parts with multiple data orderBy TS desc",
+ dpsList: []*dataPoints{dps_ts1, dps_ts2},
+ sids: []common.SeriesID{1, 2, 3},
+ minTimestamp: 1,
+ maxTimestamp: 2,
+ want: []pbv1.Result{{
+ SID: 1,
+ Timestamps: []int64{2},
+ TagFamilies: []pbv1.TagFamily{
+ {Name: "arrTag", Tags: []pbv1.Tag{
+ {Name: "strArrTag", Values: []*modelv1.TagValue{strArrTagValue([]string{"value5", "value6"})}},
+ {Name: "intArrTag", Values: []*modelv1.TagValue{int64ArrTagValue([]int64{35, 40})}},
+ }},
+ {Name: "binaryTag", Tags: []pbv1.Tag{
+ {Name: "binaryTag", Values: []*modelv1.TagValue{binaryDataTagValue(longText)}},
+ }},
+ {Name: "singleTag", Tags: []pbv1.Tag{
+ {Name: "strTag", Values: []*modelv1.TagValue{strTagValue("value3")}},
+ {Name: "intTag", Values: []*modelv1.TagValue{int64TagValue(30)}},
+ }},
+ },
+ Fields: []pbv1.Field{
+ {Name: "strField", Values: []*modelv1.FieldValue{strFieldValue("field3")}},
+ {Name: "intField", Values: []*modelv1.FieldValue{int64FieldValue(3330)}},
+ {Name: "floatField", Values: []*modelv1.FieldValue{float64FieldValue(3663699.029)}},
+ {Name: "binaryField", Values: []*modelv1.FieldValue{binaryDataFieldValue(longText)}},
+ },
+ }, {
+ SID: 2,
+ Timestamps: []int64{2},
+ TagFamilies: []pbv1.TagFamily{
+ {Name: "singleTag", Tags: []pbv1.Tag{
+ {Name: "strTag1", Values: []*modelv1.TagValue{strTagValue("tag3")}},
+ {Name: "strTag2", Values: []*modelv1.TagValue{strTagValue("tag4")}},
+ }},
+ },
+ Fields: nil,
+ }, {
+ SID: 3,
+ Timestamps: []int64{2},
+ TagFamilies: nil,
+ Fields: []pbv1.Field{
+ {Name: "intField", Values: []*modelv1.FieldValue{int64FieldValue(4440)}},
+ },
+ }, {
+ SID: 1,
+ Timestamps: []int64{1},
+ TagFamilies: []pbv1.TagFamily{
+ {Name: "arrTag", Tags: []pbv1.Tag{
+ {Name: "strArrTag", Values: []*modelv1.TagValue{strArrTagValue([]string{"value1", "value2"})}},
+ {Name: "intArrTag", Values: []*modelv1.TagValue{int64ArrTagValue([]int64{25, 30})}},
+ }},
+ {Name: "binaryTag", Tags: []pbv1.Tag{
+ {Name: "binaryTag", Values: []*modelv1.TagValue{binaryDataTagValue(longText)}},
+ }},
+ {Name: "singleTag", Tags: []pbv1.Tag{
+ {Name: "strTag", Values: []*modelv1.TagValue{strTagValue("value1")}},
+ {Name: "intTag", Values: []*modelv1.TagValue{int64TagValue(10)}},
+ }},
+ },
+ Fields: []pbv1.Field{
+ {Name: "strField", Values: []*modelv1.FieldValue{strFieldValue("field1")}},
+ {Name: "intField", Values: []*modelv1.FieldValue{int64FieldValue(1110)}},
+ {Name: "floatField", Values: []*modelv1.FieldValue{float64FieldValue(1221233.343)}},
+ {Name: "binaryField", Values: []*modelv1.FieldValue{binaryDataFieldValue(longText)}},
+ },
+ }, {
+ SID: 2,
+ Timestamps: []int64{1},
+ TagFamilies: []pbv1.TagFamily{
+ {Name: "singleTag", Tags: []pbv1.Tag{
+ {Name: "strTag1", Values: []*modelv1.TagValue{strTagValue("tag1")}},
+ {Name: "strTag2", Values: []*modelv1.TagValue{strTagValue("tag2")}},
+ }},
+ },
+ Fields: nil,
+ }, {
+ SID: 3,
+ Timestamps: []int64{1},
+ TagFamilies: nil,
+ Fields: []pbv1.Field{
+ {Name: "intField", Values: []*modelv1.FieldValue{int64FieldValue(1110)}},
+ },
+ }},
+ },
+ {
+ name: "Test with multiple parts with multiple data orderBy TS asc",
+ dpsList: []*dataPoints{dps_ts1, dps_ts2},
+ sids: []common.SeriesID{1, 2, 3},
+ ascTS: true,
+ minTimestamp: 1,
+ maxTimestamp: 2,
+ want: []pbv1.Result{{
+ SID: 1,
+ Timestamps: []int64{1},
+ TagFamilies: []pbv1.TagFamily{
+ {Name: "arrTag", Tags: []pbv1.Tag{
+ {Name: "strArrTag", Values: []*modelv1.TagValue{strArrTagValue([]string{"value1", "value2"})}},
+ {Name: "intArrTag", Values: []*modelv1.TagValue{int64ArrTagValue([]int64{25, 30})}},
+ }},
+ {Name: "binaryTag", Tags: []pbv1.Tag{
+ {Name: "binaryTag", Values: []*modelv1.TagValue{binaryDataTagValue(longText)}},
+ }},
+ {Name: "singleTag", Tags: []pbv1.Tag{
+ {Name: "strTag", Values: []*modelv1.TagValue{strTagValue("value1")}},
+ {Name: "intTag", Values: []*modelv1.TagValue{int64TagValue(10)}},
+ }},
+ },
+ Fields: []pbv1.Field{
+ {Name: "strField", Values: []*modelv1.FieldValue{strFieldValue("field1")}},
+ {Name: "intField", Values: []*modelv1.FieldValue{int64FieldValue(1110)}},
+ {Name: "floatField", Values: []*modelv1.FieldValue{float64FieldValue(1221233.343)}},
+ {Name: "binaryField", Values: []*modelv1.FieldValue{binaryDataFieldValue(longText)}},
+ },
+ }, {
+ SID: 2,
+ Timestamps: []int64{1},
+ TagFamilies: []pbv1.TagFamily{
+ {Name: "singleTag", Tags: []pbv1.Tag{
+ {Name: "strTag1", Values: []*modelv1.TagValue{strTagValue("tag1")}},
+ {Name: "strTag2", Values: []*modelv1.TagValue{strTagValue("tag2")}},
+ }},
+ },
+ Fields: nil,
+ }, {
+ SID: 3,
+ Timestamps: []int64{1},
+ TagFamilies: nil,
+ Fields: []pbv1.Field{
+ {Name: "intField", Values: []*modelv1.FieldValue{int64FieldValue(1110)}},
+ },
+ }, {
+ SID: 1,
+ Timestamps: []int64{2},
+ TagFamilies: []pbv1.TagFamily{
+ {Name: "arrTag", Tags: []pbv1.Tag{
+ {Name: "strArrTag", Values: []*modelv1.TagValue{strArrTagValue([]string{"value5", "value6"})}},
+ {Name: "intArrTag", Values: []*modelv1.TagValue{int64ArrTagValue([]int64{35, 40})}},
+ }},
+ {Name: "binaryTag", Tags: []pbv1.Tag{
+ {Name: "binaryTag", Values: []*modelv1.TagValue{binaryDataTagValue(longText)}},
+ }},
+ {Name: "singleTag", Tags: []pbv1.Tag{
+ {Name: "strTag", Values: []*modelv1.TagValue{strTagValue("value3")}},
+ {Name: "intTag", Values: []*modelv1.TagValue{int64TagValue(30)}},
+ }},
+ },
+ Fields: []pbv1.Field{
+ {Name: "strField", Values: []*modelv1.FieldValue{strFieldValue("field3")}},
+ {Name: "intField", Values: []*modelv1.FieldValue{int64FieldValue(3330)}},
+ {Name: "floatField", Values: []*modelv1.FieldValue{float64FieldValue(3663699.029)}},
+ {Name: "binaryField", Values: []*modelv1.FieldValue{binaryDataFieldValue(longText)}},
+ },
+ }, {
+ SID: 2,
+ Timestamps: []int64{2},
+ TagFamilies: []pbv1.TagFamily{
+ {Name: "singleTag", Tags: []pbv1.Tag{
+ {Name: "strTag1", Values: []*modelv1.TagValue{strTagValue("tag3")}},
+ {Name: "strTag2", Values: []*modelv1.TagValue{strTagValue("tag4")}},
+ }},
+ },
+ Fields: nil,
+ }, {
+ SID: 3,
+ Timestamps: []int64{2},
+ TagFamilies: nil,
+ Fields: []pbv1.Field{
+ {Name: "intField", Values: []*modelv1.FieldValue{int64FieldValue(4440)}},
+ },
+ }},
+ },
+ {
+ name: "Test with multiple parts with duplicated data order by Series",
+ dpsList: []*dataPoints{dps_ts1, dps_ts1},
+ sids: []common.SeriesID{1, 2, 3},
+ orderBySeries: true,
+ minTimestamp: 1,
+ maxTimestamp: 1,
+ want: []pbv1.Result{{
+ SID: 1,
+ Timestamps: []int64{1},
+ TagFamilies: []pbv1.TagFamily{
+ {Name: "arrTag", Tags: []pbv1.Tag{
+ {Name: "strArrTag", Values: []*modelv1.TagValue{strArrTagValue([]string{"value1", "value2"})}},
+ {Name: "intArrTag", Values: []*modelv1.TagValue{int64ArrTagValue([]int64{25, 30})}},
+ }},
+ {Name: "binaryTag", Tags: []pbv1.Tag{
+ {Name: "binaryTag", Values: []*modelv1.TagValue{binaryDataTagValue(longText)}},
+ }},
+ {Name: "singleTag", Tags: []pbv1.Tag{
+ {Name: "strTag", Values: []*modelv1.TagValue{strTagValue("value1")}},
+ {Name: "intTag", Values: []*modelv1.TagValue{int64TagValue(10)}},
+ }},
+ },
+ Fields: []pbv1.Field{
+ {Name: "strField", Values: []*modelv1.FieldValue{strFieldValue("field1")}},
+ {Name: "intField", Values: []*modelv1.FieldValue{int64FieldValue(1110)}},
+ {Name: "floatField", Values: []*modelv1.FieldValue{float64FieldValue(1.221233343e+06)}},
+ {Name: "binaryField", Values: []*modelv1.FieldValue{binaryDataFieldValue(longText)}},
+ },
+ }, {
+ SID: 2,
+ Timestamps: []int64{1},
+ TagFamilies: []pbv1.TagFamily{
+ {Name: "singleTag", Tags: []pbv1.Tag{
+ {Name: "strTag1", Values: []*modelv1.TagValue{strTagValue("tag1")}},
+ {Name: "strTag2", Values: []*modelv1.TagValue{strTagValue("tag2")}},
+ }},
+ },
+ Fields: nil,
+ }, {
+ SID: 3,
+ Timestamps: []int64{1},
+ TagFamilies: nil,
+ Fields: []pbv1.Field{
+ {Name: "intField", Values: []*modelv1.FieldValue{int64FieldValue(1110)}},
+ },
+ }},
+ },
+ {
+ name: "Test with multiple parts with multiple data order by Series",
+ dpsList: []*dataPoints{dps_ts1, dps_ts2},
+ sids: []common.SeriesID{2, 1, 3},
+ orderBySeries: true,
+ minTimestamp: 1,
+ maxTimestamp: 2,
+ want: []pbv1.Result{{
+ SID: 2,
+ Timestamps: []int64{1, 2},
+ TagFamilies: []pbv1.TagFamily{
+ {Name: "singleTag", Tags: []pbv1.Tag{
+ {Name: "strTag1", Values: []*modelv1.TagValue{strTagValue("tag1"), strTagValue("tag3")}},
+ {Name: "strTag2", Values: []*modelv1.TagValue{strTagValue("tag2"), strTagValue("tag4")}},
+ }},
+ },
+ Fields: nil,
+ }, {
+ SID: 1,
+ Timestamps: []int64{1, 2},
+ TagFamilies: []pbv1.TagFamily{
+ {Name: "arrTag", Tags: []pbv1.Tag{
+ {Name: "strArrTag", Values: []*modelv1.TagValue{strArrTagValue([]string{"value1", "value2"}), strArrTagValue([]string{"value5", "value6"})}},
+ {Name: "intArrTag", Values: []*modelv1.TagValue{int64ArrTagValue([]int64{25, 30}), int64ArrTagValue([]int64{35, 40})}},
+ }},
+ {Name: "binaryTag", Tags: []pbv1.Tag{
+ {Name: "binaryTag", Values: []*modelv1.TagValue{binaryDataTagValue(longText), binaryDataTagValue(longText)}},
+ }},
+ {Name: "singleTag", Tags: []pbv1.Tag{
+ {Name: "strTag", Values: []*modelv1.TagValue{strTagValue("value1"), strTagValue("value3")}},
+ {Name: "intTag", Values: []*modelv1.TagValue{int64TagValue(10), int64TagValue(30)}},
+ }},
+ },
+ Fields: []pbv1.Field{
+ {Name: "strField", Values: []*modelv1.FieldValue{strFieldValue("field1"), strFieldValue("field3")}},
+ {Name: "intField", Values: []*modelv1.FieldValue{int64FieldValue(1110), int64FieldValue(3330)}},
+ {Name: "floatField", Values: []*modelv1.FieldValue{float64FieldValue(1.221233343e+06), float64FieldValue(3663699.029)}},
+ {Name: "binaryField", Values: []*modelv1.FieldValue{binaryDataFieldValue(longText), binaryDataFieldValue(longText)}},
+ },
+ }, {
+ SID: 3,
+ Timestamps: []int64{1, 2},
+ TagFamilies: nil,
+ Fields: []pbv1.Field{
+ {Name: "intField", Values: []*modelv1.FieldValue{int64FieldValue(1110), int64FieldValue(4440)}},
+ },
+ }},
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ // Initialize a tstIter object.
+ tst := &tsTable{}
+ defer tst.Close()
+ for _, dps := range tt.dpsList {
+ tst.mustAddDataPoints(dps)
+ time.Sleep(100 * time.Millisecond)
+ }
+ pws, pp := tst.getParts(nil, nil, &QueryOptions{
+ minTimestamp: tt.minTimestamp,
+ maxTimestamp: tt.maxTimestamp,
+ })
+ defer func() {
+ for _, pw := range pws {
+ pw.decRef()
+ }
+ }()
+ sids := make([]common.SeriesID, len(tt.sids))
+ copy(sids, tt.sids)
+ sort.Slice(sids, func(i, j int) bool {
+ return sids[i] < tt.sids[j]
+ })
+ ti := &tstIter{}
+ ti.init(pp, sids, tt.minTimestamp, tt.maxTimestamp)
+
+ var result QueryResult
+ for ti.nextBlock() {
+ bc := generateBlockCursor()
+ p := ti.piHeap[0]
+ bc.init(p.p, p.curBlock, tt.minTimestamp, tt.maxTimestamp)
+ result.data = append(result.data, bc)
+ }
+ defer result.Release()
+ if tt.orderBySeries {
+ result.sidToIndex = make(map[common.SeriesID]int)
+ for i, si := range tt.sids {
+ result.sidToIndex[si] = i
+ }
+ } else {
+ result.orderByTS = true
+ result.ascTS = tt.ascTS
+ }
+ var got []pbv1.Result
+ for {
+ r := result.Pull()
+ if r == nil {
+ break
+ }
+ sort.Slice(r.TagFamilies, func(i, j int) bool {
+ return r.TagFamilies[i].Name < r.TagFamilies[j].Name
+ })
+ got = append(got, *r)
+ }
+
+ if ti.Error() != tt.wantErr {
+ t.Errorf("Unexpected error: got %v, want %v", ti.err, tt.wantErr)
+ }
+
+ if diff := cmp.Diff(got, tt.want,
+ protocmp.IgnoreUnknown(), protocmp.Transform()); diff != "" {
+ t.Errorf("Unexpected []pbv1.Result (-got +want):\n%s", diff)
+ }
+ })
+ }
+}
+
+func strField(value string) *modelv1.FieldValue {
+ return &modelv1.FieldValue{
+ Value: &modelv1.FieldValue_Str{
+ Str: &modelv1.Str{
+ Value: value,
+ },
+ },
+ }
+}
diff --git a/banyand/measure/measure_topn.go b/banyand/measure/topn.go
similarity index 100%
rename from banyand/measure/measure_topn.go
rename to banyand/measure/topn.go
diff --git a/banyand/measure/tstable_test.go b/banyand/measure/tstable_test.go
index 0f7ee9c4..6787a424 100644
--- a/banyand/measure/tstable_test.go
+++ b/banyand/measure/tstable_test.go
@@ -100,7 +100,7 @@ func Test_tsTable_mustAddDataPoints(t *testing.T) {
}
}
-func Test_tstIter_init(t *testing.T) {
+func Test_tstIter(t *testing.T) {
tests := []struct {
name string
dpsList []*dataPoints
@@ -204,8 +204,8 @@ func Test_tstIter_init(t *testing.T) {
}
var dps_ts1 = &dataPoints{
- seriesIDs: []common.SeriesID{1, 1, 2, 2, 3, 3},
- timestamps: []int64{1, 1, 1, 1, 1, 1},
+ seriesIDs: []common.SeriesID{1, 2, 3},
+ timestamps: []int64{1, 1, 1},
tagFamilies: [][]nameValues{
{
{
@@ -221,45 +221,17 @@ var dps_ts1 = &dataPoints{
"singleTag", []*nameValue{
{"strTag", pbv1.ValueTypeStr, []byte("value1"), nil},
{"intTag", pbv1.ValueTypeInt64, convert.Int64ToBytes(10), nil},
- {"floatTag", pbv1.ValueTypeFloat64, convert.Float64ToBytes(12233.343), nil},
},
},
},
{
{
- "arrTag", []*nameValue{
- {"strArrTag", pbv1.ValueTypeStrArr, nil, [][]byte{[]byte("value3"), []byte("value4")}},
- {"intArrTag", pbv1.ValueTypeInt64Arr, nil, [][]byte{convert.Int64ToBytes(50), convert.Int64ToBytes(60)}},
- },
- }, {
- "binaryTag", []*nameValue{
- {"binaryTag", pbv1.ValueTypeBinaryData, []byte(longText), nil},
- },
- }, {
"singleTag", []*nameValue{
- {"strTag", pbv1.ValueTypeStr, []byte("value2"), nil},
- {"intTag", pbv1.ValueTypeInt64, convert.Int64ToBytes(20), nil},
- {"floatTag", pbv1.ValueTypeFloat64, convert.Float64ToBytes(24466.686), nil},
+ {"strTag1", pbv1.ValueTypeStr, []byte("tag1"), nil},
+ {"strTag2", pbv1.ValueTypeStr, []byte("tag2"), nil},
},
},
},
- {
- {
- "singleTag", []*nameValue{
- {"strTag", pbv1.ValueTypeStr, []byte("tag1"), nil},
- {"strTag", pbv1.ValueTypeInt64, []byte("tag2"), nil},
- },
- },
- },
- {
- {
- "singleTag", []*nameValue{
- {"strTag", pbv1.ValueTypeStr, []byte("tag11"), nil},
- {"strTag", pbv1.ValueTypeInt64, []byte("tag22"), nil},
- },
- },
- },
- {},
{}, // empty tagFamilies for seriesID 3
},
fields: []nameValues{
@@ -271,32 +243,18 @@ var dps_ts1 = &dataPoints{
{"binaryField", pbv1.ValueTypeBinaryData, []byte(longText), nil},
},
},
- {
- "skipped", []*nameValue{
- {"strField", pbv1.ValueTypeStr, []byte("field2"), nil},
- {"intField", pbv1.ValueTypeInt64, convert.Int64ToBytes(2220), nil},
- {"floatField", pbv1.ValueTypeFloat64, convert.Float64ToBytes(2442466.686), nil},
- {"binaryField", pbv1.ValueTypeBinaryData, []byte(longText), nil},
- },
- },
- {},
{}, // empty fields for seriesID 2
{
"onlyFields", []*nameValue{
{"intField", pbv1.ValueTypeInt64, convert.Int64ToBytes(1110), nil},
},
},
- {
- "onlyFields", []*nameValue{
- {"intField", pbv1.ValueTypeInt64, convert.Int64ToBytes(2220), nil},
- },
- },
},
}
var dps_ts2 = &dataPoints{
- seriesIDs: []common.SeriesID{1, 1, 2, 2, 3, 3},
- timestamps: []int64{2, 2, 2, 2, 2, 2},
+ seriesIDs: []common.SeriesID{1, 2, 3},
+ timestamps: []int64{2, 2, 2},
tagFamilies: [][]nameValues{
{
{
@@ -312,45 +270,17 @@ var dps_ts2 = &dataPoints{
"singleTag", []*nameValue{
{"strTag", pbv1.ValueTypeStr, []byte("value3"), nil},
{"intTag", pbv1.ValueTypeInt64, convert.Int64ToBytes(30), nil},
- {"floatTag", pbv1.ValueTypeFloat64, convert.Float64ToBytes(36699.029), nil},
- },
- },
- },
- {
- {
- "arrTag", []*nameValue{
- {"strArrTag", pbv1.ValueTypeStrArr, nil, [][]byte{[]byte("value7"), []byte("value8")}},
- {"intArrTag", pbv1.ValueTypeInt64Arr, nil, [][]byte{convert.Int64ToBytes(70), convert.Int64ToBytes(80)}},
- },
- }, {
- "binaryTag", []*nameValue{
- {"binaryTag", pbv1.ValueTypeBinaryData, []byte(longText), nil},
- },
- }, {
- "singleTag", []*nameValue{
- {"strTag", pbv1.ValueTypeStr, []byte("value4"), nil},
- {"intTag", pbv1.ValueTypeInt64, convert.Int64ToBytes(40), nil},
- {"floatTag", pbv1.ValueTypeFloat64, convert.Float64ToBytes(48999.372), nil},
},
},
},
{
{
"singleTag", []*nameValue{
- {"strTag", pbv1.ValueTypeStr, []byte("tag3"), nil},
- {"strTag", pbv1.ValueTypeInt64, []byte("tag4"), nil},
+ {"strTag1", pbv1.ValueTypeStr, []byte("tag3"), nil},
+ {"strTag2", pbv1.ValueTypeStr, []byte("tag4"), nil},
},
},
},
- {
- {
- "singleTag", []*nameValue{
- {"strTag", pbv1.ValueTypeStr, []byte("tag33"), nil},
- {"strTag", pbv1.ValueTypeInt64, []byte("tag44"), nil},
- },
- },
- },
- {},
{}, // empty tagFamilies for seriesID 6
},
fields: []nameValues{
@@ -362,21 +292,7 @@ var dps_ts2 = &dataPoints{
{"binaryField", pbv1.ValueTypeBinaryData, []byte(longText), nil},
},
},
- {
- "skipped", []*nameValue{
- {"strField", pbv1.ValueTypeStr, []byte("field4"), nil},
- {"intField", pbv1.ValueTypeInt64, convert.Int64ToBytes(4440), nil},
- {"floatField", pbv1.ValueTypeFloat64, convert.Float64ToBytes(4899899.372), nil},
- {"binaryField", pbv1.ValueTypeBinaryData, []byte(longText), nil},
- },
- },
- {},
{}, // empty fields for seriesID 5
- {
- "onlyFields", []*nameValue{
- {"intField", pbv1.ValueTypeInt64, convert.Int64ToBytes(3330), nil},
- },
- },
{
"onlyFields", []*nameValue{
{"intField", pbv1.ValueTypeInt64, convert.Int64ToBytes(4440), nil},
diff --git a/banyand/measure/measure_write.go b/banyand/measure/write.go
similarity index 100%
rename from banyand/measure/measure_write.go
rename to banyand/measure/write.go
diff --git a/pkg/pb/v1/metadata.go b/pkg/pb/v1/metadata.go
index 3881b82a..87dda12f 100644
--- a/pkg/pb/v1/metadata.go
+++ b/pkg/pb/v1/metadata.go
@@ -19,6 +19,7 @@
package v1
import (
+ "github.com/apache/skywalking-banyandb/api/common"
databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
"github.com/apache/skywalking-banyandb/pkg/index"
@@ -97,6 +98,7 @@ type Field struct {
}
type Result struct {
+ SID common.SeriesID
Timestamps []int64
TagFamilies []TagFamily
Fields []Field
(skywalking-banyandb) 01/07: Finish test block
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch storage-column
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit 835b36fec113395d391771dca398f55e62ba05c7
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Wed Dec 13 05:53:43 2023 +0800
Finish test block
Signed-off-by: Gao Hongtao <ha...@gmail.com>
---
banyand/measure/block.go | 71 +++++++++--------
banyand/measure/block_test.go | 158 +++++++++++++++++++++++++++++--------
banyand/measure/block_writer.go | 33 ++++----
banyand/measure/column_metadata.go | 2 +-
4 files changed, 181 insertions(+), 83 deletions(-)
diff --git a/banyand/measure/block.go b/banyand/measure/block.go
index 98e8e811..2c0407ce 100644
--- a/banyand/measure/block.go
+++ b/banyand/measure/block.go
@@ -18,6 +18,7 @@
package measure
import (
+ "sort"
"sync"
"github.com/apache/skywalking-banyandb/api/common"
@@ -124,26 +125,25 @@ func (b *block) Len() int {
return len(b.timestamps)
}
-func (b *block) mustWriteTo(sid common.SeriesID, bh *blockMetadata, sw *writers) {
+func (b *block) mustWriteTo(sid common.SeriesID, bm *blockMetadata, ww *writers) {
b.validate()
- bh.reset()
+ bm.reset()
- bh.seriesID = sid
- bh.uncompressedSizeBytes = b.uncompressedSizeBytes()
- bh.count = uint64(b.Len())
+ bm.seriesID = sid
+ bm.uncompressedSizeBytes = b.uncompressedSizeBytes()
+ bm.count = uint64(b.Len())
- mustWriteTimestampsTo(&bh.timestamps, b.timestamps, sw.timestampsWriter)
+ mustWriteTimestampsTo(&bm.timestamps, b.timestamps, ww.timestampsWriter)
- tff := b.tagFamilies
- for ti := range tff {
- b.marshalTagFamily(tff[ti], bh, sw)
+ for ti := range b.tagFamilies {
+ b.marshalTagFamily(b.tagFamilies[ti], bm, ww)
}
f := b.field
cc := f.Columns
- chh := bh.field.resizeColumnMetadata(len(cc))
+ cmm := bm.field.resizeColumnMetadata(len(cc))
for i := range cc {
- cc[i].mustWriteTo(&chh[i], &sw.fieldValuesWriter)
+ cc[i].mustWriteTo(&cmm[i], &ww.fieldValuesWriter)
}
}
@@ -173,30 +173,30 @@ func (b *block) validate() {
}
}
-func (b *block) marshalTagFamily(tf ColumnFamily, bh *blockMetadata, sw *writers) {
- hw, w := sw.getColumnMetadataWriterAndColumnWriter(tf.Name)
+func (b *block) marshalTagFamily(tf ColumnFamily, bm *blockMetadata, ww *writers) {
+ hw, w := ww.getColumnMetadataWriterAndColumnWriter(tf.Name)
cc := tf.Columns
- cfh := generateColumnFamilyMetadata()
- chh := cfh.resizeColumnMetadata(len(cc))
+ cfm := generateColumnFamilyMetadata()
+ cmm := cfm.resizeColumnMetadata(len(cc))
for i := range cc {
- cc[i].mustWriteTo(&chh[i], w)
+ cc[i].mustWriteTo(&cmm[i], w)
}
bb := bigValuePool.Generate()
defer bigValuePool.Release(bb)
- bb.Buf = cfh.marshal(bb.Buf)
- releaseColumnFamilyMetadata(cfh)
- tfh := bh.getTagFamilyMetadata(tf.Name)
- tfh.offset = w.bytesWritten
- tfh.size = uint64(len(bb.Buf))
- if tfh.size > maxTagFamiliesMetadataSize {
- logger.Panicf("too big columnFamilyMetadataSize: %d bytes; mustn't exceed %d bytes", tfh.size, maxTagFamiliesMetadataSize)
+ bb.Buf = cfm.marshal(bb.Buf)
+ releaseColumnFamilyMetadata(cfm)
+ tfm := bm.getTagFamilyMetadata(tf.Name)
+ tfm.offset = hw.bytesWritten
+ tfm.size = uint64(len(bb.Buf))
+ if tfm.size > maxTagFamiliesMetadataSize {
+ logger.Panicf("too big columnFamilyMetadataSize: %d bytes; mustn't exceed %d bytes", tfm.size, maxTagFamiliesMetadataSize)
}
hw.MustWrite(bb.Buf)
}
func (b *block) unmarshalTagFamily(decoder *encoding.BytesBlockDecoder, tfIndex int, name string, columnFamilyMetadataBlock *dataBlock, metaReader, valueReader fs.Reader) {
bb := bigValuePool.Generate()
- bytes.ResizeExact(bb.Buf, int(columnFamilyMetadataBlock.size))
+ bb.Buf = bytes.ResizeExact(bb.Buf, int(columnFamilyMetadataBlock.size))
fs.MustReadData(metaReader, int64(columnFamilyMetadataBlock.offset), bb.Buf)
cfm := generateColumnFamilyMetadata()
defer releaseColumnFamilyMetadata(cfm)
@@ -205,10 +205,10 @@ func (b *block) unmarshalTagFamily(decoder *encoding.BytesBlockDecoder, tfIndex
logger.Panicf("%s: cannot unmarshal columnFamilyMetadata: %v", metaReader.Path(), err)
}
bigValuePool.Release(bb)
- tf := b.tagFamilies[tfIndex]
- cc := tf.resizeColumns(len(cfm.columnMetadata))
- for i, c := range cc {
- c.mustReadValues(decoder, valueReader, cfm.columnMetadata[i], uint64(b.Len()))
+ b.tagFamilies[tfIndex].Name = name
+ cc := b.tagFamilies[tfIndex].resizeColumns(len(cfm.columnMetadata))
+ for i := range cc {
+ cc[i].mustReadValues(decoder, valueReader, cfm.columnMetadata[i], uint64(b.Len()))
}
}
@@ -244,7 +244,7 @@ func (b *block) uncompressedSizeBytes() uint64 {
return n
}
-func (b *block) mustReadFrom(decoder *encoding.BytesBlockDecoder, p *part, bm *blockMetadata, opts *QueryOptions) {
+func (b *block) mustReadFrom(decoder *encoding.BytesBlockDecoder, p *part, bm *blockMetadata) {
b.reset()
b.timestamps = mustReadTimestampsFrom(b.timestamps, &bm.timestamps, int(bm.count), p.timestamps)
@@ -256,11 +256,18 @@ func (b *block) mustReadFrom(decoder *encoding.BytesBlockDecoder, p *part, bm *b
i++
}
cc := b.field.resizeColumns(len(bm.field.columnMetadata))
- for i, c := range cc {
- c.mustReadValues(decoder, p.fieldValues, bm.field.columnMetadata[i], bm.count)
+ for i := range cc {
+ cc[i].mustReadValues(decoder, p.fieldValues, bm.field.columnMetadata[i], bm.count)
}
}
+// For testing purpose only.
+func (b *block) sortTagFamilies() {
+ sort.Slice(b.tagFamilies, func(i, j int) bool {
+ return b.tagFamilies[i].Name < b.tagFamilies[j].Name
+ })
+}
+
func mustWriteTimestampsTo(tm *timestampsMetadata, timestamps []int64, timestampsWriter writer) {
tm.reset()
@@ -345,7 +352,7 @@ func (bc *blockCursor) init(p *part, bm *blockMetadata, opts *QueryOptions) {
func (bc *blockCursor) loadData(tmpBlock *block) bool {
bc.reset()
tmpBlock.reset()
- tmpBlock.mustReadFrom(&bc.columnValuesDecoder, bc.p, bc.bm, bc.opts)
+ tmpBlock.mustReadFrom(&bc.columnValuesDecoder, bc.p, bc.bm)
start, end, ok := findRange(tmpBlock.timestamps, bc.opts.minTimestamp, bc.opts.maxTimestamp)
if !ok {
diff --git a/banyand/measure/block_test.go b/banyand/measure/block_test.go
index df2601f3..066a9020 100644
--- a/banyand/measure/block_test.go
+++ b/banyand/measure/block_test.go
@@ -23,8 +23,11 @@ import (
"testing"
"time"
+ "github.com/apache/skywalking-banyandb/api/common"
"github.com/apache/skywalking-banyandb/pkg/bytes"
"github.com/apache/skywalking-banyandb/pkg/convert"
+ "github.com/apache/skywalking-banyandb/pkg/encoding"
+ "github.com/apache/skywalking-banyandb/pkg/fs"
pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
)
@@ -68,6 +71,30 @@ func Test_block_reset(t *testing.T) {
}
}
+var conventionalBlock = block{
+ timestamps: []int64{1, 2},
+ tagFamilies: []ColumnFamily{
+ {"arrTag", []Column{
+ {"strArrTag", pbv1.ValueTypeStrArr, [][]byte{marshalStrArr([][]byte{[]byte("value1"), []byte("value2")}), marshalStrArr([][]byte{[]byte("value3"), []byte("value4")})}},
+ {"intArrTag", pbv1.ValueTypeInt64Arr, [][]byte{marshalIntArr([][]byte{convert.Int64ToBytes(25), convert.Int64ToBytes(30)}), marshalIntArr([][]byte{convert.Int64ToBytes(50), convert.Int64ToBytes(60)})}},
+ }},
+ {"binaryTag", []Column{
+ {Name: "binaryTag", ValueType: pbv1.ValueTypeBinaryData, Values: [][]byte{[]byte(longText), []byte(longText)}},
+ }},
+ {"singleTag", []Column{
+ {"strTag", pbv1.ValueTypeStr, [][]byte{[]byte("value1"), []byte("value2")}},
+ {"intTag", pbv1.ValueTypeInt64, [][]byte{convert.Int64ToBytes(10), convert.Int64ToBytes(20)}},
+ {"floatTag", pbv1.ValueTypeFloat64, [][]byte{convert.Float64ToBytes(12233.343), convert.Float64ToBytes(24466.686)}},
+ }},
+ },
+ field: ColumnFamily{Columns: []Column{
+ {"strField", pbv1.ValueTypeStr, [][]byte{[]byte("field1"), []byte("field2")}},
+ {"intField", pbv1.ValueTypeInt64, [][]byte{convert.Int64ToBytes(1110), convert.Int64ToBytes(2220)}},
+ {"floatField", pbv1.ValueTypeFloat64, [][]byte{convert.Float64ToBytes(1221233.343), convert.Float64ToBytes(2442466.686)}},
+ {Name: "binaryField", ValueType: pbv1.ValueTypeBinaryData, Values: [][]byte{[]byte(longText), []byte(longText)}},
+ }},
+}
+
func Test_block_mustInitFromDataPoints(t *testing.T) {
type args struct {
timestamps []int64
@@ -85,12 +112,6 @@ func Test_block_mustInitFromDataPoints(t *testing.T) {
timestamps: []int64{1, 2},
tagFamilies: [][]nameValues{{
{
- "singleTag", []*nameValue{
- {"strTag", pbv1.ValueTypeStr, []byte("value1"), nil},
- {"intTag", pbv1.ValueTypeInt64, convert.Int64ToBytes(10), nil},
- {"floatTag", pbv1.ValueTypeFloat64, convert.Float64ToBytes(12233.343), nil},
- },
- }, {
"arrTag", []*nameValue{
{"strArrTag", pbv1.ValueTypeStrArr, nil, [][]byte{[]byte("value1"), []byte("value2")}},
{"intArrTag", pbv1.ValueTypeInt64Arr, nil, [][]byte{convert.Int64ToBytes(25), convert.Int64ToBytes(30)}},
@@ -99,15 +120,15 @@ func Test_block_mustInitFromDataPoints(t *testing.T) {
"binaryTag", []*nameValue{
{"binaryTag", pbv1.ValueTypeBinaryData, []byte(longText), nil},
},
+ }, {
+ "singleTag", []*nameValue{
+ {"strTag", pbv1.ValueTypeStr, []byte("value1"), nil},
+ {"intTag", pbv1.ValueTypeInt64, convert.Int64ToBytes(10), nil},
+ {"floatTag", pbv1.ValueTypeFloat64, convert.Float64ToBytes(12233.343), nil},
+ },
},
}, {
{
- "singleTag", []*nameValue{
- {"strTag", pbv1.ValueTypeStr, []byte("value2"), nil},
- {"intTag", pbv1.ValueTypeInt64, convert.Int64ToBytes(20), nil},
- {"floatTag", pbv1.ValueTypeFloat64, convert.Float64ToBytes(24466.686), nil},
- },
- }, {
"arrTag", []*nameValue{
{"strArrTag", pbv1.ValueTypeStrArr, nil, [][]byte{[]byte("value3"), []byte("value4")}},
{"intArrTag", pbv1.ValueTypeInt64Arr, nil, [][]byte{convert.Int64ToBytes(50), convert.Int64ToBytes(60)}},
@@ -116,6 +137,12 @@ func Test_block_mustInitFromDataPoints(t *testing.T) {
"binaryTag", []*nameValue{
{"binaryTag", pbv1.ValueTypeBinaryData, []byte(longText), nil},
},
+ }, {
+ "singleTag", []*nameValue{
+ {"strTag", pbv1.ValueTypeStr, []byte("value2"), nil},
+ {"intTag", pbv1.ValueTypeInt64, convert.Int64ToBytes(20), nil},
+ {"floatTag", pbv1.ValueTypeFloat64, convert.Float64ToBytes(24466.686), nil},
+ },
},
}},
fields: []nameValues{{
@@ -134,29 +161,7 @@ func Test_block_mustInitFromDataPoints(t *testing.T) {
},
}},
},
- want: block{
- timestamps: []int64{1, 2},
- tagFamilies: []ColumnFamily{
- {"singleTag", []Column{
- {"strTag", pbv1.ValueTypeStr, [][]byte{[]byte("value1"), []byte("value2")}},
- {"intTag", pbv1.ValueTypeInt64, [][]byte{convert.Int64ToBytes(10), convert.Int64ToBytes(20)}},
- {"floatTag", pbv1.ValueTypeFloat64, [][]byte{convert.Float64ToBytes(12233.343), convert.Float64ToBytes(24466.686)}},
- }},
- {"arrTag", []Column{
- {"strArrTag", pbv1.ValueTypeStrArr, [][]byte{marshalStrArr([][]byte{[]byte("value1"), []byte("value2")}), marshalStrArr([][]byte{[]byte("value3"), []byte("value4")})}},
- {"intArrTag", pbv1.ValueTypeInt64Arr, [][]byte{marshalIntArr([][]byte{convert.Int64ToBytes(25), convert.Int64ToBytes(30)}), marshalIntArr([][]byte{convert.Int64ToBytes(50), convert.Int64ToBytes(60)})}},
- }},
- {"binaryTag", []Column{
- {Name: "binaryTag", ValueType: pbv1.ValueTypeBinaryData, Values: [][]byte{[]byte(longText), []byte(longText)}},
- }},
- },
- field: ColumnFamily{Columns: []Column{
- {"strField", pbv1.ValueTypeStr, [][]byte{[]byte("field1"), []byte("field2")}},
- {"intField", pbv1.ValueTypeInt64, [][]byte{convert.Int64ToBytes(1110), convert.Int64ToBytes(2220)}},
- {"floatField", pbv1.ValueTypeFloat64, [][]byte{convert.Float64ToBytes(1221233.343), convert.Float64ToBytes(2442466.686)}},
- {Name: "binaryField", ValueType: pbv1.ValueTypeBinaryData, Values: [][]byte{[]byte(longText), []byte(longText)}},
- }},
- },
+ want: conventionalBlock,
},
}
for _, tt := range tests {
@@ -237,3 +242,86 @@ func getBitInt64Arr() []int64 {
}
return randSlice
}
+
+func Test_marshalAndUnmarshalTagFamily(t *testing.T) {
+ metaBuffer, dataBuffer := &bytes.Buffer{}, &bytes.Buffer{}
+ ww := &writers{
+ mustCreateTagFamilyWriters: func(name string) (fs.Writer, fs.Writer) {
+ return metaBuffer, dataBuffer
+ },
+ tagFamilyMetadataWriters: make(map[string]*writer),
+ tagFamilyWriters: make(map[string]*writer),
+ }
+ b := &conventionalBlock
+ tfIndex := 0
+ name := "arrTag"
+ decoder := &encoding.BytesBlockDecoder{}
+ bm := &blockMetadata{}
+
+ b.marshalTagFamily(b.tagFamilies[tfIndex], bm, ww)
+
+ metaWriter, ok1 := ww.tagFamilyMetadataWriters[name]
+ valueWriter, ok2 := ww.tagFamilyWriters[name]
+ if !ok1 || !ok2 {
+ t.Fatalf("Writers not correctly added to maps")
+ }
+ if metaWriter.w != metaBuffer || valueWriter.w != dataBuffer {
+ t.Fatalf("Writers not correctly added to maps")
+ }
+
+ unmarshaled := generateBlock()
+ defer releaseBlock(unmarshaled)
+ // set the timestamps to the same length as the original block
+ // the data size in a block depends on the timestamps length
+ unmarshaled.timestamps = make([]int64, len(b.timestamps))
+ unmarshaled.resizeTagFamilies(1)
+
+ unmarshaled.unmarshalTagFamily(decoder, tfIndex, name, bm.getTagFamilyMetadata(name), metaBuffer, dataBuffer)
+
+ if !reflect.DeepEqual(b.tagFamilies[0], unmarshaled.tagFamilies[0]) {
+ t.Errorf("block.unmarshalTagFamily() = %+v, want %+v", unmarshaled.tagFamilies, b.tagFamilies)
+ }
+}
+
+func Test_marshalAndUnmarshalBlock(t *testing.T) {
+ timestampBuffer, fieldBuffer := &bytes.Buffer{}, &bytes.Buffer{}
+ ww := &writers{
+ mustCreateTagFamilyWriters: func(name string) (fs.Writer, fs.Writer) {
+ return &bytes.Buffer{}, &bytes.Buffer{}
+ },
+ tagFamilyMetadataWriters: make(map[string]*writer),
+ tagFamilyWriters: make(map[string]*writer),
+ timestampsWriter: writer{w: timestampBuffer},
+ fieldValuesWriter: writer{w: fieldBuffer},
+ }
+ p := &part{
+ timestamps: timestampBuffer,
+ fieldValues: fieldBuffer,
+ }
+ b := &conventionalBlock
+ decoder := &encoding.BytesBlockDecoder{}
+ sid := common.SeriesID(1)
+ bm := &blockMetadata{}
+
+ b.mustWriteTo(sid, bm, ww)
+
+ tagFamilyMetadataReaders := make(map[string]fs.Reader)
+ tagFamilyReaders := make(map[string]fs.Reader)
+
+ for k, w := range ww.tagFamilyMetadataWriters {
+ tagFamilyMetadataReaders[k] = w.w.(*bytes.Buffer)
+ tagFamilyReaders[k] = ww.tagFamilyWriters[k].w.(*bytes.Buffer)
+ }
+ p.tagFamilyMetadata = tagFamilyMetadataReaders
+ p.tagFamilies = tagFamilyReaders
+
+ unmarshaled := generateBlock()
+ defer releaseBlock(unmarshaled)
+ unmarshaled.mustReadFrom(decoder, p, bm)
+ // blockMetadata is using a map, so the order of tag families is not guaranteed
+ unmarshaled.sortTagFamilies()
+
+ if !reflect.DeepEqual(b, unmarshaled) {
+ t.Errorf("block.mustReadFrom() = %+v, want %+v", unmarshaled, b)
+ }
+}
diff --git a/banyand/measure/block_writer.go b/banyand/measure/block_writer.go
index 553baf41..4fd50190 100644
--- a/banyand/measure/block_writer.go
+++ b/banyand/measure/block_writer.go
@@ -126,7 +126,7 @@ func (sw *writers) getColumnMetadataWriterAndColumnWriter(columnName string) (*w
}
type blockWriter struct {
- streamWriters writers
+ writers writers
sidLast common.SeriesID
@@ -158,7 +158,7 @@ type blockWriter struct {
}
func (bw *blockWriter) reset() {
- bw.streamWriters.reset()
+ bw.writers.reset()
bw.sidLast = 0
bw.sidFirst = 0
bw.minTimestampLast = 0
@@ -177,13 +177,11 @@ func (bw *blockWriter) reset() {
func (bsw *blockWriter) MustInitForMemPart(mp *memPart) {
bsw.reset()
- bsw.streamWriters = writers{
- mustCreateTagFamilyWriters: mp.mustCreateMemTagFamilyWriters,
- }
- bsw.streamWriters.metaWriter.init(&mp.meta)
- bsw.streamWriters.primaryWriter.init(&mp.primary)
- bsw.streamWriters.timestampsWriter.init(&mp.timestamps)
- bsw.streamWriters.fieldValuesWriter.init(&mp.fieldValues)
+ bsw.writers.mustCreateTagFamilyWriters = mp.mustCreateMemTagFamilyWriters
+ bsw.writers.metaWriter.init(&mp.meta)
+ bsw.writers.primaryWriter.init(&mp.primary)
+ bsw.writers.timestampsWriter.init(&mp.timestamps)
+ bsw.writers.fieldValuesWriter.init(&mp.fieldValues)
}
func (bsw *blockWriter) MustWriteDataPoints(sid common.SeriesID, timestamps []int64, tagFamilies [][]nameValues, fields []nameValues) {
@@ -209,7 +207,7 @@ func (bsw *blockWriter) MustWriteDataPoints(sid common.SeriesID, timestamps []in
bsw.sidLast = sid
bh := generateBlockMetadata()
- b.mustWriteTo(sid, bh, &bsw.streamWriters)
+ b.mustWriteTo(sid, bh, &bsw.writers)
th := &bh.timestamps
if bsw.totalCount == 0 || th.min < bsw.totalMinTimestamp {
bsw.totalMinTimestamp = th.min
@@ -242,7 +240,7 @@ func (bsw *blockWriter) MustWriteDataPoints(sid common.SeriesID, timestamps []in
func (bsw *blockWriter) mustFlushPrimaryBlock(data []byte) {
if len(data) > 0 {
- bsw.primaryBlockMetadata.mustWriteBlock(data, bsw.sidFirst, bsw.minTimestamp, bsw.maxTimestamp, &bsw.streamWriters)
+ bsw.primaryBlockMetadata.mustWriteBlock(data, bsw.sidFirst, bsw.minTimestamp, bsw.maxTimestamp, &bsw.writers)
bsw.metaData = bsw.primaryBlockMetadata.marshal(bsw.metaData)
}
bsw.hasWrittenBlocks = false
@@ -262,19 +260,24 @@ func (bsw *blockWriter) Flush(ph *partMetadata) {
bb := bigValuePool.Generate()
bb.Buf = zstd.Compress(bb.Buf[:0], bsw.metaData, 1)
- bsw.streamWriters.metaWriter.MustWrite(bb.Buf)
+ bsw.writers.metaWriter.MustWrite(bb.Buf)
bigValuePool.Release(bb)
- ph.CompressedSizeBytes = bsw.streamWriters.totalBytesWritten()
+ ph.CompressedSizeBytes = bsw.writers.totalBytesWritten()
- bsw.streamWriters.MustClose()
+ bsw.writers.MustClose()
bsw.reset()
}
func generateBlockWriter() *blockWriter {
v := blockWriterPool.Get()
if v == nil {
- return &blockWriter{}
+ return &blockWriter{
+ writers: writers{
+ tagFamilyMetadataWriters: make(map[string]*writer),
+ tagFamilyWriters: make(map[string]*writer),
+ },
+ }
}
return v.(*blockWriter)
}
diff --git a/banyand/measure/column_metadata.go b/banyand/measure/column_metadata.go
index f678e559..7fda131b 100644
--- a/banyand/measure/column_metadata.go
+++ b/banyand/measure/column_metadata.go
@@ -56,7 +56,7 @@ func (cm *columnMetadata) unmarshal(src []byte) ([]byte, error) {
if err != nil {
return nil, fmt.Errorf("cannot unmarshal columnMetadata.name: %w", err)
}
- cm.name = convert.BytesToString(nameBytes)
+ cm.name = string(nameBytes)
if len(src) < 1 {
return nil, fmt.Errorf("cannot unmarshal columnMetadata.valueType: src is too short")
}
(skywalking-banyandb) 07/07: Pass integration test
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch storage-column
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit 7f157e8040c33f310810132f8bcc5fccbf2a5271
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Mon Dec 18 05:45:33 2023 +0800
Pass integration test
Signed-off-by: Gao Hongtao <ha...@gmail.com>
---
banyand/measure/block.go | 19 ++++++++++++++++---
banyand/measure/query.go | 4 ++++
pkg/index/inverted/inverted_series.go | 11 +++++++++--
pkg/index/inverted/inverted_series_test.go | 1 +
pkg/query/logical/index_filter.go | 2 +-
.../logical/measure/measure_plan_indexscan_local.go | 20 +++++++++++---------
test/cases/measure/measure.go | 4 ++--
7 files changed, 44 insertions(+), 17 deletions(-)
diff --git a/banyand/measure/block.go b/banyand/measure/block.go
index fd0661e1..1730d022 100644
--- a/banyand/measure/block.go
+++ b/banyand/measure/block.go
@@ -213,9 +213,10 @@ func (b *block) unmarshalTagFamily(decoder *encoding.BytesBlockDecoder, tfIndex
cc := b.tagFamilies[tfIndex].resizeColumns(len(tagProjection))
for j := range tagProjection {
- for i := range cc {
+ for i := range cfm.columnMetadata {
if tagProjection[j] == cfm.columnMetadata[i].name {
- cc[i].mustReadValues(decoder, valueReader, cfm.columnMetadata[i], uint64(b.Len()))
+ cc[j].mustReadValues(decoder, valueReader, cfm.columnMetadata[i], uint64(b.Len()))
+ break
}
}
}
@@ -372,7 +373,7 @@ func (bc *blockCursor) init(p *part, bm blockMetadata, queryOpts QueryOptions) {
func (bc *blockCursor) copyAllTo(r *pbv1.Result) {
r.SID = bc.bm.seriesID
- r.Timestamps = bc.timestamps
+ r.Timestamps = append(r.Timestamps, bc.timestamps...)
for _, cf := range bc.tagFamilies {
tf := pbv1.TagFamily{
Name: cf.Name,
@@ -479,6 +480,12 @@ func (bc *blockCursor) loadData(tmpBlock *block) bool {
Name: cf.Columns[i].Name,
ValueType: cf.Columns[i].ValueType,
}
+ if len(cf.Columns[i].Values) == 0 {
+ continue
+ }
+ if len(cf.Columns[i].Values) != len(tmpBlock.timestamps) {
+ logger.Panicf("unexpected number of values for tags %q: got %d; want %d", cf.Columns[i].Name, len(cf.Columns[i].Values), len(tmpBlock.timestamps))
+ }
column.Values = append(column.Values, cf.Columns[i].Values[start:end]...)
tf.Columns = append(tf.Columns, column)
}
@@ -486,6 +493,12 @@ func (bc *blockCursor) loadData(tmpBlock *block) bool {
}
bc.fields.Name = tmpBlock.field.Name
for i := range tmpBlock.field.Columns {
+ if len(tmpBlock.field.Columns[i].Values) == 0 {
+ continue
+ }
+ if len(tmpBlock.field.Columns[i].Values) != len(tmpBlock.timestamps) {
+ logger.Panicf("unexpected number of values for fields %q: got %d; want %d", tmpBlock.field.Columns[i].Name, len(tmpBlock.field.Columns[i].Values), len(tmpBlock.timestamps))
+ }
column := Column{
Name: tmpBlock.field.Columns[i].Name,
ValueType: tmpBlock.field.Columns[i].ValueType,
diff --git a/banyand/measure/query.go b/banyand/measure/query.go
index 15a9193f..6a41ca45 100644
--- a/banyand/measure/query.go
+++ b/banyand/measure/query.go
@@ -70,6 +70,9 @@ func (s *measure) Query(ctx context.Context, mqo pbv1.MeasureQueryOptions) (pbv1
if mqo.TimeRange == nil || mqo.Entity == nil {
return nil, errors.New("invalid query options: timeRange and series are required")
}
+ if len(mqo.TagProjection) == 0 && len(mqo.FieldProjection) == 0 {
+ return nil, errors.New("invalid query options: tagProjection or fieldProjection is required")
+ }
tsdb := s.databaseSupplier.SupplyTSDB().(storage.TSDB[*tsTable])
tabWrappers := tsdb.SelectTSTables(*mqo.TimeRange)
sl, err := tsdb.IndexDB().Search(ctx, &pbv1.Series{Subject: mqo.Name, EntityValues: mqo.Entity}, mqo.Filter, mqo.Order)
@@ -123,6 +126,7 @@ func (s *measure) Query(ctx context.Context, mqo pbv1.MeasureQueryOptions) (pbv1
if mqo.Order.Sort == modelv1.Sort_SORT_ASC || mqo.Order.Sort == modelv1.Sort_SORT_UNSPECIFIED {
result.ascTS = true
}
+ return &result, nil
}
result.sidToIndex = make(map[common.SeriesID]int)
diff --git a/pkg/index/inverted/inverted_series.go b/pkg/index/inverted/inverted_series.go
index f3a477ed..31d56406 100644
--- a/pkg/index/inverted/inverted_series.go
+++ b/pkg/index/inverted/inverted_series.go
@@ -105,11 +105,16 @@ func (s *store) SearchWildcard(wildcard []byte) ([]index.Series, error) {
func parseResult(dmi search.DocumentMatchIterator) ([]index.Series, error) {
result := make([]index.Series, 0, 10)
next, err := dmi.Next()
+ docIDMap := make(map[uint64]struct{})
for err == nil && next != nil {
var series index.Series
err = next.VisitStoredFields(func(field string, value []byte) bool {
if field == docIDField {
- series.ID = common.SeriesID(convert.BytesToUint64(value))
+ id := convert.BytesToUint64(value)
+ if _, ok := docIDMap[id]; !ok {
+ series.ID = common.SeriesID(convert.BytesToUint64(value))
+ docIDMap[id] = struct{}{}
+ }
}
if field == entityField {
series.EntityValues = value
@@ -119,7 +124,9 @@ func parseResult(dmi search.DocumentMatchIterator) ([]index.Series, error) {
if err != nil {
return nil, errors.WithMessage(err, "visit stored fields")
}
- result = append(result, series)
+ if series.ID > 0 {
+ result = append(result, series)
+ }
next, err = dmi.Next()
}
if err != nil {
diff --git a/pkg/index/inverted/inverted_series_test.go b/pkg/index/inverted/inverted_series_test.go
index 64daaaa2..609f70bd 100644
--- a/pkg/index/inverted/inverted_series_test.go
+++ b/pkg/index/inverted/inverted_series_test.go
@@ -210,5 +210,6 @@ func setupData(tester *assert.Assertions, s index.SeriesStore) {
EntityValues: []byte("test3"),
}
tester.NoError(s.Create(series3))
+ tester.NoError(s.Create(series3)) // duplicated data
s.(*store).flush()
}
diff --git a/pkg/query/logical/index_filter.go b/pkg/query/logical/index_filter.go
index 78a210c5..ac2df54e 100644
--- a/pkg/query/logical/index_filter.go
+++ b/pkg/query/logical/index_filter.go
@@ -512,7 +512,7 @@ func parseEntities(op modelv1.LogicalExpression_LogicalOp, input []*modelv1.TagV
anyEntity := func(entities [][]*modelv1.TagValue) bool {
for _, entity := range entities {
for _, entry := range entity {
- if entry == pbv1.AnyTagValue {
+ if entry != pbv1.AnyTagValue {
return false
}
}
diff --git a/pkg/query/logical/measure/measure_plan_indexscan_local.go b/pkg/query/logical/measure/measure_plan_indexscan_local.go
index 620a17bc..e4b1ccc3 100644
--- a/pkg/query/logical/measure/measure_plan_indexscan_local.go
+++ b/pkg/query/logical/measure/measure_plan_indexscan_local.go
@@ -137,15 +137,11 @@ func (i *localIndexScan) Sort(order *logical.OrderBy) {
func (i *localIndexScan) Execute(ctx context.Context) (mit executor.MIterator, err error) {
var orderBy *pbv1.OrderBy
- if i.order.Index != nil {
+ if i.order != nil {
orderBy = &pbv1.OrderBy{
Index: i.order.Index,
Sort: i.order.Sort,
}
- } else if i.groupByEntity {
- orderBy = &pbv1.OrderBy{
- Sort: i.order.Sort,
- }
}
ec := executor.FromMeasureExecutionContext(ctx)
var results []pbv1.MeasureQueryResult
@@ -205,15 +201,20 @@ func indexScan(startTime, endTime time.Time, metadata *commonv1.Metadata, projec
}
type resultMIterator struct {
- results []pbv1.MeasureQueryResult
- current []*measurev1.DataPoint
- index int
+ results []pbv1.MeasureQueryResult
+ current []*measurev1.DataPoint
+ index int
+ currentIndex int
}
func (ei *resultMIterator) Next() bool {
if ei.index >= len(ei.results) {
return false
}
+ ei.currentIndex++
+ if ei.currentIndex < len(ei.current) {
+ return true
+ }
r := ei.results[ei.index].Pull()
if r == nil {
@@ -221,6 +222,7 @@ func (ei *resultMIterator) Next() bool {
return ei.Next()
}
ei.current = ei.current[:0]
+ ei.currentIndex = 0
for i := range r.Timestamps {
dp := &measurev1.DataPoint{
Timestamp: timestamppb.New(time.Unix(0, int64(r.Timestamps[i]))),
@@ -251,7 +253,7 @@ func (ei *resultMIterator) Next() bool {
}
func (ei *resultMIterator) Current() []*measurev1.DataPoint {
- return ei.current
+ return []*measurev1.DataPoint{ei.current[ei.currentIndex]}
}
func (ei *resultMIterator) Close() error {
diff --git a/test/cases/measure/measure.go b/test/cases/measure/measure.go
index bfcb6642..f011fe29 100644
--- a/test/cases/measure/measure.go
+++ b/test/cases/measure/measure.go
@@ -39,9 +39,9 @@ var (
}
)
-var _ = g.DescribeTable("Scanning Measures", verify,
+var _ = g.FDescribeTable("Scanning Measures", verify,
g.Entry("all", helpers.Args{Input: "all", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
- g.FEntry("all only fields", helpers.Args{Input: "all_only_fields", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
+ g.Entry("all only fields", helpers.Args{Input: "all_only_fields", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
g.Entry("filter by tag", helpers.Args{Input: "tag_filter", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
g.Entry("filter by a integer tag", helpers.Args{Input: "tag_filter_int", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
g.Entry("filter by an unknown tag", helpers.Args{Input: "tag_filter_unknown", Duration: 25 * time.Minute, Offset: -20 * time.Minute, WantEmpty: true}),