You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ha...@apache.org on 2023/12/19 21:47:23 UTC
(skywalking-banyandb) 03/04: Merge remote-tracking branch 'origin/main' into storage-column
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch storage-column
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit 71fda92a8e7f08d00dcd9ff78ec64d2057f054e5
Merge: 12c0466f 234fe3e2
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Tue Dec 19 11:49:49 2023 +0000
Merge remote-tracking branch 'origin/main' into storage-column
.github/workflows/e2e.storage.yml | 2 +-
.github/workflows/publish-docker.yml | 2 +-
CHANGES.md | 1 +
Makefile | 18 +++++++++++++-----
banyand/Makefile | 1 -
banyand/measure/block_test.go | 4 +++-
bydbctl/Makefile | 3 ---
bydbctl/internal/cmd/health_check.go | 8 +++++++-
bydbctl/internal/cmd/health_check_test.go | 21 +++++++++++++++------
.dockerignore => docker/.dockerignore | 0
{banyand => docker}/Dockerfile | 6 +++++-
bydbctl/Dockerfile => docker/Makefile | 20 ++------------------
test/docker/base-compose.yml | 9 +++++++--
test/stress/docker-compose.yaml | 3 +--
test/stress/env | 1 -
test/stress/env.dev | 1 -
16 files changed, 56 insertions(+), 44 deletions(-)
diff --cc banyand/measure/block_test.go
index f989b6db,00000000..e7b776c1
mode 100644,000000..100644
--- a/banyand/measure/block_test.go
+++ b/banyand/measure/block_test.go
@@@ -1,464 -1,0 +1,466 @@@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package measure
+
+import (
+ "crypto/rand"
+ "encoding/binary"
+ "reflect"
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+
+ "github.com/apache/skywalking-banyandb/api/common"
+ "github.com/apache/skywalking-banyandb/pkg/bytes"
+ "github.com/apache/skywalking-banyandb/pkg/convert"
+ "github.com/apache/skywalking-banyandb/pkg/encoding"
+ "github.com/apache/skywalking-banyandb/pkg/fs"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
+)
+
+func Test_block_reset(t *testing.T) {
+ type fields struct {
+ timestamps []int64
+ tagFamilies []columnFamily
+ field columnFamily
+ }
+ tests := []struct {
+ name string
+ fields fields
+ want block
+ }{
+ {
+ name: "Test reset",
+ fields: fields{
+ timestamps: []int64{1, 2, 3},
+ tagFamilies: []columnFamily{{}, {}, {}},
+ field: columnFamily{columns: []column{{}, {}}},
+ },
+ want: block{
+ timestamps: []int64{},
+ tagFamilies: []columnFamily{},
+ field: columnFamily{columns: []column{}},
+ },
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ b := &block{
+ timestamps: tt.fields.timestamps,
+ tagFamilies: tt.fields.tagFamilies,
+ field: tt.fields.field,
+ }
+ b.reset()
+ if !reflect.DeepEqual(*b, tt.want) {
+ t.Errorf("block.reset() = %+v, want %+v", *b, tt.want)
+ }
+ })
+ }
+}
+
+func toTagProjection(b block) map[string][]string {
+ result := make(map[string][]string, len(b.tagFamilies))
+ for i := range b.tagFamilies {
+ names := make([]string, len(b.tagFamilies[i].columns))
+ for i2 := range b.tagFamilies[i].columns {
+ names[i2] = b.tagFamilies[i].columns[i2].name
+ }
+ result[b.tagFamilies[i].name] = names
+ }
+ return result
+}
+
+var conventionalBlock = block{
+ timestamps: []int64{1, 2},
+ tagFamilies: []columnFamily{
+ {
+ name: "arrTag",
+ columns: []column{
+ {
+ name: "strArrTag", valueType: pbv1.ValueTypeStrArr,
+ values: [][]byte{marshalStrArr([][]byte{[]byte("value1"), []byte("value2")}), marshalStrArr([][]byte{[]byte("value3"), []byte("value4")})},
+ },
+ {
+ name: "intArrTag", valueType: pbv1.ValueTypeInt64Arr,
+ values: [][]byte{
+ marshalIntArr([][]byte{convert.Int64ToBytes(25), convert.Int64ToBytes(30)}),
+ marshalIntArr([][]byte{convert.Int64ToBytes(50), convert.Int64ToBytes(60)}),
+ },
+ },
+ },
+ },
+ {
+ name: "binaryTag",
+ columns: []column{
+ {name: "binaryTag", valueType: pbv1.ValueTypeBinaryData, values: [][]byte{longText, longText}},
+ },
+ },
+ {
+ name: "singleTag",
+ columns: []column{
+ {name: "strTag", valueType: pbv1.ValueTypeStr, values: [][]byte{[]byte("value1"), []byte("value2")}},
+ {name: "intTag", valueType: pbv1.ValueTypeInt64, values: [][]byte{convert.Int64ToBytes(10), convert.Int64ToBytes(20)}},
+ },
+ },
+ },
+ field: columnFamily{
+ columns: []column{
+ {name: "strField", valueType: pbv1.ValueTypeStr, values: [][]byte{[]byte("field1"), []byte("field2")}},
+ {name: "intField", valueType: pbv1.ValueTypeInt64, values: [][]byte{convert.Int64ToBytes(1110), convert.Int64ToBytes(2220)}},
+ {name: "floatField", valueType: pbv1.ValueTypeFloat64, values: [][]byte{convert.Float64ToBytes(1221233.343), convert.Float64ToBytes(2442466.686)}},
+ {name: "binaryField", valueType: pbv1.ValueTypeBinaryData, values: [][]byte{longText, longText}},
+ },
+ },
+}
+
+func Test_block_mustInitFromDataPoints(t *testing.T) {
+ type args struct {
+ timestamps []int64
+ tagFamilies [][]nameValues
+ fields []nameValues
+ }
+ tests := []struct {
+ name string
+ args args
+ want block
+ }{
+ {
+ name: "Test mustInitFromDataPoints",
+ args: args{
+ timestamps: []int64{1, 2},
+ tagFamilies: [][]nameValues{
+ {
+ {
+ "arrTag", []*nameValue{
+ {name: "strArrTag", valueType: pbv1.ValueTypeStrArr, value: nil, valueArr: [][]byte{[]byte("value1"), []byte("value2")}},
+ {name: "intArrTag", valueType: pbv1.ValueTypeInt64Arr, value: nil, valueArr: [][]byte{convert.Int64ToBytes(25), convert.Int64ToBytes(30)}},
+ },
+ },
+ {
+ "binaryTag", []*nameValue{
+ {name: "binaryTag", valueType: pbv1.ValueTypeBinaryData, value: longText, valueArr: nil},
+ },
+ },
+ {
+ "singleTag", []*nameValue{
+ {name: "strTag", valueType: pbv1.ValueTypeStr, value: []byte("value1"), valueArr: nil},
+ {name: "intTag", valueType: pbv1.ValueTypeInt64, value: convert.Int64ToBytes(10), valueArr: nil},
+ },
+ },
+ },
+ {
+ {
+ "arrTag", []*nameValue{
+ {name: "strArrTag", valueType: pbv1.ValueTypeStrArr, value: nil, valueArr: [][]byte{[]byte("value3"), []byte("value4")}},
+ {name: "intArrTag", valueType: pbv1.ValueTypeInt64Arr, value: nil, valueArr: [][]byte{convert.Int64ToBytes(50), convert.Int64ToBytes(60)}},
+ },
+ },
+ {
+ "binaryTag", []*nameValue{
+ {name: "binaryTag", valueType: pbv1.ValueTypeBinaryData, value: longText, valueArr: nil},
+ },
+ },
+ {
+ "singleTag", []*nameValue{
+ {name: "strTag", valueType: pbv1.ValueTypeStr, value: []byte("value2"), valueArr: nil},
+ {name: "intTag", valueType: pbv1.ValueTypeInt64, value: convert.Int64ToBytes(20), valueArr: nil},
+ },
+ },
+ },
+ },
+ fields: []nameValues{
+ {
+ "skipped", []*nameValue{
+ {name: "strField", valueType: pbv1.ValueTypeStr, value: []byte("field1"), valueArr: nil},
+ {name: "intField", valueType: pbv1.ValueTypeInt64, value: convert.Int64ToBytes(1110), valueArr: nil},
+ {name: "floatField", valueType: pbv1.ValueTypeFloat64, value: convert.Float64ToBytes(1221233.343), valueArr: nil},
+ {name: "binaryField", valueType: pbv1.ValueTypeBinaryData, value: longText, valueArr: nil},
+ },
+ },
+ {
+ "skipped", []*nameValue{
+ {name: "strField", valueType: pbv1.ValueTypeStr, value: []byte("field2"), valueArr: nil},
+ {name: "intField", valueType: pbv1.ValueTypeInt64, value: convert.Int64ToBytes(2220), valueArr: nil},
+ {name: "floatField", valueType: pbv1.ValueTypeFloat64, value: convert.Float64ToBytes(2442466.686), valueArr: nil},
+ {name: "binaryField", valueType: pbv1.ValueTypeBinaryData, value: longText, valueArr: nil},
+ },
+ },
+ },
+ },
+ want: conventionalBlock,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ b := &block{}
+ b.mustInitFromDataPoints(tt.args.timestamps, tt.args.tagFamilies, tt.args.fields)
+ if !reflect.DeepEqual(*b, tt.want) {
+ t.Errorf("block.mustInitFromDataPoints() = %+v, want %+v", *b, tt.want)
+ }
+ })
+ }
+}
+
+var longText = []byte(`
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec a diam lectus. Sed sit amet ipsum mauris.
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec a diam lectus. Sed sit amet ipsum mauris.
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec a diam lectus. Sed sit amet ipsum mauris.
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec a diam lectus. Sed sit amet ipsum mauris.
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec a diam lectus. Sed sit amet ipsum mauris.
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec a diam lectus. Sed sit amet ipsum mauris.
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec a diam lectus. Sed sit amet ipsum mauris.
+`)
+
+func marshalStrArr(arr [][]byte) []byte {
+ nv := &nameValue{valueType: pbv1.ValueTypeStrArr, valueArr: arr}
+ return nv.marshal()
+}
+
+func marshalIntArr(arr [][]byte) []byte {
+ nv := &nameValue{valueType: pbv1.ValueTypeInt64Arr, valueArr: arr}
+ return nv.marshal()
+}
+
+func Test_mustWriteAndReadTimestamps(t *testing.T) {
+ tests := []struct {
+ name string
+ args []int64
+ wantPanic bool
+ wantTM timestampsMetadata
+ }{
+ {
+ name: "Test mustWriteAndReadTimestamps",
+ args: []int64{1, 2, 3, 4, 5},
+ wantPanic: false,
+ },
+ {
+ name: "Test mustWriteAndReadTimestamps with panic",
+ args: getBitInt64Arr(),
+ wantPanic: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ defer func() {
+ r := recover()
+ if (r != nil) != tt.wantPanic {
+ t.Errorf("mustWriteTimestampsTo() recover = %v, wantPanic = %v", r, tt.wantPanic)
+ }
+ }()
+ tm := ×tampsMetadata{}
+ b := &bytes.Buffer{}
+ mustWriteTimestampsTo(tm, tt.args, writer{w: b})
+ timestamps := mustReadTimestampsFrom(nil, tm, len(tt.args), b)
+ if !reflect.DeepEqual(timestamps, tt.args) {
+ t.Errorf("mustReadTimestampsFrom() = %v, want %v", timestamps, tt.args)
+ }
+ })
+ }
+}
+
+func getBitInt64Arr() []int64 {
+ size := maxTimestampsBlockSize + 1
+ randSlice := make([]int64, size)
+ for i := range randSlice {
+ b := make([]byte, 8)
+ _, _ = rand.Read(b)
+ randSlice[i] = int64(binary.BigEndian.Uint64(b))
+ }
+ return randSlice
+}
+
+func Test_marshalAndUnmarshalTagFamily(t *testing.T) {
+ metaBuffer, dataBuffer := &bytes.Buffer{}, &bytes.Buffer{}
+ ww := &writers{
+ mustCreateTagFamilyWriters: func(name string) (fs.Writer, fs.Writer) {
+ return metaBuffer, dataBuffer
+ },
+ tagFamilyMetadataWriters: make(map[string]*writer),
+ tagFamilyWriters: make(map[string]*writer),
+ }
+ b := &conventionalBlock
+ tagProjection := toTagProjection(*b)
+ tfIndex := 0
+ name := "arrTag"
+ decoder := &encoding.BytesBlockDecoder{}
+ bm := &blockMetadata{}
+
+ b.marshalTagFamily(b.tagFamilies[tfIndex], bm, ww)
+
+ metaWriter, ok1 := ww.tagFamilyMetadataWriters[name]
+ valueWriter, ok2 := ww.tagFamilyWriters[name]
+ if !ok1 || !ok2 {
+ t.Fatalf("Writers not correctly added to maps")
+ }
+ if metaWriter.w != metaBuffer || valueWriter.w != dataBuffer {
+ t.Fatalf("Writers not correctly added to maps")
+ }
+
+ unmarshaled := generateBlock()
+ defer releaseBlock(unmarshaled)
+ // set the timestamps to the same length as the original block
+ // the data size in a block depends on the timestamps length
+ unmarshaled.timestamps = make([]int64, len(b.timestamps))
+ unmarshaled.resizeTagFamilies(1)
+
+ unmarshaled.unmarshalTagFamily(decoder, tfIndex, name, bm.getTagFamilyMetadata(name), tagProjection[name], metaBuffer, dataBuffer)
+
- if diff := cmp.Diff(unmarshaled.tagFamilies[0], b.tagFamilies[0]); diff != "" {
++ if diff := cmp.Diff(unmarshaled.tagFamilies[0], b.tagFamilies[0],
++ cmp.AllowUnexported(columnFamily{}, column{}),
++ ); diff != "" {
+ t.Errorf("block.unmarshalTagFamily() (-got +want):\n%s", diff)
+ }
+}
+
+func Test_marshalAndUnmarshalBlock(t *testing.T) {
+ timestampBuffer, fieldBuffer := &bytes.Buffer{}, &bytes.Buffer{}
+ ww := &writers{
+ mustCreateTagFamilyWriters: func(name string) (fs.Writer, fs.Writer) {
+ return &bytes.Buffer{}, &bytes.Buffer{}
+ },
+ tagFamilyMetadataWriters: make(map[string]*writer),
+ tagFamilyWriters: make(map[string]*writer),
+ timestampsWriter: writer{w: timestampBuffer},
+ fieldValuesWriter: writer{w: fieldBuffer},
+ }
+ p := &part{
+ timestamps: timestampBuffer,
+ fieldValues: fieldBuffer,
+ }
+ b := &conventionalBlock
+ tagProjection := toTagProjection(*b)
+ decoder := &encoding.BytesBlockDecoder{}
+ sid := common.SeriesID(1)
+ bm := blockMetadata{}
+
+ b.mustWriteTo(sid, &bm, ww)
+
+ tagFamilyMetadataReaders := make(map[string]fs.Reader)
+ tagFamilyReaders := make(map[string]fs.Reader)
+
+ for k, w := range ww.tagFamilyMetadataWriters {
+ tagFamilyMetadataReaders[k] = w.w.(*bytes.Buffer)
+ tagFamilyReaders[k] = ww.tagFamilyWriters[k].w.(*bytes.Buffer)
+ }
+ p.tagFamilyMetadata = tagFamilyMetadataReaders
+ p.tagFamilies = tagFamilyReaders
+
+ unmarshaled := generateBlock()
+ defer releaseBlock(unmarshaled)
+
+ var tp []pbv1.TagProjection
+ for family, names := range tagProjection {
+ tp = append(tp, pbv1.TagProjection{
+ Family: family,
+ Names: names,
+ })
+ }
+ bm.tagProjection = tp
+ unmarshaled.mustReadFrom(decoder, p, bm)
+ // blockMetadata is using a map, so the order of tag families is not guaranteed
+ unmarshaled.sortTagFamilies()
+
+ if !reflect.DeepEqual(b, unmarshaled) {
+ t.Errorf("block.mustReadFrom() = %+v, want %+v", unmarshaled, b)
+ }
+}
+
+func Test_findRange(t *testing.T) {
+ type args struct {
+ timestamps []int64
+ min int64
+ max int64
+ }
+ tests := []struct {
+ name string
+ args args
+ wantStart int
+ wantEnd int
+ wantExist bool
+ }{
+ {
+ name: "Test with empty timestamps",
+ args: args{
+ timestamps: []int64{},
+ min: 1,
+ max: 10,
+ },
+ wantStart: 0,
+ wantEnd: 0,
+ wantExist: false,
+ },
+ {
+ name: "Test with single timestamp",
+ args: args{
+ timestamps: []int64{1},
+ min: 1,
+ max: 1,
+ },
+ wantStart: 0,
+ wantEnd: 1,
+ wantExist: true,
+ },
+ {
+ name: "Test with range not in timestamps",
+ args: args{
+ timestamps: []int64{1, 2, 3, 4, 5},
+ min: 6,
+ max: 10,
+ },
+ wantStart: 0,
+ wantEnd: 0,
+ wantExist: false,
+ },
+ {
+ name: "Test with range in timestamps",
+ args: args{
+ timestamps: []int64{1, 2, 3, 4, 5},
+ min: 2,
+ max: 4,
+ },
+ wantStart: 1,
+ wantEnd: 4,
+ wantExist: true,
+ },
+ {
+ name: "Test with range as timestamps",
+ args: args{
+ timestamps: []int64{1, 2, 3, 4, 5},
+ min: 1,
+ max: 5,
+ },
+ wantStart: 0,
+ wantEnd: 5,
+ wantExist: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ gotStart, gotEnd, gotExist := findRange(tt.args.timestamps, tt.args.min, tt.args.max)
+ if gotStart != tt.wantStart {
+ t.Errorf("findRange() gotStart = %v, want %v", gotStart, tt.wantStart)
+ }
+ if gotEnd != tt.wantEnd {
+ t.Errorf("findRange() gotEnd = %v, want %v", gotEnd, tt.wantEnd)
+ }
+ if gotExist != tt.wantExist {
+ t.Errorf("findRange() gotExist = %v, want %v", gotExist, tt.wantExist)
+ }
+ })
+ }
+}