You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by wu...@apache.org on 2024/05/29 14:41:35 UTC
(skywalking-banyandb) branch main updated: Fix several issues of topN pre-calculation (#455)
This is an automated email from the ASF dual-hosted git repository.
wusheng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
The following commit(s) were added to refs/heads/main by this push:
new 8906b3ca Fix several issues of topN pre-calculation (#455)
8906b3ca is described below
commit 8906b3caf276aed936854688434407415da3f6cc
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Wed May 29 22:41:30 2024 +0800
Fix several issues of topN pre-calculation (#455)
---
CHANGES.md | 7 ++-
banyand/internal/storage/index.go | 6 +-
banyand/measure/block_metadata_test.go | 14 ++---
banyand/measure/block_writer.go | 24 ++++----
banyand/measure/column.go | 16 +++---
banyand/measure/topn.go | 62 +++++++++------------
banyand/query/processor_topn.go | 3 +-
banyand/queue/sub/sub.go | 14 ++++-
banyand/stream/block_metadata.go | 64 +++++++++++-----------
banyand/stream/block_metadata_test.go | 14 ++---
banyand/stream/block_writer.go | 24 ++++----
banyand/stream/tag.go | 16 +++---
pkg/flow/streaming/sliding_window.go | 57 +++++++++++++++----
pkg/flow/streaming/sliding_window_test.go | 4 +-
pkg/flow/streaming/streaming.go | 8 ++-
pkg/flow/streaming/streaming_test.go | 12 ++--
pkg/flow/streaming/topn.go | 24 ++++++++
pkg/flow/streaming/topn_test.go | 2 +
pkg/flow/types.go | 7 +++
pkg/pb/v1/series.go | 15 ++++-
pkg/pb/v1/series_test.go | 8 +++
pkg/pb/v1/value.go | 18 +++---
pkg/query/logical/measure/topn_analyzer.go | 15 +++--
pkg/schema/init.go | 18 ++----
...nstance_endpoint_cpm_minute_top_bottom_100.json | 2 +-
pkg/test/setup/setup.go | 2 +-
.../service_instance_endpoint_cpm_minute_data.json | 8 +--
...service_instance_endpoint_cpm_minute_data1.json | 10 ++--
...service_instance_endpoint_cpm_minute_data2.json | 4 +-
test/cases/topn/data/want/null_group.yaml | 16 +++---
30 files changed, 297 insertions(+), 197 deletions(-)
diff --git a/CHANGES.md b/CHANGES.md
index 29496416..420b4f8a 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -2,18 +2,23 @@
Release Notes.
-## 0.7.0
+## 0.6.1
- Add benchmarks for stream filtering and sorting.
### Features
+- Limit the max pre-calculation result flush interval to 1 minute.
+- Use both datapoint timestamp and server time to trigger the flush of topN pre-calculation result.
+
### Bugs
- Fix the bug that topN query doesn't return when an error occurs.
- Data race in the hot series index selection.
- Remove SetSchema from measure cache which could change the schema in the cache.
- Fix duplicated items in the query aggregation top-n list.
+- Fix non-"value" field in topN pre-calculation result measure is lack of data.
+- Encode escaped characters to int64 bytes to fix the malformed data.
## 0.6.0
diff --git a/banyand/internal/storage/index.go b/banyand/internal/storage/index.go
index 1ac969f4..6980369d 100644
--- a/banyand/internal/storage/index.go
+++ b/banyand/internal/storage/index.go
@@ -103,7 +103,7 @@ func (s *seriesIndex) searchPrimary(ctx context.Context, series []*pbv1.Series)
}
result, err := convertIndexSeriesToSeriesList(ss)
if err != nil {
- return nil, errors.WithMessagef(err, "failed to convert index series to series list, matchers: %v, result:%v", seriesMatchers, ss)
+ return nil, errors.WithMessagef(err, "failed to convert index series to series list, matchers: %v, matched: %d", seriesMatchers, len(ss))
}
return result, nil
}
@@ -135,7 +135,7 @@ func convertEntityValuesToSeriesMatcher(series *pbv1.Series) (index.SeriesMatche
if hasAny {
if hasWildcard {
- if err = series.Marshal(); err != nil {
+ if err = series.MarshalWithWildcard(); err != nil {
return emptySeriesMatcher, err
}
return index.SeriesMatcher{
@@ -167,7 +167,7 @@ func convertIndexSeriesToSeriesList(indexSeries []index.Series) (pbv1.SeriesList
var series pbv1.Series
series.ID = s.ID
if err := series.Unmarshal(s.EntityValues); err != nil {
- return nil, err
+ return nil, errors.WithMessagef(err, "failed to unmarshal series: %s", s.EntityValues)
}
seriesList = append(seriesList, &series)
}
diff --git a/banyand/measure/block_metadata_test.go b/banyand/measure/block_metadata_test.go
index 81da440b..59709b73 100644
--- a/banyand/measure/block_metadata_test.go
+++ b/banyand/measure/block_metadata_test.go
@@ -75,7 +75,7 @@ func Test_dataBlock_marshal_unmarshal(t *testing.T) {
}
func Test_timestampsMetadata_reset(t *testing.T) {
- th := ×tampsMetadata{
+ tm := ×tampsMetadata{
dataBlock: dataBlock{
offset: 1,
size: 1,
@@ -85,13 +85,13 @@ func Test_timestampsMetadata_reset(t *testing.T) {
encodeType: encoding.EncodeTypeConst,
}
- th.reset()
+ tm.reset()
- assert.Equal(t, uint64(0), th.dataBlock.offset)
- assert.Equal(t, uint64(0), th.dataBlock.size)
- assert.Equal(t, int64(0), th.min)
- assert.Equal(t, int64(0), th.max)
- assert.Equal(t, encoding.EncodeTypeUnknown, th.encodeType)
+ assert.Equal(t, uint64(0), tm.dataBlock.offset)
+ assert.Equal(t, uint64(0), tm.dataBlock.size)
+ assert.Equal(t, int64(0), tm.min)
+ assert.Equal(t, int64(0), tm.max)
+ assert.Equal(t, encoding.EncodeTypeUnknown, tm.encodeType)
}
func Test_timestampsMetadata_copyFrom(t *testing.T) {
diff --git a/banyand/measure/block_writer.go b/banyand/measure/block_writer.go
index 8f97c066..760a63ad 100644
--- a/banyand/measure/block_writer.go
+++ b/banyand/measure/block_writer.go
@@ -214,23 +214,23 @@ func (bw *blockWriter) mustWriteBlock(sid common.SeriesID, b *block) {
bm := generateBlockMetadata()
b.mustWriteTo(sid, bm, &bw.writers)
- th := &bm.timestamps
- if bw.totalCount == 0 || th.min < bw.totalMinTimestamp {
- bw.totalMinTimestamp = th.min
+ tm := &bm.timestamps
+ if bw.totalCount == 0 || tm.min < bw.totalMinTimestamp {
+ bw.totalMinTimestamp = tm.min
}
- if bw.totalCount == 0 || th.max > bw.totalMaxTimestamp {
- bw.totalMaxTimestamp = th.max
+ if bw.totalCount == 0 || tm.max > bw.totalMaxTimestamp {
+ bw.totalMaxTimestamp = tm.max
}
- if !hasWrittenBlocks || th.min < bw.minTimestamp {
- bw.minTimestamp = th.min
+ if !hasWrittenBlocks || tm.min < bw.minTimestamp {
+ bw.minTimestamp = tm.min
}
- if !hasWrittenBlocks || th.max > bw.maxTimestamp {
- bw.maxTimestamp = th.max
+ if !hasWrittenBlocks || tm.max > bw.maxTimestamp {
+ bw.maxTimestamp = tm.max
}
- if isSeenSid && th.min < bw.minTimestampLast {
- logger.Panicf("the block for sid=%d cannot contain timestamp smaller than %d, but it contains timestamp %d", sid, bw.minTimestampLast, th.min)
+ if isSeenSid && tm.min < bw.minTimestampLast {
+ logger.Panicf("the block for sid=%d cannot contain timestamp smaller than %d, but it contains timestamp %d", sid, bw.minTimestampLast, tm.min)
}
- bw.minTimestampLast = th.min
+ bw.minTimestampLast = tm.min
bw.totalUncompressedSizeBytes += bm.uncompressedSizeBytes
bw.totalCount += bm.count
diff --git a/banyand/measure/column.go b/banyand/measure/column.go
index 2f17d186..f208786c 100644
--- a/banyand/measure/column.go
+++ b/banyand/measure/column.go
@@ -51,11 +51,11 @@ func (c *column) resizeValues(valuesLen int) [][]byte {
return values
}
-func (c *column) mustWriteTo(ch *columnMetadata, columnWriter *writer) {
- ch.reset()
+func (c *column) mustWriteTo(cm *columnMetadata, columnWriter *writer) {
+ cm.reset()
- ch.name = c.name
- ch.valueType = c.valueType
+ cm.name = c.name
+ cm.valueType = c.valueType
// TODO: encoding values based on value type
@@ -64,11 +64,11 @@ func (c *column) mustWriteTo(ch *columnMetadata, columnWriter *writer) {
// marshal values
bb.Buf = encoding.EncodeBytesBlock(bb.Buf[:0], c.values)
- ch.size = uint64(len(bb.Buf))
- if ch.size > maxValuesBlockSize {
- logger.Panicf("too valuesSize: %d bytes; mustn't exceed %d bytes", ch.size, maxValuesBlockSize)
+ cm.size = uint64(len(bb.Buf))
+ if cm.size > maxValuesBlockSize {
+ logger.Panicf("too valuesSize: %d bytes; mustn't exceed %d bytes", cm.size, maxValuesBlockSize)
}
- ch.offset = columnWriter.bytesWritten
+ cm.offset = columnWriter.bytesWritten
columnWriter.MustWrite(bb.Buf)
}
diff --git a/banyand/measure/topn.go b/banyand/measure/topn.go
index 1ec55c8e..85460d5d 100644
--- a/banyand/measure/topn.go
+++ b/banyand/measure/topn.go
@@ -52,6 +52,7 @@ import (
const (
timeBucketFormat = "200601021504"
resultPersistencyTimeout = 10 * time.Second
+ maxFlushInterval = time.Minute
)
var (
@@ -132,7 +133,6 @@ func (t *topNStreamingProcessor) writeStreamRecord(record flow.StreamRecord) err
}
// down-sample the start of the timeWindow to a time-bucket
eventTime := t.downSampleTimeBucket(record.TimestampMillis())
- timeBucket := eventTime.Format(timeBucketFormat)
var err error
publisher := t.pipeline.NewBatchPublisher(resultPersistencyTimeout)
defer publisher.Close()
@@ -146,14 +146,14 @@ func (t *topNStreamingProcessor) writeStreamRecord(record flow.StreamRecord) err
for rankNum, tuple := range tuples {
fieldValue := tuple.V1.(int64)
data := tuple.V2.(flow.StreamRecord).Data().(flow.Data)
- err = multierr.Append(err, t.writeData(publisher, eventTime, timeBucket, fieldValue, group, data, rankNum))
+ err = multierr.Append(err, t.writeData(publisher, eventTime, fieldValue, data, rankNum))
}
}
return err
}
-func (t *topNStreamingProcessor) writeData(publisher queue.BatchPublisher, eventTime time.Time, timeBucket string, fieldValue int64,
- group string, data flow.Data, rankNum int,
+func (t *topNStreamingProcessor) writeData(publisher queue.BatchPublisher, eventTime time.Time, fieldValue int64,
+ data flow.Data, rankNum int,
) error {
var tagValues []*modelv1.TagValue
if len(t.topNSchema.GetGroupByTagNames()) > 0 {
@@ -172,11 +172,6 @@ func (t *topNStreamingProcessor) writeData(publisher queue.BatchPublisher, event
return err
}
- // measureID is consist of three parts,
- // 1. groupValues
- // 2. rankNumber
- // 3. timeBucket
- measureID := group + "_" + strconv.Itoa(rankNum) + "_" + timeBucket
iwr := &measurev1.InternalWriteRequest{
Request: &measurev1.WriteRequest{
MessageId: uint64(time.Now().UnixNano()),
@@ -185,17 +180,10 @@ func (t *topNStreamingProcessor) writeData(publisher queue.BatchPublisher, event
Timestamp: timestamppb.New(eventTime),
TagFamilies: []*modelv1.TagFamilyForWrite{
{
- Tags: append([]*modelv1.TagValue{
- // MeasureID
- {
- Value: &modelv1.TagValue_Str{
- Str: &modelv1.Str{
- Value: measureID,
- },
- },
- },
+ Tags: append(
+ data[0].([]*modelv1.TagValue),
// SortDirection
- {
+ &modelv1.TagValue{
Value: &modelv1.TagValue_Int{
Int: &modelv1.Int{
Value: int64(t.sortDirection),
@@ -203,14 +191,14 @@ func (t *topNStreamingProcessor) writeData(publisher queue.BatchPublisher, event
},
},
// RankNumber
- {
+ &modelv1.TagValue{
Value: &modelv1.TagValue_Int{
Int: &modelv1.Int{
Value: int64(rankNum),
},
},
},
- }, data[0].([]*modelv1.TagValue)...),
+ ),
},
},
Fields: []*modelv1.FieldValue{
@@ -240,40 +228,35 @@ func (t *topNStreamingProcessor) locate(tagValues []*modelv1.TagValue, rankNum i
if len(tagValues) != 0 && len(t.topNSchema.GetGroupByTagNames()) != len(tagValues) {
return nil, 0, errors.New("no enough tag values for the entity")
}
- // Subject: topN aggregation Name
- //
- // entity prefix
- //
- // 1) sort direction
- // 2) rank number
- // >3) group tag values if needed
series := &pbv1.Series{
Subject: t.topNSchema.GetMetadata().GetName(),
EntityValues: make([]*modelv1.TagValue, 1+1+len(tagValues)),
}
- // entity prefix
- series.EntityValues[0] = &modelv1.TagValue{
+ copy(series.EntityValues, tagValues)
+ series.EntityValues[len(series.EntityValues)-2] = &modelv1.TagValue{
Value: &modelv1.TagValue_Int{
Int: &modelv1.Int{
Value: int64(t.sortDirection),
},
},
}
- series.EntityValues[1] = &modelv1.TagValue{
+ series.EntityValues[len(series.EntityValues)-1] = &modelv1.TagValue{
Value: &modelv1.TagValue_Int{
Int: &modelv1.Int{
Value: int64(rankNum),
},
},
}
- // measureID as sharding key
- for idx, tagVal := range tagValues {
- series.EntityValues[idx+2] = tagVal
- }
if err := series.Marshal(); err != nil {
return nil, 0, fmt.Errorf("fail to marshal series: %w", err)
}
+ src := make([]byte, len(series.Buffer))
+ copy(src, series.Buffer)
+ var s1 pbv1.Series
+ if err := s1.Unmarshal(src); err != nil {
+ return nil, 0, fmt.Errorf("fail to unmarshal series encoded:[%s] tagValues:[%s]: %w", series.Buffer, series.EntityValues, err)
+ }
id, err := partition.ShardID(series.Buffer, t.m.shardNum)
if err != nil {
return nil, 0, err
@@ -282,7 +265,11 @@ func (t *topNStreamingProcessor) locate(tagValues []*modelv1.TagValue, rankNum i
}
func (t *topNStreamingProcessor) start() *topNStreamingProcessor {
- t.errCh = t.streamingFlow.Window(streaming.NewTumblingTimeWindows(t.interval)).
+ flushInterval := t.interval
+ if flushInterval > maxFlushInterval {
+ flushInterval = maxFlushInterval
+ }
+ t.errCh = t.streamingFlow.Window(streaming.NewTumblingTimeWindows(t.interval, flushInterval)).
AllowedMaxWindows(int(t.topNSchema.GetLruSize())).
TopN(int(t.topNSchema.GetCountersNumber()),
streaming.WithSortKeyExtractor(func(record flow.StreamRecord) int64 {
@@ -364,7 +351,8 @@ func (manager *topNProcessorManager) start() error {
for i, sortDirection := range sortDirections {
srcCh := make(chan interface{})
src, _ := sources.NewChannel(srcCh)
- streamingFlow := streaming.New(src)
+ name := strings.Join([]string{topNSchema.GetMetadata().Group, topNSchema.GetMetadata().Name, modelv1.Sort_name[int32(sortDirection)]}, "-")
+ streamingFlow := streaming.New(name, src)
filters, buildErr := manager.buildFilter(topNSchema.GetCriteria())
if buildErr != nil {
diff --git a/banyand/query/processor_topn.go b/banyand/query/processor_topn.go
index de54a1da..d734e18b 100644
--- a/banyand/query/processor_topn.go
+++ b/banyand/query/processor_topn.go
@@ -105,7 +105,7 @@ func (t *topNQueryProcessor) Rev(message bus.Message) (resp bus.Message) {
Str("topN", topNMetadata.GetName()).
Msg("fail to build schema")
}
- plan, err := logical_measure.TopNAnalyze(context.TODO(), request, topNResultMeasure.GetSchema(), sourceMeasure.GetSchema(), topNSchema, s)
+ plan, err := logical_measure.TopNAnalyze(context.TODO(), request, topNResultMeasure.GetSchema(), sourceMeasure.GetSchema(), s)
if err != nil {
resp = bus.NewMessage(bus.MessageID(now), common.NewError("fail to analyze the query request for topn %s: %v", topNMetadata.GetName(), err))
return
@@ -256,7 +256,6 @@ func (aggr *postAggregationProcessor) Put(entityValues pbv1.EntityValues, val in
key := entityValues.String()
if item, found := aggr.cache[key]; found {
item.int64Func.In(val)
- aggr.tryEnqueue(key, item)
return nil
}
diff --git a/banyand/queue/sub/sub.go b/banyand/queue/sub/sub.go
index 9846c4db..bcdc6ab1 100644
--- a/banyand/queue/sub/sub.go
+++ b/banyand/queue/sub/sub.go
@@ -19,6 +19,7 @@
package sub
import (
+ "fmt"
"io"
"github.com/pkg/errors"
@@ -27,6 +28,7 @@ import (
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
+ "github.com/apache/skywalking-banyandb/api/common"
"github.com/apache/skywalking-banyandb/api/data"
clusterv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/cluster/v1"
"github.com/apache/skywalking-banyandb/pkg/bus"
@@ -120,9 +122,15 @@ func (s *server) Send(stream clusterv1.Service_SendServer) error {
}
continue
}
- message, ok := m.Data().(proto.Message)
- if !ok {
- reply(writeEntity, err, "invalid response")
+ var message proto.Message
+ switch d := m.Data().(type) {
+ case proto.Message:
+ message = d
+ case common.Error:
+ reply(writeEntity, nil, d.Msg())
+ continue
+ default:
+ reply(writeEntity, nil, fmt.Sprintf("invalid response: %T", d))
continue
}
anyMessage, err := anypb.New(message)
diff --git a/banyand/stream/block_metadata.go b/banyand/stream/block_metadata.go
index 3906f603..41527229 100644
--- a/banyand/stream/block_metadata.go
+++ b/banyand/stream/block_metadata.go
@@ -245,38 +245,38 @@ type timestampsMetadata struct {
encodeType encoding.EncodeType
}
-func (th *timestampsMetadata) reset() {
- th.dataBlock.reset()
- th.min = 0
- th.max = 0
- th.encodeType = 0
+func (tm *timestampsMetadata) reset() {
+ tm.dataBlock.reset()
+ tm.min = 0
+ tm.max = 0
+ tm.encodeType = 0
}
-func (th *timestampsMetadata) copyFrom(src *timestampsMetadata) {
- th.dataBlock.copyFrom(&src.dataBlock)
- th.min = src.min
- th.max = src.max
- th.encodeType = src.encodeType
+func (tm *timestampsMetadata) copyFrom(src *timestampsMetadata) {
+ tm.dataBlock.copyFrom(&src.dataBlock)
+ tm.min = src.min
+ tm.max = src.max
+ tm.encodeType = src.encodeType
}
-func (th *timestampsMetadata) marshal(dst []byte) []byte {
- dst = th.dataBlock.marshal(dst)
- dst = encoding.Uint64ToBytes(dst, uint64(th.min))
- dst = encoding.Uint64ToBytes(dst, uint64(th.max))
- dst = append(dst, byte(th.encodeType))
+func (tm *timestampsMetadata) marshal(dst []byte) []byte {
+ dst = tm.dataBlock.marshal(dst)
+ dst = encoding.Uint64ToBytes(dst, uint64(tm.min))
+ dst = encoding.Uint64ToBytes(dst, uint64(tm.max))
+ dst = append(dst, byte(tm.encodeType))
return dst
}
-func (th *timestampsMetadata) unmarshal(src []byte) ([]byte, error) {
- src, err := th.dataBlock.unmarshal(src)
+func (tm *timestampsMetadata) unmarshal(src []byte) ([]byte, error) {
+ src, err := tm.dataBlock.unmarshal(src)
if err != nil {
return nil, fmt.Errorf("cannot unmarshal dataBlock: %w", err)
}
- th.min = int64(encoding.BytesToUint64(src))
+ tm.min = int64(encoding.BytesToUint64(src))
src = src[8:]
- th.max = int64(encoding.BytesToUint64(src))
+ tm.max = int64(encoding.BytesToUint64(src))
src = src[8:]
- th.encodeType = encoding.EncodeType(src[0])
+ tm.encodeType = encoding.EncodeType(src[0])
return src[1:], nil
}
@@ -285,28 +285,28 @@ type elementIDsMetadata struct {
encodeType encoding.EncodeType
}
-func (th *elementIDsMetadata) reset() {
- th.dataBlock.reset()
- th.encodeType = 0
+func (em *elementIDsMetadata) reset() {
+ em.dataBlock.reset()
+ em.encodeType = 0
}
-func (th *elementIDsMetadata) copyFrom(src *elementIDsMetadata) {
- th.dataBlock.copyFrom(&src.dataBlock)
- th.encodeType = src.encodeType
+func (em *elementIDsMetadata) copyFrom(src *elementIDsMetadata) {
+ em.dataBlock.copyFrom(&src.dataBlock)
+ em.encodeType = src.encodeType
}
-func (th *elementIDsMetadata) marshal(dst []byte) []byte {
- dst = th.dataBlock.marshal(dst)
- dst = append(dst, byte(th.encodeType))
+func (em *elementIDsMetadata) marshal(dst []byte) []byte {
+ dst = em.dataBlock.marshal(dst)
+ dst = append(dst, byte(em.encodeType))
return dst
}
-func (th *elementIDsMetadata) unmarshal(src []byte) ([]byte, error) {
- src, err := th.dataBlock.unmarshal(src)
+func (em *elementIDsMetadata) unmarshal(src []byte) ([]byte, error) {
+ src, err := em.dataBlock.unmarshal(src)
if err != nil {
return nil, fmt.Errorf("cannot unmarshal dataBlock: %w", err)
}
- th.encodeType = encoding.EncodeType(src[0])
+ em.encodeType = encoding.EncodeType(src[0])
return src[1:], nil
}
diff --git a/banyand/stream/block_metadata_test.go b/banyand/stream/block_metadata_test.go
index 67c7e8e3..5c721b7d 100644
--- a/banyand/stream/block_metadata_test.go
+++ b/banyand/stream/block_metadata_test.go
@@ -74,7 +74,7 @@ func Test_dataBlock_marshal_unmarshal(t *testing.T) {
}
func Test_timestampsMetadata_reset(t *testing.T) {
- th := ×tampsMetadata{
+ tm := ×tampsMetadata{
dataBlock: dataBlock{
offset: 1,
size: 1,
@@ -84,13 +84,13 @@ func Test_timestampsMetadata_reset(t *testing.T) {
encodeType: encoding.EncodeTypeConst,
}
- th.reset()
+ tm.reset()
- assert.Equal(t, uint64(0), th.dataBlock.offset)
- assert.Equal(t, uint64(0), th.dataBlock.size)
- assert.Equal(t, int64(0), th.min)
- assert.Equal(t, int64(0), th.max)
- assert.Equal(t, encoding.EncodeTypeUnknown, th.encodeType)
+ assert.Equal(t, uint64(0), tm.dataBlock.offset)
+ assert.Equal(t, uint64(0), tm.dataBlock.size)
+ assert.Equal(t, int64(0), tm.min)
+ assert.Equal(t, int64(0), tm.max)
+ assert.Equal(t, encoding.EncodeTypeUnknown, tm.encodeType)
}
func Test_timestampsMetadata_copyFrom(t *testing.T) {
diff --git a/banyand/stream/block_writer.go b/banyand/stream/block_writer.go
index ad6a69e3..2c981c3d 100644
--- a/banyand/stream/block_writer.go
+++ b/banyand/stream/block_writer.go
@@ -213,23 +213,23 @@ func (bw *blockWriter) mustWriteBlock(sid common.SeriesID, b *block) {
bm := generateBlockMetadata()
b.mustWriteTo(sid, bm, &bw.writers)
- th := &bm.timestamps
- if bw.totalCount == 0 || th.min < bw.totalMinTimestamp {
- bw.totalMinTimestamp = th.min
+ tm := &bm.timestamps
+ if bw.totalCount == 0 || tm.min < bw.totalMinTimestamp {
+ bw.totalMinTimestamp = tm.min
}
- if bw.totalCount == 0 || th.max > bw.totalMaxTimestamp {
- bw.totalMaxTimestamp = th.max
+ if bw.totalCount == 0 || tm.max > bw.totalMaxTimestamp {
+ bw.totalMaxTimestamp = tm.max
}
- if !hasWrittenBlocks || th.min < bw.minTimestamp {
- bw.minTimestamp = th.min
+ if !hasWrittenBlocks || tm.min < bw.minTimestamp {
+ bw.minTimestamp = tm.min
}
- if !hasWrittenBlocks || th.max > bw.maxTimestamp {
- bw.maxTimestamp = th.max
+ if !hasWrittenBlocks || tm.max > bw.maxTimestamp {
+ bw.maxTimestamp = tm.max
}
- if isSeenSid && th.min < bw.minTimestampLast {
- logger.Panicf("the block for sid=%d cannot contain timestamp smaller than %d, but it contains timestamp %d", sid, bw.minTimestampLast, th.min)
+ if isSeenSid && tm.min < bw.minTimestampLast {
+ logger.Panicf("the block for sid=%d cannot contain timestamp smaller than %d, but it contains timestamp %d", sid, bw.minTimestampLast, tm.min)
}
- bw.minTimestampLast = th.min
+ bw.minTimestampLast = tm.min
bw.totalUncompressedSizeBytes += bm.uncompressedSizeBytes
bw.totalCount += bm.count
diff --git a/banyand/stream/tag.go b/banyand/stream/tag.go
index f9ad8740..5d595043 100644
--- a/banyand/stream/tag.go
+++ b/banyand/stream/tag.go
@@ -51,11 +51,11 @@ func (t *tag) resizeValues(valuesLen int) [][]byte {
return values
}
-func (t *tag) mustWriteTo(ch *tagMetadata, tagWriter *writer) {
- ch.reset()
+func (t *tag) mustWriteTo(tm *tagMetadata, tagWriter *writer) {
+ tm.reset()
- ch.name = t.name
- ch.valueType = t.valueType
+ tm.name = t.name
+ tm.valueType = t.valueType
// TODO: encoding values based on value type
@@ -64,11 +64,11 @@ func (t *tag) mustWriteTo(ch *tagMetadata, tagWriter *writer) {
// marshal values
bb.Buf = encoding.EncodeBytesBlock(bb.Buf[:0], t.values)
- ch.size = uint64(len(bb.Buf))
- if ch.size > maxValuesBlockSize {
- logger.Panicf("too valuesSize: %d bytes; mustn't exceed %d bytes", ch.size, maxValuesBlockSize)
+ tm.size = uint64(len(bb.Buf))
+ if tm.size > maxValuesBlockSize {
+ logger.Panicf("too valuesSize: %d bytes; mustn't exceed %d bytes", tm.size, maxValuesBlockSize)
}
- ch.offset = tagWriter.bytesWritten
+ tm.offset = tagWriter.bytesWritten
tagWriter.MustWrite(bb.Buf)
}
diff --git a/pkg/flow/streaming/sliding_window.go b/pkg/flow/streaming/sliding_window.go
index 860f193f..ed012084 100644
--- a/pkg/flow/streaming/sliding_window.go
+++ b/pkg/flow/streaming/sliding_window.go
@@ -28,6 +28,7 @@ import (
"github.com/pkg/errors"
"github.com/apache/skywalking-banyandb/pkg/flow"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
)
type triggerResult bool
@@ -49,6 +50,7 @@ func (f *streamingFlow) Window(w flow.WindowAssigner) flow.WindowedFlow {
switch v := w.(type) {
case *tumblingTimeWindows:
v.errorHandler = f.drainErr
+ v.l = f.l
f.ops = append(f.ops, v)
default:
f.drainErr(errors.New("window type is not supported"))
@@ -57,6 +59,7 @@ func (f *streamingFlow) Window(w flow.WindowAssigner) flow.WindowedFlow {
return &windowedFlow{
f: f,
wa: w,
+ l: f.l,
}
}
@@ -71,16 +74,19 @@ func (s *windowedFlow) AllowedMaxWindows(windowCnt int) flow.WindowedFlow {
}
type tumblingTimeWindows struct {
- errorHandler func(error)
+ l *logger.Logger
snapshots *lru.Cache
timerHeap *flow.DedupPriorityQueue
aggregationFactory flow.AggregationOpFactory
in chan flow.StreamRecord
out chan flow.StreamRecord
+ errorHandler func(error)
flow.ComponentState
- windowSize int64
windowCount int
currentWatermark int64
+ lastFlushTime int64
+ flushInterval int64
+ windowSize int64
timerMu sync.Mutex
}
@@ -98,7 +104,10 @@ func (s *tumblingTimeWindows) Setup(_ context.Context) (err error) {
s.windowCount = defaultCacheSize
}
s.snapshots, err = lru.NewWithEvict(s.windowCount, func(key interface{}, value interface{}) {
- s.flushSnapshot(key.(timeWindow), value.(flow.AggregationOp))
+ flushed := s.flushSnapshot(key.(timeWindow), value.(flow.AggregationOp))
+ if e := s.l.Debug(); e.Enabled() {
+ e.Stringer("window", key.(timeWindow)).Bool("flushed", flushed).Msg("evict window on lru cache is full")
+ }
})
if err != nil {
return err
@@ -111,15 +120,20 @@ func (s *tumblingTimeWindows) Setup(_ context.Context) (err error) {
return
}
-func (s *tumblingTimeWindows) flushSnapshot(w timeWindow, snapshot flow.AggregationOp) {
+func (s *tumblingTimeWindows) flushSnapshot(w timeWindow, snapshot flow.AggregationOp) bool {
if snapshot.Dirty() {
s.out <- flow.NewStreamRecord(snapshot.Snapshot(), w.start)
+ return true
}
+ return false
}
func (s *tumblingTimeWindows) flushWindow(w timeWindow) {
if snapshot, ok := s.snapshots.Get(w); ok {
- s.flushSnapshot(w, snapshot.(flow.AggregationOp))
+ flushed := s.flushSnapshot(w, snapshot.(flow.AggregationOp))
+ if e := s.l.Debug(); e.Enabled() {
+ e.Stringer("window", w).Bool("flushed", flushed).Msg("flush window")
+ }
}
}
@@ -170,6 +184,9 @@ func (s *tumblingTimeWindows) receive() {
newAggr := s.aggregationFactory()
newAggr.Add([]flow.StreamRecord{elem})
s.snapshots.Add(tw, newAggr)
+ if e := s.l.Debug(); e.Enabled() {
+ e.Stringer("window", tw).Msg("create new window")
+ }
}
result := ctx.OnElement(elem)
@@ -180,7 +197,13 @@ func (s *tumblingTimeWindows) receive() {
// even if the incoming elements do not follow strict order,
// the watermark could increase monotonically.
- if pastDur := elem.TimestampMillis() - s.currentWatermark; pastDur > 0 {
+ now := time.Now().UnixNano() / int64(time.Millisecond)
+ pastDataDur := elem.TimestampMillis() - s.currentWatermark
+ if s.lastFlushTime == 0 {
+ s.lastFlushTime = now
+ }
+ pastDur := now - s.lastFlushTime
+ if pastDur > 0 || pastDataDur > 0 {
previousWaterMark := s.currentWatermark
s.currentWatermark = elem.TimestampMillis()
@@ -197,8 +220,9 @@ func (s *tumblingTimeWindows) receive() {
// | 40% | 40% | 20% |
// | flush flush |
// |---------------------------------|
- // TODO: how to determine the threshold
- if previousWaterMark > 0 && float64(pastDur) > float64(s.windowSize)*0.4 {
+ // the max flush interval is 1 minute.
+ if (pastDur > s.flushInterval) || (previousWaterMark > 0 && pastDataDur > s.flushInterval) {
+ s.lastFlushTime = now
s.flushDirtyWindows()
}
}
@@ -226,15 +250,22 @@ func (s *tumblingTimeWindows) Exec(downstream flow.Inlet) {
}
// NewTumblingTimeWindows return tumbling-time windows.
-func NewTumblingTimeWindows(size time.Duration) flow.WindowAssigner {
+func NewTumblingTimeWindows(size time.Duration, maxFlushInterval time.Duration) flow.WindowAssigner {
+ ws := size.Milliseconds()
+ mfi := maxFlushInterval.Milliseconds()
+ it := int64(float64(ws) * 0.4)
+ if it > mfi {
+ it = mfi
+ }
return &tumblingTimeWindows{
- windowSize: size.Milliseconds(),
+ windowSize: ws,
timerHeap: flow.NewPriorityQueue(func(a, b interface{}) int {
return int(a.(*internalTimer).triggerTimeMillis - b.(*internalTimer).triggerTimeMillis)
}, false),
in: make(chan flow.StreamRecord),
out: make(chan flow.StreamRecord),
currentWatermark: 0,
+ flushInterval: it,
}
}
@@ -247,6 +278,12 @@ func (t timeWindow) MaxTimestamp() int64 {
return t.end - 1
}
+func (t timeWindow) String() string {
+ st := time.Unix(0, t.start*int64(time.Millisecond)).String()
+ et := time.Unix(0, t.end*int64(time.Millisecond)).String()
+ return st + " - " + et
+}
+
// AssignWindows assigns windows according to the given timestamp.
func (s *tumblingTimeWindows) AssignWindows(timestamp int64) ([]flow.Window, error) {
if timestamp > math.MinInt64 {
diff --git a/pkg/flow/streaming/sliding_window_test.go b/pkg/flow/streaming/sliding_window_test.go
index 2a3366b0..cee7973d 100644
--- a/pkg/flow/streaming/sliding_window_test.go
+++ b/pkg/flow/streaming/sliding_window_test.go
@@ -25,6 +25,7 @@ import (
"github.com/onsi/gomega"
"github.com/apache/skywalking-banyandb/pkg/flow"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
"github.com/apache/skywalking-banyandb/pkg/test/flags"
)
@@ -72,9 +73,10 @@ var _ = g.Describe("Sliding Window", func() {
g.JustBeforeEach(func() {
snk = newSlice()
- slidingWindows = NewTumblingTimeWindows(time.Second * 15).(*tumblingTimeWindows)
+ slidingWindows = NewTumblingTimeWindows(time.Second*15, time.Second*15).(*tumblingTimeWindows)
slidingWindows.aggregationFactory = aggrFactory
slidingWindows.windowCount = 2
+ slidingWindows.l = logger.GetLogger("tumblingTimeWindows")
gomega.Expect(slidingWindows.Setup(context.TODO())).Should(gomega.Succeed())
gomega.Expect(snk.Setup(context.TODO())).Should(gomega.Succeed())
diff --git a/pkg/flow/streaming/streaming.go b/pkg/flow/streaming/streaming.go
index 63b02edf..f144aebd 100644
--- a/pkg/flow/streaming/streaming.go
+++ b/pkg/flow/streaming/streaming.go
@@ -25,6 +25,7 @@ import (
"go.uber.org/multierr"
"github.com/apache/skywalking-banyandb/pkg/flow"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
)
var _ flow.Flow = (*streamingFlow)(nil)
@@ -34,15 +35,17 @@ type streamingFlow struct {
source flow.Source
sink flow.Sink
drain chan error
+ l *logger.Logger
ops []flow.Operator
}
// New returns a new streaming flow.
-func New(source flow.Source) flow.Flow {
+func New(name string, source flow.Source) flow.Flow {
return &streamingFlow{
source: source,
ops: make([]flow.Operator, 0),
drain: make(chan error),
+ l: logger.GetLogger(name),
}
}
@@ -66,6 +69,7 @@ func (f *streamingFlow) To(sink flow.Sink) flow.Flow {
}
func (f *streamingFlow) Close() error {
+ f.l.Info().Msg("flow is closing")
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
err := f.source.Teardown(ctx)
@@ -114,6 +118,8 @@ func (f *streamingFlow) Open() <-chan error {
// finally connect sources and the first operator
f.source.Exec(f.ops[0])
+ f.l.Info().Msg("flow is running")
+
return f.drain
}
diff --git a/pkg/flow/streaming/streaming_test.go b/pkg/flow/streaming/streaming_test.go
index 57d99c5e..796022e5 100644
--- a/pkg/flow/streaming/streaming_test.go
+++ b/pkg/flow/streaming/streaming_test.go
@@ -61,7 +61,7 @@ var _ = g.Describe("Streaming", func() {
g.JustBeforeEach(func() {
snk = newSlice()
- f = New(input).
+ f = New("test", input).
Filter(filter).
To(snk)
errCh = f.Open()
@@ -98,7 +98,7 @@ var _ = g.Describe("Streaming", func() {
g.JustBeforeEach(func() {
snk = newSlice()
- f = New(input).
+ f = New("test", input).
Map(mapper).
To(snk)
errCh = f.Open()
@@ -143,12 +143,12 @@ var _ = g.Describe("Streaming", func() {
g.JustBeforeEach(func() {
snk = newSlice()
- f = New(flowTest.NewSlice(input)).
+ f = New("test", flowTest.NewSlice(input)).
Map(flow.UnaryFunc[any](func(_ context.Context, item interface{}) interface{} {
// groupBy
return flow.Data{item.(*record).service, int64(item.(*record).value)}
})).
- Window(NewTumblingTimeWindows(15*time.Second)).
+ Window(NewTumblingTimeWindows(15*time.Second, 15*time.Second)).
TopN(3, WithSortKeyExtractor(func(record flow.StreamRecord) int64 {
return record.Data().(flow.Data)[1].(int64)
}), OrderBy(ASC), WithGroupKeyExtractor(func(record flow.StreamRecord) string {
@@ -206,12 +206,12 @@ var _ = g.Describe("Streaming", func() {
g.JustBeforeEach(func() {
snk = newSlice()
- f = New(flowTest.NewSlice(input)).
+ f = New("test", flowTest.NewSlice(input)).
Map(flow.UnaryFunc[any](func(_ context.Context, item interface{}) interface{} {
// groupBy
return flow.Data{item.(*record).service, int64(item.(*record).value)}
})).
- Window(NewTumblingTimeWindows(15*time.Second)).
+ Window(NewTumblingTimeWindows(15*time.Second, 15*time.Second)).
TopN(3, WithSortKeyExtractor(func(record flow.StreamRecord) int64 {
return record.Data().(flow.Data)[1].(int64)
}), WithGroupKeyExtractor(func(record flow.StreamRecord) string {
diff --git a/pkg/flow/streaming/topn.go b/pkg/flow/streaming/topn.go
index 746faa66..eaaafe24 100644
--- a/pkg/flow/streaming/topn.go
+++ b/pkg/flow/streaming/topn.go
@@ -18,11 +18,16 @@
package streaming
import (
+ "strconv"
+ "strings"
+ "time"
+
"github.com/emirpasic/gods/maps/treemap"
"github.com/emirpasic/gods/utils"
"github.com/pkg/errors"
"github.com/apache/skywalking-banyandb/pkg/flow"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
)
// TopNSort defines the order of sorting.
@@ -37,6 +42,7 @@ const (
type windowedFlow struct {
f *streamingFlow
wa flow.WindowAssigner
+ l *logger.Logger
}
func (s *windowedFlow) TopN(topNum int, opts ...any) flow.Flow {
@@ -44,6 +50,7 @@ func (s *windowedFlow) TopN(topNum int, opts ...any) flow.Flow {
topNAggrFunc := &topNAggregatorGroup{
cacheSize: topNum,
sort: DESC,
+ l: s.l,
}
// apply user customized options
for _, opt := range opts {
@@ -72,6 +79,7 @@ type topNAggregatorGroup struct {
sortKeyExtractor func(flow.StreamRecord) int64
groupKeyExtractor func(flow.StreamRecord) string
comparator utils.Comparator
+ l *logger.Logger
cacheSize int
sort TopNSort
}
@@ -113,6 +121,9 @@ func (t *topNAggregatorGroup) Add(input []flow.StreamRecord) {
groupKey := t.groupKeyExtractor(item)
aggregator := t.getOrCreateGroup(groupKey)
if aggregator.checkSortKeyInBufferRange(sortKey) {
+ if e := t.l.Debug(); e.Enabled() {
+ e.Str("group", groupKey).Time("elem_ts", time.Unix(0, item.TimestampMillis()*int64(time.Millisecond))).Msg("put into topN buffer")
+ }
aggregator.put(sortKey, item)
aggregator.doCleanUp()
}
@@ -136,6 +147,19 @@ func (t *topNAggregatorGroup) Snapshot() interface{} {
}
groupRanks[group] = items
}
+ if len(groupRanks) > 0 {
+ if e := t.l.Debug(); e.Enabled() {
+ sb := strings.Builder{}
+ for g, item := range groupRanks {
+ sb.WriteString("{")
+ sb.WriteString(g)
+ sb.WriteString(":")
+ sb.WriteString(strconv.Itoa(len(item)))
+ sb.WriteString("}")
+ }
+ t.l.Debug().Interface("snapshot", sb.String()).Msg("taken a topN snapshot")
+ }
+ }
return groupRanks
}
diff --git a/pkg/flow/streaming/topn_test.go b/pkg/flow/streaming/topn_test.go
index 988fe825..16781838 100644
--- a/pkg/flow/streaming/topn_test.go
+++ b/pkg/flow/streaming/topn_test.go
@@ -25,6 +25,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/apache/skywalking-banyandb/pkg/flow"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
)
func TestFlow_TopN_Aggregator(t *testing.T) {
@@ -117,6 +118,7 @@ func TestFlow_TopN_Aggregator(t *testing.T) {
groupKeyExtractor: func(record flow.StreamRecord) string {
return record.Data().(flow.Data)[0].(string)
},
+ l: logger.GetLogger("test"),
}
topN.Add(input)
snapshot := topN.Snapshot()
diff --git a/pkg/flow/types.go b/pkg/flow/types.go
index 538ce849..6e8576a4 100644
--- a/pkg/flow/types.go
+++ b/pkg/flow/types.go
@@ -20,8 +20,10 @@ package flow
import (
"context"
+ "fmt"
"io"
"sync"
+ "time"
"github.com/google/go-cmp/cmp"
"google.golang.org/protobuf/types/known/timestamppb"
@@ -142,6 +144,11 @@ func (sr StreamRecord) Equal(other StreamRecord) bool {
return sr.ts == other.ts && cmp.Equal(sr.data, other.data)
}
+func (sr StreamRecord) String() string {
+ t := time.Unix(0, sr.ts*int64(time.Millisecond))
+ return fmt.Sprintf("data: %s, ts: %v", sr.data, t)
+}
+
// Inlet represents a type that exposes one open input.
//
//go:generate mockgen -destination=./inlet_mock.go -package=flow github.com/apache/skywalking-banyandb/pkg/flow Inlet
diff --git a/pkg/pb/v1/series.go b/pkg/pb/v1/series.go
index 3f747bf8..f6d47397 100644
--- a/pkg/pb/v1/series.go
+++ b/pkg/pb/v1/series.go
@@ -51,6 +51,19 @@ func (s *Series) Marshal() error {
return nil
}
+// MarshalWithWildcard encodes series to internal Buffer and generates ID with wildcard.
+func (s *Series) MarshalWithWildcard() error {
+ s.Buffer = marshalEntityValue(s.Buffer, convert.StringToBytes(s.Subject))
+ var err error
+ for _, tv := range s.EntityValues {
+ if s.Buffer, err = marshalTagValueWithWildcard(s.Buffer, tv); err != nil {
+ return errors.WithMessage(err, "marshal subject and entity values")
+ }
+ }
+ s.ID = common.SeriesID(convert.Hash(s.Buffer))
+ return nil
+}
+
// Unmarshal decodes series from internal Buffer.
func (s *Series) Unmarshal(src []byte) error {
var err error
@@ -63,7 +76,7 @@ func (s *Series) Unmarshal(src []byte) error {
s.Buffer = s.Buffer[:0]
var tv *modelv1.TagValue
if s.Buffer, src, tv, err = unmarshalTagValue(s.Buffer, src); err != nil {
- return errors.WithMessage(err, "unmarshal tag value")
+ return errors.WithMessagef(err, "unmarshal tag value [%s], marshaled %s", src, s.EntityValues)
}
s.EntityValues = append(s.EntityValues, tv)
}
diff --git a/pkg/pb/v1/series_test.go b/pkg/pb/v1/series_test.go
index c44c5a89..0399d045 100644
--- a/pkg/pb/v1/series_test.go
+++ b/pkg/pb/v1/series_test.go
@@ -92,6 +92,14 @@ func TestMarshalAndUnmarshalSeries(t *testing.T) {
{Value: &modelv1.TagValue_Int{Int: &modelv1.Int{Value: 123}}},
}},
},
+ {
+ name: "topn",
+ src: &Series{Subject: "subject", EntityValues: []*modelv1.TagValue{
+ {Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "stringValue"}}},
+ {Value: &modelv1.TagValue_Int{Int: &modelv1.Int{Value: 1}}},
+ {Value: &modelv1.TagValue_Int{Int: &modelv1.Int{Value: 46}}},
+ }},
+ },
}
for _, tt := range tests {
diff --git a/pkg/pb/v1/value.go b/pkg/pb/v1/value.go
index f7c6abed..064d2022 100644
--- a/pkg/pb/v1/value.go
+++ b/pkg/pb/v1/value.go
@@ -19,6 +19,7 @@ package v1
import (
"bytes"
+ "fmt"
"github.com/pkg/errors"
@@ -63,10 +64,6 @@ func MustTagValueToValueType(tag *modelv1.TagValue) ValueType {
}
func marshalTagValue(dest []byte, tv *modelv1.TagValue) ([]byte, error) {
- if tv == AnyTagValue {
- dest = marshalEntityValue(dest, anyWildcard)
- return dest, nil
- }
dest = append(dest, byte(MustTagValueToValueType(tv)))
switch tv.Value.(type) {
case *modelv1.TagValue_Null:
@@ -74,8 +71,7 @@ func marshalTagValue(dest []byte, tv *modelv1.TagValue) ([]byte, error) {
case *modelv1.TagValue_Str:
dest = marshalEntityValue(dest, convert.StringToBytes(tv.GetStr().Value))
case *modelv1.TagValue_Int:
- dest = encoding.Int64ToBytes(dest, tv.GetInt().Value)
- dest = marshalEntityValue(dest, nil)
+ dest = marshalEntityValue(dest, encoding.Int64ToBytes(nil, tv.GetInt().Value))
case *modelv1.TagValue_BinaryData:
dest = marshalEntityValue(dest, tv.GetBinaryData())
default:
@@ -84,6 +80,14 @@ func marshalTagValue(dest []byte, tv *modelv1.TagValue) ([]byte, error) {
return dest, nil
}
+func marshalTagValueWithWildcard(dest []byte, tv *modelv1.TagValue) ([]byte, error) {
+ if tv == AnyTagValue {
+ dest = marshalEntityValue(dest, anyWildcard)
+ return dest, nil
+ }
+ return marshalTagValue(dest, tv)
+}
+
func unmarshalTagValue(dest []byte, src []byte) ([]byte, []byte, *modelv1.TagValue, error) {
if len(src) == 0 {
return nil, nil, nil, errors.New("empty tag value")
@@ -128,7 +132,7 @@ func unmarshalTagValue(dest []byte, src []byte) ([]byte, []byte, *modelv1.TagVal
},
}, nil
default:
- return nil, nil, nil, errors.New("unsupported tag value type")
+ return nil, src, nil, fmt.Errorf("unsupported tag value type %d, tag value: %s", vt, src)
}
}
diff --git a/pkg/query/logical/measure/topn_analyzer.go b/pkg/query/logical/measure/topn_analyzer.go
index 4a6e05e6..4d72787f 100644
--- a/pkg/query/logical/measure/topn_analyzer.go
+++ b/pkg/query/logical/measure/topn_analyzer.go
@@ -20,6 +20,7 @@ package measure
import (
"context"
+ "fmt"
commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
@@ -52,31 +53,33 @@ func BuildTopNSchema(md *databasev1.Measure) (logical.Schema, error) {
// TopNAnalyze converts logical expressions to executable operation tree represented by Plan.
func TopNAnalyze(_ context.Context, criteria *measurev1.TopNRequest, schema *databasev1.Measure,
- sourceMeasureSchema *databasev1.Measure, topNAggregation *databasev1.TopNAggregation, s logical.Schema,
+ sourceMeasureSchema *databasev1.Measure, s logical.Schema,
) (logical.Plan, error) {
groupByProjectionTags := sourceMeasureSchema.GetEntity().GetTagNames()
groupByTags := make([][]*logical.Tag, len(schema.GetTagFamilies()))
tagFamily := schema.GetTagFamilies()[0]
groupByTags[0] = logical.NewTags(tagFamily.GetName(), groupByProjectionTags...)
- projectionFields := make([]*logical.Field, len(schema.GetFields()))
- for i, fieldSpecProj := range schema.GetFields() {
- projectionFields[i] = logical.NewField(fieldSpecProj.GetName())
+ if len(schema.GetFields()) != 1 {
+ return nil, fmt.Errorf("topN schema fields count should be 1 but got %d", len(schema.GetFields()))
}
+ projectionFields := make([]*logical.Field, 1)
+ fieldName := schema.GetFields()[0].GetName()
+ projectionFields[0] = logical.NewField(fieldName)
// parse fields
plan := parse(criteria, schema.GetMetadata(), projectionFields, groupByTags)
if criteria.GetAgg() != 0 {
plan = newUnresolvedGroupBy(plan, groupByTags, false)
plan = newUnresolvedAggregation(plan,
- logical.NewField(topNAggregation.GetFieldName()),
+ projectionFields[0],
criteria.GetAgg(),
true)
}
plan = top(plan, &measurev1.QueryRequest_Top{
Number: criteria.GetTopN(),
- FieldName: topNAggregation.GetFieldName(),
+ FieldName: fieldName,
FieldValueSort: criteria.GetFieldValueSort(),
})
p, err := plan.Analyze(s)
diff --git a/pkg/schema/init.go b/pkg/schema/init.go
index 4a95dfa7..a158daab 100644
--- a/pkg/schema/init.go
+++ b/pkg/schema/init.go
@@ -361,7 +361,6 @@ func buildTopNSourceMeasure(topNSchema *databasev1.TopNAggregation, sourceMeasur
return nil, fmt.Errorf("fail to find tag spec %s", tagName)
}
}
-
// create a new "derived" measure for TopN result
return &databasev1.Measure{
Metadata: topNSchema.Metadata,
@@ -369,28 +368,23 @@ func buildTopNSourceMeasure(topNSchema *databasev1.TopNAggregation, sourceMeasur
TagFamilies: []*databasev1.TagFamilySpec{
{
Name: topNTagFamily,
- Tags: append([]*databasev1.TagSpec{
- {
- Name: "measure_id",
- Type: databasev1.TagType_TAG_TYPE_STRING,
- },
- {
+ Tags: append(seriesSpecs,
+ &databasev1.TagSpec{
Name: "sortDirection",
Type: databasev1.TagType_TAG_TYPE_INT,
},
- {
+ &databasev1.TagSpec{
Name: "rankNumber",
Type: databasev1.TagType_TAG_TYPE_INT,
},
- }, seriesSpecs...),
+ ),
},
},
Fields: []*databasev1.FieldSpec{topNValueFieldSpec},
Entity: &databasev1.Entity{
- TagNames: append([]string{
+ TagNames: append(topNSchema.GetGroupByTagNames(),
"sortDirection",
- "rankNumber",
- }, topNSchema.GetGroupByTagNames()...),
+ "rankNumber"),
},
}, nil
}
diff --git a/pkg/test/measure/testdata/topn_aggregations/service_instance_endpoint_cpm_minute_top_bottom_100.json b/pkg/test/measure/testdata/topn_aggregations/service_instance_endpoint_cpm_minute_top_bottom_100.json
index 65f3af15..8f1966da 100644
--- a/pkg/test/measure/testdata/topn_aggregations/service_instance_endpoint_cpm_minute_top_bottom_100.json
+++ b/pkg/test/measure/testdata/topn_aggregations/service_instance_endpoint_cpm_minute_top_bottom_100.json
@@ -7,7 +7,7 @@
"name": "service_instance_endpoint_cpm_minute",
"group": "sw_metric"
},
- "field_name": "value",
+ "field_name": "total",
"field_value_sort": 0,
"group_by_tag_names": [
"http.uri"
diff --git a/pkg/test/setup/setup.go b/pkg/test/setup/setup.go
index 8fdcbfec..a9596105 100644
--- a/pkg/test/setup/setup.go
+++ b/pkg/test/setup/setup.go
@@ -102,7 +102,7 @@ func standaloneServer(path string, ports []int, schemaLoaders []SchemaLoader, ce
endpoint := fmt.Sprintf("http://%s:%d", host, ports[2])
ff := []string{
"--logging-env=dev",
- "--logging-level=error",
+ "--logging-level=" + testflags.LogLevel,
"--grpc-host=" + host,
fmt.Sprintf("--grpc-port=%d", ports[0]),
"--http-host=" + host,
diff --git a/test/cases/measure/data/testdata/service_instance_endpoint_cpm_minute_data.json b/test/cases/measure/data/testdata/service_instance_endpoint_cpm_minute_data.json
index 86b99575..13897328 100644
--- a/test/cases/measure/data/testdata/service_instance_endpoint_cpm_minute_data.json
+++ b/test/cases/measure/data/testdata/service_instance_endpoint_cpm_minute_data.json
@@ -69,7 +69,7 @@
"fields": [
{
"int": {
- "value": 100
+ "value": 101
}
},
{
@@ -104,7 +104,7 @@
"fields": [
{
"int": {
- "value": 100
+ "value": 130
}
},
{
@@ -139,7 +139,7 @@
"fields": [
{
"int": {
- "value": 100
+ "value": 155
}
},
{
@@ -215,7 +215,7 @@
"fields": [
{
"int": {
- "value": 300
+ "value": 240
}
},
{
diff --git a/test/cases/measure/data/testdata/service_instance_endpoint_cpm_minute_data1.json b/test/cases/measure/data/testdata/service_instance_endpoint_cpm_minute_data1.json
index f4f6a118..dee1ddae 100644
--- a/test/cases/measure/data/testdata/service_instance_endpoint_cpm_minute_data1.json
+++ b/test/cases/measure/data/testdata/service_instance_endpoint_cpm_minute_data1.json
@@ -27,7 +27,7 @@
"fields": [
{
"int": {
- "value": 100
+ "value": 102
}
},
{
@@ -67,7 +67,7 @@
"fields": [
{
"int": {
- "value": 100
+ "value": 103
}
},
{
@@ -107,7 +107,7 @@
"fields": [
{
"int": {
- "value": 100
+ "value": 130
}
},
{
@@ -142,7 +142,7 @@
"fields": [
{
"int": {
- "value": 100
+ "value": 133
}
},
{
@@ -177,7 +177,7 @@
"fields": [
{
"int": {
- "value": 50
+ "value": 54
}
},
{
diff --git a/test/cases/measure/data/testdata/service_instance_endpoint_cpm_minute_data2.json b/test/cases/measure/data/testdata/service_instance_endpoint_cpm_minute_data2.json
index 59a94d56..13cf282f 100644
--- a/test/cases/measure/data/testdata/service_instance_endpoint_cpm_minute_data2.json
+++ b/test/cases/measure/data/testdata/service_instance_endpoint_cpm_minute_data2.json
@@ -29,7 +29,7 @@
"fields": [
{
"int": {
- "value": 100
+ "value": 170
}
},
{
@@ -69,7 +69,7 @@
"fields": [
{
"int": {
- "value": 100
+ "value": 172
}
},
{
diff --git a/test/cases/topn/data/want/null_group.yaml b/test/cases/topn/data/want/null_group.yaml
index 166de533..46004d64 100644
--- a/test/cases/topn/data/want/null_group.yaml
+++ b/test/cases/topn/data/want/null_group.yaml
@@ -20,15 +20,14 @@ lists:
- entity:
- key: service_id
value:
- str:
- value: svc_2
+ "null": null
- key: entity_id
value:
str:
value: entity_1
value:
int:
- value: "12"
+ value: "300"
- entity:
- key: service_id
value:
@@ -37,18 +36,19 @@ lists:
- key: entity_id
value:
str:
- value: entity_3
+ value: entity_1
value:
int:
- value: "11"
+ value: "240"
- entity:
- key: service_id
value:
- "null": null
+ str:
+ value: svc_2
- key: entity_id
value:
str:
- value: entity_2
+ value: entity_3
value:
int:
- value: "10"
\ No newline at end of file
+ value: "155"
\ No newline at end of file