You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by wu...@apache.org on 2024/05/29 14:41:35 UTC

(skywalking-banyandb) branch main updated: Fix several issues of topN pre-calculation (#455)

This is an automated email from the ASF dual-hosted git repository.

wusheng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git


The following commit(s) were added to refs/heads/main by this push:
     new 8906b3ca Fix several issues of topN pre-calculation (#455)
8906b3ca is described below

commit 8906b3caf276aed936854688434407415da3f6cc
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Wed May 29 22:41:30 2024 +0800

    Fix several issues of topN pre-calculation (#455)
---
 CHANGES.md                                         |  7 ++-
 banyand/internal/storage/index.go                  |  6 +-
 banyand/measure/block_metadata_test.go             | 14 ++---
 banyand/measure/block_writer.go                    | 24 ++++----
 banyand/measure/column.go                          | 16 +++---
 banyand/measure/topn.go                            | 62 +++++++++------------
 banyand/query/processor_topn.go                    |  3 +-
 banyand/queue/sub/sub.go                           | 14 ++++-
 banyand/stream/block_metadata.go                   | 64 +++++++++++-----------
 banyand/stream/block_metadata_test.go              | 14 ++---
 banyand/stream/block_writer.go                     | 24 ++++----
 banyand/stream/tag.go                              | 16 +++---
 pkg/flow/streaming/sliding_window.go               | 57 +++++++++++++++----
 pkg/flow/streaming/sliding_window_test.go          |  4 +-
 pkg/flow/streaming/streaming.go                    |  8 ++-
 pkg/flow/streaming/streaming_test.go               | 12 ++--
 pkg/flow/streaming/topn.go                         | 24 ++++++++
 pkg/flow/streaming/topn_test.go                    |  2 +
 pkg/flow/types.go                                  |  7 +++
 pkg/pb/v1/series.go                                | 15 ++++-
 pkg/pb/v1/series_test.go                           |  8 +++
 pkg/pb/v1/value.go                                 | 18 +++---
 pkg/query/logical/measure/topn_analyzer.go         | 15 +++--
 pkg/schema/init.go                                 | 18 ++----
 ...nstance_endpoint_cpm_minute_top_bottom_100.json |  2 +-
 pkg/test/setup/setup.go                            |  2 +-
 .../service_instance_endpoint_cpm_minute_data.json |  8 +--
 ...service_instance_endpoint_cpm_minute_data1.json | 10 ++--
 ...service_instance_endpoint_cpm_minute_data2.json |  4 +-
 test/cases/topn/data/want/null_group.yaml          | 16 +++---
 30 files changed, 297 insertions(+), 197 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index 29496416..420b4f8a 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -2,18 +2,23 @@
 
 Release Notes.
 
-## 0.7.0
+## 0.6.1
 
 - Add benchmarks for stream filtering and sorting.
 
 ### Features
 
+- Limit the max pre-calculation result flush interval to 1 minute.
+- Use both datapoint timestamp and server time to trigger the flush of topN pre-calculation result.
+
 ### Bugs
 
 - Fix the bug that topN query doesn't return when an error occurs.
 - Data race in the hot series index selection.
 - Remove SetSchema from measure cache which could change the schema in the cache.
 - Fix duplicated items in the query aggregation top-n list.
+- Fix non-"value" field in topN pre-calculation result measure is lack of data.
+- Encode escaped characters to int64 bytes to fix the malformed data.
 
 ## 0.6.0
 
diff --git a/banyand/internal/storage/index.go b/banyand/internal/storage/index.go
index 1ac969f4..6980369d 100644
--- a/banyand/internal/storage/index.go
+++ b/banyand/internal/storage/index.go
@@ -103,7 +103,7 @@ func (s *seriesIndex) searchPrimary(ctx context.Context, series []*pbv1.Series)
 	}
 	result, err := convertIndexSeriesToSeriesList(ss)
 	if err != nil {
-		return nil, errors.WithMessagef(err, "failed to convert index series to series list, matchers: %v, result:%v", seriesMatchers, ss)
+		return nil, errors.WithMessagef(err, "failed to convert index series to series list, matchers: %v, matched: %d", seriesMatchers, len(ss))
 	}
 	return result, nil
 }
@@ -135,7 +135,7 @@ func convertEntityValuesToSeriesMatcher(series *pbv1.Series) (index.SeriesMatche
 
 	if hasAny {
 		if hasWildcard {
-			if err = series.Marshal(); err != nil {
+			if err = series.MarshalWithWildcard(); err != nil {
 				return emptySeriesMatcher, err
 			}
 			return index.SeriesMatcher{
@@ -167,7 +167,7 @@ func convertIndexSeriesToSeriesList(indexSeries []index.Series) (pbv1.SeriesList
 		var series pbv1.Series
 		series.ID = s.ID
 		if err := series.Unmarshal(s.EntityValues); err != nil {
-			return nil, err
+			return nil, errors.WithMessagef(err, "failed to unmarshal series: %s", s.EntityValues)
 		}
 		seriesList = append(seriesList, &series)
 	}
diff --git a/banyand/measure/block_metadata_test.go b/banyand/measure/block_metadata_test.go
index 81da440b..59709b73 100644
--- a/banyand/measure/block_metadata_test.go
+++ b/banyand/measure/block_metadata_test.go
@@ -75,7 +75,7 @@ func Test_dataBlock_marshal_unmarshal(t *testing.T) {
 }
 
 func Test_timestampsMetadata_reset(t *testing.T) {
-	th := &timestampsMetadata{
+	tm := &timestampsMetadata{
 		dataBlock: dataBlock{
 			offset: 1,
 			size:   1,
@@ -85,13 +85,13 @@ func Test_timestampsMetadata_reset(t *testing.T) {
 		encodeType: encoding.EncodeTypeConst,
 	}
 
-	th.reset()
+	tm.reset()
 
-	assert.Equal(t, uint64(0), th.dataBlock.offset)
-	assert.Equal(t, uint64(0), th.dataBlock.size)
-	assert.Equal(t, int64(0), th.min)
-	assert.Equal(t, int64(0), th.max)
-	assert.Equal(t, encoding.EncodeTypeUnknown, th.encodeType)
+	assert.Equal(t, uint64(0), tm.dataBlock.offset)
+	assert.Equal(t, uint64(0), tm.dataBlock.size)
+	assert.Equal(t, int64(0), tm.min)
+	assert.Equal(t, int64(0), tm.max)
+	assert.Equal(t, encoding.EncodeTypeUnknown, tm.encodeType)
 }
 
 func Test_timestampsMetadata_copyFrom(t *testing.T) {
diff --git a/banyand/measure/block_writer.go b/banyand/measure/block_writer.go
index 8f97c066..760a63ad 100644
--- a/banyand/measure/block_writer.go
+++ b/banyand/measure/block_writer.go
@@ -214,23 +214,23 @@ func (bw *blockWriter) mustWriteBlock(sid common.SeriesID, b *block) {
 
 	bm := generateBlockMetadata()
 	b.mustWriteTo(sid, bm, &bw.writers)
-	th := &bm.timestamps
-	if bw.totalCount == 0 || th.min < bw.totalMinTimestamp {
-		bw.totalMinTimestamp = th.min
+	tm := &bm.timestamps
+	if bw.totalCount == 0 || tm.min < bw.totalMinTimestamp {
+		bw.totalMinTimestamp = tm.min
 	}
-	if bw.totalCount == 0 || th.max > bw.totalMaxTimestamp {
-		bw.totalMaxTimestamp = th.max
+	if bw.totalCount == 0 || tm.max > bw.totalMaxTimestamp {
+		bw.totalMaxTimestamp = tm.max
 	}
-	if !hasWrittenBlocks || th.min < bw.minTimestamp {
-		bw.minTimestamp = th.min
+	if !hasWrittenBlocks || tm.min < bw.minTimestamp {
+		bw.minTimestamp = tm.min
 	}
-	if !hasWrittenBlocks || th.max > bw.maxTimestamp {
-		bw.maxTimestamp = th.max
+	if !hasWrittenBlocks || tm.max > bw.maxTimestamp {
+		bw.maxTimestamp = tm.max
 	}
-	if isSeenSid && th.min < bw.minTimestampLast {
-		logger.Panicf("the block for sid=%d cannot contain timestamp smaller than %d, but it contains timestamp %d", sid, bw.minTimestampLast, th.min)
+	if isSeenSid && tm.min < bw.minTimestampLast {
+		logger.Panicf("the block for sid=%d cannot contain timestamp smaller than %d, but it contains timestamp %d", sid, bw.minTimestampLast, tm.min)
 	}
-	bw.minTimestampLast = th.min
+	bw.minTimestampLast = tm.min
 
 	bw.totalUncompressedSizeBytes += bm.uncompressedSizeBytes
 	bw.totalCount += bm.count
diff --git a/banyand/measure/column.go b/banyand/measure/column.go
index 2f17d186..f208786c 100644
--- a/banyand/measure/column.go
+++ b/banyand/measure/column.go
@@ -51,11 +51,11 @@ func (c *column) resizeValues(valuesLen int) [][]byte {
 	return values
 }
 
-func (c *column) mustWriteTo(ch *columnMetadata, columnWriter *writer) {
-	ch.reset()
+func (c *column) mustWriteTo(cm *columnMetadata, columnWriter *writer) {
+	cm.reset()
 
-	ch.name = c.name
-	ch.valueType = c.valueType
+	cm.name = c.name
+	cm.valueType = c.valueType
 
 	// TODO: encoding values based on value type
 
@@ -64,11 +64,11 @@ func (c *column) mustWriteTo(ch *columnMetadata, columnWriter *writer) {
 
 	// marshal values
 	bb.Buf = encoding.EncodeBytesBlock(bb.Buf[:0], c.values)
-	ch.size = uint64(len(bb.Buf))
-	if ch.size > maxValuesBlockSize {
-		logger.Panicf("too valuesSize: %d bytes; mustn't exceed %d bytes", ch.size, maxValuesBlockSize)
+	cm.size = uint64(len(bb.Buf))
+	if cm.size > maxValuesBlockSize {
+		logger.Panicf("too valuesSize: %d bytes; mustn't exceed %d bytes", cm.size, maxValuesBlockSize)
 	}
-	ch.offset = columnWriter.bytesWritten
+	cm.offset = columnWriter.bytesWritten
 	columnWriter.MustWrite(bb.Buf)
 }
 
diff --git a/banyand/measure/topn.go b/banyand/measure/topn.go
index 1ec55c8e..85460d5d 100644
--- a/banyand/measure/topn.go
+++ b/banyand/measure/topn.go
@@ -52,6 +52,7 @@ import (
 const (
 	timeBucketFormat         = "200601021504"
 	resultPersistencyTimeout = 10 * time.Second
+	maxFlushInterval         = time.Minute
 )
 
 var (
@@ -132,7 +133,6 @@ func (t *topNStreamingProcessor) writeStreamRecord(record flow.StreamRecord) err
 	}
 	// down-sample the start of the timeWindow to a time-bucket
 	eventTime := t.downSampleTimeBucket(record.TimestampMillis())
-	timeBucket := eventTime.Format(timeBucketFormat)
 	var err error
 	publisher := t.pipeline.NewBatchPublisher(resultPersistencyTimeout)
 	defer publisher.Close()
@@ -146,14 +146,14 @@ func (t *topNStreamingProcessor) writeStreamRecord(record flow.StreamRecord) err
 		for rankNum, tuple := range tuples {
 			fieldValue := tuple.V1.(int64)
 			data := tuple.V2.(flow.StreamRecord).Data().(flow.Data)
-			err = multierr.Append(err, t.writeData(publisher, eventTime, timeBucket, fieldValue, group, data, rankNum))
+			err = multierr.Append(err, t.writeData(publisher, eventTime, fieldValue, data, rankNum))
 		}
 	}
 	return err
 }
 
-func (t *topNStreamingProcessor) writeData(publisher queue.BatchPublisher, eventTime time.Time, timeBucket string, fieldValue int64,
-	group string, data flow.Data, rankNum int,
+func (t *topNStreamingProcessor) writeData(publisher queue.BatchPublisher, eventTime time.Time, fieldValue int64,
+	data flow.Data, rankNum int,
 ) error {
 	var tagValues []*modelv1.TagValue
 	if len(t.topNSchema.GetGroupByTagNames()) > 0 {
@@ -172,11 +172,6 @@ func (t *topNStreamingProcessor) writeData(publisher queue.BatchPublisher, event
 		return err
 	}
 
-	// measureID is consist of three parts,
-	// 1. groupValues
-	// 2. rankNumber
-	// 3. timeBucket
-	measureID := group + "_" + strconv.Itoa(rankNum) + "_" + timeBucket
 	iwr := &measurev1.InternalWriteRequest{
 		Request: &measurev1.WriteRequest{
 			MessageId: uint64(time.Now().UnixNano()),
@@ -185,17 +180,10 @@ func (t *topNStreamingProcessor) writeData(publisher queue.BatchPublisher, event
 				Timestamp: timestamppb.New(eventTime),
 				TagFamilies: []*modelv1.TagFamilyForWrite{
 					{
-						Tags: append([]*modelv1.TagValue{
-							// MeasureID
-							{
-								Value: &modelv1.TagValue_Str{
-									Str: &modelv1.Str{
-										Value: measureID,
-									},
-								},
-							},
+						Tags: append(
+							data[0].([]*modelv1.TagValue),
 							// SortDirection
-							{
+							&modelv1.TagValue{
 								Value: &modelv1.TagValue_Int{
 									Int: &modelv1.Int{
 										Value: int64(t.sortDirection),
@@ -203,14 +191,14 @@ func (t *topNStreamingProcessor) writeData(publisher queue.BatchPublisher, event
 								},
 							},
 							// RankNumber
-							{
+							&modelv1.TagValue{
 								Value: &modelv1.TagValue_Int{
 									Int: &modelv1.Int{
 										Value: int64(rankNum),
 									},
 								},
 							},
-						}, data[0].([]*modelv1.TagValue)...),
+						),
 					},
 				},
 				Fields: []*modelv1.FieldValue{
@@ -240,40 +228,35 @@ func (t *topNStreamingProcessor) locate(tagValues []*modelv1.TagValue, rankNum i
 	if len(tagValues) != 0 && len(t.topNSchema.GetGroupByTagNames()) != len(tagValues) {
 		return nil, 0, errors.New("no enough tag values for the entity")
 	}
-	// Subject: topN aggregation Name
-	//
-	// entity prefix
-	//
-	// 1) sort direction
-	// 2) rank number
-	// >3) group tag values if needed
 	series := &pbv1.Series{
 		Subject:      t.topNSchema.GetMetadata().GetName(),
 		EntityValues: make([]*modelv1.TagValue, 1+1+len(tagValues)),
 	}
 
-	// entity prefix
-	series.EntityValues[0] = &modelv1.TagValue{
+	copy(series.EntityValues, tagValues)
+	series.EntityValues[len(series.EntityValues)-2] = &modelv1.TagValue{
 		Value: &modelv1.TagValue_Int{
 			Int: &modelv1.Int{
 				Value: int64(t.sortDirection),
 			},
 		},
 	}
-	series.EntityValues[1] = &modelv1.TagValue{
+	series.EntityValues[len(series.EntityValues)-1] = &modelv1.TagValue{
 		Value: &modelv1.TagValue_Int{
 			Int: &modelv1.Int{
 				Value: int64(rankNum),
 			},
 		},
 	}
-	// measureID as sharding key
-	for idx, tagVal := range tagValues {
-		series.EntityValues[idx+2] = tagVal
-	}
 	if err := series.Marshal(); err != nil {
 		return nil, 0, fmt.Errorf("fail to marshal series: %w", err)
 	}
+	src := make([]byte, len(series.Buffer))
+	copy(src, series.Buffer)
+	var s1 pbv1.Series
+	if err := s1.Unmarshal(src); err != nil {
+		return nil, 0, fmt.Errorf("fail to unmarshal series encoded:[%s] tagValues:[%s]: %w", series.Buffer, series.EntityValues, err)
+	}
 	id, err := partition.ShardID(series.Buffer, t.m.shardNum)
 	if err != nil {
 		return nil, 0, err
@@ -282,7 +265,11 @@ func (t *topNStreamingProcessor) locate(tagValues []*modelv1.TagValue, rankNum i
 }
 
 func (t *topNStreamingProcessor) start() *topNStreamingProcessor {
-	t.errCh = t.streamingFlow.Window(streaming.NewTumblingTimeWindows(t.interval)).
+	flushInterval := t.interval
+	if flushInterval > maxFlushInterval {
+		flushInterval = maxFlushInterval
+	}
+	t.errCh = t.streamingFlow.Window(streaming.NewTumblingTimeWindows(t.interval, flushInterval)).
 		AllowedMaxWindows(int(t.topNSchema.GetLruSize())).
 		TopN(int(t.topNSchema.GetCountersNumber()),
 			streaming.WithSortKeyExtractor(func(record flow.StreamRecord) int64 {
@@ -364,7 +351,8 @@ func (manager *topNProcessorManager) start() error {
 		for i, sortDirection := range sortDirections {
 			srcCh := make(chan interface{})
 			src, _ := sources.NewChannel(srcCh)
-			streamingFlow := streaming.New(src)
+			name := strings.Join([]string{topNSchema.GetMetadata().Group, topNSchema.GetMetadata().Name, modelv1.Sort_name[int32(sortDirection)]}, "-")
+			streamingFlow := streaming.New(name, src)
 
 			filters, buildErr := manager.buildFilter(topNSchema.GetCriteria())
 			if buildErr != nil {
diff --git a/banyand/query/processor_topn.go b/banyand/query/processor_topn.go
index de54a1da..d734e18b 100644
--- a/banyand/query/processor_topn.go
+++ b/banyand/query/processor_topn.go
@@ -105,7 +105,7 @@ func (t *topNQueryProcessor) Rev(message bus.Message) (resp bus.Message) {
 			Str("topN", topNMetadata.GetName()).
 			Msg("fail to build schema")
 	}
-	plan, err := logical_measure.TopNAnalyze(context.TODO(), request, topNResultMeasure.GetSchema(), sourceMeasure.GetSchema(), topNSchema, s)
+	plan, err := logical_measure.TopNAnalyze(context.TODO(), request, topNResultMeasure.GetSchema(), sourceMeasure.GetSchema(), s)
 	if err != nil {
 		resp = bus.NewMessage(bus.MessageID(now), common.NewError("fail to analyze the query request for topn %s: %v", topNMetadata.GetName(), err))
 		return
@@ -256,7 +256,6 @@ func (aggr *postAggregationProcessor) Put(entityValues pbv1.EntityValues, val in
 	key := entityValues.String()
 	if item, found := aggr.cache[key]; found {
 		item.int64Func.In(val)
-		aggr.tryEnqueue(key, item)
 		return nil
 	}
 
diff --git a/banyand/queue/sub/sub.go b/banyand/queue/sub/sub.go
index 9846c4db..bcdc6ab1 100644
--- a/banyand/queue/sub/sub.go
+++ b/banyand/queue/sub/sub.go
@@ -19,6 +19,7 @@
 package sub
 
 import (
+	"fmt"
 	"io"
 
 	"github.com/pkg/errors"
@@ -27,6 +28,7 @@ import (
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
 
+	"github.com/apache/skywalking-banyandb/api/common"
 	"github.com/apache/skywalking-banyandb/api/data"
 	clusterv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/cluster/v1"
 	"github.com/apache/skywalking-banyandb/pkg/bus"
@@ -120,9 +122,15 @@ func (s *server) Send(stream clusterv1.Service_SendServer) error {
 			}
 			continue
 		}
-		message, ok := m.Data().(proto.Message)
-		if !ok {
-			reply(writeEntity, err, "invalid response")
+		var message proto.Message
+		switch d := m.Data().(type) {
+		case proto.Message:
+			message = d
+		case common.Error:
+			reply(writeEntity, nil, d.Msg())
+			continue
+		default:
+			reply(writeEntity, nil, fmt.Sprintf("invalid response: %T", d))
 			continue
 		}
 		anyMessage, err := anypb.New(message)
diff --git a/banyand/stream/block_metadata.go b/banyand/stream/block_metadata.go
index 3906f603..41527229 100644
--- a/banyand/stream/block_metadata.go
+++ b/banyand/stream/block_metadata.go
@@ -245,38 +245,38 @@ type timestampsMetadata struct {
 	encodeType encoding.EncodeType
 }
 
-func (th *timestampsMetadata) reset() {
-	th.dataBlock.reset()
-	th.min = 0
-	th.max = 0
-	th.encodeType = 0
+func (tm *timestampsMetadata) reset() {
+	tm.dataBlock.reset()
+	tm.min = 0
+	tm.max = 0
+	tm.encodeType = 0
 }
 
-func (th *timestampsMetadata) copyFrom(src *timestampsMetadata) {
-	th.dataBlock.copyFrom(&src.dataBlock)
-	th.min = src.min
-	th.max = src.max
-	th.encodeType = src.encodeType
+func (tm *timestampsMetadata) copyFrom(src *timestampsMetadata) {
+	tm.dataBlock.copyFrom(&src.dataBlock)
+	tm.min = src.min
+	tm.max = src.max
+	tm.encodeType = src.encodeType
 }
 
-func (th *timestampsMetadata) marshal(dst []byte) []byte {
-	dst = th.dataBlock.marshal(dst)
-	dst = encoding.Uint64ToBytes(dst, uint64(th.min))
-	dst = encoding.Uint64ToBytes(dst, uint64(th.max))
-	dst = append(dst, byte(th.encodeType))
+func (tm *timestampsMetadata) marshal(dst []byte) []byte {
+	dst = tm.dataBlock.marshal(dst)
+	dst = encoding.Uint64ToBytes(dst, uint64(tm.min))
+	dst = encoding.Uint64ToBytes(dst, uint64(tm.max))
+	dst = append(dst, byte(tm.encodeType))
 	return dst
 }
 
-func (th *timestampsMetadata) unmarshal(src []byte) ([]byte, error) {
-	src, err := th.dataBlock.unmarshal(src)
+func (tm *timestampsMetadata) unmarshal(src []byte) ([]byte, error) {
+	src, err := tm.dataBlock.unmarshal(src)
 	if err != nil {
 		return nil, fmt.Errorf("cannot unmarshal dataBlock: %w", err)
 	}
-	th.min = int64(encoding.BytesToUint64(src))
+	tm.min = int64(encoding.BytesToUint64(src))
 	src = src[8:]
-	th.max = int64(encoding.BytesToUint64(src))
+	tm.max = int64(encoding.BytesToUint64(src))
 	src = src[8:]
-	th.encodeType = encoding.EncodeType(src[0])
+	tm.encodeType = encoding.EncodeType(src[0])
 	return src[1:], nil
 }
 
@@ -285,28 +285,28 @@ type elementIDsMetadata struct {
 	encodeType encoding.EncodeType
 }
 
-func (th *elementIDsMetadata) reset() {
-	th.dataBlock.reset()
-	th.encodeType = 0
+func (em *elementIDsMetadata) reset() {
+	em.dataBlock.reset()
+	em.encodeType = 0
 }
 
-func (th *elementIDsMetadata) copyFrom(src *elementIDsMetadata) {
-	th.dataBlock.copyFrom(&src.dataBlock)
-	th.encodeType = src.encodeType
+func (em *elementIDsMetadata) copyFrom(src *elementIDsMetadata) {
+	em.dataBlock.copyFrom(&src.dataBlock)
+	em.encodeType = src.encodeType
 }
 
-func (th *elementIDsMetadata) marshal(dst []byte) []byte {
-	dst = th.dataBlock.marshal(dst)
-	dst = append(dst, byte(th.encodeType))
+func (em *elementIDsMetadata) marshal(dst []byte) []byte {
+	dst = em.dataBlock.marshal(dst)
+	dst = append(dst, byte(em.encodeType))
 	return dst
 }
 
-func (th *elementIDsMetadata) unmarshal(src []byte) ([]byte, error) {
-	src, err := th.dataBlock.unmarshal(src)
+func (em *elementIDsMetadata) unmarshal(src []byte) ([]byte, error) {
+	src, err := em.dataBlock.unmarshal(src)
 	if err != nil {
 		return nil, fmt.Errorf("cannot unmarshal dataBlock: %w", err)
 	}
-	th.encodeType = encoding.EncodeType(src[0])
+	em.encodeType = encoding.EncodeType(src[0])
 	return src[1:], nil
 }
 
diff --git a/banyand/stream/block_metadata_test.go b/banyand/stream/block_metadata_test.go
index 67c7e8e3..5c721b7d 100644
--- a/banyand/stream/block_metadata_test.go
+++ b/banyand/stream/block_metadata_test.go
@@ -74,7 +74,7 @@ func Test_dataBlock_marshal_unmarshal(t *testing.T) {
 }
 
 func Test_timestampsMetadata_reset(t *testing.T) {
-	th := &timestampsMetadata{
+	tm := &timestampsMetadata{
 		dataBlock: dataBlock{
 			offset: 1,
 			size:   1,
@@ -84,13 +84,13 @@ func Test_timestampsMetadata_reset(t *testing.T) {
 		encodeType: encoding.EncodeTypeConst,
 	}
 
-	th.reset()
+	tm.reset()
 
-	assert.Equal(t, uint64(0), th.dataBlock.offset)
-	assert.Equal(t, uint64(0), th.dataBlock.size)
-	assert.Equal(t, int64(0), th.min)
-	assert.Equal(t, int64(0), th.max)
-	assert.Equal(t, encoding.EncodeTypeUnknown, th.encodeType)
+	assert.Equal(t, uint64(0), tm.dataBlock.offset)
+	assert.Equal(t, uint64(0), tm.dataBlock.size)
+	assert.Equal(t, int64(0), tm.min)
+	assert.Equal(t, int64(0), tm.max)
+	assert.Equal(t, encoding.EncodeTypeUnknown, tm.encodeType)
 }
 
 func Test_timestampsMetadata_copyFrom(t *testing.T) {
diff --git a/banyand/stream/block_writer.go b/banyand/stream/block_writer.go
index ad6a69e3..2c981c3d 100644
--- a/banyand/stream/block_writer.go
+++ b/banyand/stream/block_writer.go
@@ -213,23 +213,23 @@ func (bw *blockWriter) mustWriteBlock(sid common.SeriesID, b *block) {
 
 	bm := generateBlockMetadata()
 	b.mustWriteTo(sid, bm, &bw.writers)
-	th := &bm.timestamps
-	if bw.totalCount == 0 || th.min < bw.totalMinTimestamp {
-		bw.totalMinTimestamp = th.min
+	tm := &bm.timestamps
+	if bw.totalCount == 0 || tm.min < bw.totalMinTimestamp {
+		bw.totalMinTimestamp = tm.min
 	}
-	if bw.totalCount == 0 || th.max > bw.totalMaxTimestamp {
-		bw.totalMaxTimestamp = th.max
+	if bw.totalCount == 0 || tm.max > bw.totalMaxTimestamp {
+		bw.totalMaxTimestamp = tm.max
 	}
-	if !hasWrittenBlocks || th.min < bw.minTimestamp {
-		bw.minTimestamp = th.min
+	if !hasWrittenBlocks || tm.min < bw.minTimestamp {
+		bw.minTimestamp = tm.min
 	}
-	if !hasWrittenBlocks || th.max > bw.maxTimestamp {
-		bw.maxTimestamp = th.max
+	if !hasWrittenBlocks || tm.max > bw.maxTimestamp {
+		bw.maxTimestamp = tm.max
 	}
-	if isSeenSid && th.min < bw.minTimestampLast {
-		logger.Panicf("the block for sid=%d cannot contain timestamp smaller than %d, but it contains timestamp %d", sid, bw.minTimestampLast, th.min)
+	if isSeenSid && tm.min < bw.minTimestampLast {
+		logger.Panicf("the block for sid=%d cannot contain timestamp smaller than %d, but it contains timestamp %d", sid, bw.minTimestampLast, tm.min)
 	}
-	bw.minTimestampLast = th.min
+	bw.minTimestampLast = tm.min
 
 	bw.totalUncompressedSizeBytes += bm.uncompressedSizeBytes
 	bw.totalCount += bm.count
diff --git a/banyand/stream/tag.go b/banyand/stream/tag.go
index f9ad8740..5d595043 100644
--- a/banyand/stream/tag.go
+++ b/banyand/stream/tag.go
@@ -51,11 +51,11 @@ func (t *tag) resizeValues(valuesLen int) [][]byte {
 	return values
 }
 
-func (t *tag) mustWriteTo(ch *tagMetadata, tagWriter *writer) {
-	ch.reset()
+func (t *tag) mustWriteTo(tm *tagMetadata, tagWriter *writer) {
+	tm.reset()
 
-	ch.name = t.name
-	ch.valueType = t.valueType
+	tm.name = t.name
+	tm.valueType = t.valueType
 
 	// TODO: encoding values based on value type
 
@@ -64,11 +64,11 @@ func (t *tag) mustWriteTo(ch *tagMetadata, tagWriter *writer) {
 
 	// marshal values
 	bb.Buf = encoding.EncodeBytesBlock(bb.Buf[:0], t.values)
-	ch.size = uint64(len(bb.Buf))
-	if ch.size > maxValuesBlockSize {
-		logger.Panicf("too valuesSize: %d bytes; mustn't exceed %d bytes", ch.size, maxValuesBlockSize)
+	tm.size = uint64(len(bb.Buf))
+	if tm.size > maxValuesBlockSize {
+		logger.Panicf("too valuesSize: %d bytes; mustn't exceed %d bytes", tm.size, maxValuesBlockSize)
 	}
-	ch.offset = tagWriter.bytesWritten
+	tm.offset = tagWriter.bytesWritten
 	tagWriter.MustWrite(bb.Buf)
 }
 
diff --git a/pkg/flow/streaming/sliding_window.go b/pkg/flow/streaming/sliding_window.go
index 860f193f..ed012084 100644
--- a/pkg/flow/streaming/sliding_window.go
+++ b/pkg/flow/streaming/sliding_window.go
@@ -28,6 +28,7 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/apache/skywalking-banyandb/pkg/flow"
+	"github.com/apache/skywalking-banyandb/pkg/logger"
 )
 
 type triggerResult bool
@@ -49,6 +50,7 @@ func (f *streamingFlow) Window(w flow.WindowAssigner) flow.WindowedFlow {
 	switch v := w.(type) {
 	case *tumblingTimeWindows:
 		v.errorHandler = f.drainErr
+		v.l = f.l
 		f.ops = append(f.ops, v)
 	default:
 		f.drainErr(errors.New("window type is not supported"))
@@ -57,6 +59,7 @@ func (f *streamingFlow) Window(w flow.WindowAssigner) flow.WindowedFlow {
 	return &windowedFlow{
 		f:  f,
 		wa: w,
+		l:  f.l,
 	}
 }
 
@@ -71,16 +74,19 @@ func (s *windowedFlow) AllowedMaxWindows(windowCnt int) flow.WindowedFlow {
 }
 
 type tumblingTimeWindows struct {
-	errorHandler       func(error)
+	l                  *logger.Logger
 	snapshots          *lru.Cache
 	timerHeap          *flow.DedupPriorityQueue
 	aggregationFactory flow.AggregationOpFactory
 	in                 chan flow.StreamRecord
 	out                chan flow.StreamRecord
+	errorHandler       func(error)
 	flow.ComponentState
-	windowSize       int64
 	windowCount      int
 	currentWatermark int64
+	lastFlushTime    int64
+	flushInterval    int64
+	windowSize       int64
 	timerMu          sync.Mutex
 }
 
@@ -98,7 +104,10 @@ func (s *tumblingTimeWindows) Setup(_ context.Context) (err error) {
 			s.windowCount = defaultCacheSize
 		}
 		s.snapshots, err = lru.NewWithEvict(s.windowCount, func(key interface{}, value interface{}) {
-			s.flushSnapshot(key.(timeWindow), value.(flow.AggregationOp))
+			flushed := s.flushSnapshot(key.(timeWindow), value.(flow.AggregationOp))
+			if e := s.l.Debug(); e.Enabled() {
+				e.Stringer("window", key.(timeWindow)).Bool("flushed", flushed).Msg("evict window on lru cache is full")
+			}
 		})
 		if err != nil {
 			return err
@@ -111,15 +120,20 @@ func (s *tumblingTimeWindows) Setup(_ context.Context) (err error) {
 	return
 }
 
-func (s *tumblingTimeWindows) flushSnapshot(w timeWindow, snapshot flow.AggregationOp) {
+func (s *tumblingTimeWindows) flushSnapshot(w timeWindow, snapshot flow.AggregationOp) bool {
 	if snapshot.Dirty() {
 		s.out <- flow.NewStreamRecord(snapshot.Snapshot(), w.start)
+		return true
 	}
+	return false
 }
 
 func (s *tumblingTimeWindows) flushWindow(w timeWindow) {
 	if snapshot, ok := s.snapshots.Get(w); ok {
-		s.flushSnapshot(w, snapshot.(flow.AggregationOp))
+		flushed := s.flushSnapshot(w, snapshot.(flow.AggregationOp))
+		if e := s.l.Debug(); e.Enabled() {
+			e.Stringer("window", w).Bool("flushed", flushed).Msg("flush window")
+		}
 	}
 }
 
@@ -170,6 +184,9 @@ func (s *tumblingTimeWindows) receive() {
 				newAggr := s.aggregationFactory()
 				newAggr.Add([]flow.StreamRecord{elem})
 				s.snapshots.Add(tw, newAggr)
+				if e := s.l.Debug(); e.Enabled() {
+					e.Stringer("window", tw).Msg("create new window")
+				}
 			}
 
 			result := ctx.OnElement(elem)
@@ -180,7 +197,13 @@ func (s *tumblingTimeWindows) receive() {
 
 		// even if the incoming elements do not follow strict order,
 		// the watermark could increase monotonically.
-		if pastDur := elem.TimestampMillis() - s.currentWatermark; pastDur > 0 {
+		now := time.Now().UnixNano() / int64(time.Millisecond)
+		pastDataDur := elem.TimestampMillis() - s.currentWatermark
+		if s.lastFlushTime == 0 {
+			s.lastFlushTime = now
+		}
+		pastDur := now - s.lastFlushTime
+		if pastDur > 0 || pastDataDur > 0 {
 			previousWaterMark := s.currentWatermark
 			s.currentWatermark = elem.TimestampMillis()
 
@@ -197,8 +220,9 @@ func (s *tumblingTimeWindows) receive() {
 			// |    40%     |    40%     |  20%  |
 			// |          flush        flush     |
 			// |---------------------------------|
-			// TODO: how to determine the threshold
-			if previousWaterMark > 0 && float64(pastDur) > float64(s.windowSize)*0.4 {
+			// the max flush interval is 1 minute.
+			if (pastDur > s.flushInterval) || (previousWaterMark > 0 && pastDataDur > s.flushInterval) {
+				s.lastFlushTime = now
 				s.flushDirtyWindows()
 			}
 		}
@@ -226,15 +250,22 @@ func (s *tumblingTimeWindows) Exec(downstream flow.Inlet) {
 }
 
 // NewTumblingTimeWindows return tumbling-time windows.
-func NewTumblingTimeWindows(size time.Duration) flow.WindowAssigner {
+func NewTumblingTimeWindows(size time.Duration, maxFlushInterval time.Duration) flow.WindowAssigner {
+	ws := size.Milliseconds()
+	mfi := maxFlushInterval.Milliseconds()
+	it := int64(float64(ws) * 0.4)
+	if it > mfi {
+		it = mfi
+	}
 	return &tumblingTimeWindows{
-		windowSize: size.Milliseconds(),
+		windowSize: ws,
 		timerHeap: flow.NewPriorityQueue(func(a, b interface{}) int {
 			return int(a.(*internalTimer).triggerTimeMillis - b.(*internalTimer).triggerTimeMillis)
 		}, false),
 		in:               make(chan flow.StreamRecord),
 		out:              make(chan flow.StreamRecord),
 		currentWatermark: 0,
+		flushInterval:    it,
 	}
 }
 
@@ -247,6 +278,12 @@ func (t timeWindow) MaxTimestamp() int64 {
 	return t.end - 1
 }
 
+func (t timeWindow) String() string {
+	st := time.Unix(0, t.start*int64(time.Millisecond)).String()
+	et := time.Unix(0, t.end*int64(time.Millisecond)).String()
+	return st + " - " + et
+}
+
 // AssignWindows assigns windows according to the given timestamp.
 func (s *tumblingTimeWindows) AssignWindows(timestamp int64) ([]flow.Window, error) {
 	if timestamp > math.MinInt64 {
diff --git a/pkg/flow/streaming/sliding_window_test.go b/pkg/flow/streaming/sliding_window_test.go
index 2a3366b0..cee7973d 100644
--- a/pkg/flow/streaming/sliding_window_test.go
+++ b/pkg/flow/streaming/sliding_window_test.go
@@ -25,6 +25,7 @@ import (
 	"github.com/onsi/gomega"
 
 	"github.com/apache/skywalking-banyandb/pkg/flow"
+	"github.com/apache/skywalking-banyandb/pkg/logger"
 	"github.com/apache/skywalking-banyandb/pkg/test/flags"
 )
 
@@ -72,9 +73,10 @@ var _ = g.Describe("Sliding Window", func() {
 	g.JustBeforeEach(func() {
 		snk = newSlice()
 
-		slidingWindows = NewTumblingTimeWindows(time.Second * 15).(*tumblingTimeWindows)
+		slidingWindows = NewTumblingTimeWindows(time.Second*15, time.Second*15).(*tumblingTimeWindows)
 		slidingWindows.aggregationFactory = aggrFactory
 		slidingWindows.windowCount = 2
+		slidingWindows.l = logger.GetLogger("tumblingTimeWindows")
 
 		gomega.Expect(slidingWindows.Setup(context.TODO())).Should(gomega.Succeed())
 		gomega.Expect(snk.Setup(context.TODO())).Should(gomega.Succeed())
diff --git a/pkg/flow/streaming/streaming.go b/pkg/flow/streaming/streaming.go
index 63b02edf..f144aebd 100644
--- a/pkg/flow/streaming/streaming.go
+++ b/pkg/flow/streaming/streaming.go
@@ -25,6 +25,7 @@ import (
 	"go.uber.org/multierr"
 
 	"github.com/apache/skywalking-banyandb/pkg/flow"
+	"github.com/apache/skywalking-banyandb/pkg/logger"
 )
 
 var _ flow.Flow = (*streamingFlow)(nil)
@@ -34,15 +35,17 @@ type streamingFlow struct {
 	source flow.Source
 	sink   flow.Sink
 	drain  chan error
+	l      *logger.Logger
 	ops    []flow.Operator
 }
 
 // New returns a new streaming flow.
-func New(source flow.Source) flow.Flow {
+func New(name string, source flow.Source) flow.Flow {
 	return &streamingFlow{
 		source: source,
 		ops:    make([]flow.Operator, 0),
 		drain:  make(chan error),
+		l:      logger.GetLogger(name),
 	}
 }
 
@@ -66,6 +69,7 @@ func (f *streamingFlow) To(sink flow.Sink) flow.Flow {
 }
 
 func (f *streamingFlow) Close() error {
+	f.l.Info().Msg("flow is closing")
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()
 	err := f.source.Teardown(ctx)
@@ -114,6 +118,8 @@ func (f *streamingFlow) Open() <-chan error {
 	// finally connect sources and the first operator
 	f.source.Exec(f.ops[0])
 
+	f.l.Info().Msg("flow is running")
+
 	return f.drain
 }
 
diff --git a/pkg/flow/streaming/streaming_test.go b/pkg/flow/streaming/streaming_test.go
index 57d99c5e..796022e5 100644
--- a/pkg/flow/streaming/streaming_test.go
+++ b/pkg/flow/streaming/streaming_test.go
@@ -61,7 +61,7 @@ var _ = g.Describe("Streaming", func() {
 
 		g.JustBeforeEach(func() {
 			snk = newSlice()
-			f = New(input).
+			f = New("test", input).
 				Filter(filter).
 				To(snk)
 			errCh = f.Open()
@@ -98,7 +98,7 @@ var _ = g.Describe("Streaming", func() {
 
 		g.JustBeforeEach(func() {
 			snk = newSlice()
-			f = New(input).
+			f = New("test", input).
 				Map(mapper).
 				To(snk)
 			errCh = f.Open()
@@ -143,12 +143,12 @@ var _ = g.Describe("Streaming", func() {
 		g.JustBeforeEach(func() {
 			snk = newSlice()
 
-			f = New(flowTest.NewSlice(input)).
+			f = New("test", flowTest.NewSlice(input)).
 				Map(flow.UnaryFunc[any](func(_ context.Context, item interface{}) interface{} {
 					// groupBy
 					return flow.Data{item.(*record).service, int64(item.(*record).value)}
 				})).
-				Window(NewTumblingTimeWindows(15*time.Second)).
+				Window(NewTumblingTimeWindows(15*time.Second, 15*time.Second)).
 				TopN(3, WithSortKeyExtractor(func(record flow.StreamRecord) int64 {
 					return record.Data().(flow.Data)[1].(int64)
 				}), OrderBy(ASC), WithGroupKeyExtractor(func(record flow.StreamRecord) string {
@@ -206,12 +206,12 @@ var _ = g.Describe("Streaming", func() {
 		g.JustBeforeEach(func() {
 			snk = newSlice()
 
-			f = New(flowTest.NewSlice(input)).
+			f = New("test", flowTest.NewSlice(input)).
 				Map(flow.UnaryFunc[any](func(_ context.Context, item interface{}) interface{} {
 					// groupBy
 					return flow.Data{item.(*record).service, int64(item.(*record).value)}
 				})).
-				Window(NewTumblingTimeWindows(15*time.Second)).
+				Window(NewTumblingTimeWindows(15*time.Second, 15*time.Second)).
 				TopN(3, WithSortKeyExtractor(func(record flow.StreamRecord) int64 {
 					return record.Data().(flow.Data)[1].(int64)
 				}), WithGroupKeyExtractor(func(record flow.StreamRecord) string {
diff --git a/pkg/flow/streaming/topn.go b/pkg/flow/streaming/topn.go
index 746faa66..eaaafe24 100644
--- a/pkg/flow/streaming/topn.go
+++ b/pkg/flow/streaming/topn.go
@@ -18,11 +18,16 @@
 package streaming
 
 import (
+	"strconv"
+	"strings"
+	"time"
+
 	"github.com/emirpasic/gods/maps/treemap"
 	"github.com/emirpasic/gods/utils"
 	"github.com/pkg/errors"
 
 	"github.com/apache/skywalking-banyandb/pkg/flow"
+	"github.com/apache/skywalking-banyandb/pkg/logger"
 )
 
 // TopNSort defines the order of sorting.
@@ -37,6 +42,7 @@ const (
 type windowedFlow struct {
 	f  *streamingFlow
 	wa flow.WindowAssigner
+	l  *logger.Logger
 }
 
 func (s *windowedFlow) TopN(topNum int, opts ...any) flow.Flow {
@@ -44,6 +50,7 @@ func (s *windowedFlow) TopN(topNum int, opts ...any) flow.Flow {
 		topNAggrFunc := &topNAggregatorGroup{
 			cacheSize: topNum,
 			sort:      DESC,
+			l:         s.l,
 		}
 		// apply user customized options
 		for _, opt := range opts {
@@ -72,6 +79,7 @@ type topNAggregatorGroup struct {
 	sortKeyExtractor  func(flow.StreamRecord) int64
 	groupKeyExtractor func(flow.StreamRecord) string
 	comparator        utils.Comparator
+	l                 *logger.Logger
 	cacheSize         int
 	sort              TopNSort
 }
@@ -113,6 +121,9 @@ func (t *topNAggregatorGroup) Add(input []flow.StreamRecord) {
 		groupKey := t.groupKeyExtractor(item)
 		aggregator := t.getOrCreateGroup(groupKey)
 		if aggregator.checkSortKeyInBufferRange(sortKey) {
+			if e := t.l.Debug(); e.Enabled() {
+				e.Str("group", groupKey).Time("elem_ts", time.Unix(0, item.TimestampMillis()*int64(time.Millisecond))).Msg("put into topN buffer")
+			}
 			aggregator.put(sortKey, item)
 			aggregator.doCleanUp()
 		}
@@ -136,6 +147,19 @@ func (t *topNAggregatorGroup) Snapshot() interface{} {
 		}
 		groupRanks[group] = items
 	}
+	if len(groupRanks) > 0 {
+		if e := t.l.Debug(); e.Enabled() {
+			sb := strings.Builder{}
+			for g, item := range groupRanks {
+				sb.WriteString("{")
+				sb.WriteString(g)
+				sb.WriteString(":")
+				sb.WriteString(strconv.Itoa(len(item)))
+				sb.WriteString("}")
+			}
+			t.l.Debug().Interface("snapshot", sb.String()).Msg("taken a topN snapshot")
+		}
+	}
 	return groupRanks
 }
 
diff --git a/pkg/flow/streaming/topn_test.go b/pkg/flow/streaming/topn_test.go
index 988fe825..16781838 100644
--- a/pkg/flow/streaming/topn_test.go
+++ b/pkg/flow/streaming/topn_test.go
@@ -25,6 +25,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/apache/skywalking-banyandb/pkg/flow"
+	"github.com/apache/skywalking-banyandb/pkg/logger"
 )
 
 func TestFlow_TopN_Aggregator(t *testing.T) {
@@ -117,6 +118,7 @@ func TestFlow_TopN_Aggregator(t *testing.T) {
 				groupKeyExtractor: func(record flow.StreamRecord) string {
 					return record.Data().(flow.Data)[0].(string)
 				},
+				l: logger.GetLogger("test"),
 			}
 			topN.Add(input)
 			snapshot := topN.Snapshot()
diff --git a/pkg/flow/types.go b/pkg/flow/types.go
index 538ce849..6e8576a4 100644
--- a/pkg/flow/types.go
+++ b/pkg/flow/types.go
@@ -20,8 +20,10 @@ package flow
 
 import (
 	"context"
+	"fmt"
 	"io"
 	"sync"
+	"time"
 
 	"github.com/google/go-cmp/cmp"
 	"google.golang.org/protobuf/types/known/timestamppb"
@@ -142,6 +144,11 @@ func (sr StreamRecord) Equal(other StreamRecord) bool {
 	return sr.ts == other.ts && cmp.Equal(sr.data, other.data)
 }
 
+func (sr StreamRecord) String() string {
+	t := time.Unix(0, sr.ts*int64(time.Millisecond))
+	return fmt.Sprintf("data: %s, ts: %v", sr.data, t)
+}
+
 // Inlet represents a type that exposes one open input.
 //
 //go:generate mockgen -destination=./inlet_mock.go -package=flow github.com/apache/skywalking-banyandb/pkg/flow Inlet
diff --git a/pkg/pb/v1/series.go b/pkg/pb/v1/series.go
index 3f747bf8..f6d47397 100644
--- a/pkg/pb/v1/series.go
+++ b/pkg/pb/v1/series.go
@@ -51,6 +51,19 @@ func (s *Series) Marshal() error {
 	return nil
 }
 
+// MarshalWithWildcard encodes series to internal Buffer and generates ID with wildcard.
+func (s *Series) MarshalWithWildcard() error {
+	s.Buffer = marshalEntityValue(s.Buffer, convert.StringToBytes(s.Subject))
+	var err error
+	for _, tv := range s.EntityValues {
+		if s.Buffer, err = marshalTagValueWithWildcard(s.Buffer, tv); err != nil {
+			return errors.WithMessage(err, "marshal subject and entity values")
+		}
+	}
+	s.ID = common.SeriesID(convert.Hash(s.Buffer))
+	return nil
+}
+
 // Unmarshal decodes series from internal Buffer.
 func (s *Series) Unmarshal(src []byte) error {
 	var err error
@@ -63,7 +76,7 @@ func (s *Series) Unmarshal(src []byte) error {
 		s.Buffer = s.Buffer[:0]
 		var tv *modelv1.TagValue
 		if s.Buffer, src, tv, err = unmarshalTagValue(s.Buffer, src); err != nil {
-			return errors.WithMessage(err, "unmarshal tag value")
+			return errors.WithMessagef(err, "unmarshal tag value [%s], marshaled %s", src, s.EntityValues)
 		}
 		s.EntityValues = append(s.EntityValues, tv)
 	}
diff --git a/pkg/pb/v1/series_test.go b/pkg/pb/v1/series_test.go
index c44c5a89..0399d045 100644
--- a/pkg/pb/v1/series_test.go
+++ b/pkg/pb/v1/series_test.go
@@ -92,6 +92,14 @@ func TestMarshalAndUnmarshalSeries(t *testing.T) {
 				{Value: &modelv1.TagValue_Int{Int: &modelv1.Int{Value: 123}}},
 			}},
 		},
+		{
+			name: "topn",
+			src: &Series{Subject: "subject", EntityValues: []*modelv1.TagValue{
+				{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "stringValue"}}},
+				{Value: &modelv1.TagValue_Int{Int: &modelv1.Int{Value: 1}}},
+				{Value: &modelv1.TagValue_Int{Int: &modelv1.Int{Value: 46}}},
+			}},
+		},
 	}
 
 	for _, tt := range tests {
diff --git a/pkg/pb/v1/value.go b/pkg/pb/v1/value.go
index f7c6abed..064d2022 100644
--- a/pkg/pb/v1/value.go
+++ b/pkg/pb/v1/value.go
@@ -19,6 +19,7 @@ package v1
 
 import (
 	"bytes"
+	"fmt"
 
 	"github.com/pkg/errors"
 
@@ -63,10 +64,6 @@ func MustTagValueToValueType(tag *modelv1.TagValue) ValueType {
 }
 
 func marshalTagValue(dest []byte, tv *modelv1.TagValue) ([]byte, error) {
-	if tv == AnyTagValue {
-		dest = marshalEntityValue(dest, anyWildcard)
-		return dest, nil
-	}
 	dest = append(dest, byte(MustTagValueToValueType(tv)))
 	switch tv.Value.(type) {
 	case *modelv1.TagValue_Null:
@@ -74,8 +71,7 @@ func marshalTagValue(dest []byte, tv *modelv1.TagValue) ([]byte, error) {
 	case *modelv1.TagValue_Str:
 		dest = marshalEntityValue(dest, convert.StringToBytes(tv.GetStr().Value))
 	case *modelv1.TagValue_Int:
-		dest = encoding.Int64ToBytes(dest, tv.GetInt().Value)
-		dest = marshalEntityValue(dest, nil)
+		dest = marshalEntityValue(dest, encoding.Int64ToBytes(nil, tv.GetInt().Value))
 	case *modelv1.TagValue_BinaryData:
 		dest = marshalEntityValue(dest, tv.GetBinaryData())
 	default:
@@ -84,6 +80,14 @@ func marshalTagValue(dest []byte, tv *modelv1.TagValue) ([]byte, error) {
 	return dest, nil
 }
 
+func marshalTagValueWithWildcard(dest []byte, tv *modelv1.TagValue) ([]byte, error) {
+	if tv == AnyTagValue {
+		dest = marshalEntityValue(dest, anyWildcard)
+		return dest, nil
+	}
+	return marshalTagValue(dest, tv)
+}
+
 func unmarshalTagValue(dest []byte, src []byte) ([]byte, []byte, *modelv1.TagValue, error) {
 	if len(src) == 0 {
 		return nil, nil, nil, errors.New("empty tag value")
@@ -128,7 +132,7 @@ func unmarshalTagValue(dest []byte, src []byte) ([]byte, []byte, *modelv1.TagVal
 			},
 		}, nil
 	default:
-		return nil, nil, nil, errors.New("unsupported tag value type")
+		return nil, src, nil, fmt.Errorf("unsupported tag value type %d, tag value: %s", vt, src)
 	}
 }
 
diff --git a/pkg/query/logical/measure/topn_analyzer.go b/pkg/query/logical/measure/topn_analyzer.go
index 4a6e05e6..4d72787f 100644
--- a/pkg/query/logical/measure/topn_analyzer.go
+++ b/pkg/query/logical/measure/topn_analyzer.go
@@ -20,6 +20,7 @@ package measure
 
 import (
 	"context"
+	"fmt"
 
 	commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
 	databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
@@ -52,31 +53,33 @@ func BuildTopNSchema(md *databasev1.Measure) (logical.Schema, error) {
 
 // TopNAnalyze converts logical expressions to executable operation tree represented by Plan.
 func TopNAnalyze(_ context.Context, criteria *measurev1.TopNRequest, schema *databasev1.Measure,
-	sourceMeasureSchema *databasev1.Measure, topNAggregation *databasev1.TopNAggregation, s logical.Schema,
+	sourceMeasureSchema *databasev1.Measure, s logical.Schema,
 ) (logical.Plan, error) {
 	groupByProjectionTags := sourceMeasureSchema.GetEntity().GetTagNames()
 	groupByTags := make([][]*logical.Tag, len(schema.GetTagFamilies()))
 	tagFamily := schema.GetTagFamilies()[0]
 	groupByTags[0] = logical.NewTags(tagFamily.GetName(), groupByProjectionTags...)
 
-	projectionFields := make([]*logical.Field, len(schema.GetFields()))
-	for i, fieldSpecProj := range schema.GetFields() {
-		projectionFields[i] = logical.NewField(fieldSpecProj.GetName())
+	if len(schema.GetFields()) != 1 {
+		return nil, fmt.Errorf("topN schema fields count should be 1 but got %d", len(schema.GetFields()))
 	}
+	projectionFields := make([]*logical.Field, 1)
+	fieldName := schema.GetFields()[0].GetName()
+	projectionFields[0] = logical.NewField(fieldName)
 	// parse fields
 	plan := parse(criteria, schema.GetMetadata(), projectionFields, groupByTags)
 
 	if criteria.GetAgg() != 0 {
 		plan = newUnresolvedGroupBy(plan, groupByTags, false)
 		plan = newUnresolvedAggregation(plan,
-			logical.NewField(topNAggregation.GetFieldName()),
+			projectionFields[0],
 			criteria.GetAgg(),
 			true)
 	}
 
 	plan = top(plan, &measurev1.QueryRequest_Top{
 		Number:         criteria.GetTopN(),
-		FieldName:      topNAggregation.GetFieldName(),
+		FieldName:      fieldName,
 		FieldValueSort: criteria.GetFieldValueSort(),
 	})
 	p, err := plan.Analyze(s)
diff --git a/pkg/schema/init.go b/pkg/schema/init.go
index 4a95dfa7..a158daab 100644
--- a/pkg/schema/init.go
+++ b/pkg/schema/init.go
@@ -361,7 +361,6 @@ func buildTopNSourceMeasure(topNSchema *databasev1.TopNAggregation, sourceMeasur
 			return nil, fmt.Errorf("fail to find tag spec %s", tagName)
 		}
 	}
-
 	// create a new "derived" measure for TopN result
 	return &databasev1.Measure{
 		Metadata: topNSchema.Metadata,
@@ -369,28 +368,23 @@ func buildTopNSourceMeasure(topNSchema *databasev1.TopNAggregation, sourceMeasur
 		TagFamilies: []*databasev1.TagFamilySpec{
 			{
 				Name: topNTagFamily,
-				Tags: append([]*databasev1.TagSpec{
-					{
-						Name: "measure_id",
-						Type: databasev1.TagType_TAG_TYPE_STRING,
-					},
-					{
+				Tags: append(seriesSpecs,
+					&databasev1.TagSpec{
 						Name: "sortDirection",
 						Type: databasev1.TagType_TAG_TYPE_INT,
 					},
-					{
+					&databasev1.TagSpec{
 						Name: "rankNumber",
 						Type: databasev1.TagType_TAG_TYPE_INT,
 					},
-				}, seriesSpecs...),
+				),
 			},
 		},
 		Fields: []*databasev1.FieldSpec{topNValueFieldSpec},
 		Entity: &databasev1.Entity{
-			TagNames: append([]string{
+			TagNames: append(topNSchema.GetGroupByTagNames(),
 				"sortDirection",
-				"rankNumber",
-			}, topNSchema.GetGroupByTagNames()...),
+				"rankNumber"),
 		},
 	}, nil
 }
diff --git a/pkg/test/measure/testdata/topn_aggregations/service_instance_endpoint_cpm_minute_top_bottom_100.json b/pkg/test/measure/testdata/topn_aggregations/service_instance_endpoint_cpm_minute_top_bottom_100.json
index 65f3af15..8f1966da 100644
--- a/pkg/test/measure/testdata/topn_aggregations/service_instance_endpoint_cpm_minute_top_bottom_100.json
+++ b/pkg/test/measure/testdata/topn_aggregations/service_instance_endpoint_cpm_minute_top_bottom_100.json
@@ -7,7 +7,7 @@
     "name": "service_instance_endpoint_cpm_minute",
     "group": "sw_metric"
   },
-  "field_name": "value",
+  "field_name": "total",
   "field_value_sort": 0,
   "group_by_tag_names": [
     "http.uri"
diff --git a/pkg/test/setup/setup.go b/pkg/test/setup/setup.go
index 8fdcbfec..a9596105 100644
--- a/pkg/test/setup/setup.go
+++ b/pkg/test/setup/setup.go
@@ -102,7 +102,7 @@ func standaloneServer(path string, ports []int, schemaLoaders []SchemaLoader, ce
 	endpoint := fmt.Sprintf("http://%s:%d", host, ports[2])
 	ff := []string{
 		"--logging-env=dev",
-		"--logging-level=error",
+		"--logging-level=" + testflags.LogLevel,
 		"--grpc-host=" + host,
 		fmt.Sprintf("--grpc-port=%d", ports[0]),
 		"--http-host=" + host,
diff --git a/test/cases/measure/data/testdata/service_instance_endpoint_cpm_minute_data.json b/test/cases/measure/data/testdata/service_instance_endpoint_cpm_minute_data.json
index 86b99575..13897328 100644
--- a/test/cases/measure/data/testdata/service_instance_endpoint_cpm_minute_data.json
+++ b/test/cases/measure/data/testdata/service_instance_endpoint_cpm_minute_data.json
@@ -69,7 +69,7 @@
     "fields": [
       {
         "int": {
-          "value": 100
+          "value": 101
         }
       },
       {
@@ -104,7 +104,7 @@
     "fields": [
       {
         "int": {
-          "value": 100
+          "value": 130
         }
       },
       {
@@ -139,7 +139,7 @@
     "fields": [
       {
         "int": {
-          "value": 100
+          "value": 155
         }
       },
       {
@@ -215,7 +215,7 @@
     "fields": [
       {
         "int": {
-          "value": 300
+          "value": 240
         }
       },
       {
diff --git a/test/cases/measure/data/testdata/service_instance_endpoint_cpm_minute_data1.json b/test/cases/measure/data/testdata/service_instance_endpoint_cpm_minute_data1.json
index f4f6a118..dee1ddae 100644
--- a/test/cases/measure/data/testdata/service_instance_endpoint_cpm_minute_data1.json
+++ b/test/cases/measure/data/testdata/service_instance_endpoint_cpm_minute_data1.json
@@ -27,7 +27,7 @@
     "fields": [
       {
         "int": {
-          "value": 100
+          "value": 102
         }
       },
       {
@@ -67,7 +67,7 @@
     "fields": [
       {
         "int": {
-          "value": 100
+          "value": 103
         }
       },
       {
@@ -107,7 +107,7 @@
     "fields": [
       {
         "int": {
-          "value": 100
+          "value": 130
         }
       },
       {
@@ -142,7 +142,7 @@
     "fields": [
       {
         "int": {
-          "value": 100
+          "value": 133
         }
       },
       {
@@ -177,7 +177,7 @@
     "fields": [
       {
         "int": {
-          "value": 50
+          "value": 54
         }
       },
       {
diff --git a/test/cases/measure/data/testdata/service_instance_endpoint_cpm_minute_data2.json b/test/cases/measure/data/testdata/service_instance_endpoint_cpm_minute_data2.json
index 59a94d56..13cf282f 100644
--- a/test/cases/measure/data/testdata/service_instance_endpoint_cpm_minute_data2.json
+++ b/test/cases/measure/data/testdata/service_instance_endpoint_cpm_minute_data2.json
@@ -29,7 +29,7 @@
     "fields": [
       {
         "int": {
-          "value": 100
+          "value": 170
         }
       },
       {
@@ -69,7 +69,7 @@
     "fields": [
       {
         "int": {
-          "value": 100
+          "value": 172
         }
       },
       {
diff --git a/test/cases/topn/data/want/null_group.yaml b/test/cases/topn/data/want/null_group.yaml
index 166de533..46004d64 100644
--- a/test/cases/topn/data/want/null_group.yaml
+++ b/test/cases/topn/data/want/null_group.yaml
@@ -20,15 +20,14 @@ lists:
   - entity:
     - key: service_id
       value:
-        str:
-          value: svc_2
+        "null": null
     - key: entity_id
       value:
         str:
           value: entity_1
     value:
       int:
-        value: "12"
+        value: "300"
   - entity:
     - key: service_id
       value:
@@ -37,18 +36,19 @@ lists:
     - key: entity_id
       value:
         str:
-          value: entity_3
+          value: entity_1
     value:
       int:
-        value: "11"
+        value: "240"
   - entity:
     - key: service_id
       value:
-        "null": null
+        str:
+          value: svc_2
     - key: entity_id
       value:
         str:
-          value: entity_2
+          value: entity_3
     value:
       int:
-        value: "10"
\ No newline at end of file
+        value: "155"
\ No newline at end of file