You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ha...@apache.org on 2025/09/01 15:00:20 UTC
(skywalking-banyandb) 01/01: Refactor tag handling by removing 'indexed' field from tag structures and related logic. Update tests to reflect changes in tag definitions and ensure proper functionality without indexed tags. Enhance unique data tracking in QueryResponse for improved element limit enforcement.
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch trace/limit
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit 3e7429687ff51f60022d5689ac961bd3d9007cfa
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Mon Sep 1 22:59:22 2025 +0800
Refactor tag handling by removing 'indexed' field from tag structures and related logic. Update tests to reflect changes in tag definitions and ensure proper functionality without indexed tags. Enhance unique data tracking in QueryResponse for improved element limit enforcement.
---
banyand/internal/sidx/block.go | 18 +-
banyand/internal/sidx/block_scanner_test.go | 2 -
banyand/internal/sidx/element.go | 3 -
banyand/internal/sidx/element_test.go | 2 -
banyand/internal/sidx/interfaces.go | 119 ++++++--
banyand/internal/sidx/iter_test.go | 56 ++--
banyand/internal/sidx/part.go | 1 -
banyand/internal/sidx/part_iter_test.go | 19 --
banyand/internal/sidx/part_test.go | 20 --
banyand/internal/sidx/query_result.go | 60 ++--
banyand/internal/sidx/sidx.go | 8 +-
banyand/internal/sidx/sidx_test.go | 35 +--
banyand/internal/sidx/tag.go | 18 +-
banyand/internal/sidx/tag_filter_op.go | 14 +-
banyand/internal/sidx/tag_test.go | 14 +-
banyand/trace/write_standalone.go | 21 +-
test/cases/trace/data/data.go | 2 +-
.../data/input/eq_endpoint_order_duration_asc.yml | 2 +-
test/cases/trace/data/input/eq_trace_id.yml | 2 +-
.../cases/trace/data/input/order_duration_desc.yml | 2 +-
...ion_desc.yml => order_timestamp_desc_limit.yml} | 3 +-
test/cases/trace/data/testdata/sw.json | 308 ++++++++++++++++++++-
.../data/want/eq_endpoint_order_duration_asc.yml | 9 +-
.../want/eq_service_instance_order_time_asc.yml | 15 +-
.../data/want/eq_service_order_timestamp_desc.yml | 18 +-
test/cases/trace/data/want/eq_trace_id.yml | 16 +-
test/cases/trace/data/want/order_duration_desc.yml | 18 +-
.../cases/trace/data/want/order_timestamp_desc.yml | 18 +-
...time_asc.yml => order_timestamp_desc_limit.yml} | 9 +-
test/cases/trace/trace.go | 1 +
30 files changed, 584 insertions(+), 249 deletions(-)
diff --git a/banyand/internal/sidx/block.go b/banyand/internal/sidx/block.go
index a584d73b..29348a33 100644
--- a/banyand/internal/sidx/block.go
+++ b/banyand/internal/sidx/block.go
@@ -115,7 +115,6 @@ func (b *block) processTag(tagName string, elementTags [][]*tag) {
td.values = make([][]byte, len(b.userKeys))
var valueType pbv1.ValueType
- var indexed bool
// Collect values for this tag across all elements
for i, tags := range elementTags {
@@ -124,7 +123,6 @@ func (b *block) processTag(tagName string, elementTags [][]*tag) {
if tag.name == tagName {
td.values[i] = tag.value
valueType = tag.valueType
- indexed = tag.indexed
found = true
break
}
@@ -135,15 +133,12 @@ func (b *block) processTag(tagName string, elementTags [][]*tag) {
}
td.valueType = valueType
- td.indexed = indexed
// Create bloom filter for indexed tags
- if indexed {
- td.filter = generateBloomFilter(len(b.userKeys))
- for _, value := range td.values {
- if value != nil {
- td.filter.Add(value)
- }
+ td.filter = generateBloomFilter(len(b.userKeys))
+ for _, value := range td.values {
+ if value != nil {
+ td.filter.Add(value)
}
}
@@ -263,7 +258,6 @@ func (b *block) mustWriteTag(tagName string, td *tagData, bm *blockMetadata, ww
tm.name = tagName
tm.valueType = td.valueType
- tm.indexed = td.indexed
// Write tag values to data file
bb := bigValuePool.Get()
@@ -286,8 +280,8 @@ func (b *block) mustWriteTag(tagName string, td *tagData, bm *blockMetadata, ww
tm.dataBlock.size = uint64(len(bb.Buf))
tdw.MustWrite(bb.Buf)
- // Write bloom filter if indexed
- if td.indexed && td.filter != nil {
+ // Write bloom filter
+ if td.filter != nil {
filterData := encodeBloomFilter(nil, td.filter)
tm.filterBlock.offset = tfw.bytesWritten
tm.filterBlock.size = uint64(len(filterData))
diff --git a/banyand/internal/sidx/block_scanner_test.go b/banyand/internal/sidx/block_scanner_test.go
index fea166bf..6f1aa9ad 100644
--- a/banyand/internal/sidx/block_scanner_test.go
+++ b/banyand/internal/sidx/block_scanner_test.go
@@ -146,13 +146,11 @@ func TestBlockScanner_QuotaExceeded(t *testing.T) {
name: "service",
value: []byte("test-service"),
valueType: pbv1.ValueTypeStr,
- indexed: true,
},
{
name: "instance",
value: []byte("instance-" + string(rune('0'+i))),
valueType: pbv1.ValueTypeStr,
- indexed: true,
},
},
})
diff --git a/banyand/internal/sidx/element.go b/banyand/internal/sidx/element.go
index 9ca5e253..724df953 100644
--- a/banyand/internal/sidx/element.go
+++ b/banyand/internal/sidx/element.go
@@ -35,7 +35,6 @@ type tag struct {
name string
value []byte
valueType pbv1.ValueType
- indexed bool
}
// elements is a collection of elements optimized for batch operations.
@@ -51,7 +50,6 @@ func (t *tag) reset() {
t.name = ""
t.value = nil
t.valueType = pbv1.ValueTypeUnknown
- t.indexed = false
}
// reset elements collection for pooling.
@@ -170,7 +168,6 @@ func (e *elements) mustAppend(seriesID common.SeriesID, userKey int64, data []by
newTag.name = t.Name
newTag.value = append([]byte(nil), t.Value...)
newTag.valueType = t.ValueType
- newTag.indexed = t.Indexed
elementTags = append(elementTags, newTag)
}
e.tags = append(e.tags, elementTags)
diff --git a/banyand/internal/sidx/element_test.go b/banyand/internal/sidx/element_test.go
index b12b9cb5..61ef58dc 100644
--- a/banyand/internal/sidx/element_test.go
+++ b/banyand/internal/sidx/element_test.go
@@ -72,7 +72,6 @@ func TestElementReset(t *testing.T) {
tag.name = "test-tag"
tag.value = []byte("test-value")
tag.valueType = pbv1.ValueTypeStr
- tag.indexed = true
// Reset the tag
tag.reset()
@@ -81,7 +80,6 @@ func TestElementReset(t *testing.T) {
assert.Equal(t, "", tag.name, "name should be empty")
assert.Nil(t, tag.value, "value should be nil")
assert.Equal(t, pbv1.ValueTypeUnknown, tag.valueType, "valueType should be reset")
- assert.False(t, tag.indexed, "indexed should be reset")
releaseTag(tag)
})
diff --git a/banyand/internal/sidx/interfaces.go b/banyand/internal/sidx/interfaces.go
index 074bc3e5..cd1a3850 100644
--- a/banyand/internal/sidx/interfaces.go
+++ b/banyand/internal/sidx/interfaces.go
@@ -123,14 +123,13 @@ type QueryRequest struct {
// This follows BanyanDB result patterns with parallel arrays for efficiency.
// Uses individual tag-based strategy (like trace module) rather than tag-family approach (like stream module).
type QueryResponse struct {
- Error error
- uniqueDataMap map[string]struct{}
- Keys []int64
- Data [][]byte
- Tags [][]Tag
- SIDs []common.SeriesID
- Metadata ResponseMetadata
- uniqueDataCount int
+ Error error
+ uniqueTracker *UniqueDataTracker
+ Keys []int64
+ Data [][]byte
+ Tags [][]Tag
+ SIDs []common.SeriesID
+ Metadata ResponseMetadata
}
// Len returns the number of results in the QueryResponse.
@@ -146,9 +145,8 @@ func (qr *QueryResponse) Reset() {
qr.Tags = qr.Tags[:0]
qr.SIDs = qr.SIDs[:0]
qr.Metadata = ResponseMetadata{}
- // Reset unique data cache
- qr.uniqueDataMap = nil
- qr.uniqueDataCount = 0
+ // Reset unique data tracker
+ qr.uniqueTracker = nil
}
// Validate validates a QueryResponse for correctness.
@@ -181,6 +179,83 @@ func (qr *QueryResponse) Validate() error {
return nil
}
+// UniqueDataTracker tracks unique data elements for limit enforcement.
+type UniqueDataTracker struct {
+ count int
+ limit int
+ dataMap map[string]struct{}
+}
+
+// NewUniqueDataTracker creates a new UniqueDataTracker for the given limit.
+// If limit is 0 or negative, no tracking is performed.
+func NewUniqueDataTracker(limit int) *UniqueDataTracker {
+ if limit <= 0 {
+ return &UniqueDataTracker{limit: 0}
+ }
+ return &UniqueDataTracker{
+ limit: limit,
+ dataMap: make(map[string]struct{}),
+ }
+}
+
+// ShouldAdd checks if a data element should be added based on uniqueness and limit.
+// Returns true if the element should be added, false if it would exceed the limit.
+func (udt *UniqueDataTracker) ShouldAdd(data []byte) bool {
+ if udt.limit <= 0 {
+ return true // No limit, always add
+ }
+
+ dataStr := string(data)
+ if _, exists := udt.dataMap[dataStr]; !exists {
+ // New unique data element
+ if udt.count >= udt.limit {
+ return false // Would exceed limit
+ }
+ udt.dataMap[dataStr] = struct{}{}
+ udt.count++
+ }
+ return true
+}
+
+// Count returns the current count of unique data elements.
+func (udt *UniqueDataTracker) Count() int {
+ return udt.count
+}
+
+// IsLimitReached returns true if the limit has been reached.
+func (udt *UniqueDataTracker) IsLimitReached() bool {
+ return udt.limit > 0 && udt.count >= udt.limit
+}
+
+// InitUniqueTracker initializes the unique data tracker with the given limit.
+func (qr *QueryResponse) InitUniqueTracker(limit int) {
+ qr.uniqueTracker = NewUniqueDataTracker(limit)
+}
+
+// ShouldAddData checks if a data element should be added based on uniqueness and limit.
+func (qr *QueryResponse) ShouldAddData(data []byte) bool {
+ if qr.uniqueTracker == nil {
+ return true // No tracker, always add
+ }
+ return qr.uniqueTracker.ShouldAdd(data)
+}
+
+// UniqueDataCount returns the current count of unique data elements.
+func (qr *QueryResponse) UniqueDataCount() int {
+ if qr.uniqueTracker == nil {
+ return 0
+ }
+ return qr.uniqueTracker.Count()
+}
+
+// IsUniqueLimitReached returns true if the unique data limit has been reached.
+func (qr *QueryResponse) IsUniqueLimitReached() bool {
+ if qr.uniqueTracker == nil {
+ return false
+ }
+ return qr.uniqueTracker.IsLimitReached()
+}
+
// CopyFrom copies the QueryResponse from other to qr.
func (qr *QueryResponse) CopyFrom(other *QueryResponse) {
qr.Error = other.Error
@@ -215,12 +290,24 @@ func (qr *QueryResponse) CopyFrom(other *QueryResponse) {
qr.Tags[i][j].Name = tag.Name
qr.Tags[i][j].Value = append(qr.Tags[i][j].Value[:0], tag.Value...)
qr.Tags[i][j].ValueType = tag.ValueType
- qr.Tags[i][j].Indexed = tag.Indexed
}
}
// Copy metadata
qr.Metadata = other.Metadata
+
+ // Copy unique tracker if present
+ if other.uniqueTracker != nil {
+ qr.uniqueTracker = NewUniqueDataTracker(other.uniqueTracker.limit)
+ qr.uniqueTracker.count = other.uniqueTracker.count
+ // Deep copy the data map
+ qr.uniqueTracker.dataMap = make(map[string]struct{}, len(other.uniqueTracker.dataMap))
+ for k := range other.uniqueTracker.dataMap {
+ qr.uniqueTracker.dataMap[k] = struct{}{}
+ }
+ } else {
+ qr.uniqueTracker = nil
+ }
}
// Stats contains system statistics and performance metrics.
@@ -294,16 +381,14 @@ type Tag struct {
Name string
Value []byte
ValueType pbv1.ValueType
- Indexed bool
}
// NewTag creates a new Tag instance with the given values.
-func NewTag(name string, value []byte, valueType pbv1.ValueType, indexed bool) Tag {
+func NewTag(name string, value []byte, valueType pbv1.ValueType) Tag {
return Tag{
Name: name,
Value: value,
ValueType: valueType,
- Indexed: indexed,
}
}
@@ -312,7 +397,6 @@ func (t *Tag) Reset() {
t.Name = ""
t.Value = nil
t.ValueType = pbv1.ValueTypeUnknown
- t.Indexed = false
}
// Size returns the size of the tag in bytes.
@@ -331,7 +415,6 @@ func (t *Tag) Copy() Tag {
Name: t.Name,
Value: valueCopy,
ValueType: t.ValueType,
- Indexed: t.Indexed,
}
}
@@ -341,7 +424,6 @@ func (t *Tag) toInternalTag() *tag {
name: t.Name,
value: t.Value,
valueType: t.ValueType,
- indexed: t.Indexed,
}
}
@@ -351,7 +433,6 @@ func fromInternalTag(t *tag) Tag {
Name: t.name,
Value: t.value,
ValueType: t.valueType,
- Indexed: t.indexed,
}
}
diff --git a/banyand/internal/sidx/iter_test.go b/banyand/internal/sidx/iter_test.go
index 93c14e81..5cd74f62 100644
--- a/banyand/internal/sidx/iter_test.go
+++ b/banyand/internal/sidx/iter_test.go
@@ -56,7 +56,7 @@ func TestIterComprehensive(t *testing.T) {
userKey: 100,
data: []byte("data1"),
tags: []tag{
- {name: "service", value: []byte("order-service"), valueType: pbv1.ValueTypeStr, indexed: true},
+ {name: "service", value: []byte("order-service"), valueType: pbv1.ValueTypeStr},
},
},
},
@@ -80,7 +80,7 @@ func TestIterComprehensive(t *testing.T) {
userKey: 100,
data: []byte("data1"),
tags: []tag{
- {name: "service", value: []byte("order-service"), valueType: pbv1.ValueTypeStr, indexed: true},
+ {name: "service", value: []byte("order-service"), valueType: pbv1.ValueTypeStr},
},
},
{
@@ -88,7 +88,7 @@ func TestIterComprehensive(t *testing.T) {
userKey: 300,
data: []byte("data3"),
tags: []tag{
- {name: "service", value: []byte("user-service"), valueType: pbv1.ValueTypeStr, indexed: true},
+ {name: "service", value: []byte("user-service"), valueType: pbv1.ValueTypeStr},
},
},
},
@@ -99,7 +99,7 @@ func TestIterComprehensive(t *testing.T) {
userKey: 200,
data: []byte("data2"),
tags: []tag{
- {name: "service", value: []byte("payment-service"), valueType: pbv1.ValueTypeStr, indexed: true},
+ {name: "service", value: []byte("payment-service"), valueType: pbv1.ValueTypeStr},
},
},
{
@@ -107,7 +107,7 @@ func TestIterComprehensive(t *testing.T) {
userKey: 400,
data: []byte("data4"),
tags: []tag{
- {name: "service", value: []byte("notification-service"), valueType: pbv1.ValueTypeStr, indexed: true},
+ {name: "service", value: []byte("notification-service"), valueType: pbv1.ValueTypeStr},
},
},
},
@@ -129,15 +129,15 @@ func TestIterComprehensive(t *testing.T) {
parts: [][]testElement{
// Part 1
{
- {seriesID: 1, userKey: 100, data: []byte("data1"), tags: []tag{{name: "service", value: []byte("order-service"), valueType: pbv1.ValueTypeStr, indexed: true}}},
- {seriesID: 2, userKey: 200, data: []byte("data2"), tags: []tag{{name: "service", value: []byte("payment-service"), valueType: pbv1.ValueTypeStr, indexed: true}}},
+ {seriesID: 1, userKey: 100, data: []byte("data1"), tags: []tag{{name: "service", value: []byte("order-service"), valueType: pbv1.ValueTypeStr}}},
+ {seriesID: 2, userKey: 200, data: []byte("data2"), tags: []tag{{name: "service", value: []byte("payment-service"), valueType: pbv1.ValueTypeStr}}},
},
// Part 2
{
- {seriesID: 3, userKey: 300, data: []byte("data3"), tags: []tag{{name: "service", value: []byte("user-service"), valueType: pbv1.ValueTypeStr, indexed: true}}},
+ {seriesID: 3, userKey: 300, data: []byte("data3"), tags: []tag{{name: "service", value: []byte("user-service"), valueType: pbv1.ValueTypeStr}}},
{
seriesID: 4, userKey: 400, data: []byte("data4"),
- tags: []tag{{name: "service", value: []byte("notification-service"), valueType: pbv1.ValueTypeStr, indexed: true}},
+ tags: []tag{{name: "service", value: []byte("notification-service"), valueType: pbv1.ValueTypeStr}},
},
},
},
@@ -156,15 +156,15 @@ func TestIterComprehensive(t *testing.T) {
parts: [][]testElement{
// Part 1
{
- {seriesID: 1, userKey: 50, data: []byte("data1"), tags: []tag{{name: "service", value: []byte("order-service"), valueType: pbv1.ValueTypeStr, indexed: true}}},
- {seriesID: 2, userKey: 150, data: []byte("data2"), tags: []tag{{name: "service", value: []byte("payment-service"), valueType: pbv1.ValueTypeStr, indexed: true}}},
+ {seriesID: 1, userKey: 50, data: []byte("data1"), tags: []tag{{name: "service", value: []byte("order-service"), valueType: pbv1.ValueTypeStr}}},
+ {seriesID: 2, userKey: 150, data: []byte("data2"), tags: []tag{{name: "service", value: []byte("payment-service"), valueType: pbv1.ValueTypeStr}}},
},
// Part 2
{
- {seriesID: 3, userKey: 250, data: []byte("data3"), tags: []tag{{name: "service", value: []byte("user-service"), valueType: pbv1.ValueTypeStr, indexed: true}}},
+ {seriesID: 3, userKey: 250, data: []byte("data3"), tags: []tag{{name: "service", value: []byte("user-service"), valueType: pbv1.ValueTypeStr}}},
{
seriesID: 4, userKey: 350, data: []byte("data4"),
- tags: []tag{{name: "service", value: []byte("notification-service"), valueType: pbv1.ValueTypeStr, indexed: true}},
+ tags: []tag{{name: "service", value: []byte("notification-service"), valueType: pbv1.ValueTypeStr}},
},
},
},
@@ -182,8 +182,8 @@ func TestIterComprehensive(t *testing.T) {
name: "with_block_filter_allow_all",
parts: [][]testElement{
{
- {seriesID: 1, userKey: 100, data: []byte("data1"), tags: []tag{{name: "service", value: []byte("order-service"), valueType: pbv1.ValueTypeStr, indexed: true}}},
- {seriesID: 2, userKey: 200, data: []byte("data2"), tags: []tag{{name: "service", value: []byte("payment-service"), valueType: pbv1.ValueTypeStr, indexed: true}}},
+ {seriesID: 1, userKey: 100, data: []byte("data1"), tags: []tag{{name: "service", value: []byte("order-service"), valueType: pbv1.ValueTypeStr}}},
+ {seriesID: 2, userKey: 200, data: []byte("data2"), tags: []tag{{name: "service", value: []byte("payment-service"), valueType: pbv1.ValueTypeStr}}},
},
},
querySids: []common.SeriesID{1, 2},
@@ -200,8 +200,8 @@ func TestIterComprehensive(t *testing.T) {
name: "with_block_filter_skip_all",
parts: [][]testElement{
{
- {seriesID: 1, userKey: 100, data: []byte("data1"), tags: []tag{{name: "service", value: []byte("order-service"), valueType: pbv1.ValueTypeStr, indexed: true}}},
- {seriesID: 2, userKey: 200, data: []byte("data2"), tags: []tag{{name: "service", value: []byte("payment-service"), valueType: pbv1.ValueTypeStr, indexed: true}}},
+ {seriesID: 1, userKey: 100, data: []byte("data1"), tags: []tag{{name: "service", value: []byte("order-service"), valueType: pbv1.ValueTypeStr}}},
+ {seriesID: 2, userKey: 200, data: []byte("data2"), tags: []tag{{name: "service", value: []byte("payment-service"), valueType: pbv1.ValueTypeStr}}},
},
},
querySids: []common.SeriesID{1, 2},
@@ -268,7 +268,7 @@ func TestIterEdgeCases(t *testing.T) {
t.Run("empty_series_list", func(t *testing.T) {
// Create a part with data
elements := createTestElements([]testElement{
- {seriesID: 1, userKey: 100, data: []byte("data1"), tags: []tag{{name: "service", value: []byte("test-service"), valueType: pbv1.ValueTypeStr, indexed: true}}},
+ {seriesID: 1, userKey: 100, data: []byte("data1"), tags: []tag{{name: "service", value: []byte("test-service"), valueType: pbv1.ValueTypeStr}}},
})
defer releaseElements(elements)
@@ -295,12 +295,12 @@ func TestIterEdgeCases(t *testing.T) {
t.Run("no_matching_key_range", func(t *testing.T) {
// Create parts with data at key 100-300
elements1 := createTestElements([]testElement{
- {seriesID: 1, userKey: 100, data: []byte("data1"), tags: []tag{{name: "service", value: []byte("test-service1"), valueType: pbv1.ValueTypeStr, indexed: true}}},
+ {seriesID: 1, userKey: 100, data: []byte("data1"), tags: []tag{{name: "service", value: []byte("test-service1"), valueType: pbv1.ValueTypeStr}}},
})
defer releaseElements(elements1)
elements2 := createTestElements([]testElement{
- {seriesID: 2, userKey: 300, data: []byte("data2"), tags: []tag{{name: "service", value: []byte("test-service2"), valueType: pbv1.ValueTypeStr, indexed: true}}},
+ {seriesID: 2, userKey: 300, data: []byte("data2"), tags: []tag{{name: "service", value: []byte("test-service2"), valueType: pbv1.ValueTypeStr}}},
})
defer releaseElements(elements2)
@@ -336,7 +336,7 @@ func TestIterEdgeCases(t *testing.T) {
t.Run("single_part_single_block", func(t *testing.T) {
elements := createTestElements([]testElement{
- {seriesID: 1, userKey: 100, data: []byte("data1"), tags: []tag{{name: "service", value: []byte("test-service"), valueType: pbv1.ValueTypeStr, indexed: true}}},
+ {seriesID: 1, userKey: 100, data: []byte("data1"), tags: []tag{{name: "service", value: []byte("test-service"), valueType: pbv1.ValueTypeStr}}},
})
defer releaseElements(elements)
@@ -364,7 +364,7 @@ func TestIterEdgeCases(t *testing.T) {
t.Run("block_filter_error", func(t *testing.T) {
elements := createTestElements([]testElement{
- {seriesID: 1, userKey: 100, data: []byte("data1"), tags: []tag{{name: "service", value: []byte("test-service"), valueType: pbv1.ValueTypeStr, indexed: true}}},
+ {seriesID: 1, userKey: 100, data: []byte("data1"), tags: []tag{{name: "service", value: []byte("test-service"), valueType: pbv1.ValueTypeStr}}},
})
defer releaseElements(elements)
@@ -400,17 +400,17 @@ func TestIterOrdering(t *testing.T) {
t.Run("interleaved_series_ordering", func(t *testing.T) {
// Part 1: series 1, 3, 5
elements1 := createTestElements([]testElement{
- {seriesID: 1, userKey: 100, data: []byte("data1"), tags: []tag{{name: "service", value: []byte("service1"), valueType: pbv1.ValueTypeStr, indexed: true}}},
- {seriesID: 3, userKey: 300, data: []byte("data3"), tags: []tag{{name: "service", value: []byte("service3"), valueType: pbv1.ValueTypeStr, indexed: true}}},
- {seriesID: 5, userKey: 500, data: []byte("data5"), tags: []tag{{name: "service", value: []byte("service5"), valueType: pbv1.ValueTypeStr, indexed: true}}},
+ {seriesID: 1, userKey: 100, data: []byte("data1"), tags: []tag{{name: "service", value: []byte("service1"), valueType: pbv1.ValueTypeStr}}},
+ {seriesID: 3, userKey: 300, data: []byte("data3"), tags: []tag{{name: "service", value: []byte("service3"), valueType: pbv1.ValueTypeStr}}},
+ {seriesID: 5, userKey: 500, data: []byte("data5"), tags: []tag{{name: "service", value: []byte("service5"), valueType: pbv1.ValueTypeStr}}},
})
defer releaseElements(elements1)
// Part 2: series 2, 4, 6
elements2 := createTestElements([]testElement{
- {seriesID: 2, userKey: 200, data: []byte("data2"), tags: []tag{{name: "service", value: []byte("service2"), valueType: pbv1.ValueTypeStr, indexed: true}}},
- {seriesID: 4, userKey: 400, data: []byte("data4"), tags: []tag{{name: "service", value: []byte("service4"), valueType: pbv1.ValueTypeStr, indexed: true}}},
- {seriesID: 6, userKey: 600, data: []byte("data6"), tags: []tag{{name: "service", value: []byte("service6"), valueType: pbv1.ValueTypeStr, indexed: true}}},
+ {seriesID: 2, userKey: 200, data: []byte("data2"), tags: []tag{{name: "service", value: []byte("service2"), valueType: pbv1.ValueTypeStr}}},
+ {seriesID: 4, userKey: 400, data: []byte("data4"), tags: []tag{{name: "service", value: []byte("service4"), valueType: pbv1.ValueTypeStr}}},
+ {seriesID: 6, userKey: 600, data: []byte("data6"), tags: []tag{{name: "service", value: []byte("service6"), valueType: pbv1.ValueTypeStr}}},
})
defer releaseElements(elements2)
diff --git a/banyand/internal/sidx/part.go b/banyand/internal/sidx/part.go
index 52d69453..2555a942 100644
--- a/banyand/internal/sidx/part.go
+++ b/banyand/internal/sidx/part.go
@@ -425,7 +425,6 @@ func (p *part) readBlockTags(tagName string, bm *blockMetadata, elems *elements,
newTag.name = tagName
newTag.value = value
newTag.valueType = tm.valueType
- newTag.indexed = tm.indexed
elems.tags[i] = append(elems.tags[i], newTag)
}
diff --git a/banyand/internal/sidx/part_iter_test.go b/banyand/internal/sidx/part_iter_test.go
index 3289645e..ecd8bf13 100644
--- a/banyand/internal/sidx/part_iter_test.go
+++ b/banyand/internal/sidx/part_iter_test.go
@@ -54,7 +54,6 @@ func TestPartIterVerification(t *testing.T) {
name: "service",
value: []byte("order-service"),
valueType: pbv1.ValueTypeStr,
- indexed: true,
},
},
},
@@ -76,7 +75,6 @@ func TestPartIterVerification(t *testing.T) {
name: "service",
value: []byte("order-service"),
valueType: pbv1.ValueTypeStr,
- indexed: true,
},
},
},
@@ -89,7 +87,6 @@ func TestPartIterVerification(t *testing.T) {
name: "service",
value: []byte("order-service"),
valueType: pbv1.ValueTypeStr,
- indexed: true,
},
},
},
@@ -111,7 +108,6 @@ func TestPartIterVerification(t *testing.T) {
name: "service",
value: []byte("order-service"),
valueType: pbv1.ValueTypeStr,
- indexed: true,
},
},
},
@@ -124,7 +120,6 @@ func TestPartIterVerification(t *testing.T) {
name: "service",
value: []byte("payment-service"),
valueType: pbv1.ValueTypeStr,
- indexed: true,
},
},
},
@@ -137,7 +132,6 @@ func TestPartIterVerification(t *testing.T) {
name: "service",
value: []byte("user-service"),
valueType: pbv1.ValueTypeStr,
- indexed: true,
},
},
},
@@ -159,7 +153,6 @@ func TestPartIterVerification(t *testing.T) {
name: "service",
value: []byte("order-service"),
valueType: pbv1.ValueTypeStr,
- indexed: true,
},
},
},
@@ -172,7 +165,6 @@ func TestPartIterVerification(t *testing.T) {
name: "service",
value: []byte("order-service"),
valueType: pbv1.ValueTypeStr,
- indexed: true,
},
},
},
@@ -185,7 +177,6 @@ func TestPartIterVerification(t *testing.T) {
name: "service",
value: []byte("order-service"),
valueType: pbv1.ValueTypeStr,
- indexed: true,
},
},
},
@@ -207,7 +198,6 @@ func TestPartIterVerification(t *testing.T) {
name: "service",
value: []byte("order-service"),
valueType: pbv1.ValueTypeStr,
- indexed: true,
},
},
},
@@ -220,7 +210,6 @@ func TestPartIterVerification(t *testing.T) {
name: "service",
value: []byte("payment-service"),
valueType: pbv1.ValueTypeStr,
- indexed: true,
},
},
},
@@ -233,7 +222,6 @@ func TestPartIterVerification(t *testing.T) {
name: "service",
value: []byte("user-service"),
valueType: pbv1.ValueTypeStr,
- indexed: true,
},
},
},
@@ -375,7 +363,6 @@ func TestPartIterEdgeCases(t *testing.T) {
name: "service",
value: []byte("test-service"),
valueType: pbv1.ValueTypeStr,
- indexed: true,
},
},
},
@@ -417,7 +404,6 @@ func TestPartIterEdgeCases(t *testing.T) {
name: "service",
value: []byte("test-service"),
valueType: pbv1.ValueTypeStr,
- indexed: true,
},
},
},
@@ -459,7 +445,6 @@ func TestPartIterEdgeCases(t *testing.T) {
name: "service",
value: []byte("test-service"),
valueType: pbv1.ValueTypeStr,
- indexed: true,
},
},
},
@@ -506,7 +491,6 @@ func TestPartIterBlockFilter(t *testing.T) {
name: "service",
value: []byte("test-service"),
valueType: pbv1.ValueTypeStr,
- indexed: true,
},
},
},
@@ -549,7 +533,6 @@ func TestPartIterBlockFilter(t *testing.T) {
name: "service",
value: []byte("test-service"),
valueType: pbv1.ValueTypeStr,
- indexed: true,
},
},
},
@@ -595,7 +578,6 @@ func TestPartIterBlockFilter(t *testing.T) {
name: "service",
value: []byte("test-service"),
valueType: pbv1.ValueTypeStr,
- indexed: true,
},
},
},
@@ -641,7 +623,6 @@ func TestPartIterBlockFilter(t *testing.T) {
name: "service",
value: []byte("test-service"),
valueType: pbv1.ValueTypeStr,
- indexed: true,
},
},
},
diff --git a/banyand/internal/sidx/part_test.go b/banyand/internal/sidx/part_test.go
index c2ce8a5c..0fdb3878 100644
--- a/banyand/internal/sidx/part_test.go
+++ b/banyand/internal/sidx/part_test.go
@@ -164,7 +164,6 @@ func TestMemPartInitialization(t *testing.T) {
name: "service",
value: []byte("order-service"),
valueType: pbv1.ValueTypeStr,
- indexed: true,
},
},
},
@@ -231,7 +230,6 @@ func TestMemPartFlushAndReadAllRoundTrip(t *testing.T) {
name: "service",
value: []byte("order-service"),
valueType: pbv1.ValueTypeStr,
- indexed: true,
},
},
},
@@ -249,13 +247,11 @@ func TestMemPartFlushAndReadAllRoundTrip(t *testing.T) {
name: "service",
value: []byte("order-service"),
valueType: pbv1.ValueTypeStr,
- indexed: true,
},
{
name: "method",
value: []byte("POST"),
valueType: pbv1.ValueTypeStr,
- indexed: false,
},
},
},
@@ -268,13 +264,11 @@ func TestMemPartFlushAndReadAllRoundTrip(t *testing.T) {
name: "service",
value: []byte("order-service"),
valueType: pbv1.ValueTypeStr,
- indexed: true,
},
{
name: "method",
value: []byte("GET"),
valueType: pbv1.ValueTypeStr,
- indexed: false,
},
},
},
@@ -292,13 +286,11 @@ func TestMemPartFlushAndReadAllRoundTrip(t *testing.T) {
name: "service",
value: []byte("order-service"),
valueType: pbv1.ValueTypeStr,
- indexed: true,
},
{
name: "duration",
value: []byte{0, 0, 0, 0, 0, 0, 0, 100}, // int64: 100
valueType: pbv1.ValueTypeInt64,
- indexed: false,
},
},
},
@@ -311,13 +303,11 @@ func TestMemPartFlushAndReadAllRoundTrip(t *testing.T) {
name: "service",
value: []byte("order-service"),
valueType: pbv1.ValueTypeStr,
- indexed: true,
},
{
name: "duration",
value: []byte{0, 0, 0, 0, 0, 0, 0, 150}, // int64: 150
valueType: pbv1.ValueTypeInt64,
- indexed: false,
},
},
},
@@ -330,13 +320,11 @@ func TestMemPartFlushAndReadAllRoundTrip(t *testing.T) {
name: "service",
value: []byte("payment-service"),
valueType: pbv1.ValueTypeStr,
- indexed: true,
},
{
name: "duration",
value: []byte{0, 0, 0, 0, 0, 0, 0, 75}, // int64: 75
valueType: pbv1.ValueTypeInt64,
- indexed: false,
},
},
},
@@ -354,7 +342,6 @@ func TestMemPartFlushAndReadAllRoundTrip(t *testing.T) {
name: "service",
value: []byte("order-service"),
valueType: pbv1.ValueTypeStr,
- indexed: true,
},
},
},
@@ -367,13 +354,11 @@ func TestMemPartFlushAndReadAllRoundTrip(t *testing.T) {
name: "service",
value: []byte("order-service"),
valueType: pbv1.ValueTypeStr,
- indexed: true,
},
{
name: "endpoint",
value: []byte("/api/orders"),
valueType: pbv1.ValueTypeStr,
- indexed: false,
},
},
},
@@ -430,7 +415,6 @@ func TestMemPartFlushAndReadAllRoundTrip(t *testing.T) {
newTag.name = t.name
newTag.value = append([]byte(nil), t.value...)
newTag.valueType = t.valueType
- newTag.indexed = t.indexed
newTagSlice = append(newTagSlice, newTag)
}
combined.tags = append(combined.tags, newTagSlice)
@@ -451,7 +435,6 @@ func TestMemPartFlushAndReadAllRoundTrip(t *testing.T) {
newTag.name = t.name
newTag.value = append([]byte(nil), t.value...)
newTag.valueType = t.valueType
- newTag.indexed = t.indexed
newTagSlice = append(newTagSlice, newTag)
}
originalCopy.tags = append(originalCopy.tags, newTagSlice)
@@ -500,7 +483,6 @@ func createTestElements(testElems []testElement) *elements {
newTag.name = t.name
newTag.value = append([]byte(nil), t.value...)
newTag.valueType = t.valueType
- newTag.indexed = t.indexed
tagsCopy = append(tagsCopy, newTag)
}
elems.tags = append(elems.tags, tagsCopy)
@@ -548,8 +530,6 @@ func compareElements(t *testing.T, expected, actual *elements) {
"tag value mismatch at element %d, tag %d (%s)", i, j, expectedTags[j].name)
assert.Equal(t, expectedTags[j].valueType, actualTags[j].valueType,
"tag valueType mismatch at element %d, tag %d (%s)", i, j, expectedTags[j].name)
- assert.Equal(t, expectedTags[j].indexed, actualTags[j].indexed,
- "tag indexed mismatch at element %d, tag %d (%s)", i, j, expectedTags[j].name)
}
}
}
diff --git a/banyand/internal/sidx/query_result.go b/banyand/internal/sidx/query_result.go
index c035870f..d21c3b3c 100644
--- a/banyand/internal/sidx/query_result.go
+++ b/banyand/internal/sidx/query_result.go
@@ -189,7 +189,6 @@ func (qr *queryResult) loadTagData(tmpBlock *block, p *part, tagName string, tag
td.name = tagName
td.valueType = tm.valueType
- td.indexed = tm.indexed
// Set min/max for int64 tags
if tm.valueType == pbv1.ValueTypeInt64 {
@@ -197,13 +196,11 @@ func (qr *queryResult) loadTagData(tmpBlock *block, p *part, tagName string, tag
td.max = tm.max
}
- // Create bloom filter for indexed tags if needed
- if tm.indexed {
- td.filter = generateBloomFilter(count)
- for _, value := range td.values {
- if value != nil {
- td.filter.Add(value)
- }
+ // Create bloom filter for indexed tags
+ td.filter = generateBloomFilter(count)
+ for _, value := range td.values {
+ if value != nil {
+ td.filter.Add(value)
}
}
@@ -216,13 +213,12 @@ func (qr *queryResult) convertBlockToResponse(block *block, seriesID common.Seri
elemCount := len(block.userKeys)
// Initialize unique Data tracking if needed
- if qr.request.MaxElementSize > 0 && result.uniqueDataMap == nil {
- result.uniqueDataMap = make(map[string]struct{})
- // Count existing unique data elements in result if any
+ if qr.request.MaxElementSize > 0 && result.uniqueTracker == nil {
+ result.InitUniqueTracker(qr.request.MaxElementSize)
+ // Initialize tracker with existing data
for _, data := range result.Data {
- result.uniqueDataMap[string(data)] = struct{}{}
+ result.uniqueTracker.ShouldAdd(data)
}
- result.uniqueDataCount = len(result.uniqueDataMap)
}
for i := 0; i < elemCount; i++ {
@@ -235,17 +231,9 @@ func (qr *queryResult) convertBlockToResponse(block *block, seriesID common.Seri
continue
}
- // Check unique Data element limit from request (only if positive)
- if qr.request.MaxElementSize > 0 {
- dataStr := string(block.data[i])
- if _, exists := result.uniqueDataMap[dataStr]; !exists {
- // New unique data element
- if result.uniqueDataCount >= qr.request.MaxElementSize {
- break
- }
- result.uniqueDataMap[dataStr] = struct{}{}
- result.uniqueDataCount++
- }
+ // Check unique Data element limit using QueryResponse's tracker
+ if !result.ShouldAddData(block.data[i]) {
+ break
}
// Copy parallel arrays
@@ -422,34 +410,24 @@ func (qrh *QueryResponseHeap) mergeWithHeap(limit int) *QueryResponse {
SIDs: make([]common.SeriesID, 0, limit),
}
+ // Initialize unique data tracker if limit is specified
+ if limit > 0 {
+ result.InitUniqueTracker(limit)
+ }
+
step := -1
if qrh.asc {
step = 1
}
- // Track unique Data elements for limit enforcement
- var uniqueDataCount int
- var uniqueDataMap map[string]struct{}
- if limit > 0 {
- uniqueDataMap = make(map[string]struct{})
- }
-
for qrh.Len() > 0 {
topCursor := qrh.cursors[0]
idx := topCursor.idx
resp := topCursor.response
// Check unique Data element limit before adding
- if limit > 0 {
- dataStr := string(resp.Data[idx])
- if _, exists := uniqueDataMap[dataStr]; !exists {
- // New unique data element
- if uniqueDataCount >= limit {
- break
- }
- uniqueDataMap[dataStr] = struct{}{}
- uniqueDataCount++
- }
+ if !result.ShouldAddData(resp.Data[idx]) {
+ break
}
// Copy element from top cursor
diff --git a/banyand/internal/sidx/sidx.go b/banyand/internal/sidx/sidx.go
index 9a312b9d..27cd8b44 100644
--- a/banyand/internal/sidx/sidx.go
+++ b/banyand/internal/sidx/sidx.go
@@ -688,6 +688,11 @@ func (bch *blockCursorHeap) merge(limit int) *QueryResponse {
SIDs: make([]common.SeriesID, 0),
}
+ // Initialize unique data tracker if limit is specified
+ if limit > 0 {
+ result.InitUniqueTracker(limit)
+ }
+
for bch.Len() > 0 {
topBC := bch.bcc[0]
if topBC.idx < 0 || topBC.idx >= len(topBC.userKeys) {
@@ -697,7 +702,8 @@ func (bch *blockCursorHeap) merge(limit int) *QueryResponse {
// Try to copy the element (returns false if filtered out by key range)
if topBC.copyTo(result) {
- if limit > 0 && result.Len() >= limit {
+ // Check unique Data element limit after copying
+ if !result.ShouldAddData(result.Data[len(result.Data)-1]) {
break
}
}
diff --git a/banyand/internal/sidx/sidx_test.go b/banyand/internal/sidx/sidx_test.go
index a668121d..e75a7e98 100644
--- a/banyand/internal/sidx/sidx_test.go
+++ b/banyand/internal/sidx/sidx_test.go
@@ -63,7 +63,6 @@ func createTestTag(name, value string) Tag {
Name: name,
Value: []byte(value),
ValueType: pbv1.ValueTypeStr,
- Indexed: true,
}
}
@@ -1094,13 +1093,13 @@ func TestQueryResult_UniqueDataCaching(t *testing.T) {
qr.convertBlockToResponse(block1, 1, result)
// Verify cache is initialized in QueryResponse
- assert.NotNil(t, result.uniqueDataMap, "uniqueDataMap should be initialized after first call")
- assert.Equal(t, 2, result.uniqueDataCount, "uniqueDataCount should be 2 after first call")
- assert.Equal(t, 2, len(result.uniqueDataMap), "uniqueDataMap should contain 2 elements")
+ assert.NotNil(t, result.uniqueTracker, "uniqueTracker should be initialized after first call")
+ assert.Equal(t, 2, result.UniqueDataCount(), "uniqueDataCount should be 2 after first call")
+ assert.Equal(t, 2, result.uniqueTracker.Count(), "uniqueTracker should contain 2 elements")
// Verify cache contains expected data
- _, hasTrace1 := result.uniqueDataMap["trace1"]
- _, hasTrace2 := result.uniqueDataMap["trace2"]
+ _, hasTrace1 := result.uniqueTracker.dataMap["trace1"]
+ _, hasTrace2 := result.uniqueTracker.dataMap["trace2"]
assert.True(t, hasTrace1, "Cache should contain trace1")
assert.True(t, hasTrace2, "Cache should contain trace2")
@@ -1113,15 +1112,15 @@ func TestQueryResult_UniqueDataCaching(t *testing.T) {
qr.convertBlockToResponse(block2, 1, result)
// Verify cache is updated correctly in QueryResponse
- assert.Equal(t, 3, result.uniqueDataCount, "uniqueDataCount should be 3 after second call")
- assert.Equal(t, 3, len(result.uniqueDataMap), "uniqueDataMap should contain 3 elements")
+ assert.Equal(t, 3, result.UniqueDataCount(), "uniqueDataCount should be 3 after second call")
+ assert.Equal(t, 3, result.uniqueTracker.Count(), "uniqueTracker should contain 3 elements")
// Verify cache contains all expected data
- _, hasTrace3 := result.uniqueDataMap["trace3"]
+ _, hasTrace3 := result.uniqueTracker.dataMap["trace3"]
assert.True(t, hasTrace3, "Cache should contain trace3")
// trace4 should not be in cache due to limit of 3
- _, hasTrace4 := result.uniqueDataMap["trace4"]
+ _, hasTrace4 := result.uniqueTracker.dataMap["trace4"]
assert.False(t, hasTrace4, "Cache should not contain trace4 due to limit")
// Verify result has correct total elements: trace1 (2x), trace2 (1x), trace3 (1x)
@@ -1142,22 +1141,24 @@ func TestQueryResponse_Reset_CacheClear(t *testing.T) {
Data: [][]byte{[]byte("trace1"), []byte("trace2")},
Tags: [][]Tag{{}, {}},
SIDs: []common.SeriesID{1, 2},
- // Manually set cache for testing
- uniqueDataMap: map[string]struct{}{"trace1": {}, "trace2": {}},
- uniqueDataCount: 2,
}
+ // Manually set cache for testing
+ result.InitUniqueTracker(10)
+ result.uniqueTracker.dataMap = map[string]struct{}{"trace1": {}, "trace2": {}}
+ result.uniqueTracker.count = 2
+
// Verify cache is set
- assert.NotNil(t, result.uniqueDataMap, "Cache should be set before reset")
- assert.Equal(t, 2, result.uniqueDataCount, "Cache count should be 2 before reset")
+ assert.NotNil(t, result.uniqueTracker, "Cache should be set before reset")
+ assert.Equal(t, 2, result.UniqueDataCount(), "Cache count should be 2 before reset")
assert.Equal(t, 2, result.Len(), "Should have 2 elements before reset")
// Reset the result
result.Reset()
// Verify cache is cleared
- assert.Nil(t, result.uniqueDataMap, "Cache should be nil after reset")
- assert.Equal(t, 0, result.uniqueDataCount, "Cache count should be 0 after reset")
+ assert.Nil(t, result.uniqueTracker, "Cache should be nil after reset")
+ assert.Equal(t, 0, result.UniqueDataCount(), "Cache count should be 0 after reset")
assert.Equal(t, 0, result.Len(), "Should have 0 elements after reset")
// Verify other fields are also reset
diff --git a/banyand/internal/sidx/tag.go b/banyand/internal/sidx/tag.go
index 9a38f381..2db4ec53 100644
--- a/banyand/internal/sidx/tag.go
+++ b/banyand/internal/sidx/tag.go
@@ -41,7 +41,6 @@ type tagMetadata struct {
dataBlock dataBlock // Offset/size in .td file
filterBlock dataBlock // Offset/size in .tf file
valueType pbv1.ValueType
- indexed bool
}
// tagData represents the runtime data for a tag with filtering capabilities.
@@ -52,7 +51,6 @@ type tagData struct {
min []byte // For int64 tags
max []byte // For int64 tags
valueType pbv1.ValueType
- indexed bool
}
var (
@@ -101,7 +99,6 @@ func releaseTagMetadata(tm *tagMetadata) {
func (td *tagData) reset() {
td.name = ""
td.valueType = pbv1.ValueTypeUnknown
- td.indexed = false
// Reset values slice
for i := range td.values {
@@ -124,7 +121,6 @@ func (td *tagData) reset() {
func (tm *tagMetadata) reset() {
tm.name = ""
tm.valueType = pbv1.ValueTypeUnknown
- tm.indexed = false
tm.dataBlock = dataBlock{}
tm.filterBlock = dataBlock{}
tm.min = nil
@@ -226,14 +222,14 @@ func (td *tagData) addValue(value []byte) {
td.values = append(td.values, value)
// Update filter for indexed tags
- if td.indexed && td.filter != nil {
+ if td.filter != nil {
td.filter.Add(value)
}
}
// hasValue checks if a value exists in the tag using the bloom filter.
func (td *tagData) hasValue(value []byte) bool {
- if !td.indexed || td.filter == nil {
+ if td.filter == nil {
// For non-indexed tags, do linear search
for _, v := range td.values {
if bytes.Equal(v, value) {
@@ -255,13 +251,6 @@ func (tm *tagMetadata) marshal(dst []byte) []byte {
dst = pkgencoding.VarUint64ToBytes(dst, tm.filterBlock.offset)
dst = pkgencoding.VarUint64ToBytes(dst, tm.filterBlock.size)
- // Write flags
- var flags uint8
- if tm.indexed {
- flags |= 1
- }
- dst = append(dst, flags)
-
dst = pkgencoding.EncodeBytes(dst, tm.min)
dst = pkgencoding.EncodeBytes(dst, tm.max)
return dst
@@ -292,9 +281,6 @@ func (tm *tagMetadata) unmarshal(src []byte) ([]byte, error) {
if len(src) < 1 {
return nil, fmt.Errorf("cannot unmarshal tagMetadata flags: src is too short")
}
- flags := src[0]
- src = src[1:]
- tm.indexed = (flags & 1) != 0
src, tm.min, err = pkgencoding.DecodeBytes(src)
if err != nil {
diff --git a/banyand/internal/sidx/tag_filter_op.go b/banyand/internal/sidx/tag_filter_op.go
index 12a3746e..8ed186cc 100644
--- a/banyand/internal/sidx/tag_filter_op.go
+++ b/banyand/internal/sidx/tag_filter_op.go
@@ -21,7 +21,6 @@ import (
"bytes"
"fmt"
- "github.com/apache/skywalking-banyandb/pkg/compress/zstd"
"github.com/apache/skywalking-banyandb/pkg/encoding"
"github.com/apache/skywalking-banyandb/pkg/filter"
"github.com/apache/skywalking-banyandb/pkg/fs"
@@ -176,18 +175,11 @@ func (tfo *tagFilterOp) readTagMetadata(tagName string, tagBlock dataBlock) (*ta
return nil, fmt.Errorf("no metadata reader for tag %s", tagName)
}
- // Read compressed metadata
- compressedData := make([]byte, tagBlock.size)
- fs.MustReadData(metaReader, int64(tagBlock.offset), compressedData)
-
- // Decompress data
- decompressedData, err := zstd.Decompress(nil, compressedData)
- if err != nil {
- return nil, fmt.Errorf("failed to decompress tag metadata: %w", err)
- }
+ data := make([]byte, tagBlock.size)
+ fs.MustReadData(metaReader, int64(tagBlock.offset), data)
// Unmarshal tag metadata
- tm, err := unmarshalTagMetadata(decompressedData)
+ tm, err := unmarshalTagMetadata(data)
if err != nil {
return nil, fmt.Errorf("failed to unmarshal tag metadata: %w", err)
}
diff --git a/banyand/internal/sidx/tag_test.go b/banyand/internal/sidx/tag_test.go
index 9bc500bf..19ef70cf 100644
--- a/banyand/internal/sidx/tag_test.go
+++ b/banyand/internal/sidx/tag_test.go
@@ -31,24 +31,21 @@ func TestTagExportedFields(t *testing.T) {
Name: "test-tag",
Value: []byte("test-value"),
ValueType: pbv1.ValueTypeStr,
- Indexed: true,
}
// Test that exported fields are accessible
assert.Equal(t, "test-tag", tag.Name)
assert.Equal(t, []byte("test-value"), tag.Value)
assert.Equal(t, pbv1.ValueTypeStr, tag.ValueType)
- assert.Equal(t, true, tag.Indexed)
}
func TestNewTag(t *testing.T) {
// Test the NewTag constructor function
- tag := NewTag("service", []byte("order-service"), pbv1.ValueTypeStr, true)
+ tag := NewTag("service", []byte("order-service"), pbv1.ValueTypeStr)
assert.Equal(t, "service", tag.Name)
assert.Equal(t, []byte("order-service"), tag.Value)
assert.Equal(t, pbv1.ValueTypeStr, tag.ValueType)
- assert.Equal(t, true, tag.Indexed)
}
func TestTagReset(t *testing.T) {
@@ -56,7 +53,6 @@ func TestTagReset(t *testing.T) {
Name: "test-tag",
Value: []byte("test-value"),
ValueType: pbv1.ValueTypeStr,
- Indexed: true,
}
tag.Reset()
@@ -64,7 +60,6 @@ func TestTagReset(t *testing.T) {
assert.Equal(t, "", tag.Name)
assert.Nil(t, tag.Value)
assert.Equal(t, pbv1.ValueTypeUnknown, tag.ValueType)
- assert.Equal(t, false, tag.Indexed)
}
func TestTagSize(t *testing.T) {
@@ -72,7 +67,6 @@ func TestTagSize(t *testing.T) {
Name: "test",
Value: []byte("value"),
ValueType: pbv1.ValueTypeStr,
- Indexed: true,
}
// Size should be len(name) + len(value) + 1 (for valueType)
@@ -85,7 +79,6 @@ func TestTagCopy(t *testing.T) {
Name: "original",
Value: []byte("original-value"),
ValueType: pbv1.ValueTypeStr,
- Indexed: true,
}
copied := original.Copy()
@@ -94,7 +87,6 @@ func TestTagCopy(t *testing.T) {
assert.Equal(t, original.Name, copied.Name)
assert.Equal(t, original.Value, copied.Value)
assert.Equal(t, original.ValueType, copied.ValueType)
- assert.Equal(t, original.Indexed, copied.Indexed)
// Test that modifying original doesn't affect copy
original.Name = "modified"
@@ -110,8 +102,8 @@ func TestTagInWriteRequest(t *testing.T) {
Key: 456,
Data: []byte("test-data"),
Tags: []Tag{
- {Name: "service", Value: []byte("order-service"), ValueType: pbv1.ValueTypeStr, Indexed: true},
- {Name: "environment", Value: []byte("prod"), ValueType: pbv1.ValueTypeStr, Indexed: false},
+ {Name: "service", Value: []byte("order-service"), ValueType: pbv1.ValueTypeStr},
+ {Name: "environment", Value: []byte("prod"), ValueType: pbv1.ValueTypeStr},
},
}
diff --git a/banyand/trace/write_standalone.go b/banyand/trace/write_standalone.go
index 429ee482..ef089e2e 100644
--- a/banyand/trace/write_standalone.go
+++ b/banyand/trace/write_standalone.go
@@ -224,7 +224,6 @@ func processTraces(schemaRepo *schemaRepo, tracesInTable *tracesInTable, writeEv
tracesInTable.traces.tags = append(tracesInTable.traces.tags, tags)
sidxTags := make([]sidx.Tag, 0, len(tags))
- // TODO: set Indexed
for _, tag := range tags {
sidxTags = append(sidxTags, sidx.Tag{
Name: tag.tag,
@@ -241,6 +240,9 @@ func processTraces(schemaRepo *schemaRepo, tracesInTable *tracesInTable, writeEv
continue
}
tv := tagMap[tagName]
+ if tv == nil {
+ continue
+ }
if tv.valueType != pbv1.ValueTypeInt64 && tv.valueType != pbv1.ValueTypeTimestamp {
return fmt.Errorf("unsupported tag value type: %s", tv.tag)
}
@@ -274,9 +276,24 @@ func processTraces(schemaRepo *schemaRepo, tracesInTable *tracesInTable, writeEv
return fmt.Errorf("cannot marshal series: %w", err)
}
+ // Filter sidxTags to remove tags that are in indexRule.Tags
+ filteredSidxTags := make([]sidx.Tag, 0, len(sidxTags))
+ for _, sidxTag := range sidxTags {
+ shouldInclude := true
+ for _, ruleTagName := range indexRule.Tags {
+ if sidxTag.Name == ruleTagName {
+ shouldInclude = false
+ break
+ }
+ }
+ if shouldInclude {
+ filteredSidxTags = append(filteredSidxTags, sidxTag)
+ }
+ }
+
writeReq := sidx.WriteRequest{
Data: []byte(traceID),
- Tags: sidxTags,
+ Tags: filteredSidxTags,
SeriesID: series.ID,
Key: key,
}
diff --git a/test/cases/trace/data/data.go b/test/cases/trace/data/data.go
index e35751eb..ee5a6cec 100644
--- a/test/cases/trace/data/data.go
+++ b/test/cases/trace/data/data.go
@@ -276,7 +276,7 @@ func marshalToJSONWithStringBytes(resp *tracev1.QueryResponse) ([]byte, error) {
}
// GetSpanDataAsString extracts the span data as a string from a Span.
-// This converts the raw bytes back to the original string value like "span5_trace_data".
+// This converts the raw bytes back to the original string value like "trace_001_span_1".
func GetSpanDataAsString(span *tracev1.Span) string {
if span == nil || len(span.Span) == 0 {
return ""
diff --git a/test/cases/trace/data/input/eq_endpoint_order_duration_asc.yml b/test/cases/trace/data/input/eq_endpoint_order_duration_asc.yml
index 0458d6d0..a5bee326 100644
--- a/test/cases/trace/data/input/eq_endpoint_order_duration_asc.yml
+++ b/test/cases/trace/data/input/eq_endpoint_order_duration_asc.yml
@@ -23,7 +23,7 @@ criteria:
op: "BINARY_OP_EQ"
value:
str:
- value: "/home_endpoint"
+ value: "/item_endpoint"
order_by:
index_rule_name: "duration"
sort: "SORT_ASC"
diff --git a/test/cases/trace/data/input/eq_trace_id.yml b/test/cases/trace/data/input/eq_trace_id.yml
index 689aac12..83f7c91c 100644
--- a/test/cases/trace/data/input/eq_trace_id.yml
+++ b/test/cases/trace/data/input/eq_trace_id.yml
@@ -24,4 +24,4 @@ criteria:
op: "BINARY_OP_EQ"
value:
str:
- value: "1"
+ value: "trace_001"
diff --git a/test/cases/trace/data/input/order_duration_desc.yml b/test/cases/trace/data/input/order_duration_desc.yml
index 48ee0751..7b7e2fc2 100644
--- a/test/cases/trace/data/input/order_duration_desc.yml
+++ b/test/cases/trace/data/input/order_duration_desc.yml
@@ -19,4 +19,4 @@ name: "sw"
groups: ["test-trace-group"]
order_by:
index_rule_name: "duration"
- sort: "SORT_DESC"
+ sort: "SORT_DESC"
\ No newline at end of file
diff --git a/test/cases/trace/data/input/order_duration_desc.yml b/test/cases/trace/data/input/order_timestamp_desc_limit.yml
similarity index 95%
copy from test/cases/trace/data/input/order_duration_desc.yml
copy to test/cases/trace/data/input/order_timestamp_desc_limit.yml
index 48ee0751..df777aee 100644
--- a/test/cases/trace/data/input/order_duration_desc.yml
+++ b/test/cases/trace/data/input/order_timestamp_desc_limit.yml
@@ -18,5 +18,6 @@
name: "sw"
groups: ["test-trace-group"]
order_by:
- index_rule_name: "duration"
+ index_rule_name: "timestamp"
sort: "SORT_DESC"
+limit: 2
\ No newline at end of file
diff --git a/test/cases/trace/data/testdata/sw.json b/test/cases/trace/data/testdata/sw.json
index 0adcc961..826032dd 100644
--- a/test/cases/trace/data/testdata/sw.json
+++ b/test/cases/trace/data/testdata/sw.json
@@ -3,7 +3,7 @@
"tags": [
{
"str": {
- "value": "1"
+ "value": "trace_001"
}
},
{
@@ -32,18 +32,18 @@
}
}
],
- "span": "span1_trace_data"
+ "span": "trace_001_span_1"
},
{
"tags": [
{
"str": {
- "value": "2"
+ "value": "trace_001"
}
},
{
"int": {
- "value": 1
+ "value": 0
}
},
{
@@ -67,13 +67,293 @@
}
}
],
- "span": "span2_trace_data"
+ "span": "trace_001_span_2"
+ },
+ {
+ "tags": [
+ {
+ "str": {
+ "value": "trace_001"
+ }
+ },
+ {
+ "int": {
+ "value": 0
+ }
+ },
+ {
+ "str": {
+ "value": "webapp_service"
+ }
+ },
+ {
+ "str": {
+ "value": "webapp_instance_1"
+ }
+ },
+ {
+ "str": {
+ "value": "/item_endpoint"
+ }
+ },
+ {
+ "int": {
+ "value": 300
+ }
+ }
+ ],
+ "span": "trace_001_span_3"
+ },
+ {
+ "tags": [
+ {
+ "str": {
+ "value": "trace_002"
+ }
+ },
+ {
+ "int": {
+ "value": 1
+ }
+ },
+ {
+ "str": {
+ "value": "webapp_service"
+ }
+ },
+ {
+ "str": {
+ "value": "webapp_instance_1"
+ }
+ },
+ {
+ "str": {
+ "value": "/home_endpoint"
+ }
+ },
+ {
+ "int": {
+ "value": 800
+ }
+ }
+ ],
+ "span": "trace_002_span_1"
+ },
+ {
+ "tags": [
+ {
+ "str": {
+ "value": "trace_002"
+ }
+ },
+ {
+ "int": {
+ "value": 0
+ }
+ },
+ {
+ "str": {
+ "value": "webapp_service"
+ }
+ },
+ {
+ "str": {
+ "value": "webapp_instance_3"
+ }
+ },
+ {
+ "str": {
+ "value": "/price_endpoint"
+ }
+ },
+ {
+ "int": {
+ "value": 200
+ }
+ }
+ ],
+ "span": "trace_002_span_2"
+ },
+ {
+ "tags": [
+ {
+ "str": {
+ "value": "trace_003"
+ }
+ },
+ {
+ "int": {
+ "value": 1
+ }
+ },
+ {
+ "str": {
+ "value": "webapp_service"
+ }
+ },
+ {
+ "str": {
+ "value": "webapp_instance_2"
+ }
+ },
+ {
+ "str": {
+ "value": "/product_endpoint"
+ }
+ },
+ {
+ "int": {
+ "value": 1200
+ }
+ }
+ ],
+ "span": "trace_003_span_1"
+ },
+ {
+ "tags": [
+ {
+ "str": {
+ "value": "trace_003"
+ }
+ },
+ {
+ "int": {
+ "value": 0
+ }
+ },
+ {
+ "str": {
+ "value": "webapp_service"
+ }
+ },
+ {
+ "str": {
+ "value": "webapp_instance_1"
+ }
+ },
+ {
+ "str": {
+ "value": "/home_endpoint"
+ }
+ },
+ {
+ "int": {
+ "value": 150
+ }
+ }
+ ],
+ "span": "trace_003_span_2"
+ },
+ {
+ "tags": [
+ {
+ "str": {
+ "value": "trace_003"
+ }
+ },
+ {
+ "int": {
+ "value": 0
+ }
+ },
+ {
+ "str": {
+ "value": "webapp_service"
+ }
+ },
+ {
+ "str": {
+ "value": "webapp_instance_3"
+ }
+ },
+ {
+ "str": {
+ "value": "/price_endpoint"
+ }
+ },
+ {
+ "int": {
+ "value": 400
+ }
+ }
+ ],
+ "span": "trace_003_span_3"
},
{
"tags": [
{
"str": {
- "value": "3"
+ "value": "trace_004"
+ }
+ },
+ {
+ "int": {
+ "value": 1
+ }
+ },
+ {
+ "str": {
+ "value": "webapp_service"
+ }
+ },
+ {
+ "str": {
+ "value": "webapp_instance_2"
+ }
+ },
+ {
+ "str": {
+ "value": "/home_endpoint"
+ }
+ },
+ {
+ "int": {
+ "value": 600
+ }
+ }
+ ],
+ "span": "trace_004_span_1"
+ },
+ {
+ "tags": [
+ {
+ "str": {
+ "value": "trace_005"
+ }
+ },
+ {
+ "int": {
+ "value": 1
+ }
+ },
+ {
+ "str": {
+ "value": "webapp_service"
+ }
+ },
+ {
+ "str": {
+ "value": "webapp_instance_2"
+ }
+ },
+ {
+ "str": {
+ "value": "/product_endpoint"
+ }
+ },
+ {
+ "int": {
+ "value": 900
+ }
+ }
+ ],
+ "span": "trace_005_span_1"
+ },
+ {
+ "tags": [
+ {
+ "str": {
+ "value": "trace_005"
}
},
{
@@ -98,17 +378,17 @@
},
{
"int": {
- "value": 30
+ "value": 250
}
}
],
- "span": "span3_trace_data"
+ "span": "trace_005_span_2"
},
{
"tags": [
{
"str": {
- "value": "4"
+ "value": "trace_005"
}
},
{
@@ -133,17 +413,17 @@
},
{
"int": {
- "value": 60
+ "value": 350
}
}
],
- "span": "span4_trace_data"
+ "span": "trace_005_span_3"
},
{
"tags": [
{
"str": {
- "value": "5"
+ "value": "trace_005"
}
},
{
@@ -168,10 +448,10 @@
},
{
"int": {
- "value": 300
+ "value": 180
}
}
],
- "span": "span5_trace_data"
+ "span": "trace_005_span_4"
}
]
\ No newline at end of file
diff --git a/test/cases/trace/data/want/eq_endpoint_order_duration_asc.yml b/test/cases/trace/data/want/eq_endpoint_order_duration_asc.yml
index 457a547b..3af289c8 100644
--- a/test/cases/trace/data/want/eq_endpoint_order_duration_asc.yml
+++ b/test/cases/trace/data/want/eq_endpoint_order_duration_asc.yml
@@ -16,5 +16,10 @@
# under the License.
spans:
- - span: span3_trace_data
- - span: span1_trace_data
+ - span: trace_005_span_1
+ - span: trace_005_span_2
+ - span: trace_005_span_3
+ - span: trace_005_span_4
+ - span: trace_001_span_1
+ - span: trace_001_span_2
+ - span: trace_001_span_3
diff --git a/test/cases/trace/data/want/eq_service_instance_order_time_asc.yml b/test/cases/trace/data/want/eq_service_instance_order_time_asc.yml
index 725839c7..a5540160 100644
--- a/test/cases/trace/data/want/eq_service_instance_order_time_asc.yml
+++ b/test/cases/trace/data/want/eq_service_instance_order_time_asc.yml
@@ -16,6 +16,15 @@
# under the License.
spans:
- - span: span1_trace_data
- - span: span3_trace_data
- - span: span5_trace_data
+ - span: trace_001_span_1
+ - span: trace_001_span_2
+ - span: trace_001_span_3
+ - span: trace_002_span_1
+ - span: trace_002_span_2
+ - span: trace_003_span_1
+ - span: trace_003_span_2
+ - span: trace_003_span_3
+ - span: trace_005_span_1
+ - span: trace_005_span_2
+ - span: trace_005_span_3
+ - span: trace_005_span_4
diff --git a/test/cases/trace/data/want/eq_service_order_timestamp_desc.yml b/test/cases/trace/data/want/eq_service_order_timestamp_desc.yml
index 2c9aa31e..791dd197 100644
--- a/test/cases/trace/data/want/eq_service_order_timestamp_desc.yml
+++ b/test/cases/trace/data/want/eq_service_order_timestamp_desc.yml
@@ -16,8 +16,16 @@
# under the License.
spans:
- - span: span5_trace_data
- - span: span4_trace_data
- - span: span3_trace_data
- - span: span2_trace_data
- - span: span1_trace_data
+ - span: trace_005_span_1
+ - span: trace_005_span_2
+ - span: trace_005_span_3
+ - span: trace_005_span_4
+ - span: trace_004_span_1
+ - span: trace_003_span_1
+ - span: trace_003_span_2
+ - span: trace_003_span_3
+ - span: trace_002_span_1
+ - span: trace_002_span_2
+ - span: trace_001_span_1
+ - span: trace_001_span_2
+ - span: trace_001_span_3
diff --git a/test/cases/trace/data/want/eq_trace_id.yml b/test/cases/trace/data/want/eq_trace_id.yml
index 79194d67..5fb3eeef 100644
--- a/test/cases/trace/data/want/eq_trace_id.yml
+++ b/test/cases/trace/data/want/eq_trace_id.yml
@@ -16,9 +16,21 @@
# under the License.
spans:
- - span: span1_trace_data
+ - span: trace_001_span_1
tags:
- key: trace_id
value:
str:
- value: "1"
+ value: "trace_001"
+ - span: trace_001_span_2
+ tags:
+ - key: trace_id
+ value:
+ str:
+ value: "trace_001"
+ - span: trace_001_span_3
+ tags:
+ - key: trace_id
+ value:
+ str:
+ value: "trace_001"
diff --git a/test/cases/trace/data/want/order_duration_desc.yml b/test/cases/trace/data/want/order_duration_desc.yml
index f5d18520..e1e03c58 100644
--- a/test/cases/trace/data/want/order_duration_desc.yml
+++ b/test/cases/trace/data/want/order_duration_desc.yml
@@ -16,8 +16,16 @@
# under the License.
spans:
- - span: span1_trace_data
- - span: span2_trace_data
- - span: span5_trace_data
- - span: span4_trace_data
- - span: span3_trace_data
+ - span: trace_003_span_1
+ - span: trace_003_span_2
+ - span: trace_003_span_3
+ - span: trace_001_span_1
+ - span: trace_001_span_2
+ - span: trace_001_span_3
+ - span: trace_005_span_1
+ - span: trace_005_span_2
+ - span: trace_005_span_3
+ - span: trace_005_span_4
+ - span: trace_002_span_1
+ - span: trace_002_span_2
+ - span: trace_004_span_1
diff --git a/test/cases/trace/data/want/order_timestamp_desc.yml b/test/cases/trace/data/want/order_timestamp_desc.yml
index 2c9aa31e..791dd197 100644
--- a/test/cases/trace/data/want/order_timestamp_desc.yml
+++ b/test/cases/trace/data/want/order_timestamp_desc.yml
@@ -16,8 +16,16 @@
# under the License.
spans:
- - span: span5_trace_data
- - span: span4_trace_data
- - span: span3_trace_data
- - span: span2_trace_data
- - span: span1_trace_data
+ - span: trace_005_span_1
+ - span: trace_005_span_2
+ - span: trace_005_span_3
+ - span: trace_005_span_4
+ - span: trace_004_span_1
+ - span: trace_003_span_1
+ - span: trace_003_span_2
+ - span: trace_003_span_3
+ - span: trace_002_span_1
+ - span: trace_002_span_2
+ - span: trace_001_span_1
+ - span: trace_001_span_2
+ - span: trace_001_span_3
diff --git a/test/cases/trace/data/want/eq_service_instance_order_time_asc.yml b/test/cases/trace/data/want/order_timestamp_desc_limit.yml
similarity index 85%
copy from test/cases/trace/data/want/eq_service_instance_order_time_asc.yml
copy to test/cases/trace/data/want/order_timestamp_desc_limit.yml
index 725839c7..6e8d4e68 100644
--- a/test/cases/trace/data/want/eq_service_instance_order_time_asc.yml
+++ b/test/cases/trace/data/want/order_timestamp_desc_limit.yml
@@ -16,6 +16,9 @@
# under the License.
spans:
- - span: span1_trace_data
- - span: span3_trace_data
- - span: span5_trace_data
+ - span: trace_005_span_1
+ - span: trace_005_span_2
+ - span: trace_005_span_3
+ - span: trace_005_span_4
+ - span: trace_004_span_1
+
diff --git a/test/cases/trace/trace.go b/test/cases/trace/trace.go
index de208516..7237ca6b 100644
--- a/test/cases/trace/trace.go
+++ b/test/cases/trace/trace.go
@@ -48,4 +48,5 @@ var _ = g.DescribeTable("Scanning Traces", func(args helpers.Args) {
g.Entry("filter by service id", helpers.Args{Input: "eq_service_order_timestamp_desc", Duration: 1 * time.Hour}),
g.Entry("filter by service instance id", helpers.Args{Input: "eq_service_instance_order_time_asc", Duration: 1 * time.Hour}),
g.Entry("filter by endpoint", helpers.Args{Input: "eq_endpoint_order_duration_asc", Duration: 1 * time.Hour}),
+ g.Entry("order by timestamp limit 2", helpers.Args{Input: "order_timestamp_desc_limit", Duration: 1 * time.Hour}),
)