You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ha...@apache.org on 2025/08/09 11:31:06 UTC
(skywalking-banyandb) branch lifecycle-seg created (now af57b941)
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a change to branch lifecycle-seg
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
at af57b941 Updated migration visitor constructors to accept target stage interval, and added utility functions for segment calculations. Introduced tests for segment boundary utilities to ensure correct segment calculations.
This branch includes the following new commits:
new af57b941 Updated migration visitor constructors to accept target stage interval, and added utility functions for segment calculations. Introduced tests for segment boundary utilities to ensure correct segment calculations.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(skywalking-banyandb) 01/01: Updated migration visitor constructors to accept target stage interval, and added utility functions for segment calculations. Introduced tests for segment boundary utilities to ensure correct segment calculations.
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch lifecycle-seg
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit af57b941ac506cc8b72d28cd9992b9ecc56b4d36
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Sat Aug 9 19:29:15 2025 +0800
Updated migration visitor constructors to accept target stage interval, and added utility functions for segment calculations. Introduced tests for segment boundary utilities to ensure correct segment calculations.
---
.../backup/lifecycle/measure_migration_visitor.go | 301 +++++++++++--------
banyand/backup/lifecycle/migration_integration.go | 14 +-
banyand/backup/lifecycle/segment_boundary_utils.go | 78 +++++
.../lifecycle/segment_boundary_utils_test.go | 138 +++++++++
.../backup/lifecycle/stream_migration_visitor.go | 321 ++++++++++++---------
banyand/liaison/grpc/measure.go | 40 +--
banyand/liaison/grpc/property.go | 137 +++++----
banyand/liaison/grpc/stream.go | 24 +-
banyand/measure/introducer.go | 1 +
banyand/measure/metadata.go | 6 +-
banyand/stream/metadata.go | 6 +-
banyand/stream/tstable.go | 3 +
banyand/stream/write_standalone.go | 2 +-
pkg/fs/local_file_system.go | 4 +
pkg/fs/local_file_system_test.go | 161 +++++++++++
.../logical/stream/stream_plan_distributed.go | 10 +-
pkg/query/logical/stream/stream_plan_merge.go | 6 +-
pkg/test/measure/testdata/groups/sw_metric.json | 3 +-
pkg/test/stream/testdata/group.json | 68 +++--
test/cases/init.go | 1 +
test/cases/stream/data/data.go | 7 -
test/cases/stream/data/want/all.yaml | 10 +-
test/cases/stream/data/want/duplicated_all.yaml | 10 +-
.../stream/data/want/duplicated_entity_filter.yaml | 4 +-
.../stream/data/want/duplicated_index_filter.yaml | 4 +-
.../data/want/duplicated_order_by_filter.yaml | 4 +-
.../data/want/duplicated_order_by_index.yaml | 10 +-
test/cases/stream/data/want/filter_no_indexed.yaml | 4 +-
.../stream/data/want/filter_no_indexed_or.yaml | 10 +-
test/cases/stream/data/want/filter_order_desc.yaml | 4 +-
test/cases/stream/data/want/filter_tag.yaml | 2 +-
test/cases/stream/data/want/global_index.yaml | 2 +-
test/cases/stream/data/want/global_indices.yaml | 6 +-
test/cases/stream/data/want/having.yaml | 4 +-
.../cases/stream/data/want/having_non_indexed.yaml | 6 +-
.../stream/data/want/having_non_indexed_arr.yaml | 4 +-
test/cases/stream/data/want/less.yaml | 6 +-
test/cases/stream/data/want/less_eq.yaml | 8 +-
test/cases/stream/data/want/limit.yaml | 6 +-
test/cases/stream/data/want/logical.yaml | 4 +-
test/cases/stream/data/want/offset.yaml | 2 +-
test/cases/stream/data/want/order_asc.yaml | 10 +-
test/cases/stream/data/want/order_desc.yaml | 10 +-
test/cases/stream/data/want/search.yaml | 4 +-
test/cases/stream/data/want/sort_desc.yaml | 10 +-
.../data/want/sort_duration_no_index_limit.yaml | 4 +-
test/cases/stream/data/want/sort_filter.yaml | 6 +-
test/cases/stream/stream.go | 6 +-
48 files changed, 991 insertions(+), 500 deletions(-)
diff --git a/banyand/backup/lifecycle/measure_migration_visitor.go b/banyand/backup/lifecycle/measure_migration_visitor.go
index f2b7d8a3..2a6588de 100644
--- a/banyand/backup/lifecycle/measure_migration_visitor.go
+++ b/banyand/backup/lifecycle/measure_migration_visitor.go
@@ -23,10 +23,12 @@ import (
"path/filepath"
"strconv"
"strings"
+ "time"
"github.com/apache/skywalking-banyandb/api/common"
"github.com/apache/skywalking-banyandb/api/data"
commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ "github.com/apache/skywalking-banyandb/banyand/internal/storage"
"github.com/apache/skywalking-banyandb/banyand/measure"
"github.com/apache/skywalking-banyandb/banyand/queue"
"github.com/apache/skywalking-banyandb/pkg/fs"
@@ -37,33 +39,35 @@ import (
// measureMigrationVisitor implements the measure.Visitor interface for file-based migration.
type measureMigrationVisitor struct {
- selector node.Selector // From parseGroup - node selector
- client queue.Client // From parseGroup - queue client
- chunkedClients map[string]queue.ChunkedSyncClient // Per-node chunked sync clients cache
- logger *logger.Logger
- progress *Progress // Progress tracker for migration states
- lfs fs.FileSystem
- group string
- targetShardNum uint32 // From parseGroup - target shard count
- replicas uint32 // From parseGroup - replica count
- chunkSize int // Chunk size for streaming data
+ selector node.Selector // From parseGroup - node selector
+ client queue.Client // From parseGroup - queue client
+ chunkedClients map[string]queue.ChunkedSyncClient // Per-node chunked sync clients cache
+ logger *logger.Logger
+ progress *Progress // Progress tracker for migration states
+ lfs fs.FileSystem
+ group string
+ targetShardNum uint32 // From parseGroup - target shard count
+ replicas uint32 // From parseGroup - replica count
+ chunkSize int // Chunk size for streaming data
+ targetStageInterval storage.IntervalRule // NEW: target stage's segment interval
}
// newMeasureMigrationVisitor creates a new file-based migration visitor.
func newMeasureMigrationVisitor(group *commonv1.Group, shardNum, replicas uint32, selector node.Selector, client queue.Client,
- l *logger.Logger, progress *Progress, chunkSize int,
+ l *logger.Logger, progress *Progress, chunkSize int, targetStageInterval storage.IntervalRule,
) *measureMigrationVisitor {
return &measureMigrationVisitor{
- group: group.Metadata.Name,
- targetShardNum: shardNum,
- replicas: replicas,
- selector: selector,
- client: client,
- chunkedClients: make(map[string]queue.ChunkedSyncClient),
- logger: l,
- progress: progress,
- chunkSize: chunkSize,
- lfs: fs.NewLocalFileSystem(),
+ group: group.Metadata.Name,
+ targetShardNum: shardNum,
+ replicas: replicas,
+ selector: selector,
+ client: client,
+ chunkedClients: make(map[string]queue.ChunkedSyncClient),
+ logger: l,
+ progress: progress,
+ chunkSize: chunkSize,
+ targetStageInterval: targetStageInterval,
+ lfs: fs.NewLocalFileSystem(),
}
}
@@ -101,87 +105,119 @@ func (mv *measureMigrationVisitor) VisitSeries(segmentTR *timestamp.TimeRange, s
// Set the total number of series segments for progress tracking
mv.SetMeasureSeriesCount(len(segmentFiles))
- // Create StreamingPartData for this segment
- files := make([]fileInfo, 0, len(segmentFiles))
- // Process each segment file
- for _, segmentFileName := range segmentFiles {
- // Extract segment ID from filename (remove .seg extension)
- fileSegmentIDStr := strings.TrimSuffix(segmentFileName, ".seg")
+ // Calculate ALL target segments this series index should go to
+ targetSegments := calculateTargetSegments(
+ segmentTR.Start.UnixNano(),
+ segmentTR.End.UnixNano(),
+ mv.targetStageInterval,
+ )
- // Parse hex segment ID
- segmentID, err := strconv.ParseUint(fileSegmentIDStr, 16, 64)
- if err != nil {
- mv.logger.Error().
+ mv.logger.Info().
+ Int("target_segments_count", len(targetSegments)).
+ Int64("series_min_ts", segmentTR.Start.UnixNano()).
+ Int64("series_max_ts", segmentTR.End.UnixNano()).
+ Str("group", mv.group).
+ Msg("migrating measure series index to multiple target segments")
+
+ // Send series index to EACH target segment that overlaps with its time range
+ for i, targetSegmentTime := range targetSegments {
+ // Create StreamingPartData for this segment
+ files := make([]fileInfo, 0, len(segmentFiles))
+ // Process each segment file
+ for _, segmentFileName := range segmentFiles {
+ // Extract segment ID from filename (remove .seg extension)
+ fileSegmentIDStr := strings.TrimSuffix(segmentFileName, ".seg")
+
+ // Parse hex segment ID
+ segmentID, err := strconv.ParseUint(fileSegmentIDStr, 16, 64)
+ if err != nil {
+ mv.logger.Error().
+ Str("filename", segmentFileName).
+ Str("id_str", fileSegmentIDStr).
+ Err(err).
+ Msg("failed to parse segment ID from filename")
+ continue
+ }
+
+ // Convert segmentID to ShardID for progress tracking
+ shardID := common.ShardID(segmentID)
+
+ // Check if this segment has already been completed for this target segment
+ segmentIDStr := getSegmentTimeRange(targetSegmentTime, mv.targetStageInterval).String()
+ if mv.progress.IsMeasureSeriesCompleted(mv.group, segmentIDStr, shardID) {
+ mv.logger.Debug().
+ Uint64("segment_id", segmentID).
+ Str("filename", segmentFileName).
+ Time("target_segment", targetSegmentTime).
+ Str("group", mv.group).
+ Msg("measure series segment already completed for this target segment, skipping")
+ continue
+ }
+
+ mv.logger.Info().
+ Uint64("segment_id", segmentID).
Str("filename", segmentFileName).
- Str("id_str", fileSegmentIDStr).
- Err(err).
- Msg("failed to parse segment ID from filename")
- continue
- }
-
- // Convert segmentID to ShardID for progress tracking
- shardID := common.ShardID(segmentID)
-
- // Check if this segment has already been completed
- segmentIDStr := segmentTR.String()
- if mv.progress.IsMeasureSeriesCompleted(mv.group, segmentIDStr, shardID) {
- mv.logger.Debug().
+ Time("target_segment", targetSegmentTime).
+ Str("group", mv.group).
+ Msg("migrating measure series segment file")
+
+ // Create file reader for the segment file
+ segmentFilePath := filepath.Join(seriesIndexPath, segmentFileName)
+ segmentFile, err := mv.lfs.OpenFile(segmentFilePath)
+ if err != nil {
+ errorMsg := fmt.Sprintf("failed to open measure segment file %s: %v", segmentFilePath, err)
+ mv.progress.MarkMeasureSeriesError(mv.group, segmentIDStr, shardID, errorMsg)
+ mv.logger.Error().
+ Str("path", segmentFilePath).
+ Err(err).
+ Msg("failed to open measure segment file")
+ return fmt.Errorf("failed to open measure segment file %s: %w", segmentFilePath, err)
+ }
+
+ // Close the file reader
+ defer func() {
+ if i == len(targetSegments)-1 { // Only close on last iteration
+ segmentFile.Close()
+ }
+ }()
+
+ files = append(files, fileInfo{
+ file: segmentFile,
+ name: segmentFileName,
+ })
+
+ mv.logger.Info().
Uint64("segment_id", segmentID).
Str("filename", segmentFileName).
+ Time("target_segment", targetSegmentTime).
Str("group", mv.group).
- Msg("measure segment already completed, skipping")
- continue
- }
-
- mv.logger.Info().
- Uint64("segment_id", segmentID).
- Str("filename", segmentFileName).
- Str("group", mv.group).
- Msg("migrating measure segment file")
-
- // Create file reader for the segment file
- segmentFilePath := filepath.Join(seriesIndexPath, segmentFileName)
- segmentFile, err := mv.lfs.OpenFile(segmentFilePath)
- if err != nil {
- errorMsg := fmt.Sprintf("failed to open measure segment file %s: %v", segmentFilePath, err)
- mv.progress.MarkMeasureSeriesError(mv.group, segmentIDStr, shardID, errorMsg)
- mv.logger.Error().
- Str("path", segmentFilePath).
- Err(err).
- Msg("failed to open measure segment file")
- return fmt.Errorf("failed to open measure segment file %s: %w", segmentFilePath, err)
+ Int("completed_segments", mv.progress.GetMeasureSeriesProgress(mv.group)).
+ Int("total_segments", mv.progress.GetMeasureSeriesCount(mv.group)).
+ Msgf("measure series segment migration completed for target segment %d/%d", i+1, len(targetSegments))
}
- // Close the file reader
- defer segmentFile.Close()
-
- files = append(files, fileInfo{
- file: segmentFile,
- name: segmentFileName,
- })
- }
-
- // Send segment file to each shard in shardIDs
- segmentIDStr := segmentTR.String()
- for _, shardID := range shardIDs {
- targetShardID := mv.calculateTargetShardID(uint32(shardID))
- partData := mv.createStreamingSegmentFromFiles(targetShardID, files, segmentTR, data.TopicMeasureSeriesSync.String())
-
- // Stream segment to target shard replicas
- if err := mv.streamPartToTargetShard(partData); err != nil {
- errorMsg := fmt.Sprintf("failed to stream measure segment to target shard %d: %v", targetShardID, err)
- mv.progress.MarkMeasureSeriesError(mv.group, segmentIDStr, shardID, errorMsg)
- return fmt.Errorf("failed to stream measure segment to target shard %d: %w", targetShardID, err)
+ // Send segment file to each shard in shardIDs for this target segment
+ segmentIDStr := getSegmentTimeRange(targetSegmentTime, mv.targetStageInterval).String()
+ for _, shardID := range shardIDs {
+ targetShardID := mv.calculateTargetShardID(uint32(shardID))
+ partData := mv.createStreamingSegmentFromFiles(targetShardID, files, segmentTR, data.TopicMeasureSeriesSync.String())
+
+ // Stream segment to target shard replicas
+ if err := mv.streamPartToTargetShard(partData); err != nil {
+ errorMsg := fmt.Sprintf("failed to stream measure segment to target shard %d: %v", targetShardID, err)
+ mv.progress.MarkMeasureSeriesError(mv.group, segmentIDStr, shardID, errorMsg)
+ return fmt.Errorf("failed to stream measure segment to target shard %d: %w", targetShardID, err)
+ }
+ // Mark segment as completed for this specific target segment
+ mv.progress.MarkMeasureSeriesCompleted(mv.group, segmentIDStr, shardID)
}
- // Mark segment as completed
- mv.progress.MarkMeasureSeriesCompleted(mv.group, segmentIDStr, shardID)
}
return nil
}
// VisitPart implements measure.Visitor - core migration logic.
-func (mv *measureMigrationVisitor) VisitPart(segmentTR *timestamp.TimeRange, sourceShardID common.ShardID, partPath string) error {
+func (mv *measureMigrationVisitor) VisitPart(_ *timestamp.TimeRange, sourceShardID common.ShardID, partPath string) error {
partData, err := measure.ParsePartMetadata(mv.lfs, partPath)
if err != nil {
return fmt.Errorf("failed to parse measure part metadata: %w", err)
@@ -191,52 +227,69 @@ func (mv *measureMigrationVisitor) VisitPart(segmentTR *timestamp.TimeRange, sou
return fmt.Errorf("failed to parse part ID from path: %w", err)
}
- // Check if this part has already been completed
- segmentIDStr := segmentTR.String()
- if mv.progress.IsMeasurePartCompleted(mv.group, segmentIDStr, sourceShardID, partID) {
- mv.logger.Warn().
- Uint64("part_id", partID).
- Uint32("source_shard", uint32(sourceShardID)).
- Str("group", mv.group).
- Msg("measure part already completed, skipping")
- return nil
- }
-
- // Calculate target shard ID based on source shard ID mapping
- targetShardID := mv.calculateTargetShardID(uint32(sourceShardID))
+ // Calculate ALL target segments this part should go to
+ targetSegments := calculateTargetSegments(
+ partData.MinTimestamp,
+ partData.MaxTimestamp,
+ mv.targetStageInterval,
+ )
mv.logger.Info().
Uint64("part_id", partID).
Uint32("source_shard", uint32(sourceShardID)).
- Uint32("target_shard", targetShardID).
- Str("part_path", partPath).
+ Int("target_segments_count", len(targetSegments)).
+ Int64("part_min_ts", partData.MinTimestamp).
+ Int64("part_max_ts", partData.MaxTimestamp).
Str("group", mv.group).
- Msg("migrating measure part")
+ Msg("migrating measure part to multiple target segments")
- files, release := measure.CreatePartFileReaderFromPath(partPath, mv.lfs)
- defer release()
+ // Send part to EACH target segment that overlaps with its time range
+ for i, targetSegmentTime := range targetSegments {
+ targetShardID := mv.calculateTargetShardID(uint32(sourceShardID))
- partData.Group = mv.group
- partData.ShardID = targetShardID
- partData.Topic = data.TopicMeasurePartSync.String()
- partData.Files = files
+ // Check if this part has already been completed for this segment
+ segmentIDStr := getSegmentTimeRange(targetSegmentTime, mv.targetStageInterval).String()
+ if mv.progress.IsMeasurePartCompleted(mv.group, segmentIDStr, sourceShardID, partID) {
+ mv.logger.Debug().
+ Uint64("part_id", partID).
+ Uint32("source_shard", uint32(sourceShardID)).
+ Time("target_segment", targetSegmentTime).
+ Str("group", mv.group).
+ Msg("measure part already completed for this target segment, skipping")
+ continue
+ }
- // Stream entire part to target shard replicas
- if err := mv.streamPartToTargetShard(partData); err != nil {
- errorMsg := fmt.Sprintf("failed to stream measure part to target shard: %v", err)
- mv.progress.MarkMeasurePartError(mv.group, segmentIDStr, sourceShardID, partID, errorMsg)
- return fmt.Errorf("failed to stream measure part to target shard: %w", err)
- }
+ // Create file readers for this part
+ files, release := measure.CreatePartFileReaderFromPath(partPath, mv.lfs)
+ defer func() {
+ if i == len(targetSegments)-1 { // Only release on last iteration
+ release()
+ }
+ }()
+
+ // Clone part data for this target segment
+ targetPartData := partData
+ targetPartData.Group = mv.group
+ targetPartData.ShardID = targetShardID
+ targetPartData.Topic = data.TopicMeasurePartSync.String()
+ targetPartData.Files = files
+
+ // Stream part to target segment
+ if err := mv.streamPartToTargetShard(targetPartData); err != nil {
+ errorMsg := fmt.Sprintf("failed to stream measure part to target segment %s: %v", targetSegmentTime.Format(time.RFC3339), err)
+ mv.progress.MarkMeasurePartError(mv.group, segmentIDStr, sourceShardID, partID, errorMsg)
+ return fmt.Errorf("failed to stream measure part to target segment: %w", err)
+ }
- // Mark part as completed in progress tracker
- mv.progress.MarkMeasurePartCompleted(mv.group, segmentIDStr, sourceShardID, partID)
+ // Mark part as completed for this specific target segment
+ mv.progress.MarkMeasurePartCompleted(mv.group, segmentIDStr, sourceShardID, partID)
- mv.logger.Info().
- Uint64("part_id", partID).
- Str("group", mv.group).
- Int("completed_parts", mv.progress.GetMeasurePartProgress(mv.group)).
- Int("total_parts", mv.progress.GetMeasurePartCount(mv.group)).
- Msg("measure part migration completed successfully")
+ mv.logger.Info().
+ Uint64("part_id", partID).
+ Time("target_segment", targetSegmentTime).
+ Str("group", mv.group).
+ Msgf("measure part migration completed for target segment %d/%d", i+1, len(targetSegments))
+ }
return nil
}
diff --git a/banyand/backup/lifecycle/migration_integration.go b/banyand/backup/lifecycle/migration_integration.go
index be9aa164..d51d9eea 100644
--- a/banyand/backup/lifecycle/migration_integration.go
+++ b/banyand/backup/lifecycle/migration_integration.go
@@ -51,6 +51,9 @@ func migrateStreamWithFileBasedAndProgress(
// Convert TTL to IntervalRule using storage.MustToIntervalRule
intervalRule := storage.MustToIntervalRule(ttl)
+ // Get target stage configuration
+ targetStageInterval := getTargetStageInterval(group)
+
// Count total parts before starting migration
totalParts, err := countStreamParts(tsdbRootPath, timeRange, intervalRule)
if err != nil {
@@ -59,8 +62,8 @@ func migrateStreamWithFileBasedAndProgress(
logger.Info().Int("total_parts", totalParts).Msg("counted stream parts for progress tracking")
}
- // Create file-based migration visitor with progress tracking
- visitor := newStreamMigrationVisitor(group, shardNum, replicas, selector, client, logger, progress, chunkSize)
+ // Create file-based migration visitor with progress tracking and target stage interval
+ visitor := newStreamMigrationVisitor(group, shardNum, replicas, selector, client, logger, progress, chunkSize, targetStageInterval)
defer visitor.Close()
// Set the total part count for progress tracking
@@ -129,6 +132,9 @@ func migrateMeasureWithFileBasedAndProgress(
// Convert TTL to IntervalRule using storage.MustToIntervalRule
intervalRule := storage.MustToIntervalRule(ttl)
+ // Get target stage configuration
+ targetStageInterval := getTargetStageInterval(group)
+
// Count total parts before starting migration
totalParts, err := countMeasureParts(tsdbRootPath, timeRange, intervalRule)
if err != nil {
@@ -137,8 +143,8 @@ func migrateMeasureWithFileBasedAndProgress(
logger.Info().Int("total_parts", totalParts).Msg("counted measure parts for progress tracking")
}
- // Create file-based migration visitor with progress tracking
- visitor := newMeasureMigrationVisitor(group, shardNum, replicas, selector, client, logger, progress, chunkSize)
+ // Create file-based migration visitor with progress tracking and target stage interval
+ visitor := newMeasureMigrationVisitor(group, shardNum, replicas, selector, client, logger, progress, chunkSize, targetStageInterval)
defer visitor.Close()
// Set the total part count for progress tracking
diff --git a/banyand/backup/lifecycle/segment_boundary_utils.go b/banyand/backup/lifecycle/segment_boundary_utils.go
new file mode 100644
index 00000000..243bb4a9
--- /dev/null
+++ b/banyand/backup/lifecycle/segment_boundary_utils.go
@@ -0,0 +1,78 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package lifecycle
+
+import (
+ "time"
+
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ "github.com/apache/skywalking-banyandb/banyand/internal/storage"
+ "github.com/apache/skywalking-banyandb/pkg/timestamp"
+)
+
+func calculateTargetSegments(partMinTS, partMaxTS int64, targetInterval storage.IntervalRule) []time.Time {
+ minTime := time.Unix(0, partMinTS).UTC()
+ maxTime := time.Unix(0, partMaxTS).UTC()
+
+ var targetSegments []time.Time
+
+ var segmentStart time.Time
+ switch targetInterval.Unit {
+ case storage.DAY:
+ daysSinceEpoch := minTime.Sub(time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC)).Hours() / 24
+ segmentIndex := int(daysSinceEpoch) / targetInterval.Num
+ segmentStart = time.Date(1970, 1, 1+segmentIndex*targetInterval.Num, 0, 0, 0, 0, time.UTC)
+ case storage.HOUR:
+ hoursSinceEpoch := minTime.Sub(time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC)).Hours()
+ segmentIndex := int(hoursSinceEpoch) / targetInterval.Num
+ segmentStart = time.Date(1970, 1, 1, segmentIndex*targetInterval.Num, 0, 0, 0, time.UTC)
+ default:
+ segmentStart = targetInterval.Unit.Standard(minTime)
+ }
+
+ current := segmentStart
+ for !current.After(maxTime) {
+ segmentEnd := targetInterval.NextTime(current)
+ if !(segmentEnd.Before(minTime) || current.After(maxTime)) {
+ targetSegments = append(targetSegments, current)
+ }
+ current = segmentEnd
+ }
+
+ return targetSegments
+}
+
+func getSegmentTimeRange(segmentStart time.Time, interval storage.IntervalRule) timestamp.TimeRange {
+ segmentEnd := interval.NextTime(segmentStart)
+ return timestamp.NewSectionTimeRange(segmentStart, segmentEnd)
+}
+
+func getTargetStageInterval(group *commonv1.Group) storage.IntervalRule {
+ if group.ResourceOpts != nil && len(group.ResourceOpts.Stages) > 0 {
+ stage := group.ResourceOpts.Stages[0]
+ if stage.SegmentInterval != nil {
+ return storage.MustToIntervalRule(stage.SegmentInterval)
+ }
+ }
+
+ if group.ResourceOpts != nil && group.ResourceOpts.SegmentInterval != nil {
+ return storage.MustToIntervalRule(group.ResourceOpts.SegmentInterval)
+ }
+
+ return storage.IntervalRule{Unit: storage.DAY, Num: 1}
+}
diff --git a/banyand/backup/lifecycle/segment_boundary_utils_test.go b/banyand/backup/lifecycle/segment_boundary_utils_test.go
new file mode 100644
index 00000000..b6ae9f0f
--- /dev/null
+++ b/banyand/backup/lifecycle/segment_boundary_utils_test.go
@@ -0,0 +1,138 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package lifecycle
+
+import (
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+
+ "github.com/apache/skywalking-banyandb/banyand/internal/storage"
+)
+
+func TestCalculateTargetSegments(t *testing.T) {
+ //nolint:govet // fieldalignment: test struct optimization not critical
+ tests := []struct {
+ expected []time.Time
+ name string
+ targetInterval storage.IntervalRule
+ partMinTS int64
+ partMaxTS int64
+ }{
+ {
+ name: "2-day to 3-day segments - part spans multiple segments",
+ partMinTS: time.Date(2024, 1, 2, 0, 0, 0, 0, time.UTC).UnixNano(), // Day 2
+ partMaxTS: time.Date(2024, 1, 4, 0, 0, 0, 0, time.UTC).UnixNano(), // Day 4
+ targetInterval: storage.IntervalRule{
+ Unit: storage.DAY,
+ Num: 3,
+ },
+ expected: []time.Time{
+ time.Date(2024, 1, 0, 0, 0, 0, 0, time.UTC), // Day 0-3
+ time.Date(2024, 1, 3, 0, 0, 0, 0, time.UTC), // Day 3-6
+ },
+ },
+ {
+ name: "2-day to 3-day segments - part fits in single segment",
+ partMinTS: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC).UnixNano(), // Day 1
+ partMaxTS: time.Date(2024, 1, 2, 0, 0, 0, 0, time.UTC).UnixNano(), // Day 2
+ targetInterval: storage.IntervalRule{
+ Unit: storage.DAY,
+ Num: 3,
+ },
+ expected: []time.Time{
+ time.Date(2024, 1, 0, 0, 0, 0, 0, time.UTC), // Day 0-3
+ },
+ },
+ {
+ name: "hour to day segments - part spans multiple segments",
+ partMinTS: time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC).UnixNano(), // Hour 12
+ partMaxTS: time.Date(2024, 1, 1, 18, 0, 0, 0, time.UTC).UnixNano(), // Hour 18
+ targetInterval: storage.IntervalRule{
+ Unit: storage.DAY,
+ Num: 1,
+ },
+ expected: []time.Time{
+ time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), // Day 1
+ },
+ },
+ {
+ name: "day to hour segments - part spans multiple segments",
+ partMinTS: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC).UnixNano(), // Day 1
+ partMaxTS: time.Date(2024, 1, 1, 23, 59, 59, 0, time.UTC).UnixNano(), // End of Day 1
+ targetInterval: storage.IntervalRule{
+ Unit: storage.HOUR,
+ Num: 6,
+ },
+ expected: []time.Time{
+ time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), // Hour 0-6
+ time.Date(2024, 1, 1, 6, 0, 0, 0, time.UTC), // Hour 6-12
+ time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC), // Hour 12-18
+ time.Date(2024, 1, 1, 18, 0, 0, 0, time.UTC), // Hour 18-24
+ },
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ result := calculateTargetSegments(tt.partMinTS, tt.partMaxTS, tt.targetInterval)
+ assert.Equal(t, tt.expected, result)
+ })
+ }
+}
+
+func TestGetSegmentTimeRange(t *testing.T) {
+ //nolint:govet // fieldalignment: test struct optimization not critical
+ tests := []struct {
+ segmentStart time.Time
+ expectedStart time.Time
+ expectedEnd time.Time
+ name string
+ interval storage.IntervalRule
+ }{
+ {
+ name: "3-day segment",
+ segmentStart: time.Date(2024, 1, 0, 0, 0, 0, 0, time.UTC),
+ interval: storage.IntervalRule{
+ Unit: storage.DAY,
+ Num: 3,
+ },
+ expectedStart: time.Date(2024, 1, 0, 0, 0, 0, 0, time.UTC),
+ expectedEnd: time.Date(2024, 1, 3, 0, 0, 0, 0, time.UTC),
+ },
+ {
+ name: "6-hour segment",
+ segmentStart: time.Date(2024, 1, 1, 6, 0, 0, 0, time.UTC),
+ interval: storage.IntervalRule{
+ Unit: storage.HOUR,
+ Num: 6,
+ },
+ expectedStart: time.Date(2024, 1, 1, 6, 0, 0, 0, time.UTC),
+ expectedEnd: time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC),
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ result := getSegmentTimeRange(tt.segmentStart, tt.interval)
+ assert.Equal(t, tt.expectedStart, result.Start)
+ assert.Equal(t, tt.expectedEnd, result.End)
+ })
+ }
+}
diff --git a/banyand/backup/lifecycle/stream_migration_visitor.go b/banyand/backup/lifecycle/stream_migration_visitor.go
index 775ca61c..6c3fff81 100644
--- a/banyand/backup/lifecycle/stream_migration_visitor.go
+++ b/banyand/backup/lifecycle/stream_migration_visitor.go
@@ -23,10 +23,12 @@ import (
"path/filepath"
"strconv"
"strings"
+ "time"
"github.com/apache/skywalking-banyandb/api/common"
"github.com/apache/skywalking-banyandb/api/data"
commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ "github.com/apache/skywalking-banyandb/banyand/internal/storage"
"github.com/apache/skywalking-banyandb/banyand/queue"
"github.com/apache/skywalking-banyandb/banyand/stream"
"github.com/apache/skywalking-banyandb/pkg/fs"
@@ -37,33 +39,35 @@ import (
// streamMigrationVisitor implements the stream.Visitor interface for file-based migration.
type streamMigrationVisitor struct {
- selector node.Selector // From parseGroup - node selector
- client queue.Client // From parseGroup - queue client
- chunkedClients map[string]queue.ChunkedSyncClient // Per-node chunked sync clients cache
- logger *logger.Logger
- progress *Progress // Progress tracker for migration states
- lfs fs.FileSystem
- group string
- targetShardNum uint32 // From parseGroup - target shard count
- replicas uint32 // From parseGroup - replica count
- chunkSize int // Chunk size for streaming data
+ selector node.Selector // From parseGroup - node selector
+ client queue.Client // From parseGroup - queue client
+ chunkedClients map[string]queue.ChunkedSyncClient // Per-node chunked sync clients cache
+ logger *logger.Logger
+ progress *Progress // Progress tracker for migration states
+ lfs fs.FileSystem
+ group string
+ targetShardNum uint32 // From parseGroup - target shard count
+ replicas uint32 // From parseGroup - replica count
+ chunkSize int // Chunk size for streaming data
+ targetStageInterval storage.IntervalRule // NEW: target stage's segment interval
}
// newStreamMigrationVisitor creates a new file-based migration visitor.
func newStreamMigrationVisitor(group *commonv1.Group, shardNum, replicas uint32, selector node.Selector, client queue.Client,
- l *logger.Logger, progress *Progress, chunkSize int,
+ l *logger.Logger, progress *Progress, chunkSize int, targetStageInterval storage.IntervalRule,
) *streamMigrationVisitor {
return &streamMigrationVisitor{
- group: group.Metadata.Name,
- targetShardNum: shardNum,
- replicas: replicas,
- selector: selector,
- client: client,
- chunkedClients: make(map[string]queue.ChunkedSyncClient),
- logger: l,
- progress: progress,
- chunkSize: chunkSize,
- lfs: fs.NewLocalFileSystem(),
+ group: group.Metadata.Name,
+ targetShardNum: shardNum,
+ replicas: replicas,
+ selector: selector,
+ client: client,
+ chunkedClients: make(map[string]queue.ChunkedSyncClient),
+ logger: l,
+ progress: progress,
+ chunkSize: chunkSize,
+ targetStageInterval: targetStageInterval,
+ lfs: fs.NewLocalFileSystem(),
}
}
@@ -101,102 +105,126 @@ func (mv *streamMigrationVisitor) VisitSeries(segmentTR *timestamp.TimeRange, se
// Set the total number of series segments for progress tracking
mv.SetStreamSeriesCount(len(segmentFiles))
- // Create StreamingPartData for this segment
- files := make([]fileInfo, 0, len(segmentFiles))
- // Process each segment file
- for _, segmentFileName := range segmentFiles {
- // Extract segment ID from filename (remove .seg extension)
- fileSegmentIDStr := strings.TrimSuffix(segmentFileName, ".seg")
+ // Calculate ALL target segments this series index should go to
+ targetSegments := calculateTargetSegments(
+ segmentTR.Start.UnixNano(),
+ segmentTR.End.UnixNano(),
+ mv.targetStageInterval,
+ )
- // Parse hex segment ID
- segmentID, err := strconv.ParseUint(fileSegmentIDStr, 16, 64)
- if err != nil {
- mv.logger.Error().
+ mv.logger.Info().
+ Int("target_segments_count", len(targetSegments)).
+ Int64("series_min_ts", segmentTR.Start.UnixNano()).
+ Int64("series_max_ts", segmentTR.End.UnixNano()).
+ Str("group", mv.group).
+ Msg("migrating series index to multiple target segments")
+
+ // Send series index to EACH target segment that overlaps with its time range
+ for i, targetSegmentTime := range targetSegments {
+ // Create StreamingPartData for this segment
+ files := make([]fileInfo, 0, len(segmentFiles))
+ // Process each segment file
+ for _, segmentFileName := range segmentFiles {
+ // Extract segment ID from filename (remove .seg extension)
+ fileSegmentIDStr := strings.TrimSuffix(segmentFileName, ".seg")
+
+ // Parse hex segment ID
+ segmentID, err := strconv.ParseUint(fileSegmentIDStr, 16, 64)
+ if err != nil {
+ mv.logger.Error().
+ Str("filename", segmentFileName).
+ Str("id_str", fileSegmentIDStr).
+ Err(err).
+ Msg("failed to parse segment ID from filename")
+ continue
+ }
+
+ // Convert segmentID to ShardID for progress tracking
+ shardID := common.ShardID(segmentID)
+
+ // Check if this segment has already been completed for this target segment
+ segmentIDStr := getSegmentTimeRange(targetSegmentTime, mv.targetStageInterval).String()
+ if mv.progress.IsStreamSeriesCompleted(mv.group, segmentIDStr, shardID) {
+ mv.logger.Debug().
+ Uint64("segment_id", segmentID).
+ Str("filename", segmentFileName).
+ Time("target_segment", targetSegmentTime).
+ Str("group", mv.group).
+ Msg("series segment already completed for this target segment, skipping")
+ continue
+ }
+
+ mv.logger.Info().
+ Uint64("segment_id", segmentID).
Str("filename", segmentFileName).
- Str("id_str", fileSegmentIDStr).
- Err(err).
- Msg("failed to parse segment ID from filename")
- continue
- }
-
- // Convert segmentID to ShardID for progress tracking
- shardID := common.ShardID(segmentID)
+ Time("target_segment", targetSegmentTime).
+ Str("group", mv.group).
+ Msg("migrating series segment file")
+
+ // Create file reader for the segment file
+ segmentFilePath := filepath.Join(seriesIndexPath, segmentFileName)
+ segmentFile, err := mv.lfs.OpenFile(segmentFilePath)
+ if err != nil {
+ errorMsg := fmt.Sprintf("failed to open segment file %s: %v", segmentFilePath, err)
+ mv.progress.MarkStreamSeriesError(mv.group, segmentIDStr, shardID, errorMsg)
+ mv.logger.Error().
+ Str("path", segmentFilePath).
+ Err(err).
+ Msg("failed to open segment file")
+ return fmt.Errorf("failed to open segment file %s: %w", segmentFilePath, err)
+ }
+
+ // Close the file reader
+ defer func() {
+ if i == len(targetSegments)-1 { // Only close on last iteration
+ segmentFile.Close()
+ }
+ }()
+
+ files = append(files, fileInfo{
+ name: segmentFileName,
+ file: segmentFile,
+ })
- // Check if this segment has already been completed
- segmentIDStr := segmentTR.String()
- if mv.progress.IsStreamSeriesCompleted(mv.group, segmentIDStr, shardID) {
- mv.logger.Debug().
+ mv.logger.Info().
Uint64("segment_id", segmentID).
Str("filename", segmentFileName).
+ Time("target_segment", targetSegmentTime).
Str("group", mv.group).
- Msg("segment already completed, skipping")
- continue
+ Int("completed_segments", mv.progress.GetStreamSeriesProgress(mv.group)).
+ Int("total_segments", mv.progress.GetStreamSeriesCount(mv.group)).
+ Msgf("series segment migration completed for target segment %d/%d", i+1, len(targetSegments))
}
- mv.logger.Info().
- Uint64("segment_id", segmentID).
- Str("filename", segmentFileName).
- Str("group", mv.group).
- Msg("migrating segment file")
-
- // Create file reader for the segment file
- segmentFilePath := filepath.Join(seriesIndexPath, segmentFileName)
- segmentFile, err := mv.lfs.OpenFile(segmentFilePath)
- if err != nil {
- errorMsg := fmt.Sprintf("failed to open segment file %s: %v", segmentFilePath, err)
- mv.progress.MarkStreamSeriesError(mv.group, segmentIDStr, shardID, errorMsg)
- mv.logger.Error().
- Str("path", segmentFilePath).
- Err(err).
- Msg("failed to open segment file")
- return fmt.Errorf("failed to open segment file %s: %w", segmentFilePath, err)
+ // Send segment file to each shard in shardIDs for this target segment
+ segmentIDStr := getSegmentTimeRange(targetSegmentTime, mv.targetStageInterval).String()
+ for _, shardID := range shardIDs {
+ targetShardID := mv.calculateTargetShardID(uint32(shardID))
+ ff := make([]queue.FileInfo, 0, len(files))
+ for _, file := range files {
+ ff = append(ff, queue.FileInfo{
+ Name: file.name,
+ Reader: file.file.SequentialRead(),
+ })
+ }
+ partData := mv.createStreamingSegmentFromFiles(targetShardID, ff, segmentTR, data.TopicStreamSeriesSync.String())
+
+ // Stream segment to target shard replicas
+ if err := mv.streamPartToTargetShard(partData); err != nil {
+ errorMsg := fmt.Sprintf("failed to stream segment to target shard %d: %v", targetShardID, err)
+ mv.progress.MarkStreamSeriesError(mv.group, segmentIDStr, shardID, errorMsg)
+ return fmt.Errorf("failed to stream segment to target shard %d: %w", targetShardID, err)
+ }
+ // Mark segment as completed for this specific target segment
+ mv.progress.MarkStreamSeriesCompleted(mv.group, segmentIDStr, shardID)
}
-
- // Close the file reader
- defer segmentFile.Close()
-
- files = append(files, fileInfo{
- name: segmentFileName,
- file: segmentFile,
- })
-
- mv.logger.Info().
- Uint64("segment_id", segmentID).
- Str("filename", segmentFileName).
- Str("group", mv.group).
- Int("completed_segments", mv.progress.GetStreamSeriesProgress(mv.group)).
- Int("total_segments", mv.progress.GetStreamSeriesCount(mv.group)).
- Msg("segment migration completed successfully")
- }
-
- // Send segment file to each shard in shardIDs
- segmentIDStr := segmentTR.String()
- for _, shardID := range shardIDs {
- targetShardID := mv.calculateTargetShardID(uint32(shardID))
- ff := make([]queue.FileInfo, 0, len(files))
- for _, file := range files {
- ff = append(ff, queue.FileInfo{
- Name: file.name,
- Reader: file.file.SequentialRead(),
- })
- }
- partData := mv.createStreamingSegmentFromFiles(targetShardID, ff, segmentTR, data.TopicStreamSeriesSync.String())
-
- // Stream segment to target shard replicas
- if err := mv.streamPartToTargetShard(partData); err != nil {
- errorMsg := fmt.Sprintf("failed to stream segment to target shard %d: %v", targetShardID, err)
- mv.progress.MarkStreamSeriesError(mv.group, segmentIDStr, shardID, errorMsg)
- return fmt.Errorf("failed to stream segment to target shard %d: %w", targetShardID, err)
- }
- // Mark segment as completed
- mv.progress.MarkStreamSeriesCompleted(mv.group, segmentIDStr, shardID)
}
return nil
}
// VisitPart implements stream.Visitor - core migration logic.
-func (mv *streamMigrationVisitor) VisitPart(segmentTR *timestamp.TimeRange, sourceShardID common.ShardID, partPath string) error {
+func (mv *streamMigrationVisitor) VisitPart(_ *timestamp.TimeRange, sourceShardID common.ShardID, partPath string) error {
partData, err := stream.ParsePartMetadata(mv.lfs, partPath)
if err != nil {
return fmt.Errorf("failed to parse part metadata: %w", err)
@@ -206,52 +234,69 @@ func (mv *streamMigrationVisitor) VisitPart(segmentTR *timestamp.TimeRange, sour
return fmt.Errorf("failed to parse part ID from path: %w", err)
}
- // Check if this part has already been completed
- segmentIDStr := segmentTR.String()
- if mv.progress.IsStreamPartCompleted(mv.group, segmentIDStr, sourceShardID, partID) {
- mv.logger.Debug().
- Uint64("part_id", partID).
- Uint32("source_shard", uint32(sourceShardID)).
- Str("group", mv.group).
- Msg("part already completed, skipping")
- return nil
- }
-
- // Calculate target shard ID based on source shard ID mapping
- targetShardID := mv.calculateTargetShardID(uint32(sourceShardID))
+ // Calculate ALL target segments this part should go to
+ targetSegments := calculateTargetSegments(
+ partData.MinTimestamp,
+ partData.MaxTimestamp,
+ mv.targetStageInterval,
+ )
mv.logger.Info().
Uint64("part_id", partID).
Uint32("source_shard", uint32(sourceShardID)).
- Uint32("target_shard", targetShardID).
- Str("part_path", partPath).
+ Int("target_segments_count", len(targetSegments)).
+ Int64("part_min_ts", partData.MinTimestamp).
+ Int64("part_max_ts", partData.MaxTimestamp).
Str("group", mv.group).
- Msg("migrating part")
+ Msg("migrating part to multiple target segments")
- files, release := stream.CreatePartFileReaderFromPath(partPath, mv.lfs)
- defer release()
+ // Send part to EACH target segment that overlaps with its time range
+ for i, targetSegmentTime := range targetSegments {
+ targetShardID := mv.calculateTargetShardID(uint32(sourceShardID))
- partData.Group = mv.group
- partData.ShardID = targetShardID
- partData.Topic = data.TopicStreamPartSync.String()
- partData.Files = files
+ // Check if this part has already been completed for this segment
+ segmentIDStr := getSegmentTimeRange(targetSegmentTime, mv.targetStageInterval).String()
+ if mv.progress.IsStreamPartCompleted(mv.group, segmentIDStr, sourceShardID, partID) {
+ mv.logger.Debug().
+ Uint64("part_id", partID).
+ Uint32("source_shard", uint32(sourceShardID)).
+ Time("target_segment", targetSegmentTime).
+ Str("group", mv.group).
+ Msg("part already completed for this target segment, skipping")
+ continue
+ }
- // Stream entire part to target shard replicas
- if err := mv.streamPartToTargetShard(partData); err != nil {
- errorMsg := fmt.Sprintf("failed to stream part to target shard: %v", err)
- mv.progress.MarkStreamPartError(mv.group, segmentIDStr, sourceShardID, partID, errorMsg)
- return fmt.Errorf("failed to stream part to target shard: %w", err)
- }
+ // Create file readers for this part
+ files, release := stream.CreatePartFileReaderFromPath(partPath, mv.lfs)
+ defer func() {
+ if i == len(targetSegments)-1 { // Only release on last iteration
+ release()
+ }
+ }()
+
+ // Clone part data for this target segment
+ targetPartData := partData
+ targetPartData.Group = mv.group
+ targetPartData.ShardID = targetShardID
+ targetPartData.Topic = data.TopicStreamPartSync.String()
+ targetPartData.Files = files
+
+ // Stream part to target segment
+ if err := mv.streamPartToTargetShard(targetPartData); err != nil {
+ errorMsg := fmt.Sprintf("failed to stream part to target segment %s: %v", targetSegmentTime.Format(time.RFC3339), err)
+ mv.progress.MarkStreamPartError(mv.group, segmentIDStr, sourceShardID, partID, errorMsg)
+ return fmt.Errorf("failed to stream part to target segment: %w", err)
+ }
- // Mark part as completed in progress tracker
- mv.progress.MarkStreamPartCompleted(mv.group, segmentIDStr, sourceShardID, partID)
+ // Mark part as completed for this specific target segment
+ mv.progress.MarkStreamPartCompleted(mv.group, segmentIDStr, sourceShardID, partID)
- mv.logger.Info().
- Uint64("part_id", partID).
- Str("group", mv.group).
- Int("completed_parts", mv.progress.GetStreamPartProgress(mv.group)).
- Int("total_parts", mv.progress.GetStreamPartCount(mv.group)).
- Msg("part migration completed successfully")
+ mv.logger.Info().
+ Uint64("part_id", partID).
+ Time("target_segment", targetSegmentTime).
+ Str("group", mv.group).
+ Msgf("part migration completed for target segment %d/%d", i+1, len(targetSegments))
+ }
return nil
}
diff --git a/banyand/liaison/grpc/measure.go b/banyand/liaison/grpc/measure.go
index d020849a..5d17d72b 100644
--- a/banyand/liaison/grpc/measure.go
+++ b/banyand/liaison/grpc/measure.go
@@ -197,38 +197,26 @@ func (ms *measureService) processAndPublishRequest(ctx context.Context, writeReq
func (ms *measureService) publishToNodes(ctx context.Context, writeRequest *measurev1.WriteRequest, iwr *measurev1.InternalWriteRequest,
publisher queue.BatchPublisher, shardID uint32, measure measurev1.MeasureService_WriteServer,
) ([]string, error) {
- copies, ok := ms.groupRepo.copies(writeRequest.Metadata.GetGroup())
- if !ok {
- ms.l.Error().RawJSON("written", logger.Proto(writeRequest)).Msg("failed to get the group copies")
+ nodeID, errPickNode := ms.nodeRegistry.Locate(writeRequest.GetMetadata().GetGroup(), writeRequest.GetMetadata().GetName(), shardID, 0)
+ if errPickNode != nil {
+ ms.l.Error().Err(errPickNode).RawJSON("written", logger.Proto(writeRequest)).Msg("failed to pick an available node")
ms.sendReply(writeRequest.GetMetadata(), modelv1.Status_STATUS_INTERNAL_ERROR, writeRequest.GetMessageId(), measure)
- return nil, errors.New("failed to get group copies")
+ return nil, errPickNode
}
- nodes := make([]string, 0, copies)
- for i := range copies {
- nodeID, errPickNode := ms.nodeRegistry.Locate(writeRequest.GetMetadata().GetGroup(), writeRequest.GetMetadata().GetName(), shardID, i)
- if errPickNode != nil {
- ms.l.Error().Err(errPickNode).RawJSON("written", logger.Proto(writeRequest)).Msg("failed to pick an available node")
- ms.sendReply(writeRequest.GetMetadata(), modelv1.Status_STATUS_INTERNAL_ERROR, writeRequest.GetMessageId(), measure)
- return nil, errPickNode
- }
-
- message := bus.NewBatchMessageWithNode(bus.MessageID(time.Now().UnixNano()), nodeID, iwr)
- _, errWritePub := publisher.Publish(ctx, data.TopicMeasureWrite, message)
- if errWritePub != nil {
- ms.l.Error().Err(errWritePub).RawJSON("written", logger.Proto(writeRequest)).Str("nodeID", nodeID).Msg("failed to send a message")
- var ce *common.Error
- if errors.As(errWritePub, &ce) {
- ms.sendReply(writeRequest.GetMetadata(), ce.Status(), writeRequest.GetMessageId(), measure)
- return nil, errWritePub
- }
- ms.sendReply(writeRequest.GetMetadata(), modelv1.Status_STATUS_INTERNAL_ERROR, writeRequest.GetMessageId(), measure)
+ message := bus.NewBatchMessageWithNode(bus.MessageID(time.Now().UnixNano()), nodeID, iwr)
+ _, errWritePub := publisher.Publish(ctx, data.TopicMeasureWrite, message)
+ if errWritePub != nil {
+ ms.l.Error().Err(errWritePub).RawJSON("written", logger.Proto(writeRequest)).Str("nodeID", nodeID).Msg("failed to send a message")
+ var ce *common.Error
+ if errors.As(errWritePub, &ce) {
+ ms.sendReply(writeRequest.GetMetadata(), ce.Status(), writeRequest.GetMessageId(), measure)
return nil, errWritePub
}
- nodes = append(nodes, nodeID)
+ ms.sendReply(writeRequest.GetMetadata(), modelv1.Status_STATUS_INTERNAL_ERROR, writeRequest.GetMessageId(), measure)
+ return nil, errWritePub
}
-
- return nodes, nil
+ return []string{nodeID}, nil
}
func (ms *measureService) sendReply(metadata *commonv1.Metadata, status modelv1.Status, messageID uint64, measure measurev1.MeasureService_WriteServer) {
diff --git a/banyand/liaison/grpc/property.go b/banyand/liaison/grpc/property.go
index 2fef362b..9fe4aa7e 100644
--- a/banyand/liaison/grpc/property.go
+++ b/banyand/liaison/grpc/property.go
@@ -15,23 +15,6 @@
// specific language governing permissions and limitations
// under the License.
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, "property",
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
package grpc
import (
@@ -76,56 +59,52 @@ type propertyServer struct {
repairQueueCount int
}
-func (ps *propertyServer) Apply(ctx context.Context, req *propertyv1.ApplyRequest) (resp *propertyv1.ApplyResponse, err error) {
- property := req.Property
+func (ps *propertyServer) validatePropertyRequest(property *propertyv1.Property) error {
if property.Metadata == nil {
- return nil, schema.BadRequest("metadata", "metadata should not be nil")
+ return schema.BadRequest("metadata", "metadata should not be nil")
}
if property.Metadata.Group == "" {
- return nil, schema.BadRequest("metadata.group", "group should not be nil")
+ return schema.BadRequest("metadata.group", "group should not be nil")
}
if property.Metadata.Name == "" {
- return nil, schema.BadRequest("metadata.name", "name should not be empty")
+ return schema.BadRequest("metadata.name", "name should not be empty")
}
if property.Id == "" {
- return nil, schema.BadRequest("id", "id should not be empty")
+ return schema.BadRequest("id", "id should not be empty")
}
if len(property.Tags) == 0 {
- return nil, schema.BadRequest("tags", "tags should not be empty")
+ return schema.BadRequest("tags", "tags should not be empty")
}
- g := req.Property.Metadata.Group
- ps.metrics.totalStarted.Inc(1, g, "property", "apply")
- start := time.Now()
- defer func() {
- ps.metrics.totalFinished.Inc(1, g, "property", "apply")
- ps.metrics.totalLatency.Inc(time.Since(start).Seconds(), g, "property", "apply")
- if err != nil {
- ps.metrics.totalErr.Inc(1, g, "property", "apply")
- }
- }()
- var group *commonv1.Group
- if group, err = ps.schemaRegistry.GroupRegistry().GetGroup(ctx, g); err != nil {
- return nil, errors.Errorf("group %s not found", g)
+ return nil
+}
+
+func (ps *propertyServer) validateGroupForProperty(ctx context.Context, groupName string) (*commonv1.Group, error) {
+ group, err := ps.schemaRegistry.GroupRegistry().GetGroup(ctx, groupName)
+ if err != nil {
+ return nil, errors.Errorf("group %s not found", groupName)
}
if group.Catalog != commonv1.Catalog_CATALOG_PROPERTY {
- return nil, errors.Errorf("group %s is not allowed to have properties", g)
+ return nil, errors.Errorf("group %s is not allowed to have properties", groupName)
}
if group.ResourceOpts == nil {
- return nil, errors.Errorf("group %s has no resource options", g)
+ return nil, errors.Errorf("group %s has no resource options", groupName)
}
if group.ResourceOpts.ShardNum == 0 {
- return nil, errors.Errorf("group %s has no shard number", g)
+ return nil, errors.Errorf("group %s has no shard number", groupName)
}
- var propSchema *databasev1.Property
- propSchema, err = ps.schemaRegistry.PropertyRegistry().GetProperty(ctx, &commonv1.Metadata{
- Group: g,
+ return group, nil
+}
+
+func (ps *propertyServer) validatePropertyTags(ctx context.Context, property *propertyv1.Property) error {
+ propSchema, err := ps.schemaRegistry.PropertyRegistry().GetProperty(ctx, &commonv1.Metadata{
+ Group: property.Metadata.Group,
Name: property.Metadata.Name,
})
if err != nil {
- return nil, err
+ return err
}
if len(propSchema.Tags) < len(property.Tags) {
- return nil, errors.Errorf("property %s tags count mismatch", property.Metadata.Name)
+ return errors.Errorf("property %s tags count mismatch", property.Metadata.Name)
}
for _, tag := range property.Tags {
found := false
@@ -133,14 +112,62 @@ func (ps *propertyServer) Apply(ctx context.Context, req *propertyv1.ApplyReques
if ts.Name == tag.Key {
typ := databasev1.TagType(pbv1.MustTagValueToValueType(tag.Value))
if typ != databasev1.TagType_TAG_TYPE_UNSPECIFIED && ts.Type != typ {
- return nil, errors.Errorf("property %s tag %s type mismatch", property.Metadata.Name, tag.Key)
+ return errors.Errorf("property %s tag %s type mismatch", property.Metadata.Name, tag.Key)
}
found = true
}
}
if !found {
- return nil, errors.Errorf("property %s tag %s not found", property.Metadata.Name, tag.Key)
+ return errors.Errorf("property %s tag %s not found", property.Metadata.Name, tag.Key)
+ }
+ }
+ return nil
+}
+
+func (ps *propertyServer) locateNodeSetForProperty(property *propertyv1.Property, shardID uint32) ([]string, error) {
+ copies, ok := ps.groupRepo.copies(property.Metadata.GetGroup())
+ if !ok {
+ return nil, errors.New("failed to get group copies")
+ }
+
+ nodeSet := make(map[string]struct{}, copies)
+ var nodeID string
+ var err error
+ for i := range copies {
+ nodeID, err = ps.nodeRegistry.Locate(property.GetMetadata().GetGroup(), property.GetMetadata().GetName(), shardID, i)
+ if err != nil {
+ return nil, err
+ }
+ nodeSet[nodeID] = struct{}{}
+ }
+ nodes := make([]string, 0, len(nodeSet))
+ for nodeID := range nodeSet {
+ nodes = append(nodes, nodeID)
+ }
+ return nodes, nil
+}
+
+func (ps *propertyServer) Apply(ctx context.Context, req *propertyv1.ApplyRequest) (resp *propertyv1.ApplyResponse, err error) {
+ property := req.Property
+ if err = ps.validatePropertyRequest(property); err != nil {
+ return nil, err
+ }
+ g := req.Property.Metadata.Group
+ ps.metrics.totalStarted.Inc(1, g, "property", "apply")
+ start := time.Now()
+ defer func() {
+ ps.metrics.totalFinished.Inc(1, g, "property", "apply")
+ ps.metrics.totalLatency.Inc(time.Since(start).Seconds(), g, "property", "apply")
+ if err != nil {
+ ps.metrics.totalErr.Inc(1, g, "property", "apply")
}
+ }()
+ var group *commonv1.Group
+ if group, err = ps.validateGroupForProperty(ctx, g); err != nil {
+ return nil, err
+ }
+ if err = ps.validatePropertyTags(ctx, property); err != nil {
+ return nil, err
}
nodeProperties, _, _, err := ps.queryProperties(ctx, &propertyv1.QueryRequest{
Groups: []string{g},
@@ -156,19 +183,9 @@ func (ps *propertyServer) Apply(ctx context.Context, req *propertyv1.ApplyReques
if err != nil {
return nil, err
}
- copies, ok := ps.groupRepo.copies(property.Metadata.GetGroup())
- if !ok {
- return nil, errors.New("failed to get group copies")
- }
-
- nodes := make([]string, 0, copies)
- var nodeID string
- for i := range copies {
- nodeID, err = ps.nodeRegistry.Locate(property.GetMetadata().GetGroup(), property.GetMetadata().GetName(), uint32(shardID), i)
- if err != nil {
- return nil, err
- }
- nodes = append(nodes, nodeID)
+ nodes, err := ps.locateNodeSetForProperty(property, uint32(shardID))
+ if err != nil {
+ return nil, err
}
var prev *propertyv1.Property
if prevPropertyWithMetadata != nil && prevPropertyWithMetadata.deletedTime <= 0 {
diff --git a/banyand/liaison/grpc/stream.go b/banyand/liaison/grpc/stream.go
index 8dc9ff5f..5f9949e3 100644
--- a/banyand/liaison/grpc/stream.go
+++ b/banyand/liaison/grpc/stream.go
@@ -117,26 +117,16 @@ func (s *streamService) publishMessages(
ShardId: uint32(shardID),
EntityValues: tagValues[1:].Encode(),
}
-
- copies, ok := s.groupRepo.copies(writeEntity.Metadata.GetGroup())
- if !ok {
- return nil, errors.New("failed to get group copies")
+ nodeID, err := s.nodeRegistry.Locate(writeEntity.GetMetadata().GetGroup(), writeEntity.GetMetadata().GetName(), uint32(shardID), 0)
+ if err != nil {
+ return nil, err
}
- nodes := make([]string, 0, copies)
- for i := range copies {
- nodeID, err := s.nodeRegistry.Locate(writeEntity.GetMetadata().GetGroup(), writeEntity.GetMetadata().GetName(), uint32(shardID), i)
- if err != nil {
- return nil, err
- }
-
- message := bus.NewBatchMessageWithNode(bus.MessageID(time.Now().UnixNano()), nodeID, iwr)
- if _, err := publisher.Publish(ctx, data.TopicStreamWrite, message); err != nil {
- return nil, err
- }
- nodes = append(nodes, nodeID)
+ message := bus.NewBatchMessageWithNode(bus.MessageID(time.Now().UnixNano()), nodeID, iwr)
+ if _, err := publisher.Publish(ctx, data.TopicStreamWrite, message); err != nil {
+ return nil, err
}
- return nodes, nil
+ return []string{nodeID}, nil
}
func (s *streamService) Write(stream streamv1.StreamService_WriteServer) error {
diff --git a/banyand/measure/introducer.go b/banyand/measure/introducer.go
index b28538ca..f8f3f811 100644
--- a/banyand/measure/introducer.go
+++ b/banyand/measure/introducer.go
@@ -75,6 +75,7 @@ func generateFlusherIntroduction() *flusherIntroduction {
}
func releaseFlusherIntroduction(i *flusherIntroduction) {
+ i.reset()
flusherIntroductionPool.Put(i)
}
diff --git a/banyand/measure/metadata.go b/banyand/measure/metadata.go
index 9402b37a..ea90301e 100644
--- a/banyand/measure/metadata.go
+++ b/banyand/measure/metadata.go
@@ -559,13 +559,17 @@ func (s *queueSupplier) OpenDB(groupSchema *commonv1.Group) (resourceSchema.DB,
SubQueueCreator: newWriteQueue,
GetNodes: func(shardID common.ShardID) []string {
copies := ro.Replicas + 1
- nodes := make([]string, 0, copies)
+ nodeSet := make(map[string]struct{}, copies)
for i := uint32(0); i < copies; i++ {
nodeID, err := s.measureDataNodeRegistry.Locate(group, "", uint32(shardID), i)
if err != nil {
s.l.Error().Err(err).Str("group", group).Uint32("shard", uint32(shardID)).Uint32("copy", i).Msg("failed to locate node")
return nil
}
+ nodeSet[nodeID] = struct{}{}
+ }
+ nodes := make([]string, 0, len(nodeSet))
+ for nodeID := range nodeSet {
nodes = append(nodes, nodeID)
}
return nodes
diff --git a/banyand/stream/metadata.go b/banyand/stream/metadata.go
index 112db8b7..4a5b6d5f 100644
--- a/banyand/stream/metadata.go
+++ b/banyand/stream/metadata.go
@@ -434,13 +434,17 @@ func (s *queueSupplier) OpenDB(groupSchema *commonv1.Group) (resourceSchema.DB,
SubQueueCreator: newWriteQueue,
GetNodes: func(shardID common.ShardID) []string {
copies := ro.Replicas + 1
- nodes := make([]string, 0, copies)
+ nodeSet := make(map[string]struct{}, copies)
for i := uint32(0); i < copies; i++ {
nodeID, err := s.streamDataNodeRegistry.Locate(group, "", uint32(shardID), i)
if err != nil {
s.l.Error().Err(err).Str("group", group).Uint32("shard", uint32(shardID)).Uint32("copy", i).Msg("failed to locate node")
return nil
}
+ nodeSet[nodeID] = struct{}{}
+ }
+ nodes := make([]string, 0, len(nodeSet))
+ for nodeID := range nodeSet {
nodes = append(nodes, nodeID)
}
return nodes
diff --git a/banyand/stream/tstable.go b/banyand/stream/tstable.go
index 817ef7eb..5809ac42 100644
--- a/banyand/stream/tstable.go
+++ b/banyand/stream/tstable.go
@@ -226,6 +226,9 @@ func initTSTable(fileSystem fs.FileSystem, rootPath string, p common.Position,
if ee[i].Name() == elementIndexFilename {
continue
}
+ if ee[i].Name() == inverted.ExternalSegmentTempDirName {
+ continue
+ }
p, err := parseEpoch(ee[i].Name())
if err != nil {
l.Info().Err(err).Msg("cannot parse part file name. skip and delete it")
diff --git a/banyand/stream/write_standalone.go b/banyand/stream/write_standalone.go
index ce4da15d..71f01961 100644
--- a/banyand/stream/write_standalone.go
+++ b/banyand/stream/write_standalone.go
@@ -166,7 +166,7 @@ func processElements(schemaRepo *schemaRepo, elements *elements, writeEvent *str
req := writeEvent.Request
elements.timestamps = append(elements.timestamps, ts)
- eID := convert.HashStr(req.Metadata.Name + "|" + req.Element.ElementId)
+ eID := convert.HashStr(req.Metadata.Group + "|" + req.Metadata.Name + "|" + req.Element.ElementId)
elements.elementIDs = append(elements.elementIDs, eID)
stm, ok := schemaRepo.loadStream(writeEvent.GetRequest().GetMetadata())
diff --git a/pkg/fs/local_file_system.go b/pkg/fs/local_file_system.go
index 92ffb433..49f24f7b 100644
--- a/pkg/fs/local_file_system.go
+++ b/pkg/fs/local_file_system.go
@@ -462,6 +462,10 @@ func (file *LocalFile) SequentialRead() SeqReader {
// Use the cached parameter from upper layer instead of local file's cached field
// This allows override when needed (e.g. metadata always cached)
+ // Reset file position to beginning to ensure each SeqReader starts from the beginning
+ // If seek fails, we'll still try to create the reader and the error will be handled when reading
+ _, _ = file.file.Seek(0, 0)
+
reader := generateReader(file.file, file.ioSize)
return &seqReader{reader: reader, file: file.file, fileName: file.file.Name(), length: 0, skipFadvise: file.cached}
}
diff --git a/pkg/fs/local_file_system_test.go b/pkg/fs/local_file_system_test.go
index c82739e1..b96dc9dc 100644
--- a/pkg/fs/local_file_system_test.go
+++ b/pkg/fs/local_file_system_test.go
@@ -32,6 +32,7 @@ import (
var _ = ginkgo.Describe("Local File System", func() {
const (
data string = "BanyanDB"
+ testData string = "Hello BanyanDB World"
dirName string = "tmpDir"
fileName string = "tmpDir/temFile"
flushFileName string = "tmpDir/tempFlushFile"
@@ -123,6 +124,166 @@ var _ = ginkgo.Describe("Local File System", func() {
}
})
+ ginkgo.It("SeqReader Multiple Complete Reads", func() {
+ // Write test data
+ testStr := testData
+ size, err := file.Write([]byte(testStr))
+ gomega.Expect(err).ToNot(gomega.HaveOccurred())
+ gomega.Expect(size == len(testStr)).To(gomega.BeTrue())
+
+ // First SeqReader - read entire file
+ reader1 := file.SequentialRead()
+ buffer1 := make([]byte, len(testStr))
+ readSize1, err := reader1.Read(buffer1)
+ gomega.Expect(err).ToNot(gomega.HaveOccurred())
+ gomega.Expect(readSize1).To(gomega.Equal(len(testStr)))
+ gomega.Expect(string(buffer1)).To(gomega.Equal(testStr))
+ err = reader1.Close()
+ gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
+ // Second SeqReader - should read entire file again (fixed behavior)
+ reader2 := file.SequentialRead()
+ buffer2 := make([]byte, len(testStr))
+ readSize2, err := reader2.Read(buffer2)
+ gomega.Expect(err).ToNot(gomega.HaveOccurred())
+ gomega.Expect(readSize2).To(gomega.Equal(len(testStr)))
+ gomega.Expect(string(buffer2)).To(gomega.Equal(testStr))
+ err = reader2.Close()
+ gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
+ // Third SeqReader - should read entire file again (fixed behavior)
+ reader3 := file.SequentialRead()
+ buffer3 := make([]byte, len(testStr))
+ readSize3, err := reader3.Read(buffer3)
+ gomega.Expect(err).ToNot(gomega.HaveOccurred())
+ gomega.Expect(readSize3).To(gomega.Equal(len(testStr)))
+ gomega.Expect(string(buffer3)).To(gomega.Equal(testStr))
+ err = reader3.Close()
+ gomega.Expect(err).ToNot(gomega.HaveOccurred())
+ })
+
+ ginkgo.It("SeqReader Partial Reads", func() {
+ // Write test data
+ testStr := testData
+ size, err := file.Write([]byte(testStr))
+ gomega.Expect(err).ToNot(gomega.HaveOccurred())
+ gomega.Expect(size == len(testStr)).To(gomega.BeTrue())
+
+ // First SeqReader - read first part
+ reader1 := file.SequentialRead()
+ buffer1 := make([]byte, 5) // Read only first 5 bytes
+ readSize1, err := reader1.Read(buffer1)
+ gomega.Expect(err).ToNot(gomega.HaveOccurred())
+ gomega.Expect(readSize1).To(gomega.Equal(5))
+ gomega.Expect(string(buffer1)).To(gomega.Equal("Hello"))
+ err = reader1.Close()
+ gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
+ // Second SeqReader - should read from beginning again (fixed behavior)
+ reader2 := file.SequentialRead()
+ buffer2 := make([]byte, 5) // Read only first 5 bytes
+ readSize2, err := reader2.Read(buffer2)
+ gomega.Expect(err).ToNot(gomega.HaveOccurred())
+ gomega.Expect(readSize2).To(gomega.Equal(5))
+ gomega.Expect(string(buffer2)).To(gomega.Equal("Hello"))
+ err = reader2.Close()
+ gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
+ // Third SeqReader - should read from beginning again (fixed behavior)
+ reader3 := file.SequentialRead()
+ buffer3 := make([]byte, 5) // Read only first 5 bytes
+ readSize3, err := reader3.Read(buffer3)
+ gomega.Expect(err).ToNot(gomega.HaveOccurred())
+ gomega.Expect(readSize3).To(gomega.Equal(5))
+ gomega.Expect(string(buffer3)).To(gomega.Equal("Hello"))
+ err = reader3.Close()
+ gomega.Expect(err).ToNot(gomega.HaveOccurred())
+ })
+
+ ginkgo.It("SeqReader Mixed Read Patterns", func() {
+ // Write test data
+ testStr := testData
+ size, err := file.Write([]byte(testStr))
+ gomega.Expect(err).ToNot(gomega.HaveOccurred())
+ gomega.Expect(size == len(testStr)).To(gomega.BeTrue())
+
+ // First SeqReader - read entire file
+ reader1 := file.SequentialRead()
+ buffer1 := make([]byte, len(testStr))
+ readSize1, err := reader1.Read(buffer1)
+ gomega.Expect(err).ToNot(gomega.HaveOccurred())
+ gomega.Expect(readSize1).To(gomega.Equal(len(testStr)))
+ gomega.Expect(string(buffer1)).To(gomega.Equal(testStr))
+ err = reader1.Close()
+ gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
+ // Second SeqReader - read only part
+ reader2 := file.SequentialRead()
+ buffer2 := make([]byte, 5) // Read only first 5 bytes
+ readSize2, err := reader2.Read(buffer2)
+ gomega.Expect(err).ToNot(gomega.HaveOccurred())
+ gomega.Expect(readSize2).To(gomega.Equal(5))
+ gomega.Expect(string(buffer2)).To(gomega.Equal("Hello"))
+ err = reader2.Close()
+ gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
+ // Third SeqReader - read entire file again
+ reader3 := file.SequentialRead()
+ buffer3 := make([]byte, len(testStr))
+ readSize3, err := reader3.Read(buffer3)
+ gomega.Expect(err).ToNot(gomega.HaveOccurred())
+ gomega.Expect(readSize3).To(gomega.Equal(len(testStr)))
+ gomega.Expect(string(buffer3)).To(gomega.Equal(testStr))
+ err = reader3.Close()
+ gomega.Expect(err).ToNot(gomega.HaveOccurred())
+ })
+
+ ginkgo.It("SeqReader File Position Reset", func() {
+ // Write test data
+ testStr := testData
+ size, err := file.Write([]byte(testStr))
+ gomega.Expect(err).ToNot(gomega.HaveOccurred())
+ gomega.Expect(size == len(testStr)).To(gomega.BeTrue())
+
+ // Cast to LocalFile to access the underlying file
+ localFile, ok := file.(*LocalFile)
+ gomega.Expect(ok).To(gomega.BeTrue())
+
+ // Reset file position to beginning for testing
+ _, err = localFile.file.Seek(0, 0) // Seek to beginning
+ gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
+ // First SeqReader - read only part of the data and close
+ reader1 := file.SequentialRead()
+ buffer1 := make([]byte, 5) // Read only first 5 bytes
+ readSize1, err := reader1.Read(buffer1)
+ gomega.Expect(err).ToNot(gomega.HaveOccurred())
+ gomega.Expect(readSize1).To(gomega.Equal(5))
+ gomega.Expect(string(buffer1)).To(gomega.Equal("Hello"))
+ err = reader1.Close()
+ gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
+ // Second SeqReader - should now read from beginning (fixed behavior)
+ reader2 := file.SequentialRead()
+
+ // Check file position after creating second reader
+ posAfterSecond, err := localFile.file.Seek(0, 1) // Get current position
+ gomega.Expect(err).ToNot(gomega.HaveOccurred())
+ // Fixed: position should be 0 (reset to beginning)
+ gomega.Expect(posAfterSecond).To(gomega.Equal(int64(0)))
+
+ buffer2 := make([]byte, len(testStr))
+ readSize2, err := reader2.Read(buffer2)
+ gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
+ // Should read the full data from beginning (fixed behavior)
+ gomega.Expect(readSize2).To(gomega.Equal(len(testStr)))
+ gomega.Expect(string(buffer2)).To(gomega.Equal(testStr))
+
+ err = reader2.Close()
+ gomega.Expect(err).ToNot(gomega.HaveOccurred())
+ })
+
ginkgo.It("Size Test", func() {
size, err := file.Write([]byte(data))
gomega.Expect(err).ToNot(gomega.HaveOccurred())
diff --git a/pkg/query/logical/stream/stream_plan_distributed.go b/pkg/query/logical/stream/stream_plan_distributed.go
index ba55e3af..7827f56f 100644
--- a/pkg/query/logical/stream/stream_plan_distributed.go
+++ b/pkg/query/logical/stream/stream_plan_distributed.go
@@ -175,11 +175,17 @@ func (t *distributedPlan) Execute(ctx context.Context) (ee []*streamv1.Element,
newSortableElements(resp.Elements, t.sortByTime, t.sortTagSpec))
}
}
- iter := sort.NewItemIter[*comparableElement](see, t.desc)
+ iter := sort.NewItemIter(see, t.desc)
var result []*streamv1.Element
+ seen := make(map[string]bool)
for iter.Next() {
- result = append(result, iter.Val().Element)
+ element := iter.Val().Element
+ if !seen[element.ElementId] {
+ seen[element.ElementId] = true
+ result = append(result, element)
+ }
}
+
return result, allErr
}
diff --git a/pkg/query/logical/stream/stream_plan_merge.go b/pkg/query/logical/stream/stream_plan_merge.go
index 18e7a3f0..b1eda4d5 100644
--- a/pkg/query/logical/stream/stream_plan_merge.go
+++ b/pkg/query/logical/stream/stream_plan_merge.go
@@ -135,11 +135,7 @@ func (m *mergePlan) Execute(ctx context.Context) ([]*streamv1.Element, error) {
continue
}
- iter := &sortableElements{
- elements: elements,
- isSortByTime: m.sortByTime,
- sortTagSpec: m.sortTagSpec,
- }
+ iter := newSortableElements(elements, m.sortByTime, m.sortTagSpec)
see = append(see, iter)
}
diff --git a/pkg/test/measure/testdata/groups/sw_metric.json b/pkg/test/measure/testdata/groups/sw_metric.json
index a91becb7..7b22cf48 100644
--- a/pkg/test/measure/testdata/groups/sw_metric.json
+++ b/pkg/test/measure/testdata/groups/sw_metric.json
@@ -12,7 +12,8 @@
"ttl": {
"unit": "UNIT_DAY",
"num": 7
- }
+ },
+ "replicas": 1
},
"updated_at": "2021-04-15T01:30:15.01Z"
}
\ No newline at end of file
diff --git a/pkg/test/stream/testdata/group.json b/pkg/test/stream/testdata/group.json
index 352560dd..e4116dd7 100644
--- a/pkg/test/stream/testdata/group.json
+++ b/pkg/test/stream/testdata/group.json
@@ -1,35 +1,39 @@
-[{
- "metadata": {
- "name": "default"
- },
- "catalog": "CATALOG_STREAM",
- "resource_opts": {
- "shard_num": 2,
- "segment_interval": {
- "unit": "UNIT_DAY",
- "num": 1
+[
+ {
+ "metadata": {
+ "name": "default"
},
- "ttl": {
- "unit": "UNIT_DAY",
- "num": 3
- }
- },
- "updated_at": "2021-04-15T01:30:15.01Z"
-},{
- "metadata": {
- "name": "updated"
- },
- "catalog": "CATALOG_STREAM",
- "resource_opts": {
- "shard_num": 2,
- "segment_interval": {
- "unit": "UNIT_DAY",
- "num": 1
+ "catalog": "CATALOG_STREAM",
+ "resource_opts": {
+ "shard_num": 2,
+ "replicas": 1,
+ "segment_interval": {
+ "unit": "UNIT_DAY",
+ "num": 1
+ },
+ "ttl": {
+ "unit": "UNIT_DAY",
+ "num": 3
+ }
},
- "ttl": {
- "unit": "UNIT_DAY",
- "num": 3
- }
+ "updated_at": "2021-04-15T01:30:15.01Z"
},
- "updated_at": "2021-04-15T01:30:15.01Z"
-}]
\ No newline at end of file
+ {
+ "metadata": {
+ "name": "updated"
+ },
+ "catalog": "CATALOG_STREAM",
+ "resource_opts": {
+ "shard_num": 2,
+ "segment_interval": {
+ "unit": "UNIT_DAY",
+ "num": 1
+ },
+ "ttl": {
+ "unit": "UNIT_DAY",
+ "num": 3
+ }
+ },
+ "updated_at": "2021-04-15T01:30:15.01Z"
+ }
+]
\ No newline at end of file
diff --git a/test/cases/init.go b/test/cases/init.go
index 6dd05a41..5e7d0543 100644
--- a/test/cases/init.go
+++ b/test/cases/init.go
@@ -59,4 +59,5 @@ func Initialize(addr string, now time.Time) {
casesmeasuredata.Write(conn, "endpoint_traffic", "sw_metric", "endpoint_traffic.json", now, interval)
casesmeasuredata.Write(conn, "duplicated", "exception", "duplicated.json", now, 0)
casesmeasuredata.Write(conn, "service_cpm_minute", "sw_updated", "service_cpm_minute_updated_data.json", now.Add(10*time.Minute), interval)
+ time.Sleep(5 * time.Second)
}
diff --git a/test/cases/stream/data/data.go b/test/cases/stream/data/data.go
index 3316179c..df44820c 100644
--- a/test/cases/stream/data/data.go
+++ b/test/cases/stream/data/data.go
@@ -22,7 +22,6 @@ import (
"context"
"embed"
"encoding/base64"
- "encoding/hex"
"encoding/json"
"fmt"
"io"
@@ -44,7 +43,6 @@ import (
databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
streamv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/stream/v1"
- "github.com/apache/skywalking-banyandb/pkg/convert"
"github.com/apache/skywalking-banyandb/pkg/test/flags"
"github.com/apache/skywalking-banyandb/pkg/test/helpers"
)
@@ -87,11 +85,6 @@ var VerifyFn = func(innerGm gm.Gomega, sharedContext helpers.SharedContext, args
innerGm.Expect(err).NotTo(gm.HaveOccurred())
want := &streamv1.QueryResponse{}
helpers.UnmarshalYAML(ww, want)
- if !args.IgnoreElementID {
- for i := range want.Elements {
- want.Elements[i].ElementId = hex.EncodeToString(convert.Uint64ToBytes(convert.HashStr(query.Name + "|" + want.Elements[i].ElementId)))
- }
- }
if args.DisOrder {
slices.SortFunc(want.Elements, func(a, b *streamv1.Element) int {
return strings.Compare(a.ElementId, b.ElementId)
diff --git a/test/cases/stream/data/want/all.yaml b/test/cases/stream/data/want/all.yaml
index ac797cbd..eeaf5b4e 100644
--- a/test/cases/stream/data/want/all.yaml
+++ b/test/cases/stream/data/want/all.yaml
@@ -16,7 +16,7 @@
# under the License.
elements:
- - elementId: "0"
+ - elementId: "0978df79cf4ed409"
tagFamilies:
- name: searchable
tags:
@@ -29,7 +29,7 @@ elements:
- key: data_binary
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
- - elementId: "1"
+ - elementId: "4dd999e3502d728d"
tagFamilies:
- name: searchable
tags:
@@ -42,7 +42,7 @@ elements:
- key: data_binary
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
- - elementId: "2"
+ - elementId: "fafaa63b403a1604"
tagFamilies:
- name: searchable
tags:
@@ -55,7 +55,7 @@ elements:
- key: data_binary
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
- - elementId: "3"
+ - elementId: "5fdc8aab6ea5c9d4"
tagFamilies:
- name: searchable
tags:
@@ -68,7 +68,7 @@ elements:
- key: data_binary
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
- - elementId: "4"
+ - elementId: "aaede9362761569a"
tagFamilies:
- name: searchable
tags:
diff --git a/test/cases/stream/data/want/duplicated_all.yaml b/test/cases/stream/data/want/duplicated_all.yaml
index 949a9574..3ed011b9 100644
--- a/test/cases/stream/data/want/duplicated_all.yaml
+++ b/test/cases/stream/data/want/duplicated_all.yaml
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
elements:
-- elementId: "3"
+- elementId: "93423d5c731c53ed"
tagFamilies:
- name: searchable
tags:
@@ -41,7 +41,7 @@ elements:
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
timestamp: "2024-07-04T10:35:00Z"
-- elementId: "4"
+- elementId: "dbc685c3e6aac8b7"
tagFamilies:
- name: searchable
tags:
@@ -67,7 +67,7 @@ elements:
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
timestamp: "2024-07-04T10:35:00Z"
-- elementId: "0"
+- elementId: "5df4d9d2f751488d"
tagFamilies:
- name: searchable
tags:
@@ -93,7 +93,7 @@ elements:
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
timestamp: "2024-07-04T10:35:00Z"
-- elementId: "1"
+- elementId: "0a6f72178f0fdbeb"
tagFamilies:
- name: searchable
tags:
@@ -119,7 +119,7 @@ elements:
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
timestamp: "2024-07-04T10:35:00Z"
-- elementId: "2"
+- elementId: "b55ed52d70bbfbe7"
tagFamilies:
- name: searchable
tags:
diff --git a/test/cases/stream/data/want/duplicated_entity_filter.yaml b/test/cases/stream/data/want/duplicated_entity_filter.yaml
index 143dcdc4..742cfedf 100644
--- a/test/cases/stream/data/want/duplicated_entity_filter.yaml
+++ b/test/cases/stream/data/want/duplicated_entity_filter.yaml
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
elements:
-- elementId: "3"
+- elementId: "93423d5c731c53ed"
tagFamilies:
- name: searchable
tags:
@@ -40,7 +40,7 @@ elements:
- key: data_binary
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
-- elementId: "4"
+- elementId: "dbc685c3e6aac8b7"
tagFamilies:
- name: searchable
tags:
diff --git a/test/cases/stream/data/want/duplicated_index_filter.yaml b/test/cases/stream/data/want/duplicated_index_filter.yaml
index 92d040ad..31da7b73 100644
--- a/test/cases/stream/data/want/duplicated_index_filter.yaml
+++ b/test/cases/stream/data/want/duplicated_index_filter.yaml
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
elements:
-- elementId: "3"
+- elementId: "93423d5c731c53ed"
tagFamilies:
- name: searchable
tags:
@@ -40,7 +40,7 @@ elements:
- key: data_binary
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
-- elementId: "0"
+- elementId: "5df4d9d2f751488d"
tagFamilies:
- name: searchable
tags:
diff --git a/test/cases/stream/data/want/duplicated_order_by_filter.yaml b/test/cases/stream/data/want/duplicated_order_by_filter.yaml
index 6556f87c..08b803ce 100644
--- a/test/cases/stream/data/want/duplicated_order_by_filter.yaml
+++ b/test/cases/stream/data/want/duplicated_order_by_filter.yaml
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
elements:
-- elementId: "0"
+- elementId: "5df4d9d2f751488d"
tagFamilies:
- name: searchable
tags:
@@ -40,7 +40,7 @@ elements:
- key: data_binary
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
-- elementId: "3"
+- elementId: "93423d5c731c53ed"
tagFamilies:
- name: searchable
tags:
diff --git a/test/cases/stream/data/want/duplicated_order_by_index.yaml b/test/cases/stream/data/want/duplicated_order_by_index.yaml
index 35209f3f..250bac23 100644
--- a/test/cases/stream/data/want/duplicated_order_by_index.yaml
+++ b/test/cases/stream/data/want/duplicated_order_by_index.yaml
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
elements:
-- elementId: "1"
+- elementId: "0a6f72178f0fdbeb"
tagFamilies:
- name: searchable
tags:
@@ -41,7 +41,7 @@ elements:
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
timestamp: "2024-07-13T04:13:00Z"
-- elementId: "0"
+- elementId: "5df4d9d2f751488d"
tagFamilies:
- name: searchable
tags:
@@ -67,7 +67,7 @@ elements:
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
timestamp: "2024-07-13T04:13:00Z"
-- elementId: "2"
+- elementId: "b55ed52d70bbfbe7"
tagFamilies:
- name: searchable
tags:
@@ -93,7 +93,7 @@ elements:
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
timestamp: "2024-07-13T04:13:00Z"
-- elementId: "4"
+- elementId: "dbc685c3e6aac8b7"
tagFamilies:
- name: searchable
tags:
@@ -119,7 +119,7 @@ elements:
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
timestamp: "2024-07-13T04:13:00Z"
-- elementId: "3"
+- elementId: "93423d5c731c53ed"
tagFamilies:
- name: searchable
tags:
diff --git a/test/cases/stream/data/want/filter_no_indexed.yaml b/test/cases/stream/data/want/filter_no_indexed.yaml
index 1df5c531..7fc4ed02 100644
--- a/test/cases/stream/data/want/filter_no_indexed.yaml
+++ b/test/cases/stream/data/want/filter_no_indexed.yaml
@@ -16,7 +16,7 @@
# under the License.
elements:
-- elementId: "0"
+- elementId: "0978df79cf4ed409"
tagFamilies:
- name: searchable
tags:
@@ -32,7 +32,7 @@ elements:
value:
int:
value: "1622933202000000000"
-- elementId: "1"
+- elementId: "4dd999e3502d728d"
tagFamilies:
- name: searchable
tags:
diff --git a/test/cases/stream/data/want/filter_no_indexed_or.yaml b/test/cases/stream/data/want/filter_no_indexed_or.yaml
index e560f7de..d4749648 100644
--- a/test/cases/stream/data/want/filter_no_indexed_or.yaml
+++ b/test/cases/stream/data/want/filter_no_indexed_or.yaml
@@ -16,7 +16,7 @@
# under the License.
elements:
- - elementId: "0"
+ - elementId: "0978df79cf4ed409"
tagFamilies:
- name: searchable
tags:
@@ -36,7 +36,7 @@ elements:
value:
int:
value: "1622933202000000000"
- - elementId: "1"
+ - elementId: "4dd999e3502d728d"
tagFamilies:
- name: searchable
tags:
@@ -56,7 +56,7 @@ elements:
value:
int:
value: "1622933202000000000"
- - elementId: "2"
+ - elementId: "fafaa63b403a1604"
tagFamilies:
- name: searchable
tags:
@@ -75,7 +75,7 @@ elements:
value:
int:
value: "1622933202000000000"
- - elementId: "3"
+ - elementId: "5fdc8aab6ea5c9d4"
tagFamilies:
- name: searchable
tags:
@@ -94,7 +94,7 @@ elements:
value:
int:
value: "1622933202000000000"
- - elementId: "4"
+ - elementId: "aaede9362761569a"
tagFamilies:
- name: searchable
tags:
diff --git a/test/cases/stream/data/want/filter_order_desc.yaml b/test/cases/stream/data/want/filter_order_desc.yaml
index 1dc6620e..bbb2fda2 100644
--- a/test/cases/stream/data/want/filter_order_desc.yaml
+++ b/test/cases/stream/data/want/filter_order_desc.yaml
@@ -16,7 +16,7 @@
# under the License.
elements:
- - elementId: "4"
+ - elementId: "aaede9362761569a"
tagFamilies:
- name: searchable
tags:
@@ -33,7 +33,7 @@ elements:
- key: data_binary
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
- - elementId: "3"
+ - elementId: "5fdc8aab6ea5c9d4"
tagFamilies:
- name: searchable
tags:
diff --git a/test/cases/stream/data/want/filter_tag.yaml b/test/cases/stream/data/want/filter_tag.yaml
index a8f4ae77..81045ca9 100644
--- a/test/cases/stream/data/want/filter_tag.yaml
+++ b/test/cases/stream/data/want/filter_tag.yaml
@@ -16,7 +16,7 @@
# under the License.
elements:
- - elementId: "0"
+ - elementId: "0978df79cf4ed409"
tagFamilies:
- name: searchable
tags:
diff --git a/test/cases/stream/data/want/global_index.yaml b/test/cases/stream/data/want/global_index.yaml
index d584585c..7a0dc8a9 100644
--- a/test/cases/stream/data/want/global_index.yaml
+++ b/test/cases/stream/data/want/global_index.yaml
@@ -16,7 +16,7 @@
# under the License.
elements:
- - elementId: "0"
+ - elementId: "0978df79cf4ed409"
tagFamilies:
- name: searchable
tags:
diff --git a/test/cases/stream/data/want/global_indices.yaml b/test/cases/stream/data/want/global_indices.yaml
index ba5e6514..1ab54abe 100644
--- a/test/cases/stream/data/want/global_indices.yaml
+++ b/test/cases/stream/data/want/global_indices.yaml
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
elements:
- - elementId: "0"
+ - elementId: "0978df79cf4ed409"
tagFamilies:
- name: searchable
tags:
@@ -28,7 +28,7 @@ elements:
- key: data_binary
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
- - elementId: "2"
+ - elementId: "fafaa63b403a1604"
tagFamilies:
- name: searchable
tags:
@@ -41,7 +41,7 @@ elements:
- key: data_binary
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
- - elementId: "4"
+ - elementId: "aaede9362761569a"
tagFamilies:
- name: searchable
tags:
diff --git a/test/cases/stream/data/want/having.yaml b/test/cases/stream/data/want/having.yaml
index c26633df..519ceb9f 100644
--- a/test/cases/stream/data/want/having.yaml
+++ b/test/cases/stream/data/want/having.yaml
@@ -16,7 +16,7 @@
# under the License.
elements:
- - elementId: "3"
+ - elementId: "5fdc8aab6ea5c9d4"
tagFamilies:
- name: searchable
tags:
@@ -35,7 +35,7 @@ elements:
- key: data_binary
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
- - elementId: "4"
+ - elementId: "aaede9362761569a"
tagFamilies:
- name: searchable
tags:
diff --git a/test/cases/stream/data/want/having_non_indexed.yaml b/test/cases/stream/data/want/having_non_indexed.yaml
index f63057e3..eee92b38 100644
--- a/test/cases/stream/data/want/having_non_indexed.yaml
+++ b/test/cases/stream/data/want/having_non_indexed.yaml
@@ -16,7 +16,7 @@
# under the License.
elements:
- - elementId: "2"
+ - elementId: "fafaa63b403a1604"
tagFamilies:
- name: searchable
tags:
@@ -34,7 +34,7 @@ elements:
- key: data_binary
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
- - elementId: "3"
+ - elementId: "5fdc8aab6ea5c9d4"
tagFamilies:
- name: searchable
tags:
@@ -53,7 +53,7 @@ elements:
- key: data_binary
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
- - elementId: "4"
+ - elementId: "aaede9362761569a"
tagFamilies:
- name: searchable
tags:
diff --git a/test/cases/stream/data/want/having_non_indexed_arr.yaml b/test/cases/stream/data/want/having_non_indexed_arr.yaml
index df924ec1..dff9f565 100644
--- a/test/cases/stream/data/want/having_non_indexed_arr.yaml
+++ b/test/cases/stream/data/want/having_non_indexed_arr.yaml
@@ -16,7 +16,7 @@
# under the License.
elements:
- - elementId: "3"
+ - elementId: "5fdc8aab6ea5c9d4"
tagFamilies:
- name: searchable
tags:
@@ -35,7 +35,7 @@ elements:
- key: data_binary
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
- - elementId: "4"
+ - elementId: "aaede9362761569a"
tagFamilies:
- name: searchable
tags:
diff --git a/test/cases/stream/data/want/less.yaml b/test/cases/stream/data/want/less.yaml
index 996bfa3b..83bd1129 100644
--- a/test/cases/stream/data/want/less.yaml
+++ b/test/cases/stream/data/want/less.yaml
@@ -16,7 +16,7 @@
# under the License.
elements:
- - elementId: "2"
+ - elementId: "fafaa63b403a1604"
tagFamilies:
- name: searchable
tags:
@@ -33,7 +33,7 @@ elements:
- key: data_binary
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
- - elementId: "3"
+ - elementId: "5fdc8aab6ea5c9d4"
tagFamilies:
- name: searchable
tags:
@@ -50,7 +50,7 @@ elements:
- key: data_binary
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
- - elementId: "4"
+ - elementId: "aaede9362761569a"
tagFamilies:
- name: searchable
tags:
diff --git a/test/cases/stream/data/want/less_eq.yaml b/test/cases/stream/data/want/less_eq.yaml
index 504f233a..56a4fef8 100644
--- a/test/cases/stream/data/want/less_eq.yaml
+++ b/test/cases/stream/data/want/less_eq.yaml
@@ -16,7 +16,7 @@
# under the License.
elements:
- - elementId: "1"
+ - elementId: "4dd999e3502d728d"
tagFamilies:
- name: searchable
tags:
@@ -33,7 +33,7 @@ elements:
- key: data_binary
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
- - elementId: "2"
+ - elementId: "fafaa63b403a1604"
tagFamilies:
- name: searchable
tags:
@@ -50,7 +50,7 @@ elements:
- key: data_binary
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
- - elementId: "3"
+ - elementId: "5fdc8aab6ea5c9d4"
tagFamilies:
- name: searchable
tags:
@@ -67,7 +67,7 @@ elements:
- key: data_binary
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
- - elementId: "4"
+ - elementId: "aaede9362761569a"
tagFamilies:
- name: searchable
tags:
diff --git a/test/cases/stream/data/want/limit.yaml b/test/cases/stream/data/want/limit.yaml
index a03ae160..db876079 100644
--- a/test/cases/stream/data/want/limit.yaml
+++ b/test/cases/stream/data/want/limit.yaml
@@ -16,7 +16,7 @@
# under the License.
elements:
- - elementId: "0"
+ - elementId: "0978df79cf4ed409"
tagFamilies:
- name: searchable
tags:
@@ -29,7 +29,7 @@ elements:
- key: data_binary
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
- - elementId: "1"
+ - elementId: "4dd999e3502d728d"
tagFamilies:
- name: searchable
tags:
@@ -42,7 +42,7 @@ elements:
- key: data_binary
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
- - elementId: "2"
+ - elementId: "fafaa63b403a1604"
tagFamilies:
- name: searchable
tags:
diff --git a/test/cases/stream/data/want/logical.yaml b/test/cases/stream/data/want/logical.yaml
index f01bcaed..aba142a9 100644
--- a/test/cases/stream/data/want/logical.yaml
+++ b/test/cases/stream/data/want/logical.yaml
@@ -16,7 +16,7 @@
# under the License.
elements:
- - elementId: "2"
+ - elementId: "fafaa63b403a1604"
tagFamilies:
- name: searchable
tags:
@@ -41,7 +41,7 @@ elements:
- key: data_binary
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
- - elementId: "3"
+ - elementId: "5fdc8aab6ea5c9d4"
tagFamilies:
- name: searchable
tags:
diff --git a/test/cases/stream/data/want/offset.yaml b/test/cases/stream/data/want/offset.yaml
index ebbe32e0..2852068f 100644
--- a/test/cases/stream/data/want/offset.yaml
+++ b/test/cases/stream/data/want/offset.yaml
@@ -16,7 +16,7 @@
# under the License.
elements:
-- elementId: "3"
+- elementId: "5fdc8aab6ea5c9d4"
tagFamilies:
- name: searchable
tags:
diff --git a/test/cases/stream/data/want/order_asc.yaml b/test/cases/stream/data/want/order_asc.yaml
index 725cf084..168b70b8 100644
--- a/test/cases/stream/data/want/order_asc.yaml
+++ b/test/cases/stream/data/want/order_asc.yaml
@@ -16,7 +16,7 @@
# under the License.
elements:
- - elementId: "0"
+ - elementId: "0978df79cf4ed409"
tagFamilies:
- name: searchable
tags:
@@ -33,7 +33,7 @@ elements:
- key: data_binary
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
- - elementId: "1"
+ - elementId: "4dd999e3502d728d"
tagFamilies:
- name: searchable
tags:
@@ -50,7 +50,7 @@ elements:
- key: data_binary
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
- - elementId: "2"
+ - elementId: "fafaa63b403a1604"
tagFamilies:
- name: searchable
tags:
@@ -67,7 +67,7 @@ elements:
- key: data_binary
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
- - elementId: "3"
+ - elementId: "5fdc8aab6ea5c9d4"
tagFamilies:
- name: searchable
tags:
@@ -84,7 +84,7 @@ elements:
- key: data_binary
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
- - elementId: "4"
+ - elementId: "aaede9362761569a"
tagFamilies:
- name: searchable
tags:
diff --git a/test/cases/stream/data/want/order_desc.yaml b/test/cases/stream/data/want/order_desc.yaml
index 055f9449..4044ab45 100644
--- a/test/cases/stream/data/want/order_desc.yaml
+++ b/test/cases/stream/data/want/order_desc.yaml
@@ -16,7 +16,7 @@
# under the License.
elements:
- - elementId: "4"
+ - elementId: "aaede9362761569a"
tagFamilies:
- name: searchable
tags:
@@ -33,7 +33,7 @@ elements:
- key: data_binary
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
- - elementId: "3"
+ - elementId: "5fdc8aab6ea5c9d4"
tagFamilies:
- name: searchable
tags:
@@ -50,7 +50,7 @@ elements:
- key: data_binary
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
- - elementId: "2"
+ - elementId: "fafaa63b403a1604"
tagFamilies:
- name: searchable
tags:
@@ -67,7 +67,7 @@ elements:
- key: data_binary
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
- - elementId: "1"
+ - elementId: "4dd999e3502d728d"
tagFamilies:
- name: searchable
tags:
@@ -84,7 +84,7 @@ elements:
- key: data_binary
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
- - elementId: "0"
+ - elementId: "0978df79cf4ed409"
tagFamilies:
- name: searchable
tags:
diff --git a/test/cases/stream/data/want/search.yaml b/test/cases/stream/data/want/search.yaml
index 05909818..382627be 100644
--- a/test/cases/stream/data/want/search.yaml
+++ b/test/cases/stream/data/want/search.yaml
@@ -16,7 +16,7 @@
# under the License.
elements:
- - elementId: "1"
+ - elementId: "4dd999e3502d728d"
tagFamilies:
- name: searchable
tags:
@@ -33,7 +33,7 @@ elements:
- key: data_binary
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
- - elementId: "2"
+ - elementId: "fafaa63b403a1604"
tagFamilies:
- name: searchable
tags:
diff --git a/test/cases/stream/data/want/sort_desc.yaml b/test/cases/stream/data/want/sort_desc.yaml
index a9f1b0ff..7e1b53e9 100644
--- a/test/cases/stream/data/want/sort_desc.yaml
+++ b/test/cases/stream/data/want/sort_desc.yaml
@@ -16,7 +16,7 @@
# under the License.
elements:
- - elementId: "0"
+ - elementId: "0978df79cf4ed409"
tagFamilies:
- name: searchable
tags:
@@ -33,7 +33,7 @@ elements:
- key: data_binary
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
- - elementId: "1"
+ - elementId: "4dd999e3502d728d"
tagFamilies:
- name: searchable
tags:
@@ -50,7 +50,7 @@ elements:
- key: data_binary
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
- - elementId: "4"
+ - elementId: "aaede9362761569a"
tagFamilies:
- name: searchable
tags:
@@ -67,7 +67,7 @@ elements:
- key: data_binary
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
- - elementId: "3"
+ - elementId: "5fdc8aab6ea5c9d4"
tagFamilies:
- name: searchable
tags:
@@ -84,7 +84,7 @@ elements:
- key: data_binary
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
- - elementId: "2"
+ - elementId: "fafaa63b403a1604"
tagFamilies:
- name: searchable
tags:
diff --git a/test/cases/stream/data/want/sort_duration_no_index_limit.yaml b/test/cases/stream/data/want/sort_duration_no_index_limit.yaml
index 9d3d8d37..f88ab904 100644
--- a/test/cases/stream/data/want/sort_duration_no_index_limit.yaml
+++ b/test/cases/stream/data/want/sort_duration_no_index_limit.yaml
@@ -16,7 +16,7 @@
# under the License.
elements:
-- elementId: "3"
+- elementId: "5fdc8aab6ea5c9d4"
tagFamilies:
- name: searchable
tags:
@@ -38,7 +38,7 @@ elements:
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
timestamp: "2024-07-13T04:01:01.500Z"
-- elementId: "2"
+- elementId: "fafaa63b403a1604"
tagFamilies:
- name: searchable
tags:
diff --git a/test/cases/stream/data/want/sort_filter.yaml b/test/cases/stream/data/want/sort_filter.yaml
index 50de8723..d7f4ae74 100644
--- a/test/cases/stream/data/want/sort_filter.yaml
+++ b/test/cases/stream/data/want/sort_filter.yaml
@@ -16,7 +16,7 @@
# under the License.
elements:
- - elementId: "4"
+ - elementId: "aaede9362761569a"
tagFamilies:
- name: searchable
tags:
@@ -33,7 +33,7 @@ elements:
- key: data_binary
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
- - elementId: "3"
+ - elementId: "5fdc8aab6ea5c9d4"
tagFamilies:
- name: searchable
tags:
@@ -50,7 +50,7 @@ elements:
- key: data_binary
value:
binaryData: YWJjMTIzIT8kKiYoKSctPUB+
- - elementId: "2"
+ - elementId: "fafaa63b403a1604"
tagFamilies:
- name: searchable
tags:
diff --git a/test/cases/stream/stream.go b/test/cases/stream/stream.go
index b7a5f0ab..b2376586 100644
--- a/test/cases/stream/stream.go
+++ b/test/cases/stream/stream.go
@@ -34,10 +34,8 @@ import (
var (
// SharedContext is the parallel execution context.
SharedContext helpers.SharedContext
- verify = func(_ gm.Gomega, args helpers.Args) {
- gm.Eventually(func(innerGm gm.Gomega) {
- stream_test_data.VerifyFn(innerGm, SharedContext, args)
- }, flags.EventuallyTimeout).Should(gm.Succeed())
+ verify = func(innerGm gm.Gomega, args helpers.Args) {
+ stream_test_data.VerifyFn(innerGm, SharedContext, args)
}
)