You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by bu...@apache.org on 2026/06/06 06:08:36 UTC
(skywalking-banyandb) branch v0.10.x updated (74fbf340c -> f45404a6b)
This is an automated email from the ASF dual-hosted git repository.
ButterBright pushed a change to branch v0.10.x
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
from 74fbf340c chore: fix lint issues after cherry-pick
new e9282923d fix(property): prevent gossip propagation from getting stuck after replica scale-up (#1127)
new 2e85f7381 fix(query): dedup IndexMode measures by series ID across data nodes (#1132)
new d16421377 align part sync timestamp to segment grid so part and sidx land in the same segment (#1136)
new e66a0808f Fix backup restore path traversal (#1135)
new b2ced812a fix: release bluge index writers on segment rotation (#13874) (#1128)
new 7cd2616f6 Fix incorrect counts, rate and missing trace fields in the lifecycle migration report (#1133)
new 7b2a552a7 Fix trace identity-tag projection and distributed span tag alignment (#1147)
new 097941a11 Thread Prometheus metric type end-to-end through FODC agent and proxy (#1153)
new 7b3227edf fix(trace): move storage metrics under trace.storage subscope (#1155)
new 5fa118632 Fix FODC agent leaving node_role unspecified on startup race (#1157)
new f45404a6b Prepare for the v0.10.3 release
The 11 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
CHANGES.md | 45 ++-
api/proto/banyandb/fodc/v1/rpc.proto | 11 +
.../backup/lifecycle/measure_migration_visitor.go | 14 +-
banyand/backup/lifecycle/migration_integration.go | 148 +++++----
banyand/backup/lifecycle/progress.go | 198 ++++++++++--
banyand/backup/lifecycle/progress_mark_test.go | 192 ++++++++++++
.../lifecycle/report_partial_failure_test.go | 255 +++++++++++++++
banyand/backup/lifecycle/service.go | 69 ++++-
.../backup/lifecycle/stream_migration_visitor.go | 21 +-
.../backup/lifecycle/trace_migration_visitor.go | 21 +-
banyand/backup/restore.go | 35 ++-
banyand/backup/restore_test.go | 97 ++++++
banyand/internal/storage/rotation.go | 11 +-
banyand/internal/storage/segment.go | 77 ++++-
banyand/internal/storage/storage.go | 2 +
banyand/internal/storage/tsdb.go | 4 +
banyand/internal/storage/tsdb_test.go | 26 +-
banyand/measure/metadata.go | 4 +-
banyand/measure/write_data.go | 3 +-
banyand/measure/write_data_segmentref_test.go | 168 +++++++++-
banyand/property/db/repair.go | 19 +-
banyand/property/db/repair_gossip.go | 58 +++-
banyand/property/db/repair_gossip_test.go | 338 ++++++++++++++++++++
banyand/property/db/shard.go | 41 +++
banyand/property/gossip/server.go | 74 +++--
banyand/property/gossip/service_test.go | 344 +++++++++++++++++++++
banyand/stream/metadata.go | 4 +-
banyand/stream/write_data.go | 3 +-
banyand/stream/write_data_segmentref_test.go | 136 +++++++-
banyand/trace/block.go | 3 +
banyand/trace/metadata.go | 6 +-
banyand/trace/metadata_test.go | 14 +-
banyand/trace/metrics.go | 5 +-
banyand/trace/query.go | 26 +-
banyand/trace/query_test.go | 15 +
banyand/trace/storage_scope_test.go | 54 ++++
banyand/trace/write_data.go | 3 +-
banyand/trace/write_data_segmentref_test.go | 136 +++++++-
docs/api-reference.md | 19 ++
fodc/agent/internal/cluster/collector.go | 48 +++
fodc/agent/internal/cluster/collector_test.go | 50 +++
fodc/agent/internal/cmd/root.go | 2 +-
fodc/agent/internal/exporter/exporter.go | 19 +-
.../internal/exporter/prometheus_format_test.go | 3 +-
fodc/agent/internal/flightrecorder/datasource.go | 20 ++
fodc/agent/internal/metrics/parse.go | 57 ++++
fodc/agent/internal/metrics/parse_test.go | 45 +++
fodc/agent/internal/proxy/client.go | 27 +-
fodc/agent/internal/proxy/client_test.go | 9 +-
fodc/agent/testhelper/flightrecorder.go | 2 +
fodc/proxy/internal/api/server.go | 221 +++++++++----
fodc/proxy/internal/api/server_test.go | 164 ++++++++++
fodc/proxy/internal/metrics/aggregator.go | 21 ++
go.mod | 4 +-
pkg/fs/remote/local/local.go | 94 +++++-
pkg/fs/remote/local/local_test.go | 307 ++++++++++++++++++
pkg/path/path.go | 14 +
pkg/path/path_test.go | 36 +++
.../logical/measure/measure_plan_distributed.go | 39 ++-
.../measure/measure_plan_distributed_test.go | 93 ++++++
pkg/query/logical/measure/schema.go | 6 +
pkg/query/logical/measure/schema_test.go | 90 ++++++
pkg/query/logical/trace/trace_plan_distributed.go | 27 +-
.../logical/trace/trace_plan_distributed_test.go | 130 ++++++++
pkg/query/logical/trace/trace_plan_tag_filter.go | 15 +-
test/cases/lifecycle/lifecycle.go | 141 ++++++++-
test/property_repair/full_data/integrated_test.go | 7 +-
test/property_repair/half_data/integrated_test.go | 7 +-
test/property_repair/same_data/integrated_test.go | 7 +-
test/property_repair/shared_utils.go | 172 ++++++++++-
70 files changed, 4257 insertions(+), 319 deletions(-)
create mode 100644 banyand/backup/lifecycle/progress_mark_test.go
create mode 100644 banyand/backup/lifecycle/report_partial_failure_test.go
create mode 100644 banyand/trace/storage_scope_test.go
create mode 100644 pkg/fs/remote/local/local_test.go
create mode 100644 pkg/query/logical/measure/schema_test.go
create mode 100644 pkg/query/logical/trace/trace_plan_distributed_test.go
(skywalking-banyandb) 08/11: Thread Prometheus metric type end-to-end through FODC agent and proxy (#1153)
Posted by bu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ButterBright pushed a commit to branch v0.10.x
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit 097941a1191276e0d0fabcb36bd165a674f217c0
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Thu Jun 4 13:55:56 2026 +0800
Thread Prometheus metric type end-to-end through FODC agent and proxy (#1153)
---
CHANGES.md | 1 +
api/proto/banyandb/fodc/v1/rpc.proto | 11 +
docs/api-reference.md | 19 ++
fodc/agent/internal/exporter/exporter.go | 19 +-
.../internal/exporter/prometheus_format_test.go | 3 +-
fodc/agent/internal/flightrecorder/datasource.go | 20 ++
fodc/agent/internal/metrics/parse.go | 57 ++++++
fodc/agent/internal/metrics/parse_test.go | 45 +++++
fodc/agent/internal/proxy/client.go | 27 ++-
fodc/agent/internal/proxy/client_test.go | 9 +-
fodc/agent/testhelper/flightrecorder.go | 2 +
fodc/proxy/internal/api/server.go | 221 +++++++++++++++------
fodc/proxy/internal/api/server_test.go | 164 +++++++++++++++
fodc/proxy/internal/metrics/aggregator.go | 21 ++
go.mod | 4 +-
15 files changed, 554 insertions(+), 69 deletions(-)
diff --git a/CHANGES.md b/CHANGES.md
index c0928ad31..bd27150f3 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -7,6 +7,7 @@ Release Notes.
### Bug Fixes
- Fix trace query identity-tag projection: when `trace_id`/`span_id` are explicitly projected, reconstruct them from span identity at response build time instead of requesting them as stored tags, and preserve tag order with null-filled per-span value alignment in the distributed trace result iterator.
+- Fix FODC proxy corrupting Prometheus metric types. The agent dropped the `# TYPE` line while parsing banyandb `/metrics`, the `StreamMetrics` proto carried no type field, and the proxy guessed the type from a name-suffix heuristic — downgrading counters to gauge, mislabeling `_count`-suffixed counters as histograms, and splitting summaries into two conflicting `# TYPE` lines. Capture the type with the Prometheus `expfmt` parser, store it in the flight recorder, thread it through a new [...]
## 0.10.2
diff --git a/api/proto/banyandb/fodc/v1/rpc.proto b/api/proto/banyandb/fodc/v1/rpc.proto
index 9315da8a6..cfc48227d 100644
--- a/api/proto/banyandb/fodc/v1/rpc.proto
+++ b/api/proto/banyandb/fodc/v1/rpc.proto
@@ -60,12 +60,23 @@ message StreamMetricsRequest {
google.protobuf.Timestamp timestamp = 2;
}
+// MetricType represents the Prometheus metric type.
+enum MetricType {
+ METRIC_TYPE_UNSPECIFIED = 0;
+ METRIC_TYPE_GAUGE = 1;
+ METRIC_TYPE_COUNTER = 2;
+ METRIC_TYPE_HISTOGRAM = 3;
+ METRIC_TYPE_SUMMARY = 4;
+ METRIC_TYPE_UNTYPED = 5;
+}
+
message Metric {
string name = 1;
map<string, string> labels = 2;
double value = 3;
string description = 4;
google.protobuf.Timestamp timestamp = 5;
+ MetricType type = 6;
}
message StreamMetricsResponse {
diff --git a/docs/api-reference.md b/docs/api-reference.md
index 2c12c584f..89b9bd6c1 100644
--- a/docs/api-reference.md
+++ b/docs/api-reference.md
@@ -324,6 +324,8 @@
- [StreamMetricsResponse](#banyandb-fodc-v1-StreamMetricsResponse)
- [Topology](#banyandb-fodc-v1-Topology)
+ - [MetricType](#banyandb-fodc-v1-MetricType)
+
- [FODCService](#banyandb-fodc-v1-FODCService)
- [GroupLifecycleService](#banyandb-fodc-v1-GroupLifecycleService)
@@ -4850,6 +4852,7 @@ Phase represents the current phase of the deletion task.
| value | [double](#double) | | |
| description | [string](#string) | | |
| timestamp | [google.protobuf.Timestamp](#google-protobuf-Timestamp) | | |
+| type | [MetricType](#banyandb-fodc-v1-MetricType) | | |
@@ -5036,6 +5039,22 @@ Phase represents the current phase of the deletion task.
+
+<a name="banyandb-fodc-v1-MetricType"></a>
+
+### MetricType
+MetricType represents the Prometheus metric type.
+
+| Name | Number | Description |
+| ---- | ------ | ----------- |
+| METRIC_TYPE_UNSPECIFIED | 0 | |
+| METRIC_TYPE_GAUGE | 1 | |
+| METRIC_TYPE_COUNTER | 2 | |
+| METRIC_TYPE_HISTOGRAM | 3 | |
+| METRIC_TYPE_SUMMARY | 4 | |
+| METRIC_TYPE_UNTYPED | 5 | |
+
+
diff --git a/fodc/agent/internal/exporter/exporter.go b/fodc/agent/internal/exporter/exporter.go
index 20569a19c..c50716a3e 100644
--- a/fodc/agent/internal/exporter/exporter.go
+++ b/fodc/agent/internal/exporter/exporter.go
@@ -87,6 +87,21 @@ func (dc *DatasourceCollector) Describe(ch chan<- *prometheus.Desc) {
}
}
+// chooseValueType maps a lowercase prometheus type string to a prometheus.ValueType
+// suitable for NewConstMetric. Histogram and summary cannot be represented as scalar
+// const metrics, so they degrade to UntypedValue.
+func chooseValueType(t string) prometheus.ValueType {
+ switch t {
+ case "counter":
+ return prometheus.CounterValue
+ case "histogram", "summary", "untyped":
+ return prometheus.UntypedValue
+ default:
+ // gauge or "" (unknown) → gauge
+ return prometheus.GaugeValue
+ }
+}
+
// Collect implements prometheus.Collector interface.
// It collects metrics from all Datasources and sends them to the channel.
func (dc *DatasourceCollector) Collect(ch chan<- prometheus.Metric) {
@@ -94,6 +109,7 @@ func (dc *DatasourceCollector) Collect(ch chan<- prometheus.Metric) {
for _, ds := range datasources {
metricsMap := ds.GetMetrics()
descriptions := ds.GetDescriptions()
+ types := ds.GetTypes()
timestamps := ds.GetTimestamps()
for metricKeyStr, metricBuffer := range metricsMap {
@@ -128,9 +144,10 @@ func (dc *DatasourceCollector) Collect(ch chan<- prometheus.Metric) {
labelValues[idx] = labelValueMap[labelName]
}
+ metricType := metrics.ResolveMetricType(types, parsedKey.name)
metric, createErr := prometheus.NewConstMetric(
desc,
- prometheus.GaugeValue,
+ chooseValueType(metricType),
currentValue,
labelValues...,
)
diff --git a/fodc/agent/internal/exporter/prometheus_format_test.go b/fodc/agent/internal/exporter/prometheus_format_test.go
index ea854a571..25665d5d2 100644
--- a/fodc/agent/internal/exporter/prometheus_format_test.go
+++ b/fodc/agent/internal/exporter/prometheus_format_test.go
@@ -44,6 +44,7 @@ func TestPrometheusFormatConversion(t *testing.T) {
Value: 100.0,
Labels: []metrics.Label{{Name: "method", Value: "GET"}, {Name: "status", Value: "200"}},
Desc: "Total number of HTTP requests",
+ Type: "counter",
},
{
Name: "cpu_usage",
@@ -126,7 +127,7 @@ func TestPrometheusFormatConversion(t *testing.T) {
// Check TYPE lines
if strings.HasPrefix(line, "# TYPE") {
- if strings.Contains(line, "http_requests_total") && strings.Contains(line, "gauge") {
+ if strings.Contains(line, "http_requests_total") && strings.Contains(line, "counter") {
hasTypeHTTPRequests = true
}
if strings.Contains(line, "cpu_usage") && strings.Contains(line, "gauge") {
diff --git a/fodc/agent/internal/flightrecorder/datasource.go b/fodc/agent/internal/flightrecorder/datasource.go
index e8afff56c..747b467b0 100644
--- a/fodc/agent/internal/flightrecorder/datasource.go
+++ b/fodc/agent/internal/flightrecorder/datasource.go
@@ -67,6 +67,7 @@ func UpdateTimestampRingBuffer(trb *TimestampRingBuffer, v int64) {
type Datasource struct {
metrics map[string]*MetricRingBuffer // Map from metric name+labels to RingBuffer storing metric values
descriptions map[string]string // Map from metric name to HELP content descriptions
+ types map[string]string // Map from metric name to Prometheus type string
timestamps *TimestampRingBuffer // RingBuffer storing timestamps for each polling cycle
mu sync.RWMutex
CapacitySize int64 // Memory limit in bytes
@@ -78,6 +79,7 @@ func NewDatasource() *Datasource {
metrics: make(map[string]*MetricRingBuffer),
timestamps: NewTimestampRingBuffer(),
descriptions: make(map[string]string),
+ types: make(map[string]string),
CapacitySize: 0,
}
}
@@ -103,6 +105,9 @@ func (ds *Datasource) Update(m *metrics.RawMetric) error {
if m.Desc != "" {
ds.descriptions[m.Name] = m.Desc
}
+ if m.Type != "" {
+ ds.types[m.Name] = m.Type
+ }
UpdateMetricRingBuffer(ds.metrics[metricKey], m.Value)
return nil
@@ -140,6 +145,9 @@ func (ds *Datasource) UpdateBatch(rawMetrics []metrics.RawMetric, timestamp int6
if m.Desc != "" {
ds.descriptions[m.Name] = m.Desc
}
+ if m.Type != "" {
+ ds.types[m.Name] = m.Type
+ }
buffer := ds.metrics[metricKey]
UpdateMetricRingBuffer(buffer, m.Value)
updatedBuffers = append(updatedBuffers, buffer)
@@ -259,3 +267,15 @@ func (ds *Datasource) GetDescriptions() map[string]string {
}
return result
}
+
+// GetTypes returns a copy of the types map.
+func (ds *Datasource) GetTypes() map[string]string {
+ ds.mu.RLock()
+ defer ds.mu.RUnlock()
+
+ result := make(map[string]string)
+ for k, v := range ds.types {
+ result[k] = v
+ }
+ return result
+}
diff --git a/fodc/agent/internal/metrics/parse.go b/fodc/agent/internal/metrics/parse.go
index 5a4375142..974e960d5 100644
--- a/fodc/agent/internal/metrics/parse.go
+++ b/fodc/agent/internal/metrics/parse.go
@@ -24,6 +24,10 @@ import (
"sort"
"strconv"
"strings"
+
+ dto "github.com/prometheus/client_model/go"
+ "github.com/prometheus/common/expfmt"
+ "github.com/prometheus/common/model"
)
// Label represents a metric label as a key-value pair.
@@ -36,6 +40,7 @@ type Label struct {
type RawMetric struct {
Name string
Desc string
+ Type string // lowercase prometheus type: "counter","gauge","histogram","summary","untyped",""
Labels []Label
Value float64
}
@@ -76,8 +81,59 @@ var (
labelRegex = regexp.MustCompile(`(\w+)="([^"]+)"`)
)
+// metricTypeToString converts a dto.MetricType to its lowercase Prometheus text representation.
+func metricTypeToString(mt dto.MetricType) string {
+ switch mt {
+ case dto.MetricType_COUNTER:
+ return "counter"
+ case dto.MetricType_GAUGE:
+ return "gauge"
+ case dto.MetricType_SUMMARY:
+ return "summary"
+ case dto.MetricType_HISTOGRAM:
+ return "histogram"
+ case dto.MetricType_UNTYPED:
+ return "untyped"
+ default:
+ return ""
+ }
+}
+
+// ResolveMetricType resolves the metric type for a given series name using the typeMap.
+// For histogram/summary component series (ending in _bucket/_sum/_count), the base family
+// name is used to look up the type. Returns "" if unknown.
+func ResolveMetricType(typeMap map[string]string, name string) string {
+ if t, ok := typeMap[name]; ok {
+ return t
+ }
+ // Trim histogram/summary suffixes to find the base family name.
+ for _, suffix := range []string{"_bucket", "_sum", "_count"} {
+ if strings.HasSuffix(name, suffix) {
+ base := strings.TrimSuffix(name, suffix)
+ if base != "" {
+ if t, ok := typeMap[base]; ok {
+ return t
+ }
+ }
+ break
+ }
+ }
+ return ""
+}
+
// ParseWithAgentLabels parses Prometheus text format metrics and returns structured RawMetric objects.
func ParseWithAgentLabels(text string, nodeRole, podName, containerName string) ([]RawMetric, error) {
+ // Build authoritative type map using expfmt.
+ // TextToMetricFamilies may return a non-nil error AND a non-empty families map
+ // simultaneously (e.g. for "unexpected end of input stream" on trailing newline).
+ // We always use whatever families were successfully parsed, ignoring parse errors.
+ typeMap := make(map[string]string)
+ parser := expfmt.NewTextParser(model.LegacyValidation)
+ families, _ := parser.TextToMetricFamilies(strings.NewReader(text))
+ for familyName, mf := range families {
+ typeMap[familyName] = metricTypeToString(mf.GetType())
+ }
+
lines := strings.Split(text, "\n")
var metrics []RawMetric
helpMap := make(map[string]string)
@@ -145,6 +201,7 @@ func ParseWithAgentLabels(text string, nodeRole, podName, containerName string)
Labels: labels,
Value: value,
Desc: desc,
+ Type: ResolveMetricType(typeMap, metricName),
})
}
diff --git a/fodc/agent/internal/metrics/parse_test.go b/fodc/agent/internal/metrics/parse_test.go
index fab8c5fdf..efb4adff4 100644
--- a/fodc/agent/internal/metrics/parse_test.go
+++ b/fodc/agent/internal/metrics/parse_test.go
@@ -1076,6 +1076,51 @@ func TestParse_LabelValueWithNumbers(t *testing.T) {
assert.Equal(t, "value123", metrics[0].Labels[0].Value)
}
+// TestParse_TypePropagation verifies that expfmt-derived types are correctly set on
+// RawMetric.Type, including base-name resolution for histogram component series.
+func TestParse_TypePropagation(t *testing.T) {
+ text := `# HELP http_requests_total Total HTTP requests
+# TYPE http_requests_total counter
+http_requests_total{method="GET"} 1234
+# HELP request_latency Request latency histogram
+# TYPE request_latency histogram
+request_latency_bucket{le="0.1"} 10
+request_latency_bucket{le="0.5"} 25
+request_latency_bucket{le="+Inf"} 30
+request_latency_sum 12.5
+request_latency_count 30
+# HELP cpu_usage CPU usage
+# TYPE cpu_usage gauge
+cpu_usage 75.5`
+
+ result, parseErr := ParseWithAgentLabels(text, "", "", "")
+
+ require.NoError(t, parseErr)
+
+ // Build a name→type map for easy assertion.
+ typeByName := make(map[string][]string)
+ for _, m := range result {
+ typeByName[m.Name] = append(typeByName[m.Name], m.Type)
+ }
+
+ // Counter: all series for http_requests_total must be "counter".
+ for _, typ := range typeByName["http_requests_total"] {
+ assert.Equal(t, "counter", typ, "http_requests_total should be counter")
+ }
+
+ // Gauge.
+ for _, typ := range typeByName["cpu_usage"] {
+ assert.Equal(t, "gauge", typ, "cpu_usage should be gauge")
+ }
+
+ // Histogram component series must all resolve to "histogram".
+ for _, name := range []string{"request_latency_bucket", "request_latency_sum", "request_latency_count"} {
+ for _, typ := range typeByName[name] {
+ assert.Equal(t, "histogram", typ, "%s should resolve to histogram via base name", name)
+ }
+ }
+}
+
func TestParse_WithAgentIdentityLabels(t *testing.T) {
text := `cpu_usage{instance="localhost"} 75.5
http_requests_total{method="GET"} 100`
diff --git a/fodc/agent/internal/proxy/client.go b/fodc/agent/internal/proxy/client.go
index 26752365d..a6ff1eae8 100644
--- a/fodc/agent/internal/proxy/client.go
+++ b/fodc/agent/internal/proxy/client.go
@@ -518,6 +518,7 @@ func (c *Client) RetrieveAndSendMetrics(_ context.Context, filter *MetricsReques
allMetrics := ds.GetMetrics()
timestamps := ds.GetTimestamps()
descriptions := ds.GetDescriptions()
+ types := ds.GetTypes()
if filter != nil && (filter.StartTime != nil || filter.EndTime != nil) {
if timestamps == nil {
@@ -562,7 +563,7 @@ func (c *Client) RetrieveAndSendMetrics(_ context.Context, filter *MetricsReques
c.streamsMu.Unlock()
return fmt.Errorf("metrics stream changed during send")
}
- sendErr := c.sendFilteredMetrics(metricsStream, allMetrics, timestampValues, descriptions, filter)
+ sendErr := c.sendFilteredMetrics(metricsStream, allMetrics, timestampValues, descriptions, types, filter)
c.streamsMu.Unlock()
return sendErr
}
@@ -572,16 +573,35 @@ func (c *Client) RetrieveAndSendMetrics(_ context.Context, filter *MetricsReques
c.streamsMu.Unlock()
return fmt.Errorf("metrics stream changed during send")
}
- sendErr := c.sendLatestMetrics(metricsStream, allMetrics, descriptions)
+ sendErr := c.sendLatestMetrics(metricsStream, allMetrics, descriptions, types)
c.streamsMu.Unlock()
return sendErr
}
+// toProtoMetricType converts a lowercase prometheus type string to the proto enum.
+func toProtoMetricType(t string) fodcv1.MetricType {
+ switch t {
+ case "counter":
+ return fodcv1.MetricType_METRIC_TYPE_COUNTER
+ case "gauge":
+ return fodcv1.MetricType_METRIC_TYPE_GAUGE
+ case "histogram":
+ return fodcv1.MetricType_METRIC_TYPE_HISTOGRAM
+ case "summary":
+ return fodcv1.MetricType_METRIC_TYPE_SUMMARY
+ case "untyped":
+ return fodcv1.MetricType_METRIC_TYPE_UNTYPED
+ default:
+ return fodcv1.MetricType_METRIC_TYPE_UNSPECIFIED
+ }
+}
+
// sendLatestMetrics sends the latest metrics (most recent values).
func (c *Client) sendLatestMetrics(
stream fodcv1.FODCService_StreamMetricsClient,
allMetrics map[string]*flightrecorder.MetricRingBuffer,
descriptions map[string]string,
+ types map[string]string,
) error {
protoMetrics := make([]*fodcv1.Metric, 0)
@@ -609,6 +629,7 @@ func (c *Client) sendLatestMetrics(
Labels: labelsMap,
Value: metricValue,
Description: descriptions[parsedKey.Name],
+ Type: toProtoMetricType(metrics.ResolveMetricType(types, parsedKey.Name)),
}
protoMetrics = append(protoMetrics, protoMetric)
@@ -632,6 +653,7 @@ func (c *Client) sendFilteredMetrics(
allMetrics map[string]*flightrecorder.MetricRingBuffer,
timestampValues []int64,
descriptions map[string]string,
+ types map[string]string,
filter *MetricsRequestFilter,
) error {
protoMetrics := make([]*fodcv1.Metric, 0)
@@ -677,6 +699,7 @@ func (c *Client) sendFilteredMetrics(
Value: metricValues[idx],
Description: description,
Timestamp: timestamppb.New(timestamp),
+ Type: toProtoMetricType(metrics.ResolveMetricType(types, parsedKey.Name)),
}
protoMetrics = append(protoMetrics, protoMetric)
diff --git a/fodc/agent/internal/proxy/client_test.go b/fodc/agent/internal/proxy/client_test.go
index 0c5346c3d..d1576d3de 100644
--- a/fodc/agent/internal/proxy/client_test.go
+++ b/fodc/agent/internal/proxy/client_test.go
@@ -1328,11 +1328,12 @@ func TestProxyClient_sendLatestMetrics_WithUnfinalizedValue(t *testing.T) {
allMetrics := ds.GetMetrics()
descriptions := ds.GetDescriptions()
+ types := ds.GetTypes()
ctx := context.Background()
mockStream := newMockStreamMetricsClient(ctx)
- err := pc.sendLatestMetrics(mockStream, allMetrics, descriptions)
+ err := pc.sendLatestMetrics(mockStream, allMetrics, descriptions, types)
require.NoError(t, err)
mockStream.mu.Lock()
@@ -1371,6 +1372,7 @@ func TestProxyClient_sendFilteredMetrics_TimeWindow(t *testing.T) {
timestamps := ds.GetTimestamps()
timestampValues := timestamps.GetAllValues()
descriptions := ds.GetDescriptions()
+ types := ds.GetTypes()
startTime := now.Add(-90 * time.Minute)
endTime := now.Add(-30 * time.Minute)
@@ -1382,7 +1384,7 @@ func TestProxyClient_sendFilteredMetrics_TimeWindow(t *testing.T) {
ctx := context.Background()
mockStream := newMockStreamMetricsClient(ctx)
- err := pc.sendFilteredMetrics(mockStream, allMetrics, timestampValues, descriptions, filter)
+ err := pc.sendFilteredMetrics(mockStream, allMetrics, timestampValues, descriptions, types, filter)
require.NoError(t, err)
mockStream.mu.Lock()
@@ -1414,6 +1416,7 @@ func TestProxyClient_sendFilteredMetrics_NoMatchingTimeWindow(t *testing.T) {
timestamps := ds.GetTimestamps()
timestampValues := timestamps.GetAllValues()
descriptions := ds.GetDescriptions()
+ types := ds.GetTypes()
// Filter for future time window (no matches)
startTime := now.Add(1 * time.Hour)
@@ -1426,7 +1429,7 @@ func TestProxyClient_sendFilteredMetrics_NoMatchingTimeWindow(t *testing.T) {
ctx := context.Background()
mockStream := newMockStreamMetricsClient(ctx)
- err := pc.sendFilteredMetrics(mockStream, allMetrics, timestampValues, descriptions, filter)
+ err := pc.sendFilteredMetrics(mockStream, allMetrics, timestampValues, descriptions, types, filter)
require.NoError(t, err)
mockStream.mu.Lock()
diff --git a/fodc/agent/testhelper/flightrecorder.go b/fodc/agent/testhelper/flightrecorder.go
index 4b3390cf2..76db6c0af 100644
--- a/fodc/agent/testhelper/flightrecorder.go
+++ b/fodc/agent/testhelper/flightrecorder.go
@@ -42,6 +42,7 @@ func NewFlightRecorder(capacitySize int64) *flightrecorder.FlightRecorder {
type RawMetric struct {
Desc string
Name string
+ Type string
Labels []Label
Value float64
}
@@ -73,6 +74,7 @@ func UpdateMetrics(fr interface{}, rawMetrics []RawMetric) error {
Name: rawMetrics[idx].Name,
Value: rawMetrics[idx].Value,
Desc: rawMetrics[idx].Desc,
+ Type: rawMetrics[idx].Type,
Labels: internalLabels,
}
}
diff --git a/fodc/proxy/internal/api/server.go b/fodc/proxy/internal/api/server.go
index f8c9338fa..3673beafd 100644
--- a/fodc/proxy/internal/api/server.go
+++ b/fodc/proxy/internal/api/server.go
@@ -265,82 +265,137 @@ func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) {
}
// formatPrometheusText formats aggregated metrics as Prometheus text format.
+//
+// Metrics with a known Type (non-empty) are emitted using the real type. Metrics with
+// Type=="" (pre-upgrade agents) fall back to the legacy suffix-heuristic path so behavior
+// is unchanged for those metrics.
func (s *Server) formatPrometheusText(aggregatedMetrics []*metrics.AggregatedMetric) string {
if len(aggregatedMetrics) == 0 {
return ""
}
- metricMap := make(map[string]*metricGroup)
- histogramBases := make(map[string]bool)
+ // Partition into typed (type known from agent) and untyped (legacy/unknown).
+ var typed []*metrics.AggregatedMetric
+ var untyped []*metrics.AggregatedMetric
+ for _, m := range aggregatedMetrics {
+ if m.Type != "" {
+ typed = append(typed, m)
+ } else {
+ untyped = append(untyped, m)
+ }
+ }
- for _, metric := range aggregatedMetrics {
- key := metric.Name
- group, exists := metricMap[key]
+ var builder strings.Builder
+ remainingUntyped := writeTypedFamilies(&builder, typed, untyped)
+ writeUntypedFamilies(&builder, remainingUntyped)
+ return builder.String()
+}
+
+// writeTypedFamilies groups typed metrics by family base and emits one "# TYPE" line per
+// family using the real type. The family base for histogram/summary component series
+// (_bucket/_sum/_count) is the name with the suffix stripped; bare summary quantile series
+// and all counter/gauge/untyped series use the name as-is. Untyped metrics whose base
+// collides with a typed family are absorbed under the authoritative typed line — this
+// prevents a mixed-version rollout from emitting two conflicting "# TYPE" lines for one
+// name. The untyped metrics with no typed-family collision are returned for the legacy path.
+func writeTypedFamilies(builder *strings.Builder, typed, untyped []*metrics.AggregatedMetric) []*metrics.AggregatedMetric {
+ typedFamilyOrder := make([]string, 0)
+ typedFamilies := make(map[string]*metricGroup) // base → group
+ for _, m := range typed {
+ base := typedFamilyBase(m.Name, m.Type)
+ grp, exists := typedFamilies[base]
if !exists {
- group = &metricGroup{
- name: metric.Name,
- description: metric.Description,
+ grp = &metricGroup{
+ name: base,
+ description: m.Description,
+ metricType: m.Type,
metrics: make([]*metrics.AggregatedMetric, 0),
}
- metricMap[key] = group
+ typedFamilies[base] = grp
+ typedFamilyOrder = append(typedFamilyOrder, base)
}
- group.metrics = append(group.metrics, metric)
-
- if strings.HasSuffix(metric.Name, "_bucket") ||
- strings.HasSuffix(metric.Name, "_sum") ||
- strings.HasSuffix(metric.Name, "_count") {
- baseName := getHistogramBaseName(metric.Name)
- if baseName != "" {
- histogramBases[baseName] = true
+ grp.metrics = append(grp.metrics, m)
+ }
+
+ remainingUntyped := make([]*metrics.AggregatedMetric, 0, len(untyped))
+ for _, m := range untyped {
+ grp := matchTypedFamily(typedFamilies, m.Name)
+ if grp == nil {
+ remainingUntyped = append(remainingUntyped, m)
+ continue
+ }
+ if grp.description == "" && m.Description != "" {
+ grp.description = m.Description
+ }
+ grp.metrics = append(grp.metrics, m)
+ }
+
+ sort.Strings(typedFamilyOrder)
+ for _, base := range typedFamilyOrder {
+ grp := typedFamilies[base]
+ if grp.description != "" {
+ builder.WriteString(fmt.Sprintf("# HELP %s %s\n", base, grp.description))
+ }
+ builder.WriteString(fmt.Sprintf("# TYPE %s %s\n", base, grp.metricType))
+ for _, m := range grp.metrics {
+ builder.WriteString(formatMetricLine(m))
+ }
+ }
+ return remainingUntyped
+}
+
+// writeUntypedFamilies emits metrics with no known type using the legacy suffix heuristic:
+// a base name with a _bucket/_sum/_count sibling is treated as a histogram, everything else
+// as a gauge. This preserves pre-typed behavior for pre-upgrade agents.
+func writeUntypedFamilies(builder *strings.Builder, untyped []*metrics.AggregatedMetric) {
+ if len(untyped) == 0 {
+ return
+ }
+ metricMap := make(map[string]*metricGroup)
+ histogramBases := make(map[string]bool)
+ for _, m := range untyped {
+ grp, exists := metricMap[m.Name]
+ if !exists {
+ grp = &metricGroup{
+ name: m.Name,
+ description: m.Description,
+ metrics: make([]*metrics.AggregatedMetric, 0),
}
+ metricMap[m.Name] = grp
+ }
+ grp.metrics = append(grp.metrics, m)
+ if baseName := getHistogramBaseName(m.Name); baseName != "" {
+ histogramBases[baseName] = true
}
}
- histogramMetrics := make(map[string][]*metrics.AggregatedMetric)
+ histogramMetricsMap := make(map[string][]*metrics.AggregatedMetric)
regularMetrics := make(map[string]*metricGroup)
-
- for name, group := range metricMap {
+ for name, grp := range metricMap {
baseName := getHistogramBaseName(name)
if baseName != "" && histogramBases[baseName] {
- histogramMetrics[baseName] = append(histogramMetrics[baseName], group.metrics...)
+ histogramMetricsMap[baseName] = append(histogramMetricsMap[baseName], grp.metrics...)
} else {
- regularMetrics[name] = group
+ regularMetrics[name] = grp
}
}
- var builder strings.Builder
-
histogramNames := make([]string, 0, len(histogramBases))
for baseName := range histogramBases {
histogramNames = append(histogramNames, baseName)
}
sort.Strings(histogramNames)
-
for _, baseName := range histogramNames {
- allMetrics := histogramMetrics[baseName]
- if len(allMetrics) == 0 {
+ allM := histogramMetricsMap[baseName]
+ if len(allM) == 0 {
continue
}
-
- description := allMetrics[0].Description
- if description != "" {
+ if description := allM[0].Description; description != "" {
builder.WriteString(fmt.Sprintf("# HELP %s %s\n", baseName, description))
}
builder.WriteString(fmt.Sprintf("# TYPE %s histogram\n", baseName))
-
- for _, metric := range allMetrics {
- labelParts := make([]string, 0, len(metric.Labels))
- for key, value := range metric.Labels {
- labelParts = append(labelParts, fmt.Sprintf(`%s="%s"`, key, value))
- }
- sort.Strings(labelParts)
-
- labelStr := ""
- if len(labelParts) > 0 {
- labelStr = "{" + strings.Join(labelParts, ",") + "}"
- }
-
- builder.WriteString(fmt.Sprintf("%s%s %s\n", metric.Name, labelStr, formatFloat(metric.Value)))
+ for _, m := range allM {
+ builder.WriteString(formatMetricLine(m))
}
}
@@ -349,31 +404,76 @@ func (s *Server) formatPrometheusText(aggregatedMetrics []*metrics.AggregatedMet
regularNames = append(regularNames, name)
}
sort.Strings(regularNames)
-
for _, name := range regularNames {
- group := regularMetrics[name]
- if group.description != "" {
- builder.WriteString(fmt.Sprintf("# HELP %s %s\n", group.name, group.description))
+ grp := regularMetrics[name]
+ if grp.description != "" {
+ builder.WriteString(fmt.Sprintf("# HELP %s %s\n", grp.name, grp.description))
+ }
+ builder.WriteString(fmt.Sprintf("# TYPE %s gauge\n", grp.name))
+ for _, m := range grp.metrics {
+ builder.WriteString(formatMetricLine(m))
}
- builder.WriteString(fmt.Sprintf("# TYPE %s gauge\n", group.name))
+ }
+}
- for _, metric := range group.metrics {
- labelParts := make([]string, 0, len(metric.Labels))
- for key, value := range metric.Labels {
- labelParts = append(labelParts, fmt.Sprintf(`%s="%s"`, key, value))
+// matchTypedFamily returns the typed family an untyped series belongs to, or nil if none.
+// It checks the exact name first (counter/gauge/untyped series and bare summary quantile
+// series), then the histogram/summary component base after trimming a trailing
+// _bucket/_sum/_count suffix. Used to fold pre-upgrade (untyped) samples into the
+// authoritative typed family so the proxy never emits two TYPE lines for one name.
+func matchTypedFamily(typedFamilies map[string]*metricGroup, name string) *metricGroup {
+ if grp, ok := typedFamilies[name]; ok {
+ return grp
+ }
+ for _, suffix := range []string{"_bucket", "_sum", "_count"} {
+ if strings.HasSuffix(name, suffix) {
+ base := strings.TrimSuffix(name, suffix)
+ if base != "" {
+ if grp, ok := typedFamilies[base]; ok {
+ return grp
+ }
}
- sort.Strings(labelParts)
+ break
+ }
+ }
+ return nil
+}
- labelStr := ""
- if len(labelParts) > 0 {
- labelStr = "{" + strings.Join(labelParts, ",") + "}"
+// typedFamilyBase returns the Prometheus family base name for a typed series.
+// For histogram/summary component series the trailing _bucket/_sum/_count is stripped.
+// For all other types (counter, gauge, untyped) the name is used as-is.
+func typedFamilyBase(name, metricType string) string {
+ switch metricType {
+ case "histogram", "summary":
+ for _, suffix := range []string{"_bucket", "_sum", "_count"} {
+ if strings.HasSuffix(name, suffix) {
+ base := strings.TrimSuffix(name, suffix)
+ if base != "" {
+ return base
+ }
}
-
- builder.WriteString(fmt.Sprintf("%s%s %s\n", group.name, labelStr, formatFloat(metric.Value)))
}
}
+ return name
+}
- return builder.String()
+// labelValueEscaper escapes the three characters that are special inside a Prometheus
+// text-exposition label value: backslash, double-quote, and line feed. The replacements are
+// applied in a single left-to-right pass, so an escaped backslash is not re-escaped.
+var labelValueEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`, "\n", `\n`)
+
+// formatMetricLine renders a single metric sample as a Prometheus text line.
+func formatMetricLine(m *metrics.AggregatedMetric) string {
+ labelParts := make([]string, 0, len(m.Labels))
+ for k, v := range m.Labels {
+ labelParts = append(labelParts, fmt.Sprintf(`%s="%s"`, k, labelValueEscaper.Replace(v)))
+ }
+ sort.Strings(labelParts)
+ labelStr := ""
+ if len(labelParts) > 0 {
+ labelStr = "{" + strings.Join(labelParts, ",") + "}"
+ }
+ return fmt.Sprintf("%s%s %s\n", m.Name, labelStr, formatFloat(m.Value))
}
// formatMetricsWindowJSON formats aggregated metrics as JSON for metrics-windows endpoint.
@@ -475,6 +575,7 @@ func getHistogramBaseName(metricName string) string {
type metricGroup struct {
name string
description string
+ metricType string
metrics []*metrics.AggregatedMetric
}
diff --git a/fodc/proxy/internal/api/server_test.go b/fodc/proxy/internal/api/server_test.go
index a4cf4ba1f..572dc2369 100644
--- a/fodc/proxy/internal/api/server_test.go
+++ b/fodc/proxy/internal/api/server_test.go
@@ -102,6 +102,11 @@ func newTestServerOnly(t *testing.T) *Server {
}
func createTestMetric(name string, value float64, labels map[string]string, timestamp time.Time) *metrics.AggregatedMetric {
+ return createTestMetricWithType(name, value, labels, timestamp, "")
+}
+
+// createTestMetricWithType creates a test metric with an explicit Prometheus type.
+func createTestMetricWithType(name string, value float64, labels map[string]string, timestamp time.Time, metricType string) *metrics.AggregatedMetric {
if labels == nil {
labels = make(map[string]string)
}
@@ -112,6 +117,7 @@ func createTestMetric(name string, value float64, labels map[string]string, time
Timestamp: timestamp,
AgentID: "test-agent-1",
Description: "Test metric description",
+ Type: metricType,
}
}
@@ -970,6 +976,164 @@ func TestHandleClusterLifecycle_DataInfoEmitted(t *testing.T) {
require.Len(t, dataInfo, 1)
}
+// TestFormatPrometheusText_CounterType verifies that a metric with type=counter emits
+// "# TYPE foo_total counter" and is NOT mistaken for a histogram (invariant 2).
+func TestFormatPrometheusText_CounterType(t *testing.T) {
+ server := newTestServerOnly(t)
+ now := time.Now()
+ metricsList := []*metrics.AggregatedMetric{
+ createTestMetricWithType("http_requests_total", 42, map[string]string{"method": "GET"}, now, "counter"),
+ }
+
+ result := server.formatPrometheusText(metricsList)
+
+ assert.Contains(t, result, "# TYPE http_requests_total counter")
+ assert.Contains(t, result, "http_requests_total{")
+ assert.NotContains(t, result, "# TYPE http_requests_total gauge")
+ assert.NotContains(t, result, "# TYPE http_requests_total histogram")
+}
+
+// TestFormatPrometheusText_CounterWithCountSuffix verifies that a counter named foo_count
+// (not a histogram component) keeps type=counter and is NOT folded into histogram base "foo"
+// (invariant 3 — the key fix for the mislabeled-histogram defect).
+func TestFormatPrometheusText_CounterWithCountSuffix(t *testing.T) {
+ server := newTestServerOnly(t)
+ now := time.Now()
+ metricsList := []*metrics.AggregatedMetric{
+ createTestMetricWithType("banyandb_grpc_request_count", 100, map[string]string{"pod_name": "p1"}, now, "counter"),
+ }
+
+ result := server.formatPrometheusText(metricsList)
+
+ assert.Contains(t, result, "# TYPE banyandb_grpc_request_count counter")
+ // Must NOT be folded into a histogram base called "banyandb_grpc_request"
+ assert.NotContains(t, result, "# TYPE banyandb_grpc_request histogram")
+ assert.Contains(t, result, "banyandb_grpc_request_count{")
+}
+
+// TestFormatPrometheusText_HistogramTyped verifies that histogram component series with
+// type=histogram share exactly one "# TYPE foo histogram" line (invariant 4).
+func TestFormatPrometheusText_HistogramTyped(t *testing.T) {
+ server := newTestServerOnly(t)
+ now := time.Now()
+ labels := map[string]string{"pod_name": "p1"}
+ metricsList := []*metrics.AggregatedMetric{
+ createTestMetricWithType("request_latency_bucket", 10, labels, now, "histogram"),
+ createTestMetricWithType("request_latency_sum", 1.5, labels, now, "histogram"),
+ createTestMetricWithType("request_latency_count", 10, labels, now, "histogram"),
+ }
+
+ result := server.formatPrometheusText(metricsList)
+
+ // Exactly one TYPE line for the base name.
+ assert.Equal(t, 1, strings.Count(result, "# TYPE request_latency histogram"),
+ "must emit exactly one TYPE histogram line")
+ assert.Contains(t, result, "request_latency_bucket{")
+ assert.Contains(t, result, "request_latency_sum{")
+ assert.Contains(t, result, "request_latency_count{")
+ // Component names must not get their own TYPE lines.
+ assert.NotContains(t, result, "# TYPE request_latency_bucket")
+ assert.NotContains(t, result, "# TYPE request_latency_sum")
+ assert.NotContains(t, result, "# TYPE request_latency_count")
+}
+
+// TestFormatPrometheusText_SummaryTyped verifies that summary families emit exactly one
+// "# TYPE foo summary" line and all series (quantile, _sum, _count) fall under it (invariant 5).
+func TestFormatPrometheusText_SummaryTyped(t *testing.T) {
+ server := newTestServerOnly(t)
+ now := time.Now()
+ labelsQ := map[string]string{"quantile": "0.5"}
+ labelsBase := map[string]string{}
+ metricsList := []*metrics.AggregatedMetric{
+ createTestMetricWithType("go_gc_duration_seconds", 0.0001, labelsQ, now, "summary"),
+ createTestMetricWithType("go_gc_duration_seconds_sum", 0.5, labelsBase, now, "summary"),
+ createTestMetricWithType("go_gc_duration_seconds_count", 42, labelsBase, now, "summary"),
+ }
+
+ result := server.formatPrometheusText(metricsList)
+
+ // Exactly one TYPE line for the base name.
+ assert.Equal(t, 1, strings.Count(result, "# TYPE go_gc_duration_seconds summary"),
+ "must emit exactly one TYPE summary line")
+ assert.Contains(t, result, "go_gc_duration_seconds{")
+ assert.Contains(t, result, "go_gc_duration_seconds_sum")
+ assert.Contains(t, result, "go_gc_duration_seconds_count")
+ // Must NOT emit a separate gauge TYPE line for the bare name.
+ assert.NotContains(t, result, "# TYPE go_gc_duration_seconds gauge")
+}
+
+// TestFormatPrometheusText_MixedVersionDedup verifies that when the same metric family
+// arrives both typed (upgraded agent) and untyped (pre-upgrade agent) in one scrape, the
+// proxy emits exactly ONE "# TYPE" line — the typed one — and folds the untyped samples
+// under it, rather than emitting a conflicting second TYPE line that Prometheus rejects.
+func TestFormatPrometheusText_MixedVersionDedup(t *testing.T) {
+ server := newTestServerOnly(t)
+ now := time.Now()
+ metricsList := []*metrics.AggregatedMetric{
+ createTestMetricWithType("process_cpu_seconds_total", 5, map[string]string{"pod_name": "new"}, now, "counter"),
+ createTestMetricWithType("process_cpu_seconds_total", 3, map[string]string{"pod_name": "old"}, now, ""),
+ }
+
+ result := server.formatPrometheusText(metricsList)
+
+ assert.Equal(t, 1, strings.Count(result, "# TYPE process_cpu_seconds_total"),
+ "must emit exactly one TYPE line for the family")
+ assert.Contains(t, result, "# TYPE process_cpu_seconds_total counter")
+ assert.NotContains(t, result, "# TYPE process_cpu_seconds_total gauge")
+ // Both the upgraded and pre-upgrade samples must still be present.
+ assert.Contains(t, result, `pod_name="new"`)
+ assert.Contains(t, result, `pod_name="old"`)
+}
+
+// TestFormatPrometheusText_MixedVersionHistogramDedup verifies the same dedup for histogram
+// component series: untyped _bucket/_sum/_count samples from a pre-upgrade agent fold under
+// the typed histogram family rather than spawning a second (conflicting) TYPE line.
+func TestFormatPrometheusText_MixedVersionHistogramDedup(t *testing.T) {
+ server := newTestServerOnly(t)
+ now := time.Now()
+ newPod := map[string]string{"le": "0.1", "pod_name": "new"}
+ oldPod := map[string]string{"le": "0.1", "pod_name": "old"}
+ metricsList := []*metrics.AggregatedMetric{
+ createTestMetricWithType("req_latency_bucket", 10, newPod, now, "histogram"),
+ createTestMetricWithType("req_latency_sum", 1.0, map[string]string{"pod_name": "new"}, now, "histogram"),
+ createTestMetricWithType("req_latency_count", 10, map[string]string{"pod_name": "new"}, now, "histogram"),
+ createTestMetricWithType("req_latency_bucket", 8, oldPod, now, ""),
+ createTestMetricWithType("req_latency_sum", 0.8, map[string]string{"pod_name": "old"}, now, ""),
+ createTestMetricWithType("req_latency_count", 8, map[string]string{"pod_name": "old"}, now, ""),
+ }
+
+ result := server.formatPrometheusText(metricsList)
+
+ assert.Equal(t, 1, strings.Count(result, "# TYPE req_latency histogram"),
+ "must emit exactly one histogram TYPE line for the family")
+ // No component or gauge TYPE lines leaked from the legacy path.
+ assert.NotContains(t, result, "# TYPE req_latency_bucket")
+ assert.NotContains(t, result, "# TYPE req_latency_sum")
+ assert.NotContains(t, result, "# TYPE req_latency_count")
+ assert.NotContains(t, result, "# TYPE req_latency gauge")
+ // Pre-upgrade samples still present.
+ assert.Contains(t, result, `pod_name="old"`)
+}
+
+// TestFormatPrometheusText_EscapesLabelValues verifies that label values containing the
+// Prometheus-special characters (backslash, double-quote, line feed) are escaped, so a value
+// with special characters cannot produce malformed exposition output that breaks the scrape.
+func TestFormatPrometheusText_EscapesLabelValues(t *testing.T) {
+ server := newTestServerOnly(t)
+ now := time.Now()
+ metricsList := []*metrics.AggregatedMetric{
+ createTestMetricWithType("http_requests_total", 1, map[string]string{
+ "path": `a"b\c` + "\n" + "d",
+ }, now, "counter"),
+ }
+
+ result := server.formatPrometheusText(metricsList)
+
+ assert.Contains(t, result, `path="a\"b\\c\nd"`)
+ // The raw, unescaped sequence must not leak into the output.
+ assert.NotContains(t, result, `path="a"b\c`)
+}
+
func TestHandleClusterLifecycle_StatusReportsUseProtoJSON(t *testing.T) {
initTestLogger(t)
testLogger := logger.GetLogger("test", "api")
diff --git a/fodc/proxy/internal/metrics/aggregator.go b/fodc/proxy/internal/metrics/aggregator.go
index 6224cb560..986352a62 100644
--- a/fodc/proxy/internal/metrics/aggregator.go
+++ b/fodc/proxy/internal/metrics/aggregator.go
@@ -44,6 +44,7 @@ type AggregatedMetric struct {
Name string
AgentID string
Description string
+ Type string // lowercase prometheus type: "counter","gauge","histogram","summary","untyped",""
Value float64
}
@@ -88,6 +89,25 @@ func (ma *Aggregator) SetGRPCService(grpcService RequestSender) {
ma.grpcService = grpcService
}
+// protoMetricTypeToString converts a proto MetricType enum to its lowercase string representation.
+// UNSPECIFIED maps to "" (unknown, causes proxy to fall back to suffix heuristic).
+func protoMetricTypeToString(mt fodcv1.MetricType) string {
+ switch mt {
+ case fodcv1.MetricType_METRIC_TYPE_COUNTER:
+ return "counter"
+ case fodcv1.MetricType_METRIC_TYPE_GAUGE:
+ return "gauge"
+ case fodcv1.MetricType_METRIC_TYPE_HISTOGRAM:
+ return "histogram"
+ case fodcv1.MetricType_METRIC_TYPE_SUMMARY:
+ return "summary"
+ case fodcv1.MetricType_METRIC_TYPE_UNTYPED:
+ return "untyped"
+ default:
+ return ""
+ }
+}
+
// ProcessMetricsFromAgent processes metrics received from an agent.
func (ma *Aggregator) ProcessMetricsFromAgent(ctx context.Context, agentID string, agentInfo *registry.AgentInfo, req *fodcv1.StreamMetricsRequest) error {
aggregatedMetrics := make([]*AggregatedMetric, 0, len(req.Metrics))
@@ -119,6 +139,7 @@ func (ma *Aggregator) ProcessMetricsFromAgent(ctx context.Context, agentID strin
Timestamp: timestamp,
AgentID: agentID,
Description: metric.Description,
+ Type: protoMetricTypeToString(metric.Type),
}
aggregatedMetrics = append(aggregatedMetrics, aggregatedMetric)
diff --git a/go.mod b/go.mod
index b104d51c8..4a7615414 100644
--- a/go.mod
+++ b/go.mod
@@ -189,8 +189,8 @@ require (
github.com/pelletier/go-toml/v2 v2.3.0 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
- github.com/prometheus/client_model v0.6.2 // indirect
- github.com/prometheus/common v0.67.5 // indirect
+ github.com/prometheus/client_model v0.6.2
+ github.com/prometheus/common v0.67.5
github.com/prometheus/procfs v0.20.1 // indirect
github.com/robfig/cron/v3 v3.0.1
github.com/sagikazarmark/locafero v0.12.0 // indirect
(skywalking-banyandb) 06/11: Fix incorrect counts, rate and missing trace fields in the lifecycle migration report (#1133)
Posted by bu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ButterBright pushed a commit to branch v0.10.x
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit 7cd2616f66bd2da68358782dcc2510429633d496
Author: mrproliu <74...@qq.com>
AuthorDate: Fri May 22 14:04:24 2026 +0800
Fix incorrect counts, rate and missing trace fields in the lifecycle migration report (#1133)
---
CHANGES.md | 1 +
.../backup/lifecycle/measure_migration_visitor.go | 14 +-
banyand/backup/lifecycle/migration_integration.go | 148 +++++++-----
banyand/backup/lifecycle/progress.go | 198 +++++++++++++---
banyand/backup/lifecycle/progress_mark_test.go | 192 ++++++++++++++++
.../lifecycle/report_partial_failure_test.go | 255 +++++++++++++++++++++
banyand/backup/lifecycle/service.go | 69 +++++-
.../backup/lifecycle/stream_migration_visitor.go | 21 +-
.../backup/lifecycle/trace_migration_visitor.go | 21 +-
test/cases/lifecycle/lifecycle.go | 141 +++++++++++-
10 files changed, 947 insertions(+), 113 deletions(-)
diff --git a/CHANGES.md b/CHANGES.md
index 8a64b558d..0f007b0aa 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -41,6 +41,7 @@ Release Notes.
- Fix lifecycle migration where the receiving node could create segments shorter than the configured `SegmentInterval`.
- Fail fast on incompatible storage version at boot. Previously the server would start in a degraded `SERVING` state with affected groups un-loaded because the property schema-registry retry loop swallowed the version-incompatibility panic. Compatible versions are listed in `banyand/internal/storage/versions.yml`.
- Release bluge index writers on segment rotation so `analysisWorker` pools sized from `GOMAXPROCS` don't accumulate across rotations. Two layered defects kept the existing idle-segment reclaim path from running: `segmentIdleTimeout` defaulted to `0` (which disabled the 10-minute reclaim ticker), and `incRef` refreshed `lastAccessed` on every rotation tick so `closeIdleSegments` never observed an idle segment. Defaults to `time.Hour`, moves the `lastAccessed` bump to real read/write call [...]
+- Fix incorrect counts and missing trace fields in the lifecycle migration report.
### Chores
diff --git a/banyand/backup/lifecycle/measure_migration_visitor.go b/banyand/backup/lifecycle/measure_migration_visitor.go
index fd12b85ba..7a72e8394 100644
--- a/banyand/backup/lifecycle/measure_migration_visitor.go
+++ b/banyand/backup/lifecycle/measure_migration_visitor.go
@@ -102,9 +102,6 @@ func (mv *measureMigrationVisitor) VisitSeries(segmentTR *timestamp.TimeRange, s
Str("path", seriesIndexPath).
Msg("found measure segment files for migration")
- // Set the total number of series segments for progress tracking
- mv.SetMeasureSeriesCount(len(segmentFiles))
-
// Calculate ALL target segments this series index should go to
targetSegments := calculateTargetSegments(
segmentTR.Start.UnixNano(),
@@ -213,6 +210,15 @@ func (mv *measureMigrationVisitor) VisitSeries(segmentTR *timestamp.TimeRange, s
}
}
+ // Mark each source series file as fully migrated (idempotent — bumps Progress once per source).
+ for _, segmentFileName := range segmentFiles {
+ fileSegmentIDStr := strings.TrimSuffix(segmentFileName, ".seg")
+ segmentID, parseErr := strconv.ParseUint(fileSegmentIDStr, 16, 64)
+ if parseErr != nil {
+ continue
+ }
+ mv.progress.MarkSourceMeasureSeriesCompleted(mv.group, seriesIndexPath, common.ShardID(segmentID))
+ }
return nil
}
@@ -226,7 +232,6 @@ func (mv *measureMigrationVisitor) VisitPart(_ *timestamp.TimeRange, sourceShard
if err != nil {
return fmt.Errorf("failed to parse part ID from path: %w", err)
}
-
// Calculate ALL target segments this part should go to
targetSegments := calculateTargetSegments(
partData.MinTimestamp,
@@ -291,6 +296,7 @@ func (mv *measureMigrationVisitor) VisitPart(_ *timestamp.TimeRange, sourceShard
Msgf("measure part migration completed for target segment %d/%d", i+1, len(targetSegments))
}
+ mv.progress.MarkSourceMeasurePartCompleted(mv.group, partPath, sourceShardID, partID)
return nil
}
diff --git a/banyand/backup/lifecycle/migration_integration.go b/banyand/backup/lifecycle/migration_integration.go
index 8036f95d8..742bfeef6 100644
--- a/banyand/backup/lifecycle/migration_integration.go
+++ b/banyand/backup/lifecycle/migration_integration.go
@@ -18,11 +18,14 @@
package lifecycle
import (
+ "strings"
+
"github.com/apache/skywalking-banyandb/api/common"
"github.com/apache/skywalking-banyandb/banyand/internal/storage"
"github.com/apache/skywalking-banyandb/banyand/measure"
"github.com/apache/skywalking-banyandb/banyand/stream"
"github.com/apache/skywalking-banyandb/banyand/trace"
+ "github.com/apache/skywalking-banyandb/pkg/fs"
"github.com/apache/skywalking-banyandb/pkg/logger"
"github.com/apache/skywalking-banyandb/pkg/timestamp"
)
@@ -37,13 +40,16 @@ func migrateStreamWithFileBasedAndProgress(tsdbRootPath string, timeRange timest
// Get target stage configuration
targetStageInterval := getTargetStageInterval(group)
- // Count total parts before starting migration
- totalParts, segmentSuffixes, err := countStreamParts(tsdbRootPath, timeRange, segmentIntervalRule)
+ // Pre-walk source items so each resource has a fixed planned denominator.
+ counter, segmentSuffixes, err := countStreamParts(tsdbRootPath, timeRange, segmentIntervalRule)
if err != nil {
- logger.Warn().Err(err).Msg("failed to count stream parts, proceeding without part count")
+ logger.Warn().Err(err).Msg("failed to count stream source items, proceeding without planned counts")
} else {
- logger.Info().Int("total_parts", totalParts).Strs("segment_suffixes", segmentSuffixes).
- Msg("counted stream parts for progres tracking")
+ logger.Info().Int("total_parts", counter.partCount).
+ Int("total_series_files", counter.seriesFileCount).
+ Int("total_element_index_visits", counter.elementIndexCount).
+ Strs("segment_suffixes", segmentSuffixes).
+ Msg("counted stream source items for progress tracking")
}
// Create file-based migration visitor with progress tracking and target stage interval
@@ -53,9 +59,11 @@ func migrateStreamWithFileBasedAndProgress(tsdbRootPath string, timeRange timest
)
defer visitor.Close()
- // Set the total part count for progress tracking
- if totalParts > 0 {
- visitor.SetStreamPartCount(totalParts)
+ // Set the planned source-item counts for progress tracking.
+ if counter != nil {
+ visitor.SetStreamPartCount(counter.partCount)
+ visitor.SetStreamSeriesCount(counter.seriesFileCount)
+ visitor.SetStreamElementIndexCount(counter.elementIndexCount)
}
// Use the existing VisitStreamsInTimeRange function with our file-based visitor
@@ -66,27 +74,36 @@ func migrateStreamWithFileBasedAndProgress(tsdbRootPath string, timeRange timest
return segmentSuffixes, nil
}
-// countStreamParts counts the total number of parts in the given time range.
-func countStreamParts(tsdbRootPath string, timeRange timestamp.TimeRange, segmentInterval storage.IntervalRule) (int, []string, error) {
- // Create a simple visitor to count parts
- partCounter := &partCountVisitor{}
+// countStreamParts counts the total number of source items in the given time range.
+func countStreamParts(tsdbRootPath string, timeRange timestamp.TimeRange, segmentInterval storage.IntervalRule) (*partCountVisitor, []string, error) {
+ // Create a simple visitor to count source items
+ partCounter := &partCountVisitor{lfs: fs.NewLocalFileSystem()}
// Use the existing VisitStreamsInTimeRange function to count parts
segmentSuffixes, err := stream.VisitStreamsInTimeRange(tsdbRootPath, timeRange, partCounter, segmentInterval)
if err != nil {
- return 0, nil, err
+ return nil, nil, err
}
- return partCounter.partCount, segmentSuffixes, nil
+ return partCounter, segmentSuffixes, nil
}
-// partCountVisitor is a simple visitor that counts parts.
+// partCountVisitor is a simple visitor that counts source parts, series files and element-index visits.
type partCountVisitor struct {
- partCount int
+ lfs fs.FileSystem
+ partCount int
+ seriesFileCount int
+ elementIndexCount int
}
-// VisitSeries implements stream.Visitor.
-func (pcv *partCountVisitor) VisitSeries(_ *timestamp.TimeRange, _ string, _ []common.ShardID) error {
+// VisitSeries implements stream.Visitor; counts .seg files under the source series index directory.
+func (pcv *partCountVisitor) VisitSeries(_ *timestamp.TimeRange, seriesIndexPath string, _ []common.ShardID) error {
+ entries := pcv.lfs.ReadDir(seriesIndexPath)
+ for _, e := range entries {
+ if !e.IsDir() && strings.HasSuffix(e.Name(), ".seg") {
+ pcv.seriesFileCount++
+ }
+ }
return nil
}
@@ -98,6 +115,7 @@ func (pcv *partCountVisitor) VisitPart(_ *timestamp.TimeRange, _ common.ShardID,
// VisitElementIndex implements stream.Visitor.
func (pcv *partCountVisitor) VisitElementIndex(_ *timestamp.TimeRange, _ common.ShardID, _ string) error {
+ pcv.elementIndexCount++
return nil
}
@@ -111,13 +129,15 @@ func migrateMeasureWithFileBasedAndProgress(tsdbRootPath string, timeRange times
// Get target stage configuration
targetStageInterval := getTargetStageInterval(group)
- // Count total parts before starting migration
- totalParts, segmentSuffixes, err := countMeasureParts(tsdbRootPath, timeRange, segmentIntervalRule)
+ // Pre-walk source items so each resource has a fixed planned denominator.
+ counter, segmentSuffixes, err := countMeasureParts(tsdbRootPath, timeRange, segmentIntervalRule)
if err != nil {
- logger.Warn().Err(err).Msg("failed to count measure parts, proceeding without part count")
+ logger.Warn().Err(err).Msg("failed to count measure source items, proceeding without planned counts")
} else {
- logger.Info().Int("total_parts", totalParts).Strs("segment_suffixes", segmentSuffixes).
- Msg("counted measure parts for progress tracking")
+ logger.Info().Int("total_parts", counter.partCount).
+ Int("total_series_files", counter.seriesFileCount).
+ Strs("segment_suffixes", segmentSuffixes).
+ Msg("counted measure source items for progress tracking")
}
// Create file-based migration visitor with progress tracking and target stage interval
@@ -127,9 +147,10 @@ func migrateMeasureWithFileBasedAndProgress(tsdbRootPath string, timeRange times
)
defer visitor.Close()
- // Set the total part count for progress tracking
- if totalParts > 0 {
- visitor.SetMeasurePartCount(totalParts)
+ // Set the planned source-item counts for progress tracking.
+ if counter != nil {
+ visitor.SetMeasurePartCount(counter.partCount)
+ visitor.SetMeasureSeriesCount(counter.seriesFileCount)
}
// Use the existing VisitMeasuresInTimeRange function with our file-based visitor
@@ -140,27 +161,35 @@ func migrateMeasureWithFileBasedAndProgress(tsdbRootPath string, timeRange times
return segmentSuffixes, nil
}
-// countMeasureParts counts the total number of parts in the given time range.
-func countMeasureParts(tsdbRootPath string, timeRange timestamp.TimeRange, segmentInterval storage.IntervalRule) (int, []string, error) {
- // Create a simple visitor to count parts
- partCounter := &measurePartCountVisitor{}
+// countMeasureParts counts the total number of source items in the given time range.
+func countMeasureParts(tsdbRootPath string, timeRange timestamp.TimeRange, segmentInterval storage.IntervalRule) (*measurePartCountVisitor, []string, error) {
+ // Create a simple visitor to count source items
+ partCounter := &measurePartCountVisitor{lfs: fs.NewLocalFileSystem()}
// Use the existing VisitMeasuresInTimeRange function to count parts
segmentSuffixes, err := measure.VisitMeasuresInTimeRange(tsdbRootPath, timeRange, partCounter, segmentInterval)
if err != nil {
- return 0, nil, err
+ return nil, nil, err
}
- return partCounter.partCount, segmentSuffixes, nil
+ return partCounter, segmentSuffixes, nil
}
-// measurePartCountVisitor is a simple visitor that counts measure parts.
+// measurePartCountVisitor is a simple visitor that counts measure source parts and series files.
type measurePartCountVisitor struct {
- partCount int
+ lfs fs.FileSystem
+ partCount int
+ seriesFileCount int
}
-// VisitSeries implements measure.Visitor.
-func (pcv *measurePartCountVisitor) VisitSeries(_ *timestamp.TimeRange, _ string, _ []common.ShardID) error {
+// VisitSeries implements measure.Visitor; counts .seg files under the source series index directory.
+func (pcv *measurePartCountVisitor) VisitSeries(_ *timestamp.TimeRange, seriesIndexPath string, _ []common.ShardID) error {
+ entries := pcv.lfs.ReadDir(seriesIndexPath)
+ for _, e := range entries {
+ if !e.IsDir() && strings.HasSuffix(e.Name(), ".seg") {
+ pcv.seriesFileCount++
+ }
+ }
return nil
}
@@ -180,13 +209,15 @@ func migrateTraceWithFileBasedAndProgress(tsdbRootPath string, timeRange timesta
// Get target stage configuration
targetStageInterval := getTargetStageInterval(group)
- // Count total shards before starting migration
- totalShards, segmentSuffixes, err := countTraceShards(tsdbRootPath, timeRange, segmentIntervalRule)
+ // Pre-walk source items so each resource has a fixed planned denominator.
+ counter, segmentSuffixes, err := countTraceShards(tsdbRootPath, timeRange, segmentIntervalRule)
if err != nil {
- logger.Warn().Err(err).Msg("failed to count trace parts, proceeding without part count")
+ logger.Warn().Err(err).Msg("failed to count trace source items, proceeding without planned counts")
} else {
- logger.Info().Int("total_shards", totalShards).Strs("segment_suffixes", segmentSuffixes).
- Msg("counted trace parts for progress tracking")
+ logger.Info().Int("total_shards", counter.shardCount).
+ Int("total_series_files", counter.seriesFileCount).
+ Strs("segment_suffixes", segmentSuffixes).
+ Msg("counted trace source items for progress tracking")
}
// Create file-based migration visitor with progress tracking and target stage interval
@@ -196,9 +227,10 @@ func migrateTraceWithFileBasedAndProgress(tsdbRootPath string, timeRange timesta
)
defer visitor.Close()
- // Set the total part count for progress tracking
- if totalShards > 0 {
- visitor.SetTraceShardCount(totalShards)
+ // Set the planned source-item counts for progress tracking.
+ if counter != nil {
+ visitor.SetTraceShardCount(counter.shardCount)
+ visitor.SetTraceSeriesCount(counter.seriesFileCount)
}
// Use the existing VisitTracesInTimeRange function with our file-based visitor
@@ -209,27 +241,35 @@ func migrateTraceWithFileBasedAndProgress(tsdbRootPath string, timeRange timesta
return segmentSuffixes, nil
}
-// countTraceShards counts the total number of shards in the given time range.
-func countTraceShards(tsdbRootPath string, timeRange timestamp.TimeRange, segmentInterval storage.IntervalRule) (int, []string, error) {
- // Create a simple visitor to count shards
- shardCounter := &traceShardsCountVisitor{}
+// countTraceShards counts the total number of source items in the given time range.
+func countTraceShards(tsdbRootPath string, timeRange timestamp.TimeRange, segmentInterval storage.IntervalRule) (*traceShardsCountVisitor, []string, error) {
+ // Create a simple visitor to count source items
+ shardCounter := &traceShardsCountVisitor{lfs: fs.NewLocalFileSystem()}
// Use the existing VisitTracesInTimeRange function to count parts
segmentSuffixes, err := trace.VisitTracesInTimeRange(tsdbRootPath, timeRange, shardCounter, segmentInterval)
if err != nil {
- return 0, nil, err
+ return nil, nil, err
}
- return shardCounter.shardCount, segmentSuffixes, nil
+ return shardCounter, segmentSuffixes, nil
}
-// traceShardsCountVisitor is a simple visitor that counts trace shards.
+// traceShardsCountVisitor is a simple visitor that counts trace source shards and series files.
type traceShardsCountVisitor struct {
- shardCount int
+ lfs fs.FileSystem
+ shardCount int
+ seriesFileCount int
}
-// VisitSeries implements trace.Visitor.
-func (pcv *traceShardsCountVisitor) VisitSeries(_ *timestamp.TimeRange, _ string, _ []common.ShardID) error {
+// VisitSeries implements trace.Visitor; counts .seg files under the source series index directory.
+func (pcv *traceShardsCountVisitor) VisitSeries(_ *timestamp.TimeRange, seriesIndexPath string, _ []common.ShardID) error {
+ entries := pcv.lfs.ReadDir(seriesIndexPath)
+ for _, e := range entries {
+ if !e.IsDir() && strings.HasSuffix(e.Name(), ".seg") {
+ pcv.seriesFileCount++
+ }
+ }
return nil
}
diff --git a/banyand/backup/lifecycle/progress.go b/banyand/backup/lifecycle/progress.go
index 99f7083d1..91deb265f 100644
--- a/banyand/backup/lifecycle/progress.go
+++ b/banyand/backup/lifecycle/progress.go
@@ -64,11 +64,31 @@ type Progress struct {
TraceSeriesErrors map[string]map[string]map[common.ShardID]string `json:"trace_series_errors"`
TraceSeriesCounts map[string]int `json:"trace_series_counts"`
TraceSeriesProgress map[string]int `json:"trace_series_progress"`
- progressFilePath string `json:"-"`
- SnapshotStreamDir string `json:"snapshot_stream_dir"`
- SnapshotMeasureDir string `json:"snapshot_measure_dir"`
- SnapshotTraceDir string `json:"snapshot_trace_dir"`
- mu sync.Mutex `json:"-"`
+
+ // Per-source completion tracking (advances *Progress idempotently once per source item).
+ SourceCompletedStreamParts map[string]map[string]map[common.ShardID]map[uint64]bool `json:"source_completed_stream_parts"`
+ SourceCompletedStreamSeries map[string]map[string]map[common.ShardID]bool `json:"source_completed_stream_series"`
+ SourceCompletedStreamElementIndex map[string]map[string]map[common.ShardID]bool `json:"source_completed_stream_element_index"`
+ SourceCompletedMeasureParts map[string]map[string]map[common.ShardID]map[uint64]bool `json:"source_completed_measure_parts"`
+ SourceCompletedMeasureSeries map[string]map[string]map[common.ShardID]bool `json:"source_completed_measure_series"`
+ SourceCompletedTraceShards map[string]map[string]map[common.ShardID]bool `json:"source_completed_trace_shards"`
+ SourceCompletedTraceSeries map[string]map[string]map[common.ShardID]bool `json:"source_completed_trace_series"`
+
+ progressFilePath string `json:"-"`
+ SnapshotStreamDir string `json:"snapshot_stream_dir"`
+ SnapshotMeasureDir string `json:"snapshot_measure_dir"`
+ SnapshotTraceDir string `json:"snapshot_trace_dir"`
+ GroupsToProcess []string `json:"groups_to_process"`
+ mu sync.Mutex `json:"-"`
+}
+
+// SetGroupsToProcess records the set of groups picked up by this migration cycle.
+// Used as the denominator for migration_status.completion_rate so the report
+// reflects "completed / scheduled" rather than overlapping per-catalog buckets.
+func (p *Progress) SetGroupsToProcess(groups []string) {
+ p.mu.Lock()
+ defer p.mu.Unlock()
+ p.GroupsToProcess = append(p.GroupsToProcess[:0], groups...)
}
// AllGroupsNotFullyCompleted find is there have any group not fully completed.
@@ -120,8 +140,17 @@ func NewProgress(path string, l *logger.Logger) *Progress {
TraceSeriesErrors: make(map[string]map[string]map[common.ShardID]string),
TraceSeriesCounts: make(map[string]int),
TraceSeriesProgress: make(map[string]int),
- progressFilePath: path,
- logger: l,
+
+ SourceCompletedStreamParts: make(map[string]map[string]map[common.ShardID]map[uint64]bool),
+ SourceCompletedStreamSeries: make(map[string]map[string]map[common.ShardID]bool),
+ SourceCompletedStreamElementIndex: make(map[string]map[string]map[common.ShardID]bool),
+ SourceCompletedMeasureParts: make(map[string]map[string]map[common.ShardID]map[uint64]bool),
+ SourceCompletedMeasureSeries: make(map[string]map[string]map[common.ShardID]bool),
+ SourceCompletedTraceShards: make(map[string]map[string]map[common.ShardID]bool),
+ SourceCompletedTraceSeries: make(map[string]map[string]map[common.ShardID]bool),
+
+ progressFilePath: path,
+ logger: l,
}
}
@@ -266,9 +295,6 @@ func (p *Progress) MarkStreamPartCompleted(group string, segmentID string, shard
// Mark part as completed
p.CompletedStreamParts[group][segmentID][shardID][partID] = true
-
- // Update progress count
- p.StreamPartProgress[group]++
}
// IsStreamPartCompleted checks if a specific part of a stream has been completed.
@@ -354,9 +380,6 @@ func (p *Progress) MarkStreamSeriesCompleted(group string, segmentID string, sha
// Mark series segment as completed
p.CompletedStreamSeries[group][segmentID][shardID] = true
-
- // Update progress count
- p.StreamSeriesProgress[group]++
}
// IsStreamSeriesCompleted checks if a specific series segment of a stream has been completed.
@@ -466,9 +489,6 @@ func (p *Progress) MarkStreamElementIndexCompleted(group string, segmentID strin
// Mark shard as completed
p.CompletedStreamElementIndex[group][segmentID][shardID] = true
-
- // Update progress count
- p.StreamElementIndexProgress[group]++
}
// IsStreamElementIndexCompleted checks if a specific element index file of a stream has been completed.
@@ -552,9 +572,6 @@ func (p *Progress) MarkMeasurePartCompleted(group string, segmentID string, shar
// Mark part as completed
p.CompletedMeasureParts[group][segmentID][shardID][partID] = true
-
- // Update progress count
- p.MeasurePartProgress[group]++
}
// IsMeasurePartCompleted checks if a specific part of a measure has been completed.
@@ -633,6 +650,8 @@ func (p *Progress) ClearErrors() {
p.StreamElementIndexErrors = make(map[string]map[string]map[common.ShardID]string)
p.MeasurePartErrors = make(map[string]map[string]map[common.ShardID]map[uint64]string)
p.MeasureSeriesErrors = make(map[string]map[string]map[common.ShardID]string)
+ p.TraceShardErrors = make(map[string]map[string]map[common.ShardID]string)
+ p.TraceSeriesErrors = make(map[string]map[string]map[common.ShardID]string)
}
// MarkMeasureSeriesCompleted marks a specific series segment of a measure as completed.
@@ -651,9 +670,6 @@ func (p *Progress) MarkMeasureSeriesCompleted(group string, segmentID string, sh
// Mark series segment as completed
p.CompletedMeasureSeries[group][segmentID][shardID] = true
-
- // Update progress count
- p.MeasureSeriesProgress[group]++
}
// IsMeasureSeriesCompleted checks if a specific series segment of a measure has been completed.
@@ -746,9 +762,6 @@ func (p *Progress) MarkTraceShardCompleted(group string, segmentID string, shard
p.CompletedTraceShards[group][segmentID] = make(map[common.ShardID]bool)
}
p.CompletedTraceShards[group][segmentID][shardID] = true
-
- // Increment progress
- p.TraceShardProgress[group]++
}
// IsTraceShardCompleted checks if a specific part of a trace has been completed.
@@ -820,9 +833,6 @@ func (p *Progress) MarkTraceSeriesCompleted(group string, segmentID string, shar
p.CompletedTraceSeries[group][segmentID] = make(map[common.ShardID]bool)
}
p.CompletedTraceSeries[group][segmentID][shardID] = true
-
- // Increment progress
- p.TraceSeriesProgress[group]++
}
// IsTraceSeriesCompleted checks if a specific series segment of a trace has been completed.
@@ -881,3 +891,135 @@ func (p *Progress) GetTraceSeriesProgress(group string) int {
}
return 0
}
+
+// MarkSourceStreamPartCompleted records that one source stream part finished
+// every target write successfully; idempotent (++Progress on first call only).
+func (p *Progress) MarkSourceStreamPartCompleted(group string, sourceSegmentID string, sourceShardID common.ShardID, partID uint64) {
+ defer p.saveProgress()
+ p.mu.Lock()
+ defer p.mu.Unlock()
+ if p.SourceCompletedStreamParts[group] == nil {
+ p.SourceCompletedStreamParts[group] = make(map[string]map[common.ShardID]map[uint64]bool)
+ }
+ if p.SourceCompletedStreamParts[group][sourceSegmentID] == nil {
+ p.SourceCompletedStreamParts[group][sourceSegmentID] = make(map[common.ShardID]map[uint64]bool)
+ }
+ if p.SourceCompletedStreamParts[group][sourceSegmentID][sourceShardID] == nil {
+ p.SourceCompletedStreamParts[group][sourceSegmentID][sourceShardID] = make(map[uint64]bool)
+ }
+ if !p.SourceCompletedStreamParts[group][sourceSegmentID][sourceShardID][partID] {
+ p.SourceCompletedStreamParts[group][sourceSegmentID][sourceShardID][partID] = true
+ p.StreamPartProgress[group]++
+ }
+}
+
+// MarkSourceStreamSeriesCompleted records that one source stream series file finished
+// every target write successfully; idempotent.
+func (p *Progress) MarkSourceStreamSeriesCompleted(group string, sourceSegmentID string, sourceShardID common.ShardID) {
+ defer p.saveProgress()
+ p.mu.Lock()
+ defer p.mu.Unlock()
+ if p.SourceCompletedStreamSeries[group] == nil {
+ p.SourceCompletedStreamSeries[group] = make(map[string]map[common.ShardID]bool)
+ }
+ if p.SourceCompletedStreamSeries[group][sourceSegmentID] == nil {
+ p.SourceCompletedStreamSeries[group][sourceSegmentID] = make(map[common.ShardID]bool)
+ }
+ if !p.SourceCompletedStreamSeries[group][sourceSegmentID][sourceShardID] {
+ p.SourceCompletedStreamSeries[group][sourceSegmentID][sourceShardID] = true
+ p.StreamSeriesProgress[group]++
+ }
+}
+
+// MarkSourceStreamElementIndexCompleted records that one source stream element-index visit finished
+// every target write successfully; idempotent.
+func (p *Progress) MarkSourceStreamElementIndexCompleted(group string, sourceSegmentID string, sourceShardID common.ShardID) {
+ defer p.saveProgress()
+ p.mu.Lock()
+ defer p.mu.Unlock()
+ if p.SourceCompletedStreamElementIndex[group] == nil {
+ p.SourceCompletedStreamElementIndex[group] = make(map[string]map[common.ShardID]bool)
+ }
+ if p.SourceCompletedStreamElementIndex[group][sourceSegmentID] == nil {
+ p.SourceCompletedStreamElementIndex[group][sourceSegmentID] = make(map[common.ShardID]bool)
+ }
+ if !p.SourceCompletedStreamElementIndex[group][sourceSegmentID][sourceShardID] {
+ p.SourceCompletedStreamElementIndex[group][sourceSegmentID][sourceShardID] = true
+ p.StreamElementIndexProgress[group]++
+ }
+}
+
+// MarkSourceMeasurePartCompleted records that one source measure part finished
+// every target write successfully; idempotent.
+func (p *Progress) MarkSourceMeasurePartCompleted(group string, sourceSegmentID string, sourceShardID common.ShardID, partID uint64) {
+ defer p.saveProgress()
+ p.mu.Lock()
+ defer p.mu.Unlock()
+ if p.SourceCompletedMeasureParts[group] == nil {
+ p.SourceCompletedMeasureParts[group] = make(map[string]map[common.ShardID]map[uint64]bool)
+ }
+ if p.SourceCompletedMeasureParts[group][sourceSegmentID] == nil {
+ p.SourceCompletedMeasureParts[group][sourceSegmentID] = make(map[common.ShardID]map[uint64]bool)
+ }
+ if p.SourceCompletedMeasureParts[group][sourceSegmentID][sourceShardID] == nil {
+ p.SourceCompletedMeasureParts[group][sourceSegmentID][sourceShardID] = make(map[uint64]bool)
+ }
+ if !p.SourceCompletedMeasureParts[group][sourceSegmentID][sourceShardID][partID] {
+ p.SourceCompletedMeasureParts[group][sourceSegmentID][sourceShardID][partID] = true
+ p.MeasurePartProgress[group]++
+ }
+}
+
+// MarkSourceMeasureSeriesCompleted records that one source measure series file finished
+// every target write successfully; idempotent.
+func (p *Progress) MarkSourceMeasureSeriesCompleted(group string, sourceSegmentID string, sourceShardID common.ShardID) {
+ defer p.saveProgress()
+ p.mu.Lock()
+ defer p.mu.Unlock()
+ if p.SourceCompletedMeasureSeries[group] == nil {
+ p.SourceCompletedMeasureSeries[group] = make(map[string]map[common.ShardID]bool)
+ }
+ if p.SourceCompletedMeasureSeries[group][sourceSegmentID] == nil {
+ p.SourceCompletedMeasureSeries[group][sourceSegmentID] = make(map[common.ShardID]bool)
+ }
+ if !p.SourceCompletedMeasureSeries[group][sourceSegmentID][sourceShardID] {
+ p.SourceCompletedMeasureSeries[group][sourceSegmentID][sourceShardID] = true
+ p.MeasureSeriesProgress[group]++
+ }
+}
+
+// MarkSourceTraceShardCompleted records that one source trace shard finished
+// every target write successfully; idempotent.
+func (p *Progress) MarkSourceTraceShardCompleted(group string, sourceSegmentID string, sourceShardID common.ShardID) {
+ defer p.saveProgress()
+ p.mu.Lock()
+ defer p.mu.Unlock()
+ if p.SourceCompletedTraceShards[group] == nil {
+ p.SourceCompletedTraceShards[group] = make(map[string]map[common.ShardID]bool)
+ }
+ if p.SourceCompletedTraceShards[group][sourceSegmentID] == nil {
+ p.SourceCompletedTraceShards[group][sourceSegmentID] = make(map[common.ShardID]bool)
+ }
+ if !p.SourceCompletedTraceShards[group][sourceSegmentID][sourceShardID] {
+ p.SourceCompletedTraceShards[group][sourceSegmentID][sourceShardID] = true
+ p.TraceShardProgress[group]++
+ }
+}
+
+// MarkSourceTraceSeriesCompleted records that one source trace series file finished
+// every target write successfully; idempotent.
+func (p *Progress) MarkSourceTraceSeriesCompleted(group string, sourceSegmentID string, sourceShardID common.ShardID) {
+ defer p.saveProgress()
+ p.mu.Lock()
+ defer p.mu.Unlock()
+ if p.SourceCompletedTraceSeries[group] == nil {
+ p.SourceCompletedTraceSeries[group] = make(map[string]map[common.ShardID]bool)
+ }
+ if p.SourceCompletedTraceSeries[group][sourceSegmentID] == nil {
+ p.SourceCompletedTraceSeries[group][sourceSegmentID] = make(map[common.ShardID]bool)
+ }
+ if !p.SourceCompletedTraceSeries[group][sourceSegmentID][sourceShardID] {
+ p.SourceCompletedTraceSeries[group][sourceSegmentID][sourceShardID] = true
+ p.TraceSeriesProgress[group]++
+ }
+}
diff --git a/banyand/backup/lifecycle/progress_mark_test.go b/banyand/backup/lifecycle/progress_mark_test.go
new file mode 100644
index 000000000..367330d70
--- /dev/null
+++ b/banyand/backup/lifecycle/progress_mark_test.go
@@ -0,0 +1,192 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package lifecycle
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+)
+
+// TestMarkPerTargetDoesNotMoveCounters pins the contract that per-target
+// Mark*Completed / Mark*Error only maintain the per-target dedup / error
+// maps and do NOT advance Counts or Progress. Counts is set by pre-walk
+// (Set*Count) and Progress is advanced exclusively by MarkSource*Completed.
+func TestMarkPerTargetDoesNotMoveCounters(t *testing.T) {
+ p := NewProgress("", logger.GetLogger("test"))
+
+ // Stream part: per-target mark several times — counters must stay 0.
+ p.MarkStreamPartCompleted("g", "tgt-seg-1", 0, 1)
+ p.MarkStreamPartCompleted("g", "tgt-seg-2", 0, 1)
+ p.MarkStreamPartError("g", "tgt-seg-3", 0, 1, "err")
+ assert.Equal(t, 0, p.StreamPartCounts["g"])
+ assert.Equal(t, 0, p.StreamPartProgress["g"])
+
+ // Per-target dedup map populated.
+ assert.True(t, p.IsStreamPartCompleted("g", "tgt-seg-1", 0, 1))
+ assert.True(t, p.IsStreamPartCompleted("g", "tgt-seg-2", 0, 1))
+
+ // Same for series / element_index / measure / trace.
+ p.MarkStreamSeriesCompleted("g", "tgt-seg-1", 0)
+ p.MarkStreamElementIndexCompleted("g", "tgt-seg-1", 0)
+ p.MarkMeasurePartCompleted("g", "tgt-seg-1", 0, 1)
+ p.MarkMeasureSeriesCompleted("g", "tgt-seg-1", 0)
+ p.MarkTraceShardCompleted("g", "tgt-seg-1", 0)
+ p.MarkTraceSeriesCompleted("g", "tgt-seg-1", 0)
+ for _, counter := range []int{
+ p.StreamSeriesCounts["g"], p.StreamSeriesProgress["g"],
+ p.StreamElementIndexCounts["g"], p.StreamElementIndexProgress["g"],
+ p.MeasurePartCounts["g"], p.MeasurePartProgress["g"],
+ p.MeasureSeriesCounts["g"], p.MeasureSeriesProgress["g"],
+ p.TraceShardCounts["g"], p.TraceShardProgress["g"],
+ p.TraceSeriesCounts["g"], p.TraceSeriesProgress["g"],
+ } {
+ assert.Equal(t, 0, counter, "per-target marks must not move counters")
+ }
+}
+
+// TestSetCountIsThePlannedDenominator pins that Set*Count is the only writer
+// of the per-resource Counts denominator (set by pre-walk).
+func TestSetCountIsThePlannedDenominator(t *testing.T) {
+ p := NewProgress("", logger.GetLogger("test"))
+
+ p.SetStreamPartCount("g", 100)
+ p.SetStreamSeriesCount("g", 50)
+ p.SetStreamElementIndexCount("g", 10)
+ p.SetMeasurePartCount("g", 70)
+ p.SetMeasureSeriesCount("g", 35)
+ p.SetTraceShardCount("g", 5)
+ p.SetTraceSeriesCount("g", 8)
+
+ assert.Equal(t, 100, p.StreamPartCounts["g"])
+ assert.Equal(t, 50, p.StreamSeriesCounts["g"])
+ assert.Equal(t, 10, p.StreamElementIndexCounts["g"])
+ assert.Equal(t, 70, p.MeasurePartCounts["g"])
+ assert.Equal(t, 35, p.MeasureSeriesCounts["g"])
+ assert.Equal(t, 5, p.TraceShardCounts["g"])
+ assert.Equal(t, 8, p.TraceSeriesCounts["g"])
+
+ // Set is overwrite semantics — last call wins.
+ p.SetStreamPartCount("g", 200)
+ assert.Equal(t, 200, p.StreamPartCounts["g"])
+}
+
+// TestMarkSourceCompletedIdempotent pins that MarkSource*Completed advances
+// Progress exactly once per unique source key across all 7 resources.
+func TestMarkSourceCompletedIdempotent(t *testing.T) {
+ p := NewProgress("", logger.GetLogger("test"))
+
+ // Stream part: same key 3 times → Progress=1.
+ p.MarkSourceStreamPartCompleted("g", "src-seg", 0, 1)
+ p.MarkSourceStreamPartCompleted("g", "src-seg", 0, 1)
+ p.MarkSourceStreamPartCompleted("g", "src-seg", 0, 1)
+ assert.Equal(t, 1, p.StreamPartProgress["g"])
+ assert.True(t, p.SourceCompletedStreamParts["g"]["src-seg"][0][1])
+ // Different partID = different source = +1.
+ p.MarkSourceStreamPartCompleted("g", "src-seg", 0, 2)
+ assert.Equal(t, 2, p.StreamPartProgress["g"])
+
+ // Stream series, element_index, measure, trace: same pattern, +1 once.
+ p.MarkSourceStreamSeriesCompleted("g", "src-seg", 0)
+ p.MarkSourceStreamSeriesCompleted("g", "src-seg", 0)
+ assert.Equal(t, 1, p.StreamSeriesProgress["g"])
+
+ p.MarkSourceStreamElementIndexCompleted("g", "src-seg", 0)
+ p.MarkSourceStreamElementIndexCompleted("g", "src-seg", 0)
+ assert.Equal(t, 1, p.StreamElementIndexProgress["g"])
+
+ p.MarkSourceMeasurePartCompleted("g", "src-seg", 0, 1)
+ p.MarkSourceMeasurePartCompleted("g", "src-seg", 0, 1)
+ assert.Equal(t, 1, p.MeasurePartProgress["g"])
+
+ p.MarkSourceMeasureSeriesCompleted("g", "src-seg", 0)
+ p.MarkSourceMeasureSeriesCompleted("g", "src-seg", 0)
+ assert.Equal(t, 1, p.MeasureSeriesProgress["g"])
+
+ p.MarkSourceTraceShardCompleted("g", "src-seg", 0)
+ p.MarkSourceTraceShardCompleted("g", "src-seg", 0)
+ assert.Equal(t, 1, p.TraceShardProgress["g"])
+
+ p.MarkSourceTraceSeriesCompleted("g", "src-seg", 0)
+ p.MarkSourceTraceSeriesCompleted("g", "src-seg", 0)
+ assert.Equal(t, 1, p.TraceSeriesProgress["g"])
+}
+
+// TestClearErrors_AllSevenBuckets pins that ClearErrors resets every error
+// map (including trace) and does NOT touch Counts/Progress (which now
+// derive from pre-walk + MarkSource and are immutable to error replay).
+func TestClearErrors_AllSevenBuckets(t *testing.T) {
+ p := NewProgress("", logger.GetLogger("test"))
+
+ // Set planned counts (pre-walk).
+ p.SetStreamPartCount("g", 10)
+ p.SetStreamSeriesCount("g", 10)
+ p.SetStreamElementIndexCount("g", 10)
+ p.SetMeasurePartCount("g", 10)
+ p.SetMeasureSeriesCount("g", 10)
+ p.SetTraceShardCount("g", 10)
+ p.SetTraceSeriesCount("g", 10)
+
+ // Drop one error per bucket.
+ p.MarkStreamPartError("g", "s", 0, 1, "x")
+ p.MarkStreamSeriesError("g", "s", 0, "x")
+ p.MarkStreamElementIndexError("g", "s", 0, "x")
+ p.MarkMeasurePartError("g", "s", 0, 1, "x")
+ p.MarkMeasureSeriesError("g", "s", 0, "x")
+ p.MarkTraceShardError("g", "s", 0, "x")
+ p.MarkTraceSeriesError("g", "s", 0, "x")
+
+ p.ClearErrors()
+
+ assert.Empty(t, p.StreamPartErrors["g"])
+ assert.Empty(t, p.StreamSeriesErrors["g"])
+ assert.Empty(t, p.StreamElementIndexErrors["g"])
+ assert.Empty(t, p.MeasurePartErrors["g"])
+ assert.Empty(t, p.MeasureSeriesErrors["g"])
+ assert.Empty(t, p.TraceShardErrors["g"])
+ assert.Empty(t, p.TraceSeriesErrors["g"])
+
+ // Counts are pre-walk planned — must survive ClearErrors.
+ for _, c := range []int{
+ p.StreamPartCounts["g"], p.StreamSeriesCounts["g"], p.StreamElementIndexCounts["g"],
+ p.MeasurePartCounts["g"], p.MeasureSeriesCounts["g"],
+ p.TraceShardCounts["g"], p.TraceSeriesCounts["g"],
+ } {
+ assert.Equal(t, 10, c, "Counts must not be touched by ClearErrors")
+ }
+}
+
+// TestPerTargetAndPerSourceAreIndependent pins that per-target dedup and
+// per-source progress are tracked independently: per-target Mark does not
+// move source progress, and MarkSource does not populate the per-target map.
+func TestPerTargetAndPerSourceAreIndependent(t *testing.T) {
+ p := NewProgress("", logger.GetLogger("test"))
+
+ p.MarkStreamPartCompleted("g", "tgt-1", 0, 1)
+ assert.True(t, p.IsStreamPartCompleted("g", "tgt-1", 0, 1))
+ assert.False(t, p.SourceCompletedStreamParts["g"]["src-1"][0][1])
+ assert.Equal(t, 0, p.StreamPartProgress["g"])
+
+ p.MarkSourceStreamPartCompleted("g", "src-1", 0, 1)
+ assert.True(t, p.SourceCompletedStreamParts["g"]["src-1"][0][1])
+ // Per-target map untouched by source mark.
+ assert.False(t, p.IsStreamPartCompleted("g", "src-1", 0, 1))
+ assert.Equal(t, 1, p.StreamPartProgress["g"])
+}
diff --git a/banyand/backup/lifecycle/report_partial_failure_test.go b/banyand/backup/lifecycle/report_partial_failure_test.go
new file mode 100644
index 000000000..1fd3a9683
--- /dev/null
+++ b/banyand/backup/lifecycle/report_partial_failure_test.go
@@ -0,0 +1,255 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package lifecycle
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "github.com/apache/skywalking-banyandb/api/common"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+)
+
+// TestBuildMigrationReport_PartialFailure pins down the JSON shape and
+// numeric values that buildMigrationReport must produce under a partial
+// migration: one stream group has a failed part write while every other
+// group and every other resource within the failing group completes. The
+// shapes and counts are source-level — Counts comes from the pre-walk
+// (Set*Count), Progress is bumped per source by MarkSource*Completed, and
+// completion_rate = Progress / Counts is always in [0, 100] with 100 %
+// meaning every source item migrated.
+func TestBuildMigrationReport_PartialFailure(t *testing.T) {
+ p := NewProgress("", logger.GetLogger("test"))
+ p.SetGroupsToProcess([]string{"sw_a", "sw_b", "sw_metrics", "sw_trace"})
+
+ p.SnapshotStreamDir = "/tmp/stream/snapshots/example"
+ p.SnapshotMeasureDir = "/tmp/measure/snapshots/example"
+ p.SnapshotTraceDir = "/tmp/trace/snapshots/example"
+
+ // === sw_a (stream): full success on 2 source parts, 2 source series files, 1 source element_index visit. ===
+ p.SetStreamPartCount("sw_a", 2)
+ p.MarkSourceStreamPartCompleted("sw_a", "20260513", 0, 1)
+ p.MarkSourceStreamPartCompleted("sw_a", "20260513", 0, 2)
+ p.SetStreamSeriesCount("sw_a", 2)
+ p.MarkSourceStreamSeriesCompleted("sw_a", "20260513", 100)
+ p.MarkSourceStreamSeriesCompleted("sw_a", "20260513", 101)
+ p.SetStreamElementIndexCount("sw_a", 1)
+ p.MarkSourceStreamElementIndexCompleted("sw_a", "20260513", 0)
+ p.MarkGroupCompleted("sw_a")
+ p.MarkStreamGroupDeleted("sw_a")
+
+ // === sw_b (stream): 4 source parts planned, 3 complete + 1 fails (partID 7 on shard 1). ===
+ // series + element_index all complete.
+ p.SetStreamPartCount("sw_b", 4)
+ p.MarkSourceStreamPartCompleted("sw_b", "20260512", 0, 1)
+ p.MarkSourceStreamPartCompleted("sw_b", "20260512", 0, 2)
+ p.MarkSourceStreamPartCompleted("sw_b", "20260512", 1, 1)
+ // partID 7 fails on shard 1 — record the per-target error, source not advanced.
+ p.MarkStreamPartError("sw_b", "20260512", 1, 7, "failed to stream part to target node-cold-0: connection refused")
+ p.SetStreamSeriesCount("sw_b", 2)
+ p.MarkSourceStreamSeriesCompleted("sw_b", "20260512", 0)
+ p.MarkSourceStreamSeriesCompleted("sw_b", "20260512", 1)
+ p.SetStreamElementIndexCount("sw_b", 2)
+ p.MarkSourceStreamElementIndexCompleted("sw_b", "20260512", 0)
+ p.MarkSourceStreamElementIndexCompleted("sw_b", "20260512", 1)
+ // sw_b is NOT marked completed (processStreamGroup early-returns on the failure).
+
+ // === sw_metrics (measure): full success. ===
+ p.SetMeasurePartCount("sw_metrics", 3)
+ p.MarkSourceMeasurePartCompleted("sw_metrics", "20260513", 0, 1)
+ p.MarkSourceMeasurePartCompleted("sw_metrics", "20260513", 0, 2)
+ p.MarkSourceMeasurePartCompleted("sw_metrics", "20260513", 0, 3)
+ p.SetMeasureSeriesCount("sw_metrics", 1)
+ p.MarkSourceMeasureSeriesCompleted("sw_metrics", "20260513", 100)
+ p.MarkGroupCompleted("sw_metrics")
+ p.MarkMeasureGroupDeleted("sw_metrics")
+
+ // === sw_trace: full success. ===
+ p.SetTraceShardCount("sw_trace", 1)
+ p.MarkSourceTraceShardCompleted("sw_trace", "20260513", 0)
+ p.SetTraceSeriesCount("sw_trace", 1)
+ p.MarkSourceTraceSeriesCompleted("sw_trace", "20260513", 100)
+ p.MarkGroupCompleted("sw_trace")
+ p.MarkTraceGroupDeleted("sw_trace")
+
+ // --- Build report ---
+ svc := &lifecycleService{l: logger.GetLogger("test")}
+ report := svc.buildMigrationReport(p)
+
+ require.Equal(t, "2.0", report["report_version"])
+ summary, ok := report["summary"].(map[string]interface{})
+ require.True(t, ok)
+ errs, ok := report["errors"].(map[string]interface{})
+ require.True(t, ok)
+
+ // migration_status: 4 scheduled, 3 completed (sw_a, sw_metrics, sw_trace), 75 %.
+ ms, ok := summary["migration_status"].(map[string]interface{})
+ require.True(t, ok)
+ assert.Equal(t, 4, ms["total_groups"])
+ assert.Equal(t, 3, ms["completed_groups"])
+ assert.InDelta(t, 75.0, ms["completion_rate"], 1e-9)
+
+ // stream parts: 2+4=6 planned, 2+3=5 source-completed.
+ assertResource(t, summary, "stream_migration", "parts", 6, 5, 83.33333333333334, 1)
+ assertResource(t, summary, "stream_migration", "series", 4, 4, 100.0, 0)
+ assertResource(t, summary, "stream_migration", "element_index", 3, 3, 100.0, 0)
+
+ // measure
+ assertResource(t, summary, "measure_migration", "parts", 3, 3, 100.0, 0)
+ assertResource(t, summary, "measure_migration", "series", 1, 1, 100.0, 0)
+
+ // trace
+ assertResource(t, summary, "trace_migration", "parts", 1, 1, 100.0, 0)
+ assertResource(t, summary, "trace_migration", "series", 1, 1, 100.0, 0)
+
+ // errors: stream_parts has one entry for sw_b; everything else empty.
+ for _, key := range []string{
+ "stream_parts", "stream_series", "stream_element_index",
+ "measure_parts", "measure_series",
+ "trace_parts", "trace_series",
+ } {
+ v, found := errs[key]
+ require.Truef(t, found, "errors.%s must be present", key)
+ _, isMap := v.(map[string]interface{})
+ require.Truef(t, isMap, "errors.%s must be map[string]interface{}", key)
+ }
+ streamPartErrs := errs["stream_parts"].(map[string]interface{})
+ require.Lenf(t, streamPartErrs, 1, "errors.stream_parts must hold one group entry, got %v", streamPartErrs)
+ require.Contains(t, streamPartErrs, "sw_b")
+
+ for _, key := range []string{
+ "stream_series", "stream_element_index",
+ "measure_parts", "measure_series",
+ "trace_parts", "trace_series",
+ } {
+ v := errs[key].(map[string]interface{})
+ assert.Emptyf(t, v, "errors.%s must be empty for this scenario", key)
+ }
+}
+
+// TestBuildMigrationReport_CompletedScopedToScheduledSet pins the
+// resume-safety invariant: CompletedGroups carrying entries from a prior
+// cycle that are no longer in the current GroupsToProcess set must NOT
+// inflate completed_groups beyond total_groups.
+func TestBuildMigrationReport_CompletedScopedToScheduledSet(t *testing.T) {
+ p := NewProgress("", logger.GetLogger("test"))
+ p.MarkGroupCompleted("sw_a")
+ p.MarkGroupCompleted("sw_b")
+ p.MarkGroupCompleted("sw_c")
+ p.MarkGroupCompleted("sw_d")
+ p.MarkGroupCompleted("sw_e")
+ p.SetGroupsToProcess([]string{"sw_f", "sw_g", "sw_h"})
+ p.MarkGroupCompleted("sw_f")
+ p.MarkGroupCompleted("sw_g")
+ p.MarkGroupCompleted("sw_h")
+
+ svc := &lifecycleService{l: logger.GetLogger("test")}
+ report := svc.buildMigrationReport(p)
+ summary := report["summary"].(map[string]interface{})
+ ms := summary["migration_status"].(map[string]interface{})
+
+ assert.Equal(t, 3, ms["total_groups"])
+ assert.Equal(t, 3, ms["completed_groups"])
+ assert.InDelta(t, 100.0, ms["completion_rate"], 1e-9)
+}
+
+// TestBuildMigrationReport_EmptyCycleHonestTotalGroups pins that an
+// empty-snapshot cycle reports total_groups=0 / rate=0 instead of inheriting
+// a stale prior-cycle GroupsToProcess.
+func TestBuildMigrationReport_EmptyCycleHonestTotalGroups(t *testing.T) {
+ p := NewProgress("", logger.GetLogger("test"))
+ p.MarkGroupCompleted("sw_a")
+ p.MarkGroupCompleted("sw_b")
+ p.SetGroupsToProcess(nil)
+
+ svc := &lifecycleService{l: logger.GetLogger("test")}
+ report := svc.buildMigrationReport(p)
+ summary := report["summary"].(map[string]interface{})
+ ms := summary["migration_status"].(map[string]interface{})
+
+ assert.Equal(t, 0, ms["total_groups"])
+ assert.Equal(t, 0, ms["completed_groups"])
+ assert.InDelta(t, 0.0, ms["completion_rate"], 1e-9)
+}
+
+// assertResource pins down a single resource sub-block under a catalog
+// (stream | measure | trace)_migration. total = pre-walk source count;
+// completed = source items fully migrated; errors = per-target error count;
+// rate = completed / total ∈ [0, 100].
+func assertResource(t *testing.T, summary map[string]interface{}, catalog, resource string, total, completed int, rate float64, errors int) {
+ t.Helper()
+ cat, ok := summary[catalog].(map[string]interface{})
+ require.Truef(t, ok, "summary.%s must be a map", catalog)
+ res, ok := cat[resource].(map[string]interface{})
+ require.Truef(t, ok, "summary.%s.%s must be a map", catalog, resource)
+ assert.Equalf(t, total, res["total"], "summary.%s.%s.total", catalog, resource)
+ assert.Equalf(t, completed, res["completed"], "summary.%s.%s.completed", catalog, resource)
+ assert.Equalf(t, errors, res["errors"], "summary.%s.%s.errors", catalog, resource)
+ assert.InDeltaf(t, rate, res["completion_rate"], 1e-9, "summary.%s.%s.completion_rate", catalog, resource)
+}
+
+// TestCompletionRateNeverExceeds100UnderFanOut replays the original 266 %
+// regression scenario: per-target Mark*Completed is invoked once per
+// (source × target_segment × target_shard) tuple while MarkSource*Completed
+// is invoked once per source. Under Option 3 the per-target Marks must NOT
+// move counters and MarkSource bumps Progress exactly once per source, so
+// completion_rate = Progress / pre-walk-Counts stays in [0, 100] no matter
+// how large the fan-out is. This is the regression test that pins the 266 %
+// bug as fixed.
+func TestCompletionRateNeverExceeds100UnderFanOut(t *testing.T) {
+ p := NewProgress("", logger.GetLogger("test"))
+ p.SetGroupsToProcess([]string{"sw_fanout"})
+
+ // 4 source series files × 2 target segments × 3 target shards = 24
+ // per-target Mark calls. Pre-walk says 4 source items.
+ const sourceCount = 4
+ p.SetStreamSeriesCount("sw_fanout", sourceCount)
+ targetSegs := []string{"tgt-seg-0", "tgt-seg-1"}
+
+ for src := 0; src < sourceCount; src++ {
+ sourceShard := common.ShardID(src)
+ // Fan-out: emit per-target Marks for every (target_segment, target_shard).
+ for _, tgtSeg := range targetSegs {
+ for tgtShard := common.ShardID(0); tgtShard < 3; tgtShard++ {
+ p.MarkStreamSeriesCompleted("sw_fanout", tgtSeg, tgtShard)
+ }
+ }
+ // Per-source: advance Progress exactly once per source file.
+ p.MarkSourceStreamSeriesCompleted("sw_fanout", "src-segment", sourceShard)
+ }
+ p.MarkGroupCompleted("sw_fanout")
+
+ svc := &lifecycleService{l: logger.GetLogger("test")}
+ report := svc.buildMigrationReport(p)
+ summary := report["summary"].(map[string]interface{})
+
+ // Counts comes from pre-walk Set*Count, not from per-target Marks.
+ // Progress is one bump per source MarkSource, not one per fan-out Mark.
+ // rate = 4/4 = 100 % (NOT 24/4 = 600 %).
+ streamMigration := summary["stream_migration"].(map[string]interface{})
+ series := streamMigration["series"].(map[string]interface{})
+ assert.Equal(t, sourceCount, series["total"], "Counts must come from Set*Count, not from per-target Marks")
+ assert.Equal(t, sourceCount, series["completed"], "Progress must be one bump per source, not per fan-out target")
+
+ rate := series["completion_rate"].(float64)
+ assert.GreaterOrEqualf(t, rate, 0.0, "completion_rate must be >= 0, got %v", rate)
+ assert.LessOrEqualf(t, rate, 100.0, "completion_rate must stay <= 100%% under fan-out (266%% regression), got %v", rate)
+ assert.InDelta(t, 100.0, rate, 1e-9)
+}
diff --git a/banyand/backup/lifecycle/service.go b/banyand/backup/lifecycle/service.go
index 85a78bf01..61eb8eae6 100644
--- a/banyand/backup/lifecycle/service.go
+++ b/banyand/backup/lifecycle/service.go
@@ -429,6 +429,10 @@ func (l *lifecycleService) action() error {
}
if streamDir == "" && measureDir == "" && traceDir == "" {
l.l.Warn().Msg("no snapshots found, skipping lifecycle migration")
+ // Clear any GroupsToProcess persisted from a prior cycle so the
+ // emitted report honestly reports total_groups=0 for this empty
+ // cycle instead of inheriting a stale denominator.
+ progress.SetGroupsToProcess(nil)
l.generateReport(progress)
return nil
}
@@ -437,6 +441,10 @@ func (l *lifecycleService) action() error {
Str("measure_snapshot", measureDir).
Str("trace_snapshot", traceDir).
Msg("created snapshots")
+ // Record the scheduled group set only after snapshots are confirmed so
+ // the report distinguishes "no work this cycle" (total_groups=0) from a
+ // real cycle that processed N groups (total_groups=N).
+ progress.SetGroupsToProcess(getGroupNames(groups))
progress.Save(l.progressFilePath, l.l)
nodes, err := l.metadata.NodeRegistry().ListNode(ctx, databasev1.Role_ROLE_DATA)
@@ -446,7 +454,6 @@ func (l *lifecycleService) action() error {
}
labels := common.ParseNodeFlags()
- allGroupsCompleted := true
for _, g := range groups {
switch g.Catalog {
case commonv1.Catalog_CATALOG_STREAM:
@@ -478,13 +485,18 @@ func (l *lifecycleService) action() error {
// Only remove progress file if ALL groups are fully completed
notCompleteGroups := progress.AllGroupsNotFullyCompleted(groups)
- if allGroupsCompleted && len(notCompleteGroups) == 0 {
+ if len(notCompleteGroups) == 0 {
progress.Remove(l.progressFilePath, l.l)
l.l.Info().Msg("lifecycle migration completed successfully")
l.generateReport(progress)
return nil
}
+ // Partial-failure path: also emit a report so operators can inspect
+ // errors.* and per-resource completion_rate without parsing raw
+ // progress.json. The report distinguishes itself from a clean cycle
+ // by having errors.* non-empty and completion_rate < 100.
l.l.Info().Msg("lifecycle migration partially completed, progress file retained")
+ l.generateReport(progress)
return fmt.Errorf("lifecycle migration partially completed, progress file retained; %v groups not fully completed", notCompleteGroups)
}
@@ -531,6 +543,7 @@ func (l *lifecycleService) buildMigrationReport(p *Progress) map[string]interfac
"snapshot_info": map[string]interface{}{
"stream_dir": p.SnapshotStreamDir,
"measure_dir": p.SnapshotMeasureDir,
+ "trace_dir": p.SnapshotTraceDir,
},
}
@@ -539,8 +552,20 @@ func (l *lifecycleService) buildMigrationReport(p *Progress) map[string]interfac
// buildSummaryStats creates overall migration statistics.
func (l *lifecycleService) buildSummaryStats(p *Progress) map[string]interface{} {
- totalGroups := len(p.CompletedGroups) + len(p.DeletedStreamGroups) + len(p.DeletedMeasureGroups)
- completedGroups := len(p.CompletedGroups)
+ // total_groups reflects the cycle's scheduled set captured at the top of
+ // action(); completed_groups counts groups that ran the full
+ // processXxxGroup → MarkGroupCompleted path. The intersection guards
+ // against resume from a partial-failure progress.json where
+ // CompletedGroups carries entries from prior cycles that are not in
+ // the current scheduled set, which would otherwise yield
+ // completed_groups > total_groups.
+ totalGroups := len(p.GroupsToProcess)
+ completedGroups := 0
+ for _, name := range p.GroupsToProcess {
+ if p.CompletedGroups[name] {
+ completedGroups++
+ }
+ }
// Calculate total parts and series across all groups
totalStreamParts, completedStreamParts := l.calculateTotalCounts(p.StreamPartCounts, p.StreamPartProgress)
@@ -548,6 +573,8 @@ func (l *lifecycleService) buildSummaryStats(p *Progress) map[string]interface{}
totalStreamElementIndex, completedStreamElementIndex := l.calculateTotalCounts(p.StreamElementIndexCounts, p.StreamElementIndexProgress)
totalMeasureParts, completedMeasureParts := l.calculateTotalCounts(p.MeasurePartCounts, p.MeasurePartProgress)
totalMeasureSeries, completedMeasureSeries := l.calculateTotalCounts(p.MeasureSeriesCounts, p.MeasureSeriesProgress)
+ totalTraceShards, completedTraceShards := l.calculateTotalCounts(p.TraceShardCounts, p.TraceShardProgress)
+ totalTraceSeries, completedTraceSeries := l.calculateTotalCounts(p.TraceSeriesCounts, p.TraceSeriesProgress)
// Calculate error counts
streamPartErrors := l.countErrors(p.StreamPartErrors)
@@ -555,6 +582,8 @@ func (l *lifecycleService) buildSummaryStats(p *Progress) map[string]interface{}
streamElementIndexErrors := l.countErrors(p.StreamElementIndexErrors)
measurePartErrors := l.countErrors(p.MeasurePartErrors)
measureSeriesErrors := l.countErrors(p.MeasureSeriesErrors)
+ traceShardErrors := l.countErrors(p.TraceShardErrors)
+ traceSeriesErrors := l.countErrors(p.TraceSeriesErrors)
return map[string]interface{}{
"migration_status": map[string]interface{}{
@@ -596,20 +625,37 @@ func (l *lifecycleService) buildSummaryStats(p *Progress) map[string]interface{}
"completion_rate": l.calculatePercentage(completedMeasureSeries, totalMeasureSeries),
},
},
+ // Field names parts/series mirror stream_migration / measure_migration.
+ // "parts" is fed by the per-shard TraceShard counters because trace
+ // migration is shard-batched (sidx + core parts streamed together).
+ "trace_migration": map[string]interface{}{
+ "parts": map[string]interface{}{
+ "total": totalTraceShards,
+ "completed": completedTraceShards,
+ "errors": traceShardErrors,
+ "completion_rate": l.calculatePercentage(completedTraceShards, totalTraceShards),
+ },
+ "series": map[string]interface{}{
+ "total": totalTraceSeries,
+ "completed": completedTraceSeries,
+ "errors": traceSeriesErrors,
+ "completion_rate": l.calculatePercentage(completedTraceSeries, totalTraceSeries),
+ },
+ },
}
}
// buildErrorSummary creates detailed error information.
func (l *lifecycleService) buildErrorSummary(p *Progress) map[string]interface{} {
- errors := map[string]interface{}{
+ return map[string]interface{}{
"stream_parts": l.buildErrorDetails(p.StreamPartErrors),
"stream_series": l.buildErrorDetails(p.StreamSeriesErrors),
"stream_element_index": l.buildErrorDetails(p.StreamElementIndexErrors),
"measure_parts": l.buildErrorDetails(p.MeasurePartErrors),
"measure_series": l.buildErrorDetails(p.MeasureSeriesErrors),
+ "trace_parts": l.buildErrorDetails(p.TraceShardErrors),
+ "trace_series": l.buildErrorDetails(p.TraceSeriesErrors),
}
-
- return errors
}
// Helper functions.
@@ -684,7 +730,14 @@ func (l *lifecycleService) buildErrorDetails(errorMaps interface{}) map[string]i
segmentDetails := make(map[string]interface{})
for shardID, parts := range shards {
if len(parts) > 0 {
- segmentDetails[fmt.Sprintf("shard_%d", shardID)] = parts
+ // Copy the inner map so the report does not alias
+ // Progress state; aliasing would race with a
+ // concurrent Mark*Error during JSON marshaling.
+ partDetails := make(map[uint64]string, len(parts))
+ for partID, errorMsg := range parts {
+ partDetails[partID] = errorMsg
+ }
+ segmentDetails[fmt.Sprintf("shard_%d", shardID)] = partDetails
}
}
if len(segmentDetails) > 0 {
diff --git a/banyand/backup/lifecycle/stream_migration_visitor.go b/banyand/backup/lifecycle/stream_migration_visitor.go
index 71b03e5fd..42ca53405 100644
--- a/banyand/backup/lifecycle/stream_migration_visitor.go
+++ b/banyand/backup/lifecycle/stream_migration_visitor.go
@@ -102,9 +102,6 @@ func (mv *streamMigrationVisitor) VisitSeries(segmentTR *timestamp.TimeRange, se
Str("path", seriesIndexPath).
Msg("found segment files for migration")
- // Set the total number of series segments for progress tracking
- mv.SetStreamSeriesCount(len(segmentFiles))
-
// Calculate ALL target segments this series index should go to
targetSegments := calculateTargetSegments(
segmentTR.Start.UnixNano(),
@@ -220,6 +217,15 @@ func (mv *streamMigrationVisitor) VisitSeries(segmentTR *timestamp.TimeRange, se
}
}
+ // Mark each source series file as fully migrated (idempotent — bumps Progress once per source).
+ for _, segmentFileName := range segmentFiles {
+ fileSegmentIDStr := strings.TrimSuffix(segmentFileName, ".seg")
+ segmentID, parseErr := strconv.ParseUint(fileSegmentIDStr, 16, 64)
+ if parseErr != nil {
+ continue
+ }
+ mv.progress.MarkSourceStreamSeriesCompleted(mv.group, seriesIndexPath, common.ShardID(segmentID))
+ }
return nil
}
@@ -233,7 +239,6 @@ func (mv *streamMigrationVisitor) VisitPart(_ *timestamp.TimeRange, sourceShardI
if err != nil {
return fmt.Errorf("failed to parse part ID from path: %w", err)
}
-
// Calculate ALL target segments this part should go to
targetSegments := calculateTargetSegments(
partData.MinTimestamp,
@@ -298,6 +303,7 @@ func (mv *streamMigrationVisitor) VisitPart(_ *timestamp.TimeRange, sourceShardI
Msgf("part migration completed for target segment %d/%d", i+1, len(targetSegments))
}
+ mv.progress.MarkSourceStreamPartCompleted(mv.group, partPath, sourceShardID, partID)
return nil
}
@@ -309,6 +315,9 @@ func (mv *streamMigrationVisitor) VisitElementIndex(segmentTR *timestamp.TimeRan
Str("group", mv.group).
Uint32("source_shard", uint32(sourceShardID)).
Msg("element index segment already completed, skipping")
+ // Per-target write done; ensure source progress is recorded to survive
+ // a crash between MarkStreamElementIndexCompleted and MarkSource (idempotent).
+ mv.progress.MarkSourceStreamElementIndexCompleted(mv.group, indexPath, sourceShardID)
return nil
}
@@ -337,9 +346,6 @@ func (mv *streamMigrationVisitor) VisitElementIndex(segmentTR *timestamp.TimeRan
return nil
}
- // Set the total number of element index segment files for progress tracking
- mv.SetStreamElementIndexCount(len(segmentFiles))
-
// Calculate target shard ID (using a simple approach for element index)
targetShardID := mv.calculateTargetShardID(uint32(sourceShardID))
mv.logger.Info().
@@ -411,6 +417,7 @@ func (mv *streamMigrationVisitor) VisitElementIndex(segmentTR *timestamp.TimeRan
// Mark segment as completed
mv.progress.MarkStreamElementIndexCompleted(mv.group, segmentIDStr, sourceShardID)
+ mv.progress.MarkSourceStreamElementIndexCompleted(mv.group, indexPath, sourceShardID)
return nil
}
diff --git a/banyand/backup/lifecycle/trace_migration_visitor.go b/banyand/backup/lifecycle/trace_migration_visitor.go
index 2936d7c14..10d4a2d00 100644
--- a/banyand/backup/lifecycle/trace_migration_visitor.go
+++ b/banyand/backup/lifecycle/trace_migration_visitor.go
@@ -103,9 +103,6 @@ func (mv *traceMigrationVisitor) VisitSeries(segmentTR *timestamp.TimeRange, ser
Str("path", seriesIndexPath).
Msg("found trace segment files for migration")
- // Set the total number of series segments for progress tracking
- mv.SetTraceSeriesCount(len(segmentFiles))
-
// Calculate ALL target segments this series index should go to
targetSegments := calculateTargetSegments(
segmentTR.Start.UnixNano(),
@@ -221,18 +218,33 @@ func (mv *traceMigrationVisitor) VisitSeries(segmentTR *timestamp.TimeRange, ser
}
}
+ // Mark each source series file as fully migrated (idempotent — bumps Progress once per source).
+ for _, segmentFileName := range segmentFiles {
+ fileSegmentIDStr := strings.TrimSuffix(segmentFileName, ".seg")
+ segmentID, parseErr := strconv.ParseUint(fileSegmentIDStr, 16, 64)
+ if parseErr != nil {
+ continue
+ }
+ mv.progress.MarkSourceTraceSeriesCompleted(mv.group, seriesIndexPath, common.ShardID(segmentID))
+ }
return nil
}
// VisitShard implements trace.Visitor - core and sidx migration logic.
func (mv *traceMigrationVisitor) VisitShard(timestampTR *timestamp.TimeRange, sourceShardID common.ShardID, shardPath string) error {
segmentIDStr := timestampTR.String()
- if mv.progress.IsTraceShardCompleted(mv.group, shardPath, sourceShardID) {
+ // Match the key used by Mark below; using shardPath here would always
+ // miss on resume and re-run the migration even though the shard had
+ // completed, wasting bandwidth and writing duplicates to the target.
+ if mv.progress.IsTraceShardCompleted(mv.group, segmentIDStr, sourceShardID) {
mv.logger.Debug().
Str("shard_path", shardPath).
Str("group", mv.group).
Uint32("source_shard", uint32(sourceShardID)).
Msg("trace shard already completed for this target segment, skipping")
+ // Per-target write done; ensure source progress is recorded to survive
+ // a crash between MarkTraceShardCompleted and MarkSource (idempotent).
+ mv.progress.MarkSourceTraceShardCompleted(mv.group, segmentIDStr, sourceShardID)
return nil
}
allParts := make([]queue.StreamingPartData, 0)
@@ -277,6 +289,7 @@ func (mv *traceMigrationVisitor) VisitShard(timestampTR *timestamp.TimeRange, so
// Mark shard as completed for this target segment
mv.progress.MarkTraceShardCompleted(mv.group, segmentIDStr, sourceShardID)
+ mv.progress.MarkSourceTraceShardCompleted(mv.group, segmentIDStr, sourceShardID)
mv.logger.Info().
Str("group", mv.group).
Msgf("trace shard migration completed for target segment")
diff --git a/test/cases/lifecycle/lifecycle.go b/test/cases/lifecycle/lifecycle.go
index cecc841e1..40c7c682b 100644
--- a/test/cases/lifecycle/lifecycle.go
+++ b/test/cases/lifecycle/lifecycle.go
@@ -19,6 +19,7 @@
package lifecycle_test
import (
+ "encoding/json"
"io/fs"
"os"
"path/filepath"
@@ -64,10 +65,7 @@ var _ = ginkgo.Describe("Lifecycle", func() {
gomega.Expect(err).NotTo(gomega.HaveOccurred())
verifySourceDirectoriesAfterMigration()
verifyDestinationDirectoriesAfterMigration()
- // Check report directory has files
- rEntries, err := os.ReadDir(rf)
- gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Report directory should exist")
- gomega.Expect(len(rEntries)).To(gomega.BeNumerically(">", 0), "Report directory should contain files")
+ verifyMigrationReport(rf)
conn, err := grpchelper.Conn(SharedContext.LiaisonAddr, 10*time.Second,
grpc.WithTransportCredentials(insecure.NewCredentials()))
defer func() {
@@ -131,10 +129,7 @@ var _ = ginkgo.Describe("Lifecycle", func() {
gomega.Expect(err).NotTo(gomega.HaveOccurred())
verifySourceDirectoriesAfterMigration()
verifyDestinationDirectoriesAfterMigration()
- // Check report directory has files
- rEntries, err := os.ReadDir(rf)
- gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Report directory should exist")
- gomega.Expect(len(rEntries)).To(gomega.BeNumerically(">", 0), "Report directory should contain files")
+ verifyMigrationReport(rf)
conn, err := grpchelper.Conn(SharedContext.LiaisonAddr, 10*time.Second,
grpc.WithTransportCredentials(insecure.NewCredentials()))
defer func() {
@@ -273,3 +268,133 @@ func verifyLockFileAndSegFolder(entries []fs.DirEntry) (hasLockFile bool, hasSeg
}
return hasLockFile, hasSegFolder
}
+
+// verifyMigrationReport reads every JSON report file generated under rf
+// and asserts the invariants of the comprehensive migration report:
+//
+// - migration_status.completion_rate is 100 when the cycle had scheduled
+// groups; the denominator equals the scheduled group set.
+// - Every per-resource block (parts / series / element_index) reaches
+// 100 % when the catalog had work to do, or 0 % when the cycle had
+// nothing to migrate (total == 0).
+// - The trace_migration block is present with parts + series fields
+// aligned to stream_migration / measure_migration; snapshot_info
+// includes trace_dir; errors includes trace_parts / trace_series keys.
+// - All errors.* maps are empty for a clean cycle.
+func verifyMigrationReport(rf string) {
+ rEntries, err := os.ReadDir(rf)
+ gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Report directory should exist")
+ gomega.Expect(rEntries).NotTo(gomega.BeEmpty(), "Report directory should contain files")
+
+ jsonReports := 0
+ for _, entry := range rEntries {
+ if entry.IsDir() || filepath.Ext(entry.Name()) != ".json" {
+ continue
+ }
+ jsonReports++
+ path := filepath.Join(rf, entry.Name())
+ raw, readErr := os.ReadFile(path)
+ gomega.Expect(readErr).NotTo(gomega.HaveOccurred(), "Report file %s should be readable", path)
+
+ var report map[string]interface{}
+ gomega.Expect(json.Unmarshal(raw, &report)).To(gomega.Succeed(), "Report file %s should be valid JSON", path)
+
+ gomega.Expect(report).To(gomega.HaveKey("summary"), "report %s missing summary", path)
+ gomega.Expect(report).To(gomega.HaveKey("errors"), "report %s missing errors", path)
+ gomega.Expect(report).To(gomega.HaveKey("snapshot_info"), "report %s missing snapshot_info", path)
+ gomega.Expect(report["report_version"]).To(gomega.Equal("2.0"), "report %s has unexpected report_version", path)
+
+ // snapshot_info: trace_dir must surface alongside stream/measure.
+ snap, ok := report["snapshot_info"].(map[string]interface{})
+ gomega.Expect(ok).To(gomega.BeTrue(), "snapshot_info must be a map in %s", path)
+ gomega.Expect(snap).To(gomega.HaveKey("trace_dir"), "snapshot_info must include trace_dir in %s", path)
+
+ summary, ok := report["summary"].(map[string]interface{})
+ gomega.Expect(ok).To(gomega.BeTrue(), "summary must be a map in %s", path)
+
+ // migration_status: when total_groups=0 the cycle had no scheduled
+ // work (e.g. snapshots were empty); the rate is 0 by construction.
+ // Otherwise the report reflects a fully completed cycle, so the
+ // rate must be 100.
+ ms, ok := summary["migration_status"].(map[string]interface{})
+ gomega.Expect(ok).To(gomega.BeTrue(), "summary.migration_status must be a map in %s", path)
+ if asInt(ms["total_groups"]) == 0 {
+ gomega.Expect(asFloat(ms["completion_rate"])).To(gomega.BeNumerically("~", 0.0, 1e-9),
+ "migration_status.completion_rate must be 0 when total_groups=0 in %s", path)
+ } else {
+ gomega.Expect(asFloat(ms["completion_rate"])).To(gomega.BeNumerically("~", 100.0, 1e-9),
+ "migration_status.completion_rate should be 100 in %s", path)
+ }
+
+ // Per-resource invariants.
+ verifyAllRatesAt100(summary, "stream_migration", []string{"parts", "series", "element_index"}, path)
+ verifyAllRatesAt100(summary, "measure_migration", []string{"parts", "series"}, path)
+
+ // trace_migration: block must exist with parts + series.
+ verifyAllRatesAt100(summary, "trace_migration", []string{"parts", "series"}, path)
+
+ // errors must include trace_* keys and stay empty.
+ errs, ok := report["errors"].(map[string]interface{})
+ gomega.Expect(ok).To(gomega.BeTrue(), "errors must be a map in %s", path)
+ for _, key := range []string{
+ "stream_parts", "stream_series", "stream_element_index",
+ "measure_parts", "measure_series",
+ "trace_parts", "trace_series",
+ } {
+ v, found := errs[key]
+ gomega.Expect(found).To(gomega.BeTrue(), "errors.%s must be present in %s", key, path)
+ errMap, isMap := v.(map[string]interface{})
+ gomega.Expect(isMap).To(gomega.BeTrue(), "errors.%s must be a map in %s", key, path)
+ gomega.Expect(errMap).To(gomega.BeEmpty(), "errors.%s must be empty for a clean cycle in %s", key, path)
+ }
+ }
+ gomega.Expect(jsonReports).To(gomega.BeNumerically(">", 0), "no JSON report files found under %s", rf)
+}
+
+// verifyAllRatesAt100 enforces the per-resource accounting invariant: when
+// total == 0 the cycle had no work for that resource and the rate stays at
+// 0; otherwise completed must equal total and the rate must be 100.
+func verifyAllRatesAt100(summary map[string]interface{}, catalog string, resources []string, path string) {
+ cat, ok := summary[catalog].(map[string]interface{})
+ gomega.Expect(ok).To(gomega.BeTrue(), "summary.%s must be a map in %s", catalog, path)
+ for _, r := range resources {
+ res, isMap := cat[r].(map[string]interface{})
+ gomega.Expect(isMap).To(gomega.BeTrue(), "summary.%s.%s must be a map in %s", catalog, r, path)
+ gomega.Expect(res).To(gomega.HaveKey("total"), "summary.%s.%s missing total in %s", catalog, r, path)
+ gomega.Expect(res).To(gomega.HaveKey("completed"), "summary.%s.%s missing completed in %s", catalog, r, path)
+ gomega.Expect(res).To(gomega.HaveKey("errors"), "summary.%s.%s missing errors in %s", catalog, r, path)
+ gomega.Expect(res).To(gomega.HaveKey("completion_rate"), "summary.%s.%s missing completion_rate in %s", catalog, r, path)
+
+ total := asInt(res["total"])
+ completed := asInt(res["completed"])
+ errors := asInt(res["errors"])
+ rate := asFloat(res["completion_rate"])
+
+ gomega.Expect(errors).To(gomega.Equal(0), "summary.%s.%s.errors must be 0 in %s", catalog, r, path)
+ if total == 0 {
+ gomega.Expect(rate).To(gomega.BeNumerically("~", 0.0, 1e-9),
+ "summary.%s.%s.completion_rate must be 0 when total=0 in %s", catalog, r, path)
+ continue
+ }
+ gomega.Expect(completed).To(gomega.Equal(total),
+ "summary.%s.%s.completed must equal total in %s", catalog, r, path)
+ gomega.Expect(rate).To(gomega.BeNumerically("~", 100.0, 1e-9),
+ "summary.%s.%s.completion_rate must be 100 in %s", catalog, r, path)
+ }
+}
+
+// asFloat coerces a JSON-decoded numeric value to float64.
+func asFloat(v interface{}) float64 {
+ if f, ok := v.(float64); ok {
+ return f
+ }
+ return 0
+}
+
+// asInt coerces a JSON-decoded numeric value to int.
+func asInt(v interface{}) int {
+ if f, ok := v.(float64); ok {
+ return int(f)
+ }
+ return 0
+}
(skywalking-banyandb) 03/11: align part sync timestamp to segment grid so part and sidx land in the same segment (#1136)
Posted by bu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ButterBright pushed a commit to branch v0.10.x
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit d16421377cf444efcd0c243c0dd9f4de951e1475
Author: mrproliu <74...@qq.com>
AuthorDate: Mon May 18 16:34:46 2026 +0800
align part sync timestamp to segment grid so part and sidx land in the same segment (#1136)
---
banyand/internal/storage/segment.go | 6 +
banyand/internal/storage/storage.go | 2 +
banyand/internal/storage/tsdb.go | 4 +
banyand/measure/write_data.go | 3 +-
banyand/measure/write_data_segmentref_test.go | 168 +++++++++++++++++++++++++-
banyand/stream/write_data.go | 3 +-
banyand/stream/write_data_segmentref_test.go | 136 ++++++++++++++++++++-
banyand/trace/write_data.go | 3 +-
banyand/trace/write_data_segmentref_test.go | 136 ++++++++++++++++++++-
9 files changed, 455 insertions(+), 6 deletions(-)
diff --git a/banyand/internal/storage/segment.go b/banyand/internal/storage/segment.go
index bd8c5c705..75a2994b0 100644
--- a/banyand/internal/storage/segment.go
+++ b/banyand/internal/storage/segment.go
@@ -400,6 +400,12 @@ func (sc *segmentController[T, O]) getOptions() *TSDBOpts[T, O] {
return sc.opts
}
+func (sc *segmentController[T, O]) getSegmentInterval() IntervalRule {
+ sc.optsMutex.RLock()
+ defer sc.optsMutex.RUnlock()
+ return sc.opts.SegmentInterval
+}
+
func (sc *segmentController[T, O]) updateOptions(resourceOpts *commonv1.ResourceOpts) {
sc.optsMutex.Lock()
defer sc.optsMutex.Unlock()
diff --git a/banyand/internal/storage/storage.go b/banyand/internal/storage/storage.go
index 5e54012ec..fc33a1ffa 100644
--- a/banyand/internal/storage/storage.go
+++ b/banyand/internal/storage/storage.go
@@ -112,6 +112,8 @@ type TSDB[T TSTable, O any] interface {
io.Closer
CreateSegmentIfNotExist(ts time.Time) (Segment[T, O], error)
SelectSegments(timeRange timestamp.TimeRange) ([]Segment[T, O], error)
+ // SegmentInterval returns the current segment interval rule.
+ SegmentInterval() IntervalRule
Tick(ts int64)
UpdateOptions(opts *commonv1.ResourceOpts)
TakeFileSnapshot(dst string) (bool, error)
diff --git a/banyand/internal/storage/tsdb.go b/banyand/internal/storage/tsdb.go
index 22f26569e..341db79e6 100644
--- a/banyand/internal/storage/tsdb.go
+++ b/banyand/internal/storage/tsdb.go
@@ -275,6 +275,10 @@ func (d *database[T, O]) SelectSegments(timeRange timestamp.TimeRange) ([]Segmen
return d.segmentController.selectSegments(timeRange)
}
+func (d *database[T, O]) SegmentInterval() IntervalRule {
+ return d.segmentController.getSegmentInterval()
+}
+
func (d *database[T, O]) UpdateOptions(resourceOpts *commonv1.ResourceOpts) {
if d.closed.Load() {
return
diff --git a/banyand/measure/write_data.go b/banyand/measure/write_data.go
index c37b52e60..c6079f365 100644
--- a/banyand/measure/write_data.go
+++ b/banyand/measure/write_data.go
@@ -117,7 +117,8 @@ func (s *syncCallback) CreatePartHandler(ctx *queue.ChunkedSyncPartContext) (que
s.l.Error().Err(err).Str("group", ctx.Group).Msg("failed to load TSDB for group")
return nil, err
}
- segmentTime := time.Unix(0, ctx.MinTimestamp)
+ // Align to the segment grid so part-chunk sync and sidx land in the same segment.
+ segmentTime := tsdb.SegmentInterval().Standard(time.Unix(0, ctx.MinTimestamp))
segment, err := tsdb.CreateSegmentIfNotExist(segmentTime)
if err != nil {
s.l.Error().Err(err).Str("group", ctx.Group).Time("segmentTime", segmentTime).Msg("failed to create segment")
diff --git a/banyand/measure/write_data_segmentref_test.go b/banyand/measure/write_data_segmentref_test.go
index 0606cc204..b71f5e742 100644
--- a/banyand/measure/write_data_segmentref_test.go
+++ b/banyand/measure/write_data_segmentref_test.go
@@ -122,7 +122,10 @@ func TestSyncChunkCallback_CreatePartHandler_StoresSegment(t *testing.T) {
db := openTestTSDBForRefTest(t, tmpPath, &openShardCount)
defer db.Close()
- segTime := time.Date(2026, 4, 17, 0, 0, 0, 0, time.UTC)
+ // CreatePartHandler reinterprets ctx.MinTimestamp via time.Unix in Local
+ // tz before alignment; keep the warmup segment on the same tz so both
+ // resolve to the same segment instant.
+ segTime := time.Date(2026, 4, 17, 0, 0, 0, 0, time.Local)
warmup, err := db.CreateSegmentIfNotExist(segTime)
require.NoError(t, err)
@@ -248,3 +251,166 @@ type fakeGroup struct {
func (f *fakeGroup) GetSchema() *commonv1.Group { return nil }
func (f *fakeGroup) SupplyTSDB() io.Closer { return f.tsdb }
+
+// TestSyncChunkCallback_CreatePartHandler_AlignsOffGridMinTimestamp covers the
+// regression where the part chunk-sync receiver passed the raw MinTimestamp
+// straight into CreateSegmentIfNotExist. Real data points carry a timestamp
+// anywhere inside a segment window, while the matching sidx Insert/Update
+// arrives from liaison with a grid-aligned start. Without alignment here,
+// part and sidx end up in different segments. The fix is to feed every
+// raw MinTimestamp through TSDB.SegmentInterval().Standard before creating
+// the segment so both paths converge on the same aligned start.
+func TestSyncChunkCallback_CreatePartHandler_AlignsOffGridMinTimestamp(t *testing.T) {
+ t.Run("DAY(1)", func(t *testing.T) {
+ runOffGridCase(t, storage.IntervalRule{Unit: storage.DAY, Num: 1})
+ })
+ t.Run("DAY(5)", func(t *testing.T) {
+ runOffGridCase(t, storage.IntervalRule{Unit: storage.DAY, Num: 5})
+ })
+}
+
+func runOffGridCase(t *testing.T, ir storage.IntervalRule) {
+ t.Helper()
+ tmpPath, cleanup := test.Space(require.New(t))
+ defer cleanup()
+
+ const groupName = "off-grid-group"
+ db := openTestTSDBWithInterval(t, tmpPath, groupName, ir)
+ defer db.Close()
+
+ // Pick a raw timestamp deliberately off-grid: an arbitrary 06:00 in local
+ // time inside the current grid bucket. Without the alignment fix the
+ // receiver would build a segment whose Start equals this raw ts (modulo
+ // rotation to a 0-second wall instant), which never matches the Standard
+ // grid that liaison-side sidx uses.
+ rawTS := time.Date(2026, 5, 15, 6, 0, 0, 0, time.Local)
+ wantStart := ir.Standard(rawTS)
+
+ callback := &syncCallback{
+ l: logger.GetLogger("test-offgrid"),
+ schemaRepo: newTestSchemaRepo(db, groupName),
+ }
+ ctx := &queue.ChunkedSyncPartContext{
+ ID: 1,
+ Group: groupName,
+ ShardID: 0,
+ MinTimestamp: rawTS.UnixNano(),
+ MaxTimestamp: rawTS.UnixNano(),
+ }
+ handler, err := callback.CreatePartHandler(ctx)
+ require.NoError(t, err)
+ defer func() { _ = handler.(*syncPartContext).Close() }()
+ partCtx := handler.(*syncPartContext)
+ require.NotNil(t, partCtx.segment)
+
+ gotStart := partCtx.segment.GetTimeRange().Start
+ require.True(t, gotStart.Equal(wantStart),
+ "REGRESSION: raw MinTimestamp=%s landed in segment start=%s; expected aligned start=%s (ir=%+v)",
+ rawTS, gotStart, wantStart, ir)
+ // Sanity: assert the raw ts itself would NOT have produced this start —
+ // otherwise the test passes vacuously when raw happens to land on grid.
+ require.False(t, rawTS.Equal(wantStart),
+ "test setup is degenerate: rawTS already aligned, cannot detect missing Standard call")
+}
+
+// TestSegmentCreateTS_ConsistencyAcrossPaths verifies that both the part
+// chunk-sync receiver (Path A, raw ts straight from a part's MinTimestamp)
+// and the sidx series-sync receiver (Path B, ts pre-standardized at the
+// liaison) converge on the SAME segment Start. A pre-existing aligned
+// segment in an older grid bucket simulates historical data on disk — it
+// must NOT be touched, and the two paths' new writes must land in the
+// same new bucket as each other.
+//
+// This is the regression demo for the PR #1120 fault where Path A used
+// raw ts while Path B (sidx via liaison) used Standard ts, so the same
+// real datapoint produced part data in one segment and sidx data in
+// another.
+func TestSegmentCreateTS_ConsistencyAcrossPaths(t *testing.T) {
+ tmpPath, cleanup := test.Space(require.New(t))
+ defer cleanup()
+
+ const groupName = "consistency-group"
+ ir := storage.IntervalRule{Unit: storage.DAY, Num: 5}
+ db := openTestTSDBWithInterval(t, tmpPath, groupName, ir)
+ defer db.Close()
+
+ // Historical segment: pre-create an aligned segment in a much older grid
+ // bucket so the test cluster looks like it has prior data on disk.
+ historicalRaw := time.Date(2026, 4, 10, 0, 0, 0, 0, time.Local)
+ historicalStart := ir.Standard(historicalRaw)
+ histSeg, err := db.CreateSegmentIfNotExist(historicalStart)
+ require.NoError(t, err)
+ histSeg.DecRef()
+ histSeg.DecRef() // refCount->0, idle-closed
+
+ // New raw datapoint with an off-grid (06:00 inside a 5-day bucket) ts.
+ // liaison-side sidx pre-standardizes it; data-node-side part chunk-sync
+ // also runs it through Standard now.
+ rawTS := time.Date(2026, 5, 15, 6, 0, 0, 0, time.Local)
+ wantNewStart := ir.Standard(rawTS)
+ require.False(t, wantNewStart.Equal(historicalStart),
+ "test setup degenerate: new bucket coincides with historical bucket")
+ require.False(t, wantNewStart.Equal(rawTS),
+ "test setup degenerate: rawTS already on grid, cannot detect Standard")
+
+ repo := newTestSchemaRepo(db, groupName)
+
+ // Path A: part chunk-sync receiver gets the raw MinTimestamp from the
+ // transferred part's metadata. Aligned internally by the fix.
+ chunkCallback := &syncCallback{l: logger.GetLogger("test-pathA"), schemaRepo: repo}
+ handlerA, err := chunkCallback.CreatePartHandler(&queue.ChunkedSyncPartContext{
+ ID: 1,
+ Group: groupName,
+ ShardID: 0,
+ MinTimestamp: rawTS.UnixNano(),
+ MaxTimestamp: rawTS.UnixNano(),
+ })
+ require.NoError(t, err)
+ defer func() { _ = handlerA.(*syncPartContext).Close() }()
+ gotStartA := handlerA.(*syncPartContext).segment.GetTimeRange().Start
+
+ // Path B: sidx series-sync receiver gets a MinTimestamp the liaison has
+ // already aligned via Standard. The handler uses it as-is.
+ seriesCallback := &syncSeriesCallback{l: logger.GetLogger("test-pathB"), schemaRepo: repo}
+ handlerB, err := seriesCallback.CreatePartHandler(&queue.ChunkedSyncPartContext{
+ ID: 2,
+ Group: groupName,
+ ShardID: 0,
+ MinTimestamp: wantNewStart.UnixNano(),
+ MaxTimestamp: wantNewStart.UnixNano(),
+ })
+ require.NoError(t, err)
+ defer func() { _ = handlerB.(*syncSeriesContext).Close() }()
+ gotStartB := handlerB.(*syncSeriesContext).segment.GetTimeRange().Start
+
+ require.True(t, gotStartA.Equal(gotStartB),
+ "REGRESSION: Path A (raw ts %s -> segment %s) and Path B (aligned ts %s -> segment %s) diverged",
+ rawTS, gotStartA, wantNewStart, gotStartB)
+ require.True(t, gotStartA.Equal(wantNewStart),
+ "path A segment start %s != expected aligned %s", gotStartA, wantNewStart)
+ require.False(t, gotStartA.Equal(historicalStart),
+ "new write must NOT land in historical segment %s", historicalStart)
+}
+
+func openTestTSDBWithInterval(t *testing.T, tmpPath, groupName string, ir storage.IntervalRule) storage.TSDB[*tsTable, option] {
+ t.Helper()
+ opts := storage.TSDBOpts[*tsTable, option]{
+ ShardNum: 1,
+ Location: filepath.Join(tmpPath, "tab"),
+ TSTableCreator: newTSTable,
+ SegmentInterval: ir,
+ TTL: storage.IntervalRule{Unit: ir.Unit, Num: 60},
+ Option: option{protector: protector.Nop{}, mergePolicy: newDefaultMergePolicyForTesting()},
+ }
+ require.NoError(t, os.MkdirAll(opts.Location, storage.DirPerm))
+ ctx := common.SetPosition(
+ context.WithValue(context.Background(), logger.ContextKey, logger.GetLogger("test-offgrid")),
+ func(p common.Position) common.Position {
+ p.Database = "test-offgrid"
+ return p
+ },
+ )
+ db, err := storage.OpenTSDB[*tsTable, option](ctx, opts, nil, groupName)
+ require.NoError(t, err)
+ return db
+}
diff --git a/banyand/stream/write_data.go b/banyand/stream/write_data.go
index 76f4e4ced..8506a36e7 100644
--- a/banyand/stream/write_data.go
+++ b/banyand/stream/write_data.go
@@ -116,7 +116,8 @@ func (s *syncCallback) CreatePartHandler(ctx *queue.ChunkedSyncPartContext) (que
s.l.Error().Err(err).Str("group", ctx.Group).Msg("failed to load TSDB for group")
return nil, err
}
- segmentTime := time.Unix(0, ctx.MinTimestamp)
+ // Align to the segment grid so part-chunk sync and sidx land in the same segment.
+ segmentTime := tsdb.SegmentInterval().Standard(time.Unix(0, ctx.MinTimestamp))
segment, err := tsdb.CreateSegmentIfNotExist(segmentTime)
if err != nil {
s.l.Error().Err(err).Str("group", ctx.Group).Time("segmentTime", segmentTime).Msg("failed to create segment")
diff --git a/banyand/stream/write_data_segmentref_test.go b/banyand/stream/write_data_segmentref_test.go
index 81784ac1c..509d333b9 100644
--- a/banyand/stream/write_data_segmentref_test.go
+++ b/banyand/stream/write_data_segmentref_test.go
@@ -114,6 +114,12 @@ func TestSyncReceiver_SegmentRefOwnership(t *testing.T) {
// syncCallback.CreatePartHandler through a minimal schemaRepo shim and
// asserts that segment ownership transfers to partCtx and that Close
// actually DecRefs. See trace package for rationale.
+// In each StoresSegment test, segTime must use time.Local because the
+// callback path runs time.Unix(0,ns) -> Standard -> CreateSegmentIfNotExist
+// in Local tz. If the warmup segment were created in UTC, the two calls
+// could represent the same instant but format to different segment-suffix
+// strings, causing a directory-name mismatch instead of reusing the same
+// segment.
func TestSyncChunkCallback_CreatePartHandler_StoresSegment(t *testing.T) {
tmpPath, cleanup := test.Space(require.New(t))
defer cleanup()
@@ -122,7 +128,7 @@ func TestSyncChunkCallback_CreatePartHandler_StoresSegment(t *testing.T) {
db := openTestTSDBForRefTest(t, tmpPath, &openShardCount)
defer db.Close()
- segTime := time.Date(2026, 4, 17, 0, 0, 0, 0, time.UTC)
+ segTime := time.Date(2026, 4, 17, 0, 0, 0, 0, time.Local)
warmup, err := db.CreateSegmentIfNotExist(segTime)
require.NoError(t, err)
@@ -246,3 +252,131 @@ type fakeGroup struct {
func (f *fakeGroup) GetSchema() *commonv1.Group { return nil }
func (f *fakeGroup) SupplyTSDB() io.Closer { return f.tsdb }
+
+// TestSyncChunkCallback_CreatePartHandler_AlignsOffGridMinTimestamp covers
+// the regression where the stream part chunk-sync receiver passed the raw
+// MinTimestamp straight into CreateSegmentIfNotExist. See the analogous
+// measure test for full rationale.
+func TestSyncChunkCallback_CreatePartHandler_AlignsOffGridMinTimestamp(t *testing.T) {
+ t.Run("DAY(1)", func(t *testing.T) {
+ runStreamOffGridCase(t, storage.IntervalRule{Unit: storage.DAY, Num: 1})
+ })
+ t.Run("DAY(5)", func(t *testing.T) {
+ runStreamOffGridCase(t, storage.IntervalRule{Unit: storage.DAY, Num: 5})
+ })
+}
+
+func runStreamOffGridCase(t *testing.T, ir storage.IntervalRule) {
+ t.Helper()
+ tmpPath, cleanup := test.Space(require.New(t))
+ defer cleanup()
+
+ const groupName = "off-grid-stream"
+ db := openTestTSDBWithInterval(t, tmpPath, groupName, ir)
+ defer db.Close()
+
+ rawTS := time.Date(2026, 5, 15, 6, 0, 0, 0, time.Local)
+ wantStart := ir.Standard(rawTS)
+
+ callback := &syncCallback{
+ l: logger.GetLogger("test-offgrid-stream"),
+ schemaRepo: newTestSchemaRepo(db, groupName),
+ }
+ handler, err := callback.CreatePartHandler(&queue.ChunkedSyncPartContext{
+ ID: 1,
+ Group: groupName,
+ ShardID: 0,
+ MinTimestamp: rawTS.UnixNano(),
+ MaxTimestamp: rawTS.UnixNano(),
+ })
+ require.NoError(t, err)
+ defer func() { _ = handler.(*syncPartContext).Close() }()
+ gotStart := handler.(*syncPartContext).segment.GetTimeRange().Start
+ require.True(t, gotStart.Equal(wantStart),
+ "REGRESSION: raw MinTimestamp=%s landed in segment start=%s; expected aligned %s (ir=%+v)",
+ rawTS, gotStart, wantStart, ir)
+ require.False(t, rawTS.Equal(wantStart),
+ "test setup degenerate: rawTS already aligned")
+}
+
+// TestSegmentCreateTS_ConsistencyAcrossPaths is the stream variant of the
+// Path-A vs Path-B convergence demo. A pre-existing aligned historical
+// segment must NOT be touched; both new writes must land in the same
+// new bucket.
+func TestSegmentCreateTS_ConsistencyAcrossPaths(t *testing.T) {
+ tmpPath, cleanup := test.Space(require.New(t))
+ defer cleanup()
+
+ const groupName = "consistency-stream"
+ ir := storage.IntervalRule{Unit: storage.DAY, Num: 5}
+ db := openTestTSDBWithInterval(t, tmpPath, groupName, ir)
+ defer db.Close()
+
+ historicalStart := ir.Standard(time.Date(2026, 4, 10, 0, 0, 0, 0, time.Local))
+ histSeg, err := db.CreateSegmentIfNotExist(historicalStart)
+ require.NoError(t, err)
+ histSeg.DecRef()
+ histSeg.DecRef()
+
+ rawTS := time.Date(2026, 5, 15, 6, 0, 0, 0, time.Local)
+ wantNewStart := ir.Standard(rawTS)
+ require.False(t, wantNewStart.Equal(historicalStart))
+ require.False(t, wantNewStart.Equal(rawTS))
+
+ repo := newTestSchemaRepo(db, groupName)
+
+ chunkCallback := &syncCallback{l: logger.GetLogger("test-stream-pathA"), schemaRepo: repo}
+ handlerA, err := chunkCallback.CreatePartHandler(&queue.ChunkedSyncPartContext{
+ ID: 1,
+ Group: groupName,
+ ShardID: 0,
+ MinTimestamp: rawTS.UnixNano(),
+ MaxTimestamp: rawTS.UnixNano(),
+ })
+ require.NoError(t, err)
+ defer func() { _ = handlerA.(*syncPartContext).Close() }()
+ gotStartA := handlerA.(*syncPartContext).segment.GetTimeRange().Start
+
+ seriesCallback := &syncSeriesCallback{l: logger.GetLogger("test-stream-pathB"), schemaRepo: repo}
+ handlerB, err := seriesCallback.CreatePartHandler(&queue.ChunkedSyncPartContext{
+ ID: 2,
+ Group: groupName,
+ ShardID: 0,
+ MinTimestamp: wantNewStart.UnixNano(),
+ MaxTimestamp: wantNewStart.UnixNano(),
+ })
+ require.NoError(t, err)
+ defer func() { _ = handlerB.(*syncSeriesContext).Close() }()
+ gotStartB := handlerB.(*syncSeriesContext).segment.GetTimeRange().Start
+
+ require.True(t, gotStartA.Equal(gotStartB),
+ "REGRESSION: stream Path A (raw %s -> %s) and Path B (aligned %s -> %s) diverged",
+ rawTS, gotStartA, wantNewStart, gotStartB)
+ require.True(t, gotStartA.Equal(wantNewStart),
+ "path A start %s != expected aligned %s", gotStartA, wantNewStart)
+ require.False(t, gotStartA.Equal(historicalStart),
+ "new write must NOT land in historical segment %s", historicalStart)
+}
+
+func openTestTSDBWithInterval(t *testing.T, tmpPath, groupName string, ir storage.IntervalRule) storage.TSDB[*tsTable, option] {
+ t.Helper()
+ opts := storage.TSDBOpts[*tsTable, option]{
+ ShardNum: 1,
+ Location: filepath.Join(tmpPath, "tab"),
+ TSTableCreator: newTSTable,
+ SegmentInterval: ir,
+ TTL: storage.IntervalRule{Unit: ir.Unit, Num: 60},
+ Option: option{protector: protector.Nop{}, mergePolicy: newDefaultMergePolicyForTesting()},
+ }
+ require.NoError(t, os.MkdirAll(opts.Location, storage.DirPerm))
+ ctx := common.SetPosition(
+ context.WithValue(context.Background(), logger.ContextKey, logger.GetLogger("test-offgrid-stream")),
+ func(p common.Position) common.Position {
+ p.Database = "test-offgrid-stream"
+ return p
+ },
+ )
+ db, err := storage.OpenTSDB[*tsTable, option](ctx, opts, nil, groupName)
+ require.NoError(t, err)
+ return db
+}
diff --git a/banyand/trace/write_data.go b/banyand/trace/write_data.go
index 16c989b52..7a57ba4ce 100644
--- a/banyand/trace/write_data.go
+++ b/banyand/trace/write_data.go
@@ -288,7 +288,8 @@ func (s *syncChunkCallback) CreatePartHandler(ctx *queue.ChunkedSyncPartContext)
s.l.Error().Err(err).Str("group", ctx.Group).Msg("failed to load TSDB for group")
return nil, err
}
- segmentTime := time.Unix(0, ctx.MinTimestamp)
+ // Align to the segment grid so part-chunk sync and sidx land in the same segment.
+ segmentTime := tsdb.SegmentInterval().Standard(time.Unix(0, ctx.MinTimestamp))
segment, err := tsdb.CreateSegmentIfNotExist(segmentTime)
if err != nil {
s.l.Error().Err(err).Str("group", ctx.Group).Time("segmentTime", segmentTime).Msg("failed to create segment")
diff --git a/banyand/trace/write_data_segmentref_test.go b/banyand/trace/write_data_segmentref_test.go
index f332141d8..d170bb3cd 100644
--- a/banyand/trace/write_data_segmentref_test.go
+++ b/banyand/trace/write_data_segmentref_test.go
@@ -213,6 +213,12 @@ func openTestTSDBForRefTest(t *testing.T, tmpPath string, openShardCount *atomic
//
// This complements TestSyncReceiver_SegmentRefOwnership, which builds
// syncPartContext manually. Together they cover every line of the fix.
+//
+// segTime must use time.Local: the callback path runs time.Unix(0,ns)
+// -> Standard -> CreateSegmentIfNotExist in Local tz. If the warmup
+// segment were created in UTC, the two calls could represent the same
+// instant but format to different segment-suffix strings, causing a
+// directory-name mismatch instead of reusing the same segment.
func TestSyncChunkCallback_CreatePartHandler_StoresSegment(t *testing.T) {
tmpPath, cleanup := test.Space(require.New(t))
defer cleanup()
@@ -221,7 +227,7 @@ func TestSyncChunkCallback_CreatePartHandler_StoresSegment(t *testing.T) {
db := openTestTSDBForRefTest(t, tmpPath, &openShardCount)
defer db.Close()
- segTime := time.Date(2026, 4, 17, 0, 0, 0, 0, time.UTC)
+ segTime := time.Date(2026, 4, 17, 0, 0, 0, 0, time.Local)
// Drive the segment into the idle-closed state (refCount=0) so that the
// probe at the end of this test reliably exercises the `initialize`
@@ -352,3 +358,131 @@ type fakeGroup struct {
func (f *fakeGroup) GetSchema() *commonv1.Group { return nil }
func (f *fakeGroup) SupplyTSDB() io.Closer { return f.tsdb }
+
+// TestSyncChunkCallback_CreatePartHandler_AlignsOffGridMinTimestamp covers
+// the regression where the trace part chunk-sync receiver passed the raw
+// MinTimestamp straight into CreateSegmentIfNotExist. See the analogous
+// measure test for full rationale.
+func TestSyncChunkCallback_CreatePartHandler_AlignsOffGridMinTimestamp(t *testing.T) {
+ t.Run("DAY(1)", func(t *testing.T) {
+ runTraceOffGridCase(t, storage.IntervalRule{Unit: storage.DAY, Num: 1})
+ })
+ t.Run("DAY(5)", func(t *testing.T) {
+ runTraceOffGridCase(t, storage.IntervalRule{Unit: storage.DAY, Num: 5})
+ })
+}
+
+func runTraceOffGridCase(t *testing.T, ir storage.IntervalRule) {
+ t.Helper()
+ tmpPath, cleanup := test.Space(require.New(t))
+ defer cleanup()
+
+ const groupName = "off-grid-trace"
+ db := openTestTSDBWithInterval(t, tmpPath, groupName, ir)
+ defer db.Close()
+
+ rawTS := time.Date(2026, 5, 15, 6, 0, 0, 0, time.Local)
+ wantStart := ir.Standard(rawTS)
+
+ callback := &syncChunkCallback{
+ l: logger.GetLogger("test-offgrid-trace"),
+ schemaRepo: newTestSchemaRepo(db, groupName),
+ }
+ handler, err := callback.CreatePartHandler(&queue.ChunkedSyncPartContext{
+ ID: 1,
+ Group: groupName,
+ ShardID: 0,
+ MinTimestamp: rawTS.UnixNano(),
+ MaxTimestamp: rawTS.UnixNano(),
+ })
+ require.NoError(t, err)
+ defer func() { _ = handler.(*syncPartContext).Close() }()
+ gotStart := handler.(*syncPartContext).segment.GetTimeRange().Start
+ require.True(t, gotStart.Equal(wantStart),
+ "REGRESSION: raw MinTimestamp=%s landed in segment start=%s; expected aligned %s (ir=%+v)",
+ rawTS, gotStart, wantStart, ir)
+ require.False(t, rawTS.Equal(wantStart),
+ "test setup degenerate: rawTS already aligned")
+}
+
+// TestSegmentCreateTS_ConsistencyAcrossPaths is the trace variant of the
+// Path-A vs Path-B convergence demo. A pre-existing aligned historical
+// segment must NOT be touched; both new writes must land in the same
+// new bucket.
+func TestSegmentCreateTS_ConsistencyAcrossPaths(t *testing.T) {
+ tmpPath, cleanup := test.Space(require.New(t))
+ defer cleanup()
+
+ const groupName = "consistency-trace"
+ ir := storage.IntervalRule{Unit: storage.DAY, Num: 5}
+ db := openTestTSDBWithInterval(t, tmpPath, groupName, ir)
+ defer db.Close()
+
+ historicalStart := ir.Standard(time.Date(2026, 4, 10, 0, 0, 0, 0, time.Local))
+ histSeg, err := db.CreateSegmentIfNotExist(historicalStart)
+ require.NoError(t, err)
+ histSeg.DecRef()
+ histSeg.DecRef()
+
+ rawTS := time.Date(2026, 5, 15, 6, 0, 0, 0, time.Local)
+ wantNewStart := ir.Standard(rawTS)
+ require.False(t, wantNewStart.Equal(historicalStart))
+ require.False(t, wantNewStart.Equal(rawTS))
+
+ repo := newTestSchemaRepo(db, groupName)
+
+ chunkCallback := &syncChunkCallback{l: logger.GetLogger("test-trace-pathA"), schemaRepo: repo}
+ handlerA, err := chunkCallback.CreatePartHandler(&queue.ChunkedSyncPartContext{
+ ID: 1,
+ Group: groupName,
+ ShardID: 0,
+ MinTimestamp: rawTS.UnixNano(),
+ MaxTimestamp: rawTS.UnixNano(),
+ })
+ require.NoError(t, err)
+ defer func() { _ = handlerA.(*syncPartContext).Close() }()
+ gotStartA := handlerA.(*syncPartContext).segment.GetTimeRange().Start
+
+ seriesCallback := &syncSeriesCallback{l: logger.GetLogger("test-trace-pathB"), schemaRepo: repo}
+ handlerB, err := seriesCallback.CreatePartHandler(&queue.ChunkedSyncPartContext{
+ ID: 2,
+ Group: groupName,
+ ShardID: 0,
+ MinTimestamp: wantNewStart.UnixNano(),
+ MaxTimestamp: wantNewStart.UnixNano(),
+ })
+ require.NoError(t, err)
+ defer func() { _ = handlerB.(*syncSeriesContext).Close() }()
+ gotStartB := handlerB.(*syncSeriesContext).segment.GetTimeRange().Start
+
+ require.True(t, gotStartA.Equal(gotStartB),
+ "REGRESSION: trace Path A (raw %s -> %s) and Path B (aligned %s -> %s) diverged",
+ rawTS, gotStartA, wantNewStart, gotStartB)
+ require.True(t, gotStartA.Equal(wantNewStart),
+ "path A start %s != expected aligned %s", gotStartA, wantNewStart)
+ require.False(t, gotStartA.Equal(historicalStart),
+ "new write must NOT land in historical segment %s", historicalStart)
+}
+
+func openTestTSDBWithInterval(t *testing.T, tmpPath, groupName string, ir storage.IntervalRule) storage.TSDB[*tsTable, option] {
+ t.Helper()
+ opts := storage.TSDBOpts[*tsTable, option]{
+ ShardNum: 1,
+ Location: filepath.Join(tmpPath, "tab"),
+ TSTableCreator: newTSTable,
+ SegmentInterval: ir,
+ TTL: storage.IntervalRule{Unit: ir.Unit, Num: 60},
+ Option: option{protector: protector.Nop{}, mergePolicy: newDefaultMergePolicyForTesting()},
+ }
+ require.NoError(t, os.MkdirAll(opts.Location, storage.DirPerm))
+ ctx := common.SetPosition(
+ context.WithValue(context.Background(), logger.ContextKey, logger.GetLogger("test-offgrid-trace")),
+ func(p common.Position) common.Position {
+ p.Database = "test-offgrid-trace"
+ return p
+ },
+ )
+ db, err := storage.OpenTSDB[*tsTable, option](ctx, opts, nil, groupName)
+ require.NoError(t, err)
+ return db
+}
(skywalking-banyandb) 01/11: fix(property): prevent gossip propagation from getting stuck after replica scale-up (#1127)
Posted by bu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ButterBright pushed a commit to branch v0.10.x
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit e9282923d77b1000a09bc84cd0a849cacd699451
Author: mrproliu <74...@qq.com>
AuthorDate: Wed May 13 20:32:24 2026 +0800
fix(property): prevent gossip propagation from getting stuck after replica scale-up (#1127)
* fix(property): prevent gossip propagation from getting stuck after replica scale-up
* fix CI and comments
---------
Co-authored-by: Gao Hongtao <ha...@gmail.com>
---
banyand/property/db/repair.go | 19 +-
banyand/property/db/repair_gossip.go | 58 +++-
banyand/property/db/repair_gossip_test.go | 338 +++++++++++++++++++++
banyand/property/db/shard.go | 41 +++
banyand/property/gossip/server.go | 74 +++--
banyand/property/gossip/service_test.go | 344 ++++++++++++++++++++++
test/property_repair/full_data/integrated_test.go | 7 +-
test/property_repair/half_data/integrated_test.go | 7 +-
test/property_repair/same_data/integrated_test.go | 7 +-
test/property_repair/shared_utils.go | 172 ++++++++++-
10 files changed, 1014 insertions(+), 53 deletions(-)
diff --git a/banyand/property/db/repair.go b/banyand/property/db/repair.go
index a5b967908..0829421f4 100644
--- a/banyand/property/db/repair.go
+++ b/banyand/property/db/repair.go
@@ -1219,8 +1219,16 @@ type repairSchedulerMetrics struct {
totalRepairBuildTreeLatency meter.Counter
totalRepairBuildTreeConflicts meter.Counter
- totalRepairSuccessCount meter.Counter
- totalRepairFailedCount meter.Counter
+ totalRepairSuccessCount meter.Counter
+ totalRepairFailedCount meter.Counter
+ totalRepairPerPropertyTimeout meter.Counter
+ // repairSuccessLatency samples one full sync-attempt cycle whenever the
+ // per-property repair returned without error (success path AND no-op
+ // path where the remote was already newer). Failures and per-property
+ // timeouts are NOT sampled — operators relying on this metric for
+ // "is the system slow?" must read totalRepairPerPropertyTimeout in
+ // parallel.
+ repairSuccessLatency meter.Histogram
}
func newRepairSchedulerMetrics(omr observability.Factory) *repairSchedulerMetrics {
@@ -1231,7 +1239,10 @@ func newRepairSchedulerMetrics(omr observability.Factory) *repairSchedulerMetric
totalRepairBuildTreeLatency: omr.NewCounter("repair_build_tree_latency"),
totalRepairBuildTreeConflicts: omr.NewCounter("repair_build_tree_conflicts"),
- totalRepairSuccessCount: omr.NewCounter("property_repair_success_count", "group", "shard"),
- totalRepairFailedCount: omr.NewCounter("property_repair_failure_count", "group", "shard"),
+ totalRepairSuccessCount: omr.NewCounter("property_repair_success_count", "group", "shard"),
+ totalRepairFailedCount: omr.NewCounter("property_repair_failure_count", "group", "shard"),
+ totalRepairPerPropertyTimeout: omr.NewCounter("property_repair_per_property_timeout", "group", "shard"),
+ repairSuccessLatency: omr.NewHistogram("property_repair_success_latency_seconds",
+ meter.Buckets{0.005, 0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 30}, "group", "shard"),
}
}
diff --git a/banyand/property/db/repair_gossip.go b/banyand/property/db/repair_gossip.go
index 914ad08af..c9f258324 100644
--- a/banyand/property/db/repair_gossip.go
+++ b/banyand/property/db/repair_gossip.go
@@ -22,6 +22,7 @@ import (
"fmt"
"io"
"strings"
+ "time"
"github.com/pkg/errors"
grpclib "google.golang.org/grpc"
@@ -36,12 +37,52 @@ import (
var (
gossipMerkleTreeReadPageSize int64 = 10
gossipShardQueryDatabaseSize = 100
+
+ // repairPerPropertyCtxBudget bounds every ctx-aware call inside one
+ // gossip-triggered per-property repair: shard.repair -> s.search,
+ // shard.repair -> buildDeleteFromTimeDocuments -> s.search, and any
+ // nested ctx-aware lookup. updateDocuments takes no ctx and is not
+ // bounded.
+ repairPerPropertyCtxBudget = 10 * time.Second
)
type repairGossipBase struct {
scheduler *repairScheduler
}
+// executeRepairWithBudget runs syncShard.repair under repairPerPropertyCtxBudget;
+// shared by client and server paths. repairSuccessLatency samples every err == nil
+// (incl. no-op). totalRepairPerPropertyTimeout fires only when DeadlineExceeded
+// came from the child budget (parent ctx still active).
+func (b *repairGossipBase) executeRepairWithBudget(
+ ctx context.Context,
+ syncShard *shard,
+ id []byte,
+ property *propertyv1.Property,
+ deleteTime int64,
+ group string,
+) (bool, *queryProperty, error) {
+ repairCtx, repairCancel := context.WithTimeout(ctx, repairPerPropertyCtxBudget)
+ repairStart := time.Now()
+ updated, newer, err := syncShard.repair(repairCtx, id, property, deleteTime)
+ repairCancel()
+ shardLabel := fmt.Sprintf("%d", syncShard.id)
+ if err == nil {
+ b.scheduler.metrics.repairSuccessLatency.Observe(time.Since(repairStart).Seconds(), group, shardLabel)
+ }
+ // Only count per-property timeout when the child budget fired while the
+ // parent ctx was still active. If the parent was already done, the
+ // DeadlineExceeded came from outside this helper and would be a misleading
+ // signal for "single property too slow".
+ if err != nil && errors.Is(err, context.DeadlineExceeded) && ctx.Err() == nil {
+ b.scheduler.metrics.totalRepairPerPropertyTimeout.Inc(1, group, shardLabel)
+ b.scheduler.l.Warn().Str("group", group).Uint32("shardID", uint32(syncShard.id)).
+ Str("propertyID", string(id)).Dur("budget", repairPerPropertyCtxBudget).
+ Msg("per-property repair hit timeout — skipping single property")
+ }
+ return updated, newer, err
+}
+
func (b *repairGossipBase) getTreeReader(ctx context.Context, group string, shardID uint32) (repairTreeReader, bool, error) {
s, err := b.scheduler.db.loadShard(ctx, group, common.ShardID(shardID))
if err != nil {
@@ -265,6 +306,8 @@ func (r *repairGossipClient) Rev(ctx context.Context, tracer gossip.Trace, nextN
leafReader := newRepairBufferLeafReader(reader)
var currentComparingClientNode *repairTreeNode
var notProcessingClientNode *repairTreeNode
+ processed := 0
+ loopStart := time.Now()
for {
recvResp, err := stream.Recv()
if err != nil {
@@ -299,6 +342,13 @@ func (r *repairGossipClient) Rev(ctx context.Context, tracer gossip.Trace, nextN
}
differSpan.End()
case *propertyv1.RepairResponse_PropertySync:
+ processed++
+ if processed%100 == 0 {
+ r.scheduler.l.Info().Int("processed", processed).
+ Dur("elapsed", time.Since(loopStart)).
+ Str("group", request.Group).Uint32("shardID", request.ShardId).
+ Msg("repair progress")
+ }
r.scheduler.l.Debug().Msgf("received repair response from server")
// step 3: keep receiving messages from the server
// if the server still sending different nodes, we should keep reading them
@@ -307,13 +357,15 @@ func (r *repairGossipClient) Rev(ctx context.Context, tracer gossip.Trace, nextN
syncSpan := tracer.CreateSpan(startSyncSpan, "repair property")
syncSpan.Tag(gossip.TraceTagOperateType, gossip.TraceTagOperateRepairProperty)
syncSpan.Tag(gossip.TraceTagPropertyID, string(sync.Property.Id))
- updated, newer, err := syncShard.repair(ctx, sync.Property.Id, sync.Property.Property, sync.Property.DeleteTime)
+ updated, newer, err := r.executeRepairWithBudget(ctx, syncShard, sync.Property.Id, sync.Property.Property, sync.Property.DeleteTime, request.Group)
syncSpan.Tag("updated", fmt.Sprintf("%t", updated))
syncSpan.Tag("has_newer", fmt.Sprintf("%t", newer != nil))
if err != nil {
syncSpan.Error(err.Error())
r.scheduler.l.Warn().Err(err).Msgf("failed to repair property %s", sync.Property.Id)
r.scheduler.metrics.totalRepairFailedCount.Inc(1, request.Group, fmt.Sprintf("%d", request.ShardId))
+ syncSpan.End()
+ continue
}
if updated {
r.scheduler.l.Debug().Msgf("successfully repaired property %s on client side", sync.Property.Id)
@@ -350,8 +402,8 @@ func (r *repairGossipClient) Rev(ctx context.Context, tracer gossip.Trace, nextN
syncSpan.Error(err.Error())
r.scheduler.l.Warn().Err(err).Msgf("failed to send newer property sync response to server, entity: %s", newer.id)
}
- syncSpan.End()
}
+ syncSpan.End()
default:
r.scheduler.l.Warn().Msgf("unexpected response type: %T, expected DifferTreeSummary or PropertySync", resp)
}
@@ -741,7 +793,7 @@ func (r *repairGossipServer) processPropertySync(
s grpclib.BidiStreamingServer[propertyv1.RepairRequest, propertyv1.RepairResponse],
group string,
) bool {
- updated, newer, err := syncShard.repair(ctx, sync.Id, sync.Property, sync.DeleteTime)
+ updated, newer, err := r.executeRepairWithBudget(ctx, syncShard, sync.Id, sync.Property, sync.DeleteTime, group)
if err != nil {
r.scheduler.l.Warn().Err(err).Msgf("failed to repair property %s from server side", sync.Id)
r.scheduler.metrics.totalRepairFailedCount.Inc(1, group, fmt.Sprintf("%d", syncShard.id))
diff --git a/banyand/property/db/repair_gossip_test.go b/banyand/property/db/repair_gossip_test.go
index 20f428f9f..536f57920 100644
--- a/banyand/property/db/repair_gossip_test.go
+++ b/banyand/property/db/repair_gossip_test.go
@@ -19,6 +19,7 @@ package db
import (
"context"
+ "errors"
"fmt"
"net"
"path"
@@ -580,6 +581,49 @@ type nodeContext struct {
stopMutex sync.RWMutex
}
+// inspectCounter exposes Inc deltas — BypassRegistry would swallow them.
+type inspectCounter struct {
+ totals map[string]float64
+ mu sync.RWMutex
+}
+
+func newInspectCounter() *inspectCounter {
+ return &inspectCounter{totals: make(map[string]float64)}
+}
+
+// Inc satisfies meter.Counter.
+func (c *inspectCounter) Inc(delta float64, labelValues ...string) {
+ key := fmt.Sprintf("%v", labelValues)
+ c.mu.Lock()
+ c.totals[key] += delta
+ c.mu.Unlock()
+}
+
+// Delete satisfies meter.Counter / meter.Histogram (both embed Instrument).
+func (c *inspectCounter) Delete(labelValues ...string) bool {
+ key := fmt.Sprintf("%v", labelValues)
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ if _, ok := c.totals[key]; !ok {
+ return false
+ }
+ delete(c.totals, key)
+ return true
+}
+
+// Observe satisfies meter.Histogram. We treat the call count itself as the
+// observable signal — callers assert get(labels) == expected sample count.
+func (c *inspectCounter) Observe(_ float64, labelValues ...string) {
+ c.Inc(1, labelValues...)
+}
+
+func (c *inspectCounter) get(labelValues ...string) float64 {
+ key := fmt.Sprintf("%v", labelValues)
+ c.mu.RLock()
+ defer c.mu.RUnlock()
+ return c.totals[key]
+}
+
func (n *nodeContext) appendStop(f func()) {
n.stopMutex.Lock()
defer n.stopMutex.Unlock()
@@ -603,3 +647,297 @@ func nodeContextToParentSlice(ncs []*nodeContext) []node {
}
return nodes
}
+
+// Shrinking the var must force timeout; counter assertions fail if the wrap is dropped.
+// Only the server-side path (processPropertySync) is exercised here; the client-side
+// wrap inside processDifferTreeSummary is currently covered by code review only.
+func TestProcessPropertySyncRespectsPerPropertySearchTimeout(t *testing.T) {
+ prev := repairPerPropertyCtxBudget
+ repairPerPropertyCtxBudget = -time.Hour
+ t.Cleanup(func() { repairPerPropertyCtxBudget = prev })
+
+ var defers []func()
+ defer func() {
+ for _, f := range defers {
+ f()
+ }
+ }()
+
+ dataDir, dataDeferFunc, err := test.NewSpace()
+ if err != nil {
+ t.Fatal(err)
+ }
+ defers = append(defers, dataDeferFunc)
+ snapshotDir, snapshotDeferFunc, err := test.NewSpace()
+ if err != nil {
+ t.Fatal(err)
+ }
+ defers = append(defers, snapshotDeferFunc)
+
+ dbInstance, err := OpenDB(context.Background(), Config{
+ Location: dataDir,
+ MetricsScopeName: "property_test_timeout_wiring",
+ FlushInterval: 3 * time.Second,
+ ExpireToDeleteDuration: 1 * time.Hour,
+ Repair: RepairConfig{
+ Enabled: true,
+ Location: snapshotDir,
+ BuildTreeCron: "@every 10m",
+ QuickBuildTreeTime: time.Second * 10,
+ TreeSlotCount: 32,
+ },
+ }, observability.BypassRegistry, fs.NewLocalFileSystem())
+ if err != nil {
+ t.Fatal(err)
+ }
+ defers = append(defers, func() { _ = dbInstance.Close() })
+
+ db := dbInstance.(*database)
+
+ // Install inspect-able counters so the test can assert the timeout
+ // branch incremented the metric — not merely that "some error" occurred.
+ timeoutCounter := newInspectCounter()
+ failedCounter := newInspectCounter()
+ successCounter := newInspectCounter()
+ db.repairScheduler.metrics.totalRepairPerPropertyTimeout = timeoutCounter
+ db.repairScheduler.metrics.totalRepairFailedCount = failedCounter
+ db.repairScheduler.metrics.totalRepairSuccessCount = successCounter
+
+ property := generateProperty("test-id-timeout-wiring", time.Now().UnixNano(), 0)
+ if updateErr := db.Update(context.Background(), 0, GetPropertyID(property), property); updateErr != nil {
+ t.Fatal(updateErr)
+ }
+
+ syncShard, loadErr := db.loadShard(context.Background(), testPropertyGroup, common.ShardID(0))
+ if loadErr != nil {
+ t.Fatal(loadErr)
+ }
+
+ // Sanity probe: a fresh ctx with the pathological timeout must already be Done.
+ probeCtx, probeCancel := context.WithTimeout(context.Background(), repairPerPropertyCtxBudget)
+ probeCancel()
+ if !errors.Is(probeCtx.Err(), context.DeadlineExceeded) {
+ t.Fatalf("expected probe ctx to be DeadlineExceeded, got %v", probeCtx.Err())
+ }
+
+ server := newRepairGossipServer(db.repairScheduler)
+ sync := &propertyv1.PropertySync{
+ Id: GetPropertyID(property),
+ Property: property,
+ DeleteTime: 0,
+ }
+
+ // typed-nil stream is safe: the err path in processPropertySync returns
+ // before touching it. If a future refactor reorders the code so the
+ // stream is used before the err check, this test would panic — that
+ // panic is the desired alarm.
+ var nilStream grpc.BidiStreamingServer[propertyv1.RepairRequest, propertyv1.RepairResponse]
+ result := server.processPropertySync(context.Background(), syncShard, sync, nilStream, testPropertyGroup)
+ if result {
+ t.Fatalf("expected processPropertySync to return false when per-property search timeout fires, got true")
+ }
+
+ // The timeout-specific branch must have incremented the per-property
+ // timeout counter. Without this assertion the test would pass for any
+ // error — including non-timeout regressions like an index corruption —
+ // and the Task 4 wiring would still appear "green".
+ shardLabel := fmt.Sprintf("%d", syncShard.id)
+ if got := timeoutCounter.get(testPropertyGroup, shardLabel); got != 1 {
+ t.Fatalf("expected totalRepairPerPropertyTimeout(%s,%s) == 1, got %v",
+ testPropertyGroup, shardLabel, got)
+ }
+ if got := failedCounter.get(testPropertyGroup, shardLabel); got != 1 {
+ t.Fatalf("expected totalRepairFailedCount(%s,%s) == 1, got %v",
+ testPropertyGroup, shardLabel, got)
+ }
+ // Protective 0 check: success must not move on a forced-timeout call.
+ // If a future refactor adds a retry-on-timeout path this assertion needs
+ // to relax, but the timeout/failed counters above must still hold.
+ if got := successCounter.get(testPropertyGroup, shardLabel); got != 0 {
+ t.Fatalf("expected totalRepairSuccessCount(%s,%s) == 0, got %v",
+ testPropertyGroup, shardLabel, got)
+ }
+}
+
+// openTestRepairBase boots an in-process database and returns a fresh
+// repairGossipBase whose metrics are replaced by inspect counters for the
+// caller to assert on. Used by executeRepairWithBudget unit tests so they
+// can exercise client+server-shared timeout/latency semantics in one place.
+func openTestRepairBase(t *testing.T) (
+ base *repairGossipBase,
+ timeoutCounter *inspectCounter,
+ successLatency *inspectCounter,
+ cleanup func(),
+) {
+ t.Helper()
+ var defers []func()
+ cleanup = func() {
+ for _, f := range defers {
+ f()
+ }
+ }
+
+ dataDir, dataDeferFunc, err := test.NewSpace()
+ if err != nil {
+ cleanup()
+ t.Fatal(err)
+ }
+ defers = append(defers, dataDeferFunc)
+ snapshotDir, snapshotDeferFunc, err := test.NewSpace()
+ if err != nil {
+ cleanup()
+ t.Fatal(err)
+ }
+ defers = append(defers, snapshotDeferFunc)
+
+ dbInstance, err := OpenDB(context.Background(), Config{
+ Location: dataDir,
+ MetricsScopeName: "property_test_helper",
+ FlushInterval: 3 * time.Second,
+ ExpireToDeleteDuration: 1 * time.Hour,
+ Repair: RepairConfig{
+ Enabled: true,
+ Location: snapshotDir,
+ BuildTreeCron: "@every 10m",
+ QuickBuildTreeTime: time.Second * 10,
+ TreeSlotCount: 32,
+ },
+ }, observability.BypassRegistry, fs.NewLocalFileSystem())
+ if err != nil {
+ cleanup()
+ t.Fatal(err)
+ }
+ defers = append(defers, func() { _ = dbInstance.Close() })
+
+ db := dbInstance.(*database)
+ timeoutCounter = newInspectCounter()
+ successLatency = newInspectCounter()
+ db.repairScheduler.metrics.totalRepairPerPropertyTimeout = timeoutCounter
+ db.repairScheduler.metrics.repairSuccessLatency = successLatency
+ base = &repairGossipBase{scheduler: db.repairScheduler}
+ return base, timeoutCounter, successLatency, cleanup
+}
+
+// TestExecuteRepairWithBudget covers the per-property repair helper that both
+// client (processDifferTreeSummary) and server (processPropertySync) call
+// into. Each subtest pins one observability invariant; deleting the matching
+// branch in executeRepairWithBudget makes the corresponding assertion fail.
+func TestExecuteRepairWithBudget(t *testing.T) {
+ shardLabel := "0"
+
+ t.Run("child budget fires under healthy parent ctx → timeout counter +1", func(t *testing.T) {
+ prev := repairPerPropertyCtxBudget
+ repairPerPropertyCtxBudget = -time.Hour
+ t.Cleanup(func() { repairPerPropertyCtxBudget = prev })
+
+ base, timeoutCounter, successLatency, cleanup := openTestRepairBase(t)
+ defer cleanup()
+
+ syncShard, err := base.scheduler.db.loadShard(context.Background(), testPropertyGroup, common.ShardID(0))
+ if err != nil {
+ t.Fatal(err)
+ }
+ property := generateProperty("test-id-budget", time.Now().UnixNano(), 0)
+
+ _, _, repairErr := base.executeRepairWithBudget(context.Background(), syncShard,
+ GetPropertyID(property), property, 0, testPropertyGroup)
+ if repairErr == nil {
+ t.Fatalf("expected err under negative budget, got nil")
+ }
+ if !errors.Is(repairErr, context.DeadlineExceeded) {
+ t.Fatalf("expected wrapped DeadlineExceeded, got %v", repairErr)
+ }
+ if got := timeoutCounter.get(testPropertyGroup, shardLabel); got != 1 {
+ t.Fatalf("expected per-property timeout counter == 1, got %v", got)
+ }
+ if got := successLatency.get(testPropertyGroup, shardLabel); got != 0 {
+ t.Fatalf("expected success latency not sampled on err, got %v", got)
+ }
+ })
+
+ t.Run("parent ctx already done → timeout counter not incremented (parent-not-child)", func(t *testing.T) {
+ base, timeoutCounter, _, cleanup := openTestRepairBase(t)
+ defer cleanup()
+
+ syncShard, err := base.scheduler.db.loadShard(context.Background(), testPropertyGroup, common.ShardID(0))
+ if err != nil {
+ t.Fatal(err)
+ }
+ property := generateProperty("test-id-parent", time.Now().UnixNano(), 0)
+
+ // Parent ctx already expired: DeadlineExceeded reaches the helper
+ // but it is NOT from the per-property budget — must not be counted.
+ parentCtx, parentCancel := context.WithDeadline(context.Background(), time.Now().Add(-time.Hour))
+ defer parentCancel()
+
+ _, _, repairErr := base.executeRepairWithBudget(parentCtx, syncShard,
+ GetPropertyID(property), property, 0, testPropertyGroup)
+ if repairErr == nil {
+ t.Fatalf("expected err under already-expired parent ctx, got nil")
+ }
+ if got := timeoutCounter.get(testPropertyGroup, shardLabel); got != 0 {
+ t.Fatalf("parent-ctx expiry must not count as per-property timeout, got %v", got)
+ }
+ })
+
+ t.Run("success path with updated=true → latency sampled once", func(t *testing.T) {
+ base, timeoutCounter, successLatency, cleanup := openTestRepairBase(t)
+ defer cleanup()
+
+ syncShard, err := base.scheduler.db.loadShard(context.Background(), testPropertyGroup, common.ShardID(0))
+ if err != nil {
+ t.Fatal(err)
+ }
+ // Empty db → shard.repair takes the "no older properties" branch,
+ // updateDocuments runs, and returns updated == true.
+ property := generateProperty("test-id-success", time.Now().UnixNano(), 0)
+
+ updated, _, repairErr := base.executeRepairWithBudget(context.Background(), syncShard,
+ GetPropertyID(property), property, 0, testPropertyGroup)
+ if repairErr != nil {
+ t.Fatalf("expected nil err on healthy repair, got %v", repairErr)
+ }
+ if !updated {
+ t.Fatalf("expected updated == true on empty-db first repair, got false")
+ }
+ if got := successLatency.get(testPropertyGroup, shardLabel); got != 1 {
+ t.Fatalf("expected success latency sample count == 1, got %v", got)
+ }
+ if got := timeoutCounter.get(testPropertyGroup, shardLabel); got != 0 {
+ t.Fatalf("timeout counter must not move on success, got %v", got)
+ }
+ })
+
+ t.Run("no-op path with updated=false → latency still sampled (every err == nil counts)", func(t *testing.T) {
+ base, _, successLatency, cleanup := openTestRepairBase(t)
+ defer cleanup()
+
+ now := time.Now().UnixNano()
+ // Seed a newer revision so the repair finds an older-or-equal "self"
+ // already present and returns updated == false.
+ existing := generateProperty("test-id-noop", now, 0)
+ if err := base.scheduler.db.Update(context.Background(), 0, GetPropertyID(existing), existing); err != nil {
+ t.Fatal(err)
+ }
+ syncShard, err := base.scheduler.db.loadShard(context.Background(), testPropertyGroup, common.ShardID(0))
+ if err != nil {
+ t.Fatal(err)
+ }
+ // Re-repair with an older revision: helper returns nil err but
+ // updated == false. The success-latency histogram still samples
+ // because a no-op decision is still a successful sync cycle whose
+ // ctx-aware cost operators care about.
+ olderOrEqual := generateProperty("test-id-noop", now-1, 0)
+ updated, _, repairErr := base.executeRepairWithBudget(context.Background(), syncShard,
+ GetPropertyID(olderOrEqual), olderOrEqual, 0, testPropertyGroup)
+ if repairErr != nil {
+ t.Fatalf("expected nil err on no-op repair, got %v", repairErr)
+ }
+ if updated {
+ t.Fatalf("expected updated == false on older-or-equal repair, got true")
+ }
+ if got := successLatency.get(testPropertyGroup, shardLabel); got != 1 {
+ t.Fatalf("success latency must be sampled once even when updated == false (err == nil semantics), got %v", got)
+ }
+ })
+}
diff --git a/banyand/property/db/shard.go b/banyand/property/db/shard.go
index 6e8410eac..3d8915d9a 100644
--- a/banyand/property/db/shard.go
+++ b/banyand/property/db/shard.go
@@ -357,6 +357,37 @@ func (s *shard) search(ctx context.Context, q index.Query, orderBy *propertyv1.Q
}
func (s *shard) repair(ctx context.Context, id []byte, property *propertyv1.Property, deleteTime int64) (updated bool, selfNewer *queryProperty, err error) {
+ start := time.Now()
+ var (
+ search1Elapsed time.Duration
+ deletePhaseElapsed time.Duration
+ updateElapsed time.Duration
+ olderCount int
+ deleteCount int
+ )
+ defer func() {
+ elapsed := time.Since(start)
+ switch {
+ case err != nil:
+ s.l.Warn().Int64("elapsed_ms", elapsed.Milliseconds()).Str("group", property.Metadata.Group).
+ Str("name", property.Metadata.Name).Str("id", property.Id).Err(err).
+ Int64("search1_ms", search1Elapsed.Milliseconds()).
+ Int64("delete_phase_ms", deletePhaseElapsed.Milliseconds()).
+ Int64("update_ms", updateElapsed.Milliseconds()).
+ Int("older_count", olderCount).
+ Int("delete_count", deleteCount).
+ Msg("property repair failed")
+ case elapsed >= 5*time.Second:
+ s.l.Warn().Int64("elapsed_ms", elapsed.Milliseconds()).Str("group", property.Metadata.Group).
+ Str("name", property.Metadata.Name).Str("id", property.Id).
+ Int64("search1_ms", search1Elapsed.Milliseconds()).
+ Int64("delete_phase_ms", deletePhaseElapsed.Milliseconds()).
+ Int64("update_ms", updateElapsed.Milliseconds()).
+ Int("older_count", olderCount).
+ Int("delete_count", deleteCount).
+ Msg("slow property repair")
+ }
+ }()
iq, err := inverted.BuildPropertyQuery(&propertyv1.QueryRequest{
Groups: []string{property.Metadata.Group},
Name: property.Metadata.Name,
@@ -365,10 +396,13 @@ func (s *shard) repair(ctx context.Context, id []byte, property *propertyv1.Prop
if err != nil {
return false, nil, fmt.Errorf("build property query failure: %w", err)
}
+ search1Start := time.Now()
olderProperties, err := s.search(ctx, iq, nil, 100)
+ search1Elapsed = time.Since(search1Start)
if err != nil {
return false, nil, fmt.Errorf("query older properties failed: %w", err)
}
+ olderCount = len(olderProperties)
sort.Sort(queryPropertySlice(olderProperties))
// if there no older properties, we can insert the latest document.
if len(olderProperties) == 0 {
@@ -377,7 +411,9 @@ func (s *shard) repair(ctx context.Context, id []byte, property *propertyv1.Prop
if err != nil {
return false, nil, fmt.Errorf("build update document failed: %w", err)
}
+ updateStart := time.Now()
err = s.updateDocuments(index.Documents{*doc})
+ updateElapsed = time.Since(updateStart)
if err != nil {
return false, nil, fmt.Errorf("update document failed: %w", err)
}
@@ -393,10 +429,13 @@ func (s *shard) repair(ctx context.Context, id []byte, property *propertyv1.Prop
}
docIDList := s.buildNotDeletedDocIDList(olderProperties)
+ deletePhaseStart := time.Now()
deletedDocuments, err := s.buildDeleteFromTimeDocuments(ctx, docIDList, time.Now().UnixNano())
+ deletePhaseElapsed = time.Since(deletePhaseStart)
if err != nil {
return false, nil, fmt.Errorf("build delete older documents failed: %w", err)
}
+ deleteCount = len(deletedDocuments)
// update the property to mark it as delete
updateDoc, err := s.buildUpdateDocument(id, property, deleteTime)
if err != nil {
@@ -405,7 +444,9 @@ func (s *shard) repair(ctx context.Context, id []byte, property *propertyv1.Prop
result := make([]index.Document, 0, len(deletedDocuments)+1)
result = append(result, deletedDocuments...)
result = append(result, *updateDoc)
+ updateStart := time.Now()
err = s.updateDocuments(result)
+ updateElapsed = time.Since(updateStart)
if err != nil {
return false, nil, fmt.Errorf("update documents failed: %w", err)
}
diff --git a/banyand/property/gossip/server.go b/banyand/property/gossip/server.go
index cff25034f..1a6b8c2a8 100644
--- a/banyand/property/gossip/server.go
+++ b/banyand/property/gossip/server.go
@@ -86,7 +86,7 @@ func (s *service) getServiceRegisters() []func(server *grpc.Server) {
type groupWithShardPropagation struct {
latestTime time.Time
- channel chan *handlingRequest
+ pending *handlingRequest
originalNodeID string
}
@@ -117,16 +117,24 @@ func (q *protocolHandler) processPropagation() {
for {
select {
case <-q.groupNotify:
- request := q.findUnProcessRequest()
- if request == nil {
- continue
- }
- timeoutCtx, cancelFunc := context.WithTimeout(q.s.closer.Ctx(), perNodeSyncTimeout)
- err := q.handle(timeoutCtx, request)
- cancelFunc()
- if err != nil {
- q.s.log.Warn().Err(err).Stringer("request", request.PropagationRequest).
- Msgf("handle propagation request failure")
+ // Drain every pending per wake-up; re-check CloseNotify between handles.
+ for {
+ select {
+ case <-q.s.closer.CloseNotify():
+ return
+ default:
+ }
+ request := q.findUnProcessRequest()
+ if request == nil {
+ break
+ }
+ timeoutCtx, cancelFunc := context.WithTimeout(q.s.closer.Ctx(), perNodeSyncTimeout)
+ err := q.handle(timeoutCtx, request)
+ cancelFunc()
+ if err != nil {
+ q.s.log.Warn().Err(err).Stringer("request", request.PropagationRequest).
+ Msgf("handle propagation request failure")
+ }
}
case <-q.s.closer.CloseNotify():
return
@@ -138,11 +146,10 @@ func (q *protocolHandler) findUnProcessRequest() *handlingRequest {
q.mu.Lock()
defer q.mu.Unlock()
for _, g := range q.groupWithShards {
- select {
- case d := <-g.channel:
- return d
- default:
- continue
+ if g.pending != nil {
+ req := g.pending
+ g.pending = nil
+ return req
}
}
return nil
@@ -302,6 +309,8 @@ func (q *protocolHandler) contextIsDone(ctx context.Context) bool {
}
}
+// addToProcess enqueues the request with latest-wins coalesce on pending;
+// returns false when a different originator arrives within TTL.
func (q *protocolHandler) addToProcess(request *propertyv1.PropagationRequest, tracer Trace, span Span) bool {
q.mu.Lock()
defer q.mu.Unlock()
@@ -315,11 +324,10 @@ func (q *protocolHandler) addToProcess(request *propertyv1.PropagationRequest, t
}
if !exist {
groupShard = &groupWithShardPropagation{
- channel: make(chan *handlingRequest, 1),
+ pending: handlingRequestData,
originalNodeID: request.Context.OriginNode,
latestTime: time.Now(),
}
- groupShard.channel <- handlingRequestData
q.groupWithShards[shardKey] = groupShard
q.notifyNewRequest()
return true
@@ -328,23 +336,31 @@ func (q *protocolHandler) addToProcess(request *propertyv1.PropagationRequest, t
// if the latest round is out of ttl, then needs to change to current node to executing
if time.Since(groupShard.latestTime) > q.s.scheduleInterval/2 {
groupShard.originalNodeID = request.Context.OriginNode
- select {
- case groupShard.channel <- handlingRequestData:
- q.notifyNewRequest()
- default:
- q.s.log.Error().Msgf("ready to added propagation into group shard %s(%d) in a new round, but it's full", request.Group, request.ShardId)
+ if groupShard.pending != nil {
+ q.s.serverMetrics.totalCoalesced.Inc(1, request.Group)
+ q.s.log.Info().
+ Str("group", request.Group).
+ Uint32("shardID", request.ShardId).
+ Str("originNode", request.Context.OriginNode).
+ Msg("propagation request coalesced into pending (TTL takeover)")
}
+ groupShard.pending = handlingRequestData
+ q.notifyNewRequest()
return true
}
// if the original node ID are a same node, means which from the same round
if groupShard.originalNodeID == request.Context.OriginNode {
- select {
- case groupShard.channel <- handlingRequestData:
- q.notifyNewRequest()
- default:
- q.s.log.Error().Msgf("ready to added propagation into group shard %s(%d) in a same round, but it's full", request.Group, request.ShardId)
+ if groupShard.pending != nil {
+ q.s.serverMetrics.totalCoalesced.Inc(1, request.Group)
+ q.s.log.Info().
+ Str("group", request.Group).
+ Uint32("shardID", request.ShardId).
+ Str("originNode", request.Context.OriginNode).
+ Msg("propagation request coalesced into pending (same-round latest-wins)")
}
+ groupShard.pending = handlingRequestData
+ q.notifyNewRequest()
return true
}
@@ -419,6 +435,7 @@ type serverMetrics struct {
totalReceived meter.Counter
totalAddProcessed meter.Counter
totalSkipProcess meter.Counter
+ totalCoalesced meter.Counter
totalStarted meter.Counter
totalFinished meter.Counter
@@ -439,6 +456,7 @@ func newServerMetrics(factory observability.Factory) *serverMetrics {
totalReceived: factory.NewCounter("total_received", "group"),
totalAddProcessed: factory.NewCounter("total_add_processed", "group"),
totalSkipProcess: factory.NewCounter("total_skip_process", "group"),
+ totalCoalesced: factory.NewCounter("total_coalesced", "group"),
totalStarted: factory.NewCounter("total_started", "group"),
totalFinished: factory.NewCounter("total_finished", "group"),
diff --git a/banyand/property/gossip/service_test.go b/banyand/property/gossip/service_test.go
index dc747e9a8..cfcacebeb 100644
--- a/banyand/property/gossip/service_test.go
+++ b/banyand/property/gossip/service_test.go
@@ -21,6 +21,7 @@ import (
"context"
"fmt"
"net"
+ "strings"
"sync"
"testing"
"time"
@@ -142,6 +143,268 @@ var _ = ginkgo.Describe("Propagation Messenger", func() {
nodeListenFromVerify(node2, []string{node1.nodeID})
nodeListenFromVerify(node3, []string{node1.nodeID})
})
+
+ // Latest scheduled round must reach Rev when the worker is busy.
+ ginkgo.It("does not silently drop scheduled rounds when handler is slow", func() {
+ nodes = startNodes(3)
+ svc := nodes[0].messenger.(*service)
+
+ coalesced := newRecordingCounter()
+ svc.serverMetrics.totalCoalesced = coalesced
+
+ blocking := newBlockingListener()
+ svc.listenersLock.Lock()
+ svc.listeners = []MessageListener{blocking}
+ svc.listenersLock.Unlock()
+
+ nodeList := []string{nodes[0].nodeID, nodes[1].nodeID, nodes[2].nodeID}
+ const totalRounds = 5
+ buildRequest := func(roundIndex int32) *propertyv1.PropagationRequest {
+ return &propertyv1.PropagationRequest{
+ Context: &propertyv1.PropagationContext{
+ Nodes: nodeList,
+ OriginNode: nodes[0].nodeID,
+ MaxPropagationCount: int32(len(nodeList)*2 - 3),
+ CurrentPropagationCount: roundIndex,
+ },
+ Group: mockGroup,
+ ShardId: 0,
+ }
+ }
+
+ _, sendErr := svc.protocolHandler.Propagation(context.Background(), buildRequest(0))
+ gomega.Expect(sendErr).NotTo(gomega.HaveOccurred())
+ gomega.Eventually(func() bool {
+ select {
+ case <-blocking.entered:
+ return true
+ default:
+ return false
+ }
+ }, flags.EventuallyTimeout, "10ms").Should(gomega.BeTrue(),
+ "worker should have drained the first request and entered listener.Rev")
+
+ _, sendErr = svc.protocolHandler.Propagation(context.Background(), buildRequest(1))
+ gomega.Expect(sendErr).NotTo(gomega.HaveOccurred())
+ for i := int32(2); i < int32(totalRounds); i++ {
+ _, sendErr = svc.protocolHandler.Propagation(context.Background(), buildRequest(i))
+ gomega.Expect(sendErr).NotTo(gomega.HaveOccurred())
+ }
+ gomega.Expect(coalesced.get(mockGroup)).To(gomega.Equal(3.0),
+ "3 same-round overwrites of a non-nil pending must each increment totalCoalesced")
+
+ close(blocking.release)
+ gomega.Eventually(func() int32 {
+ return blocking.maxObservedCount()
+ }, flags.EventuallyTimeout, "100ms").Should(gomega.Equal(int32(4)),
+ "latest scheduled round must reach Rev under latest-wins semantics")
+ gomega.Expect(blocking.count()).To(gomega.BeNumerically(">=", 2),
+ "at least the in-flight round and the coalesced latest round should reach Rev")
+ })
+
+ // TTL takeover wires the new originator into pending when it is empty.
+ ginkgo.It("TTL-expired branch takes over to new originator with empty pending", func() {
+ nodes = startNodes(2)
+ svc := nodes[0].messenger.(*service)
+ svc.scheduleInterval = 50 * time.Millisecond
+
+ blocking := newBlockingListener()
+ svc.listenersLock.Lock()
+ svc.listeners = []MessageListener{blocking}
+ svc.listenersLock.Unlock()
+
+ nodeList := []string{nodes[0].nodeID, nodes[1].nodeID}
+ buildReq := func(origin string, idx int32) *propertyv1.PropagationRequest {
+ return &propertyv1.PropagationRequest{
+ Context: &propertyv1.PropagationContext{
+ Nodes: nodeList,
+ OriginNode: origin,
+ MaxPropagationCount: 1,
+ CurrentPropagationCount: idx,
+ },
+ Group: mockGroup,
+ ShardId: 0,
+ }
+ }
+
+ _, err := svc.protocolHandler.Propagation(context.Background(), buildReq(nodes[0].nodeID, 0))
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ gomega.Eventually(func() bool {
+ select {
+ case <-blocking.entered:
+ return true
+ default:
+ return false
+ }
+ }, flags.EventuallyTimeout, "10ms").Should(gomega.BeTrue())
+
+ time.Sleep(250 * time.Millisecond)
+
+ _, err = svc.protocolHandler.Propagation(context.Background(), buildReq(nodes[1].nodeID, 99))
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ close(blocking.release)
+ gomega.Eventually(func() int32 {
+ return blocking.maxObservedCount()
+ }, flags.EventuallyTimeout, "100ms").Should(gomega.Equal(int32(99)),
+ "round-B must reach Rev — confirms the TTL-expired branch ran and wired pending correctly")
+ })
+
+ // TTL takeover overwrites a non-nil pending and bumps totalCoalesced.
+ ginkgo.It("TTL-expired branch coalesces when pending is non-nil", func() {
+ nodes = startNodes(2)
+ svc := nodes[0].messenger.(*service)
+ svc.scheduleInterval = 50 * time.Millisecond
+ coalesced := newRecordingCounter()
+ svc.serverMetrics.totalCoalesced = coalesced
+
+ blocking := newBlockingListener()
+ svc.listenersLock.Lock()
+ svc.listeners = []MessageListener{blocking}
+ svc.listenersLock.Unlock()
+
+ nodeList := []string{nodes[0].nodeID, nodes[1].nodeID}
+ buildReq := func(origin string, idx int32) *propertyv1.PropagationRequest {
+ return &propertyv1.PropagationRequest{
+ Context: &propertyv1.PropagationContext{
+ Nodes: nodeList,
+ OriginNode: origin,
+ MaxPropagationCount: 1,
+ CurrentPropagationCount: idx,
+ },
+ Group: mockGroup,
+ ShardId: 0,
+ }
+ }
+
+ _, err := svc.protocolHandler.Propagation(context.Background(), buildReq(nodes[0].nodeID, 0))
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ gomega.Eventually(func() bool {
+ select {
+ case <-blocking.entered:
+ return true
+ default:
+ return false
+ }
+ }, flags.EventuallyTimeout, "10ms").Should(gomega.BeTrue())
+
+ _, err = svc.protocolHandler.Propagation(context.Background(), buildReq(nodes[0].nodeID, 1))
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ gomega.Expect(coalesced.get(mockGroup)).To(gomega.Equal(0.0),
+ "first same-origin round writes pending without coalescing")
+
+ time.Sleep(250 * time.Millisecond)
+
+ _, err = svc.protocolHandler.Propagation(context.Background(), buildReq(nodes[1].nodeID, 99))
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ gomega.Expect(coalesced.get(mockGroup)).To(gomega.Equal(1.0),
+ "TTL-expired branch must increment totalCoalesced when overwriting a non-nil pending")
+
+ close(blocking.release)
+ gomega.Eventually(func() int32 {
+ return blocking.maxObservedCount()
+ }, flags.EventuallyTimeout, "100ms").Should(gomega.Equal(int32(99)),
+ "round-B (the latest take-over) must reach Rev")
+ })
+
+ // drain-all flushes every queued pending even when groupNotify drops signals.
+ ginkgo.It("does not strand pending entries when groupNotify is saturated", func() {
+ nodes = startNodes(2)
+ svc := nodes[0].messenger.(*service)
+
+ blocking := newBlockingListener()
+ svc.listenersLock.Lock()
+ svc.listeners = []MessageListener{blocking}
+ svc.listenersLock.Unlock()
+
+ nodeList := []string{nodes[0].nodeID, nodes[1].nodeID}
+ buildReq := func(shardID uint32, idx int32) *propertyv1.PropagationRequest {
+ return &propertyv1.PropagationRequest{
+ Context: &propertyv1.PropagationContext{
+ Nodes: nodeList,
+ OriginNode: nodes[0].nodeID,
+ MaxPropagationCount: 1,
+ CurrentPropagationCount: idx,
+ },
+ Group: mockGroup,
+ ShardId: shardID,
+ }
+ }
+
+ _, err := svc.protocolHandler.Propagation(context.Background(), buildReq(0, 0))
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ gomega.Eventually(func() bool {
+ select {
+ case <-blocking.entered:
+ return true
+ default:
+ return false
+ }
+ }, flags.EventuallyTimeout, "10ms").Should(gomega.BeTrue(),
+ "worker should drain the initial request and enter listener.Rev")
+
+ const extraShards uint32 = 12
+ for i := uint32(1); i <= extraShards; i++ {
+ _, err = svc.protocolHandler.Propagation(context.Background(), buildReq(i, int32(i)))
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ }
+
+ close(blocking.release)
+ gomega.Eventually(func() int {
+ return blocking.count()
+ }, flags.EventuallyTimeout, "100ms").Should(gomega.Equal(int(extraShards)+1),
+ "drain-all on each wake-up must flush every pending shard even when notifyNewRequest dropped some signals")
+ })
+
+ // CloseNotify must abort the drain loop between handles.
+ ginkgo.It("worker honors CloseNotify in drain loop even with backlog", func() {
+ nodes = startNodes(2)
+ // Slow listener so each handle takes a measurable amount of time.
+ nodes[0].listener.delay = 200 * time.Millisecond
+
+ svc := nodes[0].messenger.(*service)
+ listener := nodes[0].listener
+ nodeList := []string{nodes[0].nodeID, nodes[1].nodeID}
+ const backlogShards uint32 = 10
+
+ for i := uint32(0); i < backlogShards; i++ {
+ req := &propertyv1.PropagationRequest{
+ Context: &propertyv1.PropagationContext{
+ Nodes: nodeList,
+ OriginNode: nodes[0].nodeID,
+ MaxPropagationCount: 1,
+ },
+ Group: mockGroup,
+ ShardId: i,
+ }
+ _, err := svc.protocolHandler.Propagation(context.Background(), req)
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ }
+
+ // Give the worker a moment to start the first handle so the listener
+ // delay is in flight when we trigger close.
+ time.Sleep(250 * time.Millisecond)
+
+ listener.mu.RLock()
+ countAtClose := len(listener.messages)
+ listener.mu.RUnlock()
+
+ // Trigger close. Without the close check in the drain loop, the worker
+ // would keep processing every queued shard before exiting.
+ svc.GracefulStop()
+ nodes[0] = nil // suite-level AfterEach must not double-stop
+
+ // Wait long enough that an unfixed worker would have drained the full
+ // backlog: backlogShards * 200ms = 2s. Use 2.5s safety margin.
+ time.Sleep(2500 * time.Millisecond)
+
+ listener.mu.RLock()
+ extraHandles := len(listener.messages) - countAtClose
+ listener.mu.RUnlock()
+
+ gomega.Expect(extraHandles).To(gomega.BeNumerically("<=", 2),
+ "worker should exit within ~one in-flight handle after CloseNotify; saw %d extra handles after close (full backlog would be ~%d)",
+ extraHandles, int(backlogShards))
+ })
})
func nodeVerify(n *nodeContext, targets []string, messagesCount int) {
@@ -272,3 +535,84 @@ func (m *mockListener) Rev(_ context.Context, _ Trace, nextNode *grpc.ClientConn
m.fromNodes = append(m.fromNodes, req.Context.OriginNode)
return nil
}
+
+// blockingListener.Rev blocks on release; propCounts records which rounds reached Rev.
+type blockingListener struct {
+ release chan struct{}
+ entered chan struct{}
+ propCounts []int32
+ enteredOnce sync.Once
+ mu sync.RWMutex
+}
+
+func newBlockingListener() *blockingListener {
+ return &blockingListener{
+ release: make(chan struct{}),
+ entered: make(chan struct{}),
+ }
+}
+
+func (b *blockingListener) Rev(_ context.Context, _ Trace, _ *grpc.ClientConn, req *propertyv1.PropagationRequest) error {
+ b.enteredOnce.Do(func() { close(b.entered) })
+ <-b.release
+ b.mu.Lock()
+ b.propCounts = append(b.propCounts, req.Context.CurrentPropagationCount)
+ b.mu.Unlock()
+ return nil
+}
+
+func (b *blockingListener) count() int {
+ b.mu.RLock()
+ defer b.mu.RUnlock()
+ return len(b.propCounts)
+}
+
+func (b *blockingListener) maxObservedCount() int32 {
+ b.mu.RLock()
+ defer b.mu.RUnlock()
+ if len(b.propCounts) == 0 {
+ return 0
+ }
+ maxCount := b.propCounts[0]
+ for _, c := range b.propCounts[1:] {
+ if c > maxCount {
+ maxCount = c
+ }
+ }
+ return maxCount
+}
+
+// recordingCounter exposes Inc deltas — the bypass registry cannot be introspected.
+type recordingCounter struct {
+ totals map[string]float64
+ mu sync.RWMutex
+}
+
+func newRecordingCounter() *recordingCounter {
+ return &recordingCounter{totals: make(map[string]float64)}
+}
+
+func (c *recordingCounter) Inc(delta float64, labelValues ...string) {
+ key := strings.Join(labelValues, "\x00")
+ c.mu.Lock()
+ c.totals[key] += delta
+ c.mu.Unlock()
+}
+
+func (c *recordingCounter) Delete(labelValues ...string) bool {
+ key := strings.Join(labelValues, "\x00")
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ if _, ok := c.totals[key]; !ok {
+ return false
+ }
+ delete(c.totals, key)
+ return true
+}
+
+func (c *recordingCounter) get(labelValues ...string) float64 {
+ key := strings.Join(labelValues, "\x00")
+ c.mu.RLock()
+ defer c.mu.RUnlock()
+ return c.totals[key]
+}
diff --git a/test/property_repair/full_data/integrated_test.go b/test/property_repair/full_data/integrated_test.go
index 765180c77..f07c6cfa2 100644
--- a/test/property_repair/full_data/integrated_test.go
+++ b/test/property_repair/full_data/integrated_test.go
@@ -168,8 +168,11 @@ var _ = ginkgo.Describe("Property Repair Full Data Test", ginkgo.Ordered, func()
gomega.Expect(metrics.IsHealthy).To(gomega.BeTrue(),
fmt.Sprintf("Node %s should be healthy before verification: %s",
metrics.NodeName, metrics.ErrorMessage))
- fmt.Printf("- %s: total_propagation_count=%d, repair_success_count=%d\n",
- metrics.NodeName, metrics.TotalPropagationCount, metrics.RepairSuccessCount)
+ fmt.Printf("- %s: total_propagation_count=%d, repair_success_count=%d, repair_failure_count=%d, "+
+ "per_property_timeout=%d, total_coalesced=%d, success_latency_count=%d, success_latency_sum_sec=%.4f\n",
+ metrics.NodeName, metrics.TotalPropagationCount, metrics.RepairSuccessCount,
+ metrics.RepairFailureCount, metrics.RepairPerPropertyTimeoutCount, metrics.TotalCoalesced,
+ metrics.RepairSuccessLatencyCount, metrics.RepairSuccessLatencySumSeconds)
}
fmt.Println("\n=== Triggering property repair by waiting for scheduled repair cycle ===")
diff --git a/test/property_repair/half_data/integrated_test.go b/test/property_repair/half_data/integrated_test.go
index bd40fa682..ff66a1937 100644
--- a/test/property_repair/half_data/integrated_test.go
+++ b/test/property_repair/half_data/integrated_test.go
@@ -195,8 +195,11 @@ var _ = ginkgo.Describe("Property Repair Half Data Test", ginkgo.Ordered, func()
gomega.Expect(metrics.IsHealthy).To(gomega.BeTrue(),
fmt.Sprintf("Node %s should be healthy before verification: %s",
metrics.NodeName, metrics.ErrorMessage))
- fmt.Printf("- %s: total_propagation_count=%d, repair_success_count=%d\n",
- metrics.NodeName, metrics.TotalPropagationCount, metrics.RepairSuccessCount)
+ fmt.Printf("- %s: total_propagation_count=%d, repair_success_count=%d, repair_failure_count=%d, "+
+ "per_property_timeout=%d, total_coalesced=%d, success_latency_count=%d, success_latency_sum_sec=%.4f\n",
+ metrics.NodeName, metrics.TotalPropagationCount, metrics.RepairSuccessCount,
+ metrics.RepairFailureCount, metrics.RepairPerPropertyTimeoutCount, metrics.TotalCoalesced,
+ metrics.RepairSuccessLatencyCount, metrics.RepairSuccessLatencySumSeconds)
}
fmt.Println("\n=== Triggering property repair by waiting for scheduled repair cycle ===")
diff --git a/test/property_repair/same_data/integrated_test.go b/test/property_repair/same_data/integrated_test.go
index 290aaf310..f7f8b9ed9 100644
--- a/test/property_repair/same_data/integrated_test.go
+++ b/test/property_repair/same_data/integrated_test.go
@@ -153,8 +153,11 @@ var _ = ginkgo.Describe("Property Repair Same Data Test", ginkgo.Ordered, func()
gomega.Expect(metrics.IsHealthy).To(gomega.BeTrue(),
fmt.Sprintf("Node %s should be healthy before verification: %s",
metrics.NodeName, metrics.ErrorMessage))
- fmt.Printf("- %s: total_propagation_count=%d, repair_success_count=%d\n",
- metrics.NodeName, metrics.TotalPropagationCount, metrics.RepairSuccessCount)
+ fmt.Printf("- %s: total_propagation_count=%d, repair_success_count=%d, repair_failure_count=%d, "+
+ "per_property_timeout=%d, total_coalesced=%d, success_latency_count=%d, success_latency_sum_sec=%.4f\n",
+ metrics.NodeName, metrics.TotalPropagationCount, metrics.RepairSuccessCount,
+ metrics.RepairFailureCount, metrics.RepairPerPropertyTimeoutCount, metrics.TotalCoalesced,
+ metrics.RepairSuccessLatencyCount, metrics.RepairSuccessLatencySumSeconds)
}
fmt.Println("\n=== Triggering property repair by waiting for scheduled repair cycle ===")
diff --git a/test/property_repair/shared_utils.go b/test/property_repair/shared_utils.go
index ee926d305..2d3fb3591 100644
--- a/test/property_repair/shared_utils.go
+++ b/test/property_repair/shared_utils.go
@@ -53,6 +53,11 @@ const (
PropertyName = "perf-test-property"
)
+// promFloatPattern matches a Prometheus exposition value: scientific
+// notation (e.g. 1.23e+06), plain decimal, [+-]Inf or NaN. The previous
+// `\d+(?:\.\d+)?` form silently missed large counters and histogram sums.
+const promFloatPattern = `(-?(?:\d+(?:\.\d+)?(?:[eE][+-]?\d+)?|[+-]?Inf|NaN))`
+
// PrometheusEndpoints defines the prometheus endpoints for data nodes.
var PrometheusEndpoints = []string{
"http://localhost:2122/metrics", // data-node-1
@@ -62,12 +67,22 @@ var PrometheusEndpoints = []string{
// NodeMetrics represents the metrics for a data node.
type NodeMetrics struct {
- LastScrapeTime time.Time
- NodeName string
- ErrorMessage string
- TotalPropagationCount int64
- RepairSuccessCount int64
- IsHealthy bool
+ LastScrapeTime time.Time
+ NodeName string
+ ErrorMessage string
+ TotalPropagationCount int64
+ RepairSuccessCount int64
+ RepairFailureCount int64
+ RepairPerPropertyTimeoutCount int64
+ TotalCoalesced int64
+ // RepairSuccessLatencyCount / SumSeconds come from the success-path
+ // histogram: every err == nil call is sampled (including no-op cases
+ // where the remote was already newer). Failures and per-property
+ // timeouts are NOT sampled, so a round where every repair times out
+ // shows count=0 here.
+ RepairSuccessLatencyCount int64
+ RepairSuccessLatencySumSeconds float64
+ IsHealthy bool
}
// GenerateLargeData creates a string of specified size filled with random characters.
@@ -287,9 +302,19 @@ func GetNodeMetrics(endpoint string, nodeIndex int) *NodeMetrics {
content := string(body)
totalPropagationCount := parseTotalPropagationCount(content)
repairSuccessCount := parseRepairSuccessCount(content)
+ repairFailureCount := parseRepairFailureCount(content)
+ repairPerPropertyTimeoutCount := parseRepairPerPropertyTimeoutCount(content)
+ totalCoalesced := parseTotalCoalesced(content)
+ repairSuccessLatencyCount := parseRepairSuccessLatencyCount(content)
+ repairSuccessLatencySumSeconds := parseRepairSuccessLatencySum(content)
metrics.TotalPropagationCount = totalPropagationCount
metrics.RepairSuccessCount = repairSuccessCount
+ metrics.RepairFailureCount = repairFailureCount
+ metrics.RepairPerPropertyTimeoutCount = repairPerPropertyTimeoutCount
+ metrics.TotalCoalesced = totalCoalesced
+ metrics.RepairSuccessLatencyCount = repairSuccessLatencyCount
+ metrics.RepairSuccessLatencySumSeconds = repairSuccessLatencySumSeconds
metrics.IsHealthy = true
return metrics
}
@@ -297,7 +322,7 @@ func GetNodeMetrics(endpoint string, nodeIndex int) *NodeMetrics {
// parseTotalPropagationCount parses the total_propagation_count from prometheus metrics text.
func parseTotalPropagationCount(content string) int64 {
// Look for metric lines like: banyandb_property_repair_gossip_server_total_propagation_count{group="perf-test-group",original_node="data-node-1:17912"} 3
- re := regexp.MustCompile(`banyandb_property_repair_gossip_server_total_propagation_count\{[^}]+\}\s+(\d+(?:\.\d+)?)`)
+ re := regexp.MustCompile(`banyandb_property_repair_gossip_server_total_propagation_count\{[^}]+\}\s+` + promFloatPattern)
matches := re.FindAllStringSubmatch(content, -1)
var totalCount int64
@@ -317,7 +342,47 @@ func parseTotalPropagationCount(content string) int64 {
// parseRepairSuccessCount parses the repair_success_count from prometheus metrics text.
func parseRepairSuccessCount(content string) int64 {
// Look for metric lines like: banyandb_property_scheduler_property_repair_success_count{group="perf-test-group",shard="0"} 100
- re := regexp.MustCompile(`banyandb_property_scheduler_property_repair_success_count\{[^}]+\}\s+(\d+(?:\.\d+)?)`)
+ re := regexp.MustCompile(`banyandb_property_scheduler_property_repair_success_count\{[^}]+\}\s+` + promFloatPattern)
+ matches := re.FindAllStringSubmatch(content, -1)
+
+ var totalCount int64
+ for _, match := range matches {
+ if len(match) >= 2 {
+ value, err := strconv.ParseFloat(match[1], 64)
+ if err != nil {
+ continue
+ }
+ totalCount += int64(value)
+ }
+ }
+
+ return totalCount
+}
+
+// parseRepairFailureCount parses the repair_failure_count from prometheus metrics text.
+func parseRepairFailureCount(content string) int64 {
+ // Look for metric lines like: banyandb_property_scheduler_property_repair_failure_count{group="perf-test-group",shard="0"} 5
+ re := regexp.MustCompile(`banyandb_property_scheduler_property_repair_failure_count\{[^}]+\}\s+` + promFloatPattern)
+ matches := re.FindAllStringSubmatch(content, -1)
+
+ var totalCount int64
+ for _, match := range matches {
+ if len(match) >= 2 {
+ value, err := strconv.ParseFloat(match[1], 64)
+ if err != nil {
+ continue
+ }
+ totalCount += int64(value)
+ }
+ }
+
+ return totalCount
+}
+
+// parseRepairPerPropertyTimeoutCount parses the property_repair_per_property_timeout from prometheus metrics text.
+func parseRepairPerPropertyTimeoutCount(content string) int64 {
+ // Look for metric lines like: banyandb_property_scheduler_property_repair_per_property_timeout{group="perf-test-group",shard="0"} 3
+ re := regexp.MustCompile(`banyandb_property_scheduler_property_repair_per_property_timeout\{[^}]+\}\s+` + promFloatPattern)
matches := re.FindAllStringSubmatch(content, -1)
var totalCount int64
@@ -334,6 +399,66 @@ func parseRepairSuccessCount(content string) int64 {
return totalCount
}
+// parseTotalCoalesced parses the gossip total_coalesced from prometheus metrics text.
+func parseTotalCoalesced(content string) int64 {
+ // Look for metric lines like: banyandb_property_repair_gossip_server_total_coalesced{group="perf-test-group"} 7
+ re := regexp.MustCompile(`banyandb_property_repair_gossip_server_total_coalesced\{[^}]+\}\s+` + promFloatPattern)
+ matches := re.FindAllStringSubmatch(content, -1)
+
+ var totalCount int64
+ for _, match := range matches {
+ if len(match) >= 2 {
+ value, err := strconv.ParseFloat(match[1], 64)
+ if err != nil {
+ continue
+ }
+ totalCount += int64(value)
+ }
+ }
+
+ return totalCount
+}
+
+// parseRepairSuccessLatencyCount parses the histogram _count of property_repair_success_latency_seconds.
+func parseRepairSuccessLatencyCount(content string) int64 {
+ // Look for metric lines like: banyandb_property_scheduler_property_repair_success_latency_seconds_count{group="perf-test-group",shard="0"} 100
+ re := regexp.MustCompile(`banyandb_property_scheduler_property_repair_success_latency_seconds_count\{[^}]+\}\s+` + promFloatPattern)
+ matches := re.FindAllStringSubmatch(content, -1)
+
+ var totalCount int64
+ for _, match := range matches {
+ if len(match) >= 2 {
+ value, err := strconv.ParseFloat(match[1], 64)
+ if err != nil {
+ continue
+ }
+ totalCount += int64(value)
+ }
+ }
+
+ return totalCount
+}
+
+// parseRepairSuccessLatencySum parses the histogram _sum (seconds) of property_repair_success_latency_seconds.
+func parseRepairSuccessLatencySum(content string) float64 {
+ // Look for metric lines like: banyandb_property_scheduler_property_repair_success_latency_seconds_sum{group="perf-test-group",shard="0"} 12.34
+ re := regexp.MustCompile(`banyandb_property_scheduler_property_repair_success_latency_seconds_sum\{[^}]+\}\s+` + promFloatPattern)
+ matches := re.FindAllStringSubmatch(content, -1)
+
+ var totalSum float64
+ for _, match := range matches {
+ if len(match) >= 2 {
+ value, err := strconv.ParseFloat(match[1], 64)
+ if err != nil {
+ continue
+ }
+ totalSum += value
+ }
+ }
+
+ return totalSum
+}
+
// GetAllNodeMetrics fetches metrics from all data nodes concurrently.
func GetAllNodeMetrics() []*NodeMetrics {
var wg sync.WaitGroup
@@ -381,24 +506,47 @@ func VerifyPropagationCountIncreased(beforeMetrics, afterMetrics []*NodeMetrics)
// PrintMetricsComparison prints a comparison of metrics before and after.
func PrintMetricsComparison(beforeMetrics, afterMetrics []*NodeMetrics) {
fmt.Println("=== Prometheus Metrics Comparison ===")
- fmt.Printf("%-12s | %-29s | %-29s | %-7s\n", "Node", "Propagation Count", "Repair Success Count", "Healthy")
- fmt.Printf("%-12s | %-9s %-9s %-9s | %-9s %-9s %-9s | %-7s\n", "", "Before", "After", "Delta", "Before", "After", "Delta", "")
- fmt.Println(strings.Repeat("-", 85))
+ fmt.Printf("%-12s | %-29s | %-29s | %-29s | %-29s | %-29s | %-12s | %-7s\n",
+ "Node", "Propagation Count", "Repair Success Count", "Repair Failure Count",
+ "Per-Property Timeout", "Coalesced (gossip)", "Succ Mean Lat", "Healthy")
+ fmt.Printf("%-12s | %-9s %-9s %-9s | %-9s %-9s %-9s | %-9s %-9s %-9s | %-9s %-9s %-9s | %-9s %-9s %-9s | %-12s | %-7s\n",
+ "",
+ "Before", "After", "Delta",
+ "Before", "After", "Delta",
+ "Before", "After", "Delta",
+ "Before", "After", "Delta",
+ "Before", "After", "Delta",
+ "ThisRound(sec)",
+ "")
+ fmt.Println(strings.Repeat("-", 190))
for i, after := range afterMetrics {
if i < len(beforeMetrics) {
before := beforeMetrics[i]
propagationDelta := after.TotalPropagationCount - before.TotalPropagationCount
repairDelta := after.RepairSuccessCount - before.RepairSuccessCount
+ failureDelta := after.RepairFailureCount - before.RepairFailureCount
+ timeoutDelta := after.RepairPerPropertyTimeoutCount - before.RepairPerPropertyTimeoutCount
+ coalescedDelta := after.TotalCoalesced - before.TotalCoalesced
+ meanLatency := 0.0
+ countDelta := after.RepairSuccessLatencyCount - before.RepairSuccessLatencyCount
+ sumDelta := after.RepairSuccessLatencySumSeconds - before.RepairSuccessLatencySumSeconds
+ if countDelta > 0 {
+ meanLatency = sumDelta / float64(countDelta)
+ }
healthStatus := "✓"
if !after.IsHealthy {
healthStatus = "✗"
}
- fmt.Printf("%-12s | %-9d %-9d %-9d | %-9d %-9d %-9d | %-7s\n",
+ fmt.Printf("%-12s | %-9d %-9d %-9d | %-9d %-9d %-9d | %-9d %-9d %-9d | %-9d %-9d %-9d | %-9d %-9d %-9d | %-12.4f | %-7s\n",
after.NodeName,
before.TotalPropagationCount, after.TotalPropagationCount, propagationDelta,
before.RepairSuccessCount, after.RepairSuccessCount, repairDelta,
+ before.RepairFailureCount, after.RepairFailureCount, failureDelta,
+ before.RepairPerPropertyTimeoutCount, after.RepairPerPropertyTimeoutCount, timeoutDelta,
+ before.TotalCoalesced, after.TotalCoalesced, coalescedDelta,
+ meanLatency,
healthStatus)
}
}
(skywalking-banyandb) 11/11: Prepare for the v0.10.3 release
Posted by bu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ButterBright pushed a commit to branch v0.10.x
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit f45404a6be0a4667c81d43542fd86601e2c536e0
Author: ButterBright <bu...@gmail.com>
AuthorDate: Sat Jun 6 14:08:28 2026 +0800
Prepare for the v0.10.3 release
---
CHANGES.md | 38 +++++++++++++++++++++-----------------
banyand/trace/metrics.go | 1 -
2 files changed, 21 insertions(+), 18 deletions(-)
diff --git a/CHANGES.md b/CHANGES.md
index c6d75b767..ba6f48272 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -6,25 +6,43 @@ Release Notes.
### Bug Fixes
+- Persist segment end time in per-segment metadata so boundaries don't shift across restarts or config changes.
+- Fix flaky on-disk integration tests caused by Ginkgo v2 random container shuffling closing gRPC connections prematurely.
+- ui: fix query editor refresh/reset behavior and BydbQL keyword highlighting.
+- Fix flaky `file_snapshot` subtest in measure/stream/trace by waiting until every introduced mem part has been flushed to disk, instead of only checking the latest snapshot creator.
+- Fix flaky `TestCollectWithPartialClosedSegments` by raising `SegmentIdleTimeout` so wall-clock variance on slow CI does not mark still-open segments as idle.
+- Fix FODC lifecycle cache poisoning where transient `InspectAll` failures were cached for 10 minutes and masked liaison recovery; raise FODC agent and proxy timeouts from 10s to 40s.
+- Fix FODC `/cluster/lifecycle` dropping zero-valued group fields (e.g. `replicas=0`, `close=false`) under `encoding/json` + `omitempty`; switch to `protojson` so all fields are emitted (nil nested messages serialize as `null`).
+- Fix trace `block_writer` panic on out-of-order timestamps within the same traceID, which dropped one trace-write batch per panic in multi-agent SkyWalking deployments. Spans of a single trace originate from independently-clocked services, and trace storage is organized by traceID rather than timestamp, so per-traceID timestamp monotonicity is not a writer invariant.
+- Fix nil-pointer panic on cold-tier data nodes when FODC `InspectAll` raced with idle-segment cleanup.
+- Add `GroupLifecycleInfo.errors` to surface per-group collection failures from FODC `InspectAll` instead of silently dropping the affected node entry.
+- Fix `CollectDataInfo` and `CollectLiaisonInfo` not handling `CATALOG_PROPERTY` groups.
+- Close BanyanDB merge write-path durability gap that allowed torn parts to be created by a crash between data write and metadata commit. Metadata files (`metadata.json` for trace/measure/stream, `manifest.json` for sidx, plus `traceID.filter` and `tag.type`) now go through a new `WriteAtomic` (write-tmp + fsync + rename + fsync-dir) sequence; data writers (`seqWriter.Close`, `localFileSystem.Write`) now propagate fdatasync errors instead of silently dropping them. `mustOpenFilePart` / ` [...]
+- Fix lifecycle migration where the receiving node could create segments shorter than the configured `SegmentInterval`.
+- Fail fast on incompatible storage version at boot. Previously the server would start in a degraded `SERVING` state with affected groups un-loaded because the property schema-registry retry loop swallowed the version-incompatibility panic. Compatible versions are listed in `banyand/internal/storage/versions.yml`.
+- Release bluge index writers on segment rotation so `analysisWorker` pools sized from `GOMAXPROCS` don't accumulate across rotations. Two layered defects kept the existing idle-segment reclaim path from running: `segmentIdleTimeout` defaulted to `0` (which disabled the 10-minute reclaim ticker), and `incRef` refreshed `lastAccessed` on every rotation tick so `closeIdleSegments` never observed an idle segment. Defaults to `time.Hour`, moves the `lastAccessed` bump to real read/write call [...]
+- Fix incorrect counts and missing trace fields in the lifecycle migration report.
- Fix trace query identity-tag projection: when `trace_id`/`span_id` are explicitly projected, reconstruct them from span identity at response build time instead of requesting them as stored tags, and preserve tag order with null-filled per-span value alignment in the distributed trace result iterator.
- Fix FODC proxy corrupting Prometheus metric types. The agent dropped the `# TYPE` line while parsing banyandb `/metrics`, the `StreamMetrics` proto carried no type field, and the proxy guessed the type from a name-suffix heuristic — downgrading counters to gauge, mislabeling `_count`-suffixed counters as histograms, and splitting summaries into two conflicting `# TYPE` lines. Capture the type with the Prometheus `expfmt` parser, store it in the flight recorder, thread it through a new [...]
- Trace storage metrics now expose the `storage` sub-scope, matching the `stream_storage_*` naming. The `StorageMetricsFactory` for trace switched from the root `trace` scope to `trace.storage`, so per-segment inverted-index metrics (`inverted_index_total_updates`, `inverted_index_total_doc_count`, `inverted_index_total_term_searchers_started`) are now emitted as `banyandb_trace_storage_*` instead of `banyandb_trace_*`, aligning the dashboard query names. Other trace metrics (`trace_tst_ [...]
- Fix FODC agent labeling metrics with `node_role="ROLE_UNSPECIFIED"`. The agent resolved the node role exactly once at startup via a single `GetCurrentNode` poll whose endpoint retries spanned only ~1s; when the sibling lifecycle/banyandb gRPC server was not yet listening (`connect: cannot assign requested address`) the role fell back to `ROLE_UNSPECIFIED` permanently, so most nodes never reported their real `ROLE_DATA`/`ROLE_LIAISON`. Retry the initial node-role resolution with exponen [...]
+### Chores
+
+- Regenerate expired TLS test certificate with 100-year validity.
+- Set Ginkgo `--repeat` to 0 in the flaky-test workflow so the hourly run completes within the 50-minute timeout.
+
## 0.10.2
### Bug Fixes
- Fix reuse of byte arrays in min/max implementation causing data corruption.
- Fix index-mode measure queries returning documents outside requested time range.
-- Close BanyanDB merge write-path durability gap that allowed torn parts to be created by a crash between data write and metadata commit. Metadata files (`metadata.json` for trace/measure/stream, `manifest.json` for sidx, plus `traceID.filter` and `tag.type`) now go through a new `WriteAtomic` (write-tmp + fsync + rename + fsync-dir) sequence; data writers (`seqWriter.Close`, `localFileSystem.Write`) now propagate fdatasync errors instead of silently dropping them. `mustOpenFilePart` / ` [...]
- Fix bydbctl command tests using global stdout capture, which caused race-enabled runs to corrupt captured command output.
- Use `topic` instead of `session_id` as the Prometheus label on liaison `queue_sub` chunk-ordering counters to avoid unbounded metric cardinality.
- Fix flaky trace query filtering caused by non-deterministic sidx tag ordering and add consistency checks for integration query cases.
- MCP: Add validation for properties and harden the mcp server.
- Fix property schema client connection not stable after data node restarted.
-- Fix flaky on-disk integration tests caused by Ginkgo v2 random container shuffling closing gRPC connections prematurely.
-- ui: fix query editor refresh/reset behavior and BydbQL keyword highlighting.
- Disable the rotation task on warm and cold nodes to prevent incorrect segment boundaries during lifecycle migration.
- Prevent epoch-dated segment directories (seg-19700101) from being created by zero timestamps in distributed sync paths.
- Fix SIDX streaming sync sending SegmentID as MinTimestamp instead of the actual timestamp, causing sync failures on the receiving node.
@@ -39,24 +57,10 @@ Release Notes.
- Fix `FileSystemError` not satisfying `errors.Is(err, io/fs.ErrNotExist)`, which prevented the segment controller from cleaning up half-born segment directories and left groups in a permanent zombie state after a crash or partial sync.
- Fix lifecycle migration panic when a stream shard's snapshot has no element index (`idx/`) directory.
- Avoid FODC lifecycle inspection failing on busy data nodes by raising the per-broadcast `CollectDataInfo` / `CollectLiaisonInfo` deadline from 5s to 30s and parallelizing per-group inspection in the cluster-internal `InspectAll`.
-- Fix flaky `file_snapshot` subtest in measure/stream/trace by waiting until every introduced mem part has been flushed to disk, instead of only checking the latest snapshot creator.
-- Fix flaky `TestCollectWithPartialClosedSegments` by raising `SegmentIdleTimeout` so wall-clock variance on slow CI does not mark still-open segments as idle.
-- Fix FODC lifecycle cache poisoning where transient `InspectAll` failures were cached for 10 minutes and masked liaison recovery; raise FODC agent and proxy timeouts from 10s to 40s.
-- Fix FODC `/cluster/lifecycle` dropping zero-valued group fields (e.g. `replicas=0`, `close=false`) under `encoding/json` + `omitempty`; switch to `protojson` so all fields are emitted (nil nested messages serialize as `null`).
-- Fix trace `block_writer` panic on out-of-order timestamps within the same traceID, which dropped one trace-write batch per panic in multi-agent SkyWalking deployments. Spans of a single trace originate from independently-clocked services, and trace storage is organized by traceID rather than timestamp, so per-traceID timestamp monotonicity is not a writer invariant.
-- Fix nil-pointer panic on cold-tier data nodes when FODC `InspectAll` raced with idle-segment cleanup.
-- Add `GroupLifecycleInfo.errors` to surface per-group collection failures from FODC `InspectAll` instead of silently dropping the affected node entry.
-- Fix `CollectDataInfo` and `CollectLiaisonInfo` not handling `CATALOG_PROPERTY` groups.
-- Fix lifecycle migration where the receiving node could create segments shorter than the configured `SegmentInterval`.
-- Fail fast on incompatible storage version at boot. Previously the server would start in a degraded `SERVING` state with affected groups un-loaded because the property schema-registry retry loop swallowed the version-incompatibility panic. Compatible versions are listed in `banyand/internal/storage/versions.yml`.
-- Release bluge index writers on segment rotation so `analysisWorker` pools sized from `GOMAXPROCS` don't accumulate across rotations. Two layered defects kept the existing idle-segment reclaim path from running: `segmentIdleTimeout` defaulted to `0` (which disabled the 10-minute reclaim ticker), and `incRef` refreshed `lastAccessed` on every rotation tick so `closeIdleSegments` never observed an idle segment. Defaults to `time.Hour`, moves the `lastAccessed` bump to real read/write call [...]
-- Fix incorrect counts and missing trace fields in the lifecycle migration report.
### Chores
- Upgrade Go and npm dependencies including etcd to v3.6.10, OpenTelemetry to v1.43.0, AWS SDK, and Google Cloud libraries.
-- Regenerate expired TLS test certificate with 100-year validity.
-- Set Ginkgo `--repeat` to 0 in the flaky-test workflow so the hourly run completes within the 50-minute timeout.
## 0.10.0
diff --git a/banyand/trace/metrics.go b/banyand/trace/metrics.go
index 0c3594e2f..d52de056d 100644
--- a/banyand/trace/metrics.go
+++ b/banyand/trace/metrics.go
@@ -20,7 +20,6 @@ package trace
import (
"github.com/apache/skywalking-banyandb/api/common"
"github.com/apache/skywalking-banyandb/banyand/internal/storage"
- "github.com/apache/skywalking-banyandb/banyand/observability"
"github.com/apache/skywalking-banyandb/pkg/index/inverted"
"github.com/apache/skywalking-banyandb/pkg/meter"
)
(skywalking-banyandb) 04/11: Fix backup restore path traversal (#1135)
Posted by bu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ButterBright pushed a commit to branch v0.10.x
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit e66a0808fb578dfe7a1aa88d0d3e530b0f8dd376
Author: Tanay Paul <14...@users.noreply.github.com>
AuthorDate: Thu May 21 06:18:52 2026 +0530
Fix backup restore path traversal (#1135)
---
banyand/backup/restore.go | 35 +++--
banyand/backup/restore_test.go | 97 ++++++++++++
pkg/fs/remote/local/local.go | 94 +++++++++++-
pkg/fs/remote/local/local_test.go | 307 ++++++++++++++++++++++++++++++++++++++
pkg/path/path.go | 14 ++
pkg/path/path_test.go | 36 +++++
6 files changed, 567 insertions(+), 16 deletions(-)
diff --git a/banyand/backup/restore.go b/banyand/backup/restore.go
index 93a3de4b1..dd5db44b8 100644
--- a/banyand/backup/restore.go
+++ b/banyand/backup/restore.go
@@ -23,6 +23,7 @@ import (
"fmt"
"io"
"os"
+ "path"
"path/filepath"
"strings"
@@ -193,13 +194,12 @@ func restoreByName(fs remote.FS, timeDir, rootPath, catalogName string) error {
logger.Infof("Restoring %s to %s from %s, remote total %d files", catalogName, localDir, remotePrefix, len(remoteFiles))
remoteRelSet := make(map[string]bool)
- var relPath string
for _, remoteFile := range remoteFiles {
- relPath, err = filepath.Rel(timeDir, remoteFile)
- if err != nil {
- return fmt.Errorf("failed to get relative path for %s: %w", remoteFile, err)
+ relPath, relPathErr := validatedRemoteRelPath(timeDir, catalogName, remoteFile)
+ if relPathErr != nil {
+ return relPathErr
}
- remoteRelSet[filepath.ToSlash(relPath)] = true
+ remoteRelSet[path.Join(catalogName, relPath)] = true
}
localFiles, err := getAllFiles(localDir)
@@ -208,7 +208,7 @@ func restoreByName(fs remote.FS, timeDir, rootPath, catalogName string) error {
}
for _, localRelPath := range localFiles {
- localRelPathWithCatalog := filepath.Join(catalogName, localRelPath)
+ localRelPathWithCatalog := path.Join(catalogName, filepath.ToSlash(localRelPath))
if !remoteRelSet[localRelPathWithCatalog] {
localPath := filepath.Join(localDir, localRelPath)
logger.Infof("found local file: %s not exist in the remote storage, so delete it", localRelPathWithCatalog)
@@ -220,11 +220,10 @@ func restoreByName(fs remote.FS, timeDir, rootPath, catalogName string) error {
}
for _, remoteFile := range remoteFiles {
- relPath, err := filepath.Rel(filepath.Join(timeDir, catalogName), remoteFile)
+ relPath, err := validatedRemoteRelPath(timeDir, catalogName, remoteFile)
if err != nil {
- return fmt.Errorf("failed to get relative path for %s: %w", remoteFile, err)
+ return err
}
- relPath = filepath.ToSlash(relPath)
localPath := filepath.Join(rootPath, catalogName, storage.DataDir, relPath)
if !contains(localFiles, relPath) {
@@ -244,6 +243,24 @@ func restoreByName(fs remote.FS, timeDir, rootPath, catalogName string) error {
return nil
}
+func validatedRemoteRelPath(timeDir, catalogName, remoteFile string) (string, error) {
+ remotePath := filepath.ToSlash(remoteFile)
+ if path.IsAbs(remotePath) || banyandbpath.HasVolumeName(remotePath) {
+ return "", fmt.Errorf("remote file %q escapes backup prefix", remoteFile)
+ }
+ prefix := path.Clean(path.Join(filepath.ToSlash(timeDir), filepath.ToSlash(catalogName)))
+ cleanRemotePath := path.Clean(remotePath)
+ if banyandbpath.HasVolumeName(cleanRemotePath) || cleanRemotePath == "." || prefix == "." {
+ return "", fmt.Errorf("remote file %q escapes backup prefix", remoteFile)
+ }
+ prefixWithSlash := prefix + "/"
+ if !strings.HasPrefix(cleanRemotePath, prefixWithSlash) {
+ return "", fmt.Errorf("remote file %q escapes backup prefix", remoteFile)
+ }
+ relPath := strings.TrimPrefix(cleanRemotePath, prefixWithSlash)
+ return relPath, nil
+}
+
func cleanEmptyDirs(dir, stopDir string) {
for {
if dir == stopDir || dir == "." {
diff --git a/banyand/backup/restore_test.go b/banyand/backup/restore_test.go
index 086f18a23..ca76f984e 100644
--- a/banyand/backup/restore_test.go
+++ b/banyand/backup/restore_test.go
@@ -19,6 +19,7 @@ package backup
import (
"context"
+ "io"
"os"
"path/filepath"
"strings"
@@ -192,3 +193,99 @@ func TestRestoreSame(t *testing.T) {
t.Fatalf("expected extra file %q exist", extraFilePath)
}
}
+
+func TestRestoreRejectsRemotePathTraversal(t *testing.T) {
+ timeDir := testTimeDir
+ catalogName := snapshot.CatalogName(commonv1.Catalog_CATALOG_STREAM)
+ localRestoreDir := t.TempDir()
+ escapedFile := filepath.Join(localRestoreDir, catalogName, "escaped.txt")
+ fs := &restoreTraversalFS{
+ files: []string{
+ filepath.ToSlash(filepath.Join(timeDir, catalogName, "..", "escaped.txt")),
+ },
+ }
+
+ err := restoreByName(fs, timeDir, localRestoreDir, catalogName)
+ if err == nil {
+ t.Fatal("expected restoreByName to reject remote path traversal")
+ }
+ if _, statErr := os.Stat(escapedFile); !os.IsNotExist(statErr) {
+ t.Fatalf("escaped file exists or stat failed with unexpected error: %v", statErr)
+ }
+}
+
+func TestValidatedRemoteRelPath(t *testing.T) {
+ timeDir := testTimeDir
+ catalogName := snapshot.CatalogName(commonv1.Catalog_CATALOG_STREAM)
+ validRemoteFile := filepath.ToSlash(filepath.Join(timeDir, catalogName, "nested", "test.txt"))
+
+ relPath, err := validatedRemoteRelPath(timeDir, catalogName, validRemoteFile)
+ if err != nil {
+ t.Fatalf("validatedRemoteRelPath failed: %v", err)
+ }
+ if relPath != "nested/test.txt" {
+ t.Fatalf("relPath = %q, want %q", relPath, "nested/test.txt")
+ }
+}
+
+func TestValidatedRemoteRelPathRejectsInvalidPaths(t *testing.T) {
+ timeDir := testTimeDir
+ catalogName := snapshot.CatalogName(commonv1.Catalog_CATALOG_STREAM)
+ tests := []struct {
+ remoteFile string
+ name string
+ }{
+ {
+ name: "absolute path",
+ remoteFile: "/tmp/backup/file.txt",
+ },
+ {
+ name: "catalog prefix only",
+ remoteFile: filepath.ToSlash(filepath.Join(timeDir, catalogName)),
+ },
+ {
+ name: "outside catalog prefix",
+ remoteFile: filepath.ToSlash(filepath.Join(timeDir, "measure", "test.txt")),
+ },
+ {
+ name: "parent traversal",
+ remoteFile: filepath.ToSlash(filepath.Join(timeDir, catalogName, "..", "escaped.txt")),
+ },
+ {
+ name: "volume name",
+ remoteFile: `C:/backup/file.txt`,
+ },
+ }
+
+ for _, testCase := range tests {
+ t.Run(testCase.name, func(t *testing.T) {
+ if _, err := validatedRemoteRelPath(timeDir, catalogName, testCase.remoteFile); err == nil {
+ t.Fatal("expected invalid remote path to be rejected")
+ }
+ })
+ }
+}
+
+type restoreTraversalFS struct {
+ files []string
+}
+
+func (r *restoreTraversalFS) Upload(_ context.Context, _ string, _ io.Reader) error {
+ return nil
+}
+
+func (r *restoreTraversalFS) Download(_ context.Context, _ string) (io.ReadCloser, error) {
+ return io.NopCloser(strings.NewReader("escape")), nil
+}
+
+func (r *restoreTraversalFS) List(_ context.Context, _ string) ([]string, error) {
+ return r.files, nil
+}
+
+func (r *restoreTraversalFS) Delete(_ context.Context, _ string) error {
+ return nil
+}
+
+func (r *restoreTraversalFS) Close() error {
+ return nil
+}
diff --git a/pkg/fs/remote/local/local.go b/pkg/fs/remote/local/local.go
index 01f204552..176e18e2f 100644
--- a/pkg/fs/remote/local/local.go
+++ b/pkg/fs/remote/local/local.go
@@ -20,11 +20,14 @@ package local
import (
"context"
+ "fmt"
"io"
"os"
"path/filepath"
+ "strings"
"github.com/apache/skywalking-banyandb/pkg/fs/remote"
+ pathutil "github.com/apache/skywalking-banyandb/pkg/path"
)
const dirPerm = 0o755
@@ -40,14 +43,28 @@ func NewFS(baseDir string) (remote.FS, error) {
if err := os.MkdirAll(baseDir, dirPerm); err != nil {
return nil, err
}
- return &fs{baseDir: baseDir}, nil
+ cleanBaseDir, err := filepath.Abs(baseDir)
+ if err != nil {
+ return nil, err
+ }
+ realBaseDir, err := filepath.EvalSymlinks(cleanBaseDir)
+ if err != nil {
+ return nil, err
+ }
+ return &fs{baseDir: realBaseDir}, nil
}
func (l *fs) Upload(_ context.Context, path string, data io.Reader) error {
- fullPath := filepath.Join(l.baseDir, path)
- if err := os.MkdirAll(filepath.Dir(fullPath), dirPerm); err != nil {
+ fullPath, err := l.fullPath(path, false)
+ if err != nil {
return err
}
+ if mkdirErr := os.MkdirAll(filepath.Dir(fullPath), dirPerm); mkdirErr != nil {
+ return mkdirErr
+ }
+ if err = l.ensureResolvedWithinBase(filepath.Dir(fullPath)); err != nil {
+ return fmt.Errorf("path %q escapes base directory: %w", path, err)
+ }
file, err := os.Create(fullPath)
if err != nil {
@@ -60,15 +77,21 @@ func (l *fs) Upload(_ context.Context, path string, data io.Reader) error {
}
func (l *fs) Download(_ context.Context, path string) (io.ReadCloser, error) {
- fullPath := filepath.Join(l.baseDir, path)
+ fullPath, err := l.fullPath(path, false)
+ if err != nil {
+ return nil, err
+ }
return os.Open(fullPath)
}
func (l *fs) List(_ context.Context, prefix string) ([]string, error) {
var files []string
- fullPath := filepath.Join(l.baseDir, prefix)
+ fullPath, err := l.fullPath(prefix, true)
+ if err != nil {
+ return nil, err
+ }
- err := filepath.Walk(fullPath, func(path string, info os.FileInfo, err error) error {
+ err = filepath.Walk(fullPath, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
@@ -89,10 +112,67 @@ func (l *fs) List(_ context.Context, prefix string) ([]string, error) {
}
func (l *fs) Delete(_ context.Context, path string) error {
- fullPath := filepath.Join(l.baseDir, path)
+ fullPath, err := l.fullPath(path, false)
+ if err != nil {
+ return err
+ }
return os.Remove(fullPath)
}
func (l *fs) Close() error {
return nil
}
+
+func (l *fs) fullPath(path string, allowRoot bool) (string, error) {
+ if filepath.IsAbs(path) || pathutil.HasVolumeName(path) {
+ return "", fmt.Errorf("path %q escapes base directory", path)
+ }
+ cleanPath := filepath.Clean(path)
+ if !allowRoot && cleanPath == "." {
+ return "", fmt.Errorf("path %q escapes base directory", path)
+ }
+ if pathutil.HasVolumeName(cleanPath) || cleanPath == ".." || strings.HasPrefix(cleanPath, ".."+string(filepath.Separator)) {
+ return "", fmt.Errorf("path %q escapes base directory", path)
+ }
+ fullPath := filepath.Join(l.baseDir, cleanPath)
+ relPath, err := filepath.Rel(l.baseDir, fullPath)
+ if err != nil {
+ return "", err
+ }
+ if relPath == ".." || strings.HasPrefix(relPath, ".."+string(filepath.Separator)) || filepath.IsAbs(relPath) {
+ return "", fmt.Errorf("path %q escapes base directory", path)
+ }
+ if err := l.ensureResolvedWithinBase(fullPath); err != nil {
+ return "", fmt.Errorf("path %q escapes base directory: %w", path, err)
+ }
+ return fullPath, nil
+}
+
+func (l *fs) ensureResolvedWithinBase(path string) error {
+ existingPath := path
+ for {
+ if _, err := os.Lstat(existingPath); err == nil {
+ break
+ } else if !os.IsNotExist(err) {
+ return err
+ }
+ parentPath := filepath.Dir(existingPath)
+ if parentPath == existingPath {
+ return os.ErrNotExist
+ }
+ existingPath = parentPath
+ }
+
+ realPath, err := filepath.EvalSymlinks(existingPath)
+ if err != nil {
+ return err
+ }
+ relPath, err := filepath.Rel(l.baseDir, realPath)
+ if err != nil {
+ return err
+ }
+ if relPath == ".." || strings.HasPrefix(relPath, ".."+string(filepath.Separator)) || filepath.IsAbs(relPath) {
+ return fmt.Errorf("resolved path %q is outside base directory %q", realPath, l.baseDir)
+ }
+ return nil
+}
diff --git a/pkg/fs/remote/local/local_test.go b/pkg/fs/remote/local/local_test.go
new file mode 100644
index 000000000..8ad28f1ca
--- /dev/null
+++ b/pkg/fs/remote/local/local_test.go
@@ -0,0 +1,307 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package local
+
+import (
+ "context"
+ "errors"
+ "io"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+)
+
+func TestFSOperationsStayWithinBase(t *testing.T) {
+ baseDir := filepath.Join(t.TempDir(), "remote")
+ fs, err := NewFS(baseDir)
+ if err != nil {
+ t.Fatalf("NewFS failed: %v", err)
+ }
+
+ const content = "hello"
+ filePath := filepath.Join("snapshot", "data", "test.txt")
+ if err = fs.Upload(context.Background(), filePath, strings.NewReader(content)); err != nil {
+ t.Fatalf("Upload failed: %v", err)
+ }
+
+ files, err := fs.List(context.Background(), "snapshot")
+ if err != nil {
+ t.Fatalf("List failed: %v", err)
+ }
+ if len(files) != 1 || files[0] != filepath.ToSlash(filePath) {
+ t.Fatalf("files = %v, want [%s]", files, filepath.ToSlash(filePath))
+ }
+
+ reader, err := fs.Download(context.Background(), filePath)
+ if err != nil {
+ t.Fatalf("Download failed: %v", err)
+ }
+ got, err := io.ReadAll(reader)
+ closeErr := reader.Close()
+ if err != nil {
+ t.Fatalf("failed to read downloaded content: %v", err)
+ }
+ if closeErr != nil {
+ t.Fatalf("failed to close downloaded content: %v", closeErr)
+ }
+ if string(got) != content {
+ t.Fatalf("content = %q, want %q", string(got), content)
+ }
+
+ if err = fs.Delete(context.Background(), filePath); err != nil {
+ t.Fatalf("Delete failed: %v", err)
+ }
+ if _, statErr := os.Stat(filepath.Join(baseDir, filePath)); !errors.Is(statErr, os.ErrNotExist) {
+ t.Fatalf("deleted file exists or stat failed with unexpected error: %v", statErr)
+ }
+}
+
+func TestFSListMissingPrefixReturnsEmpty(t *testing.T) {
+ fs, err := NewFS(t.TempDir())
+ if err != nil {
+ t.Fatalf("NewFS failed: %v", err)
+ }
+
+ files, err := fs.List(context.Background(), "missing")
+ if err != nil {
+ t.Fatalf("List failed: %v", err)
+ }
+ if len(files) != 0 {
+ t.Fatalf("files = %v, want empty", files)
+ }
+}
+
+func TestFSListEmptyPrefixReturnsFiles(t *testing.T) {
+ baseDir := filepath.Join(t.TempDir(), "remote")
+ fs, err := NewFS(baseDir)
+ if err != nil {
+ t.Fatalf("NewFS failed: %v", err)
+ }
+
+ const filePath = "snapshot/data/test.txt"
+ if err = fs.Upload(context.Background(), filePath, strings.NewReader("hello")); err != nil {
+ t.Fatalf("Upload failed: %v", err)
+ }
+
+ files, err := fs.List(context.Background(), "")
+ if err != nil {
+ t.Fatalf("List failed: %v", err)
+ }
+ if len(files) != 1 || files[0] != filePath {
+ t.Fatalf("files = %v, want [%s]", files, filePath)
+ }
+}
+
+func TestFSRejectsRootFileOperationPaths(t *testing.T) {
+ fs, err := NewFS(t.TempDir())
+ if err != nil {
+ t.Fatalf("NewFS failed: %v", err)
+ }
+
+ tests := []struct {
+ run func(string) error
+ name string
+ path string
+ }{
+ {
+ name: "upload empty",
+ path: "",
+ run: func(path string) error {
+ return fs.Upload(context.Background(), path, strings.NewReader("root"))
+ },
+ },
+ {
+ name: "upload dot",
+ path: ".",
+ run: func(path string) error {
+ return fs.Upload(context.Background(), path, strings.NewReader("root"))
+ },
+ },
+ {
+ name: "download empty",
+ path: "",
+ run: func(path string) error {
+ reader, downloadErr := fs.Download(context.Background(), path)
+ if reader != nil {
+ reader.Close()
+ }
+ return downloadErr
+ },
+ },
+ {
+ name: "delete dot",
+ path: ".",
+ run: func(path string) error {
+ return fs.Delete(context.Background(), path)
+ },
+ },
+ }
+
+ for _, testCase := range tests {
+ t.Run(testCase.name, func(t *testing.T) {
+ if err := testCase.run(testCase.path); err == nil {
+ t.Fatal("expected root file operation path to be rejected")
+ }
+ })
+ }
+}
+
+func TestFSRejectsPathTraversal(t *testing.T) {
+ baseDir := filepath.Join(t.TempDir(), "remote")
+ fs, err := NewFS(baseDir)
+ if err != nil {
+ t.Fatalf("NewFS failed: %v", err)
+ }
+
+ escapedPath := filepath.Join("..", "escaped.txt")
+ tests := []struct {
+ run func() error
+ name string
+ }{
+ {
+ name: "upload",
+ run: func() error {
+ return fs.Upload(context.Background(), escapedPath, strings.NewReader("escape"))
+ },
+ },
+ {
+ name: "download",
+ run: func() error {
+ reader, downloadErr := fs.Download(context.Background(), escapedPath)
+ if reader != nil {
+ reader.Close()
+ }
+ return downloadErr
+ },
+ },
+ {
+ name: "list",
+ run: func() error {
+ _, listErr := fs.List(context.Background(), escapedPath)
+ return listErr
+ },
+ },
+ {
+ name: "delete",
+ run: func() error {
+ return fs.Delete(context.Background(), escapedPath)
+ },
+ },
+ }
+
+ for _, testCase := range tests {
+ t.Run(testCase.name, func(t *testing.T) {
+ if err := testCase.run(); err == nil {
+ t.Fatal("expected path traversal to be rejected")
+ }
+ if _, statErr := os.Stat(filepath.Join(filepath.Dir(baseDir), "escaped.txt")); !errors.Is(statErr, os.ErrNotExist) {
+ t.Fatalf("escaped file exists or stat failed with unexpected error: %v", statErr)
+ }
+ })
+ }
+}
+
+func TestFSRejectsAbsolutePath(t *testing.T) {
+ baseDir := filepath.Join(t.TempDir(), "remote")
+ fs, err := NewFS(baseDir)
+ if err != nil {
+ t.Fatalf("NewFS failed: %v", err)
+ }
+
+ absolutePath := filepath.Join(baseDir, "escaped.txt")
+ if err = fs.Upload(context.Background(), absolutePath, strings.NewReader("escape")); err == nil {
+ t.Fatal("expected absolute path to be rejected")
+ }
+ if _, statErr := os.Stat(absolutePath); !errors.Is(statErr, os.ErrNotExist) {
+ t.Fatalf("absolute path was written or stat failed with unexpected error: %v", statErr)
+ }
+}
+
+func TestFSRejectsVolumeName(t *testing.T) {
+ fs, err := NewFS(t.TempDir())
+ if err != nil {
+ t.Fatalf("NewFS failed: %v", err)
+ }
+
+ if err = fs.Upload(context.Background(), `C:escaped.txt`, strings.NewReader("escape")); err == nil {
+ t.Fatal("expected volume-name path to be rejected")
+ }
+}
+
+func TestFSRejectsSymlinkEscape(t *testing.T) {
+ baseDir := filepath.Join(t.TempDir(), "remote")
+ outsideDir := filepath.Join(t.TempDir(), "outside")
+ if err := os.MkdirAll(outsideDir, dirPerm); err != nil {
+ t.Fatalf("failed to create outside directory: %v", err)
+ }
+ fs, err := NewFS(baseDir)
+ if err != nil {
+ t.Fatalf("NewFS failed: %v", err)
+ }
+ if err = os.Symlink(outsideDir, filepath.Join(baseDir, "link")); err != nil {
+ t.Skipf("symlinks are not available: %v", err)
+ }
+
+ escapedPath := filepath.Join("link", "escaped.txt")
+ tests := []struct {
+ run func() error
+ name string
+ }{
+ {
+ name: "upload",
+ run: func() error {
+ return fs.Upload(context.Background(), escapedPath, strings.NewReader("escape"))
+ },
+ },
+ {
+ name: "download",
+ run: func() error {
+ reader, downloadErr := fs.Download(context.Background(), escapedPath)
+ if reader != nil {
+ reader.Close()
+ }
+ return downloadErr
+ },
+ },
+ {
+ name: "list",
+ run: func() error {
+ _, listErr := fs.List(context.Background(), "link")
+ return listErr
+ },
+ },
+ {
+ name: "delete",
+ run: func() error {
+ return fs.Delete(context.Background(), escapedPath)
+ },
+ },
+ }
+
+ for _, testCase := range tests {
+ t.Run(testCase.name, func(t *testing.T) {
+ if err := testCase.run(); err == nil {
+ t.Fatal("expected symlink escape to be rejected")
+ }
+ if _, statErr := os.Stat(filepath.Join(outsideDir, "escaped.txt")); !errors.Is(statErr, os.ErrNotExist) {
+ t.Fatalf("escaped file exists or stat failed with unexpected error: %v", statErr)
+ }
+ })
+ }
+}
diff --git a/pkg/path/path.go b/pkg/path/path.go
index 6cb49c1ed..b0590ed88 100644
--- a/pkg/path/path.go
+++ b/pkg/path/path.go
@@ -32,3 +32,17 @@ func Get(p string) (string, error) {
}
return filepath.Abs(p)
}
+
+// HasVolumeName reports whether p starts with a platform volume name or a
+// Windows drive prefix. The explicit drive-prefix check keeps validation
+// portable on non-Windows platforms.
+func HasVolumeName(p string) bool {
+ if filepath.VolumeName(p) != "" || filepath.VolumeName(filepath.FromSlash(p)) != "" {
+ return true
+ }
+ return len(p) >= 2 && isASCIILetter(p[0]) && p[1] == ':'
+}
+
+func isASCIILetter(c byte) bool {
+ return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
+}
diff --git a/pkg/path/path_test.go b/pkg/path/path_test.go
index 472b85af0..f8f92dc6e 100644
--- a/pkg/path/path_test.go
+++ b/pkg/path/path_test.go
@@ -81,3 +81,39 @@ func TestGet(t *testing.T) {
})
}
}
+
+func TestHasVolumeName(t *testing.T) {
+ tests := []struct {
+ name string
+ path string
+ want bool
+ }{
+ {
+ name: "relative path",
+ path: "snapshot/data/file.txt",
+ },
+ {
+ name: "windows drive relative",
+ path: `C:snapshot\data.txt`,
+ want: true,
+ },
+ {
+ name: "windows drive absolute slash",
+ path: "C:/snapshot/data.txt",
+ want: true,
+ },
+ {
+ name: "windows drive absolute backslash",
+ path: `C:\snapshot\data.txt`,
+ want: true,
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ if got := HasVolumeName(test.path); got != test.want {
+ t.Fatalf("HasVolumeName(%q) = %t, want %t", test.path, got, test.want)
+ }
+ })
+ }
+}
(skywalking-banyandb) 02/11: fix(query): dedup IndexMode measures by series ID across data nodes (#1132)
Posted by bu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ButterBright pushed a commit to branch v0.10.x
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit 2e85f73813baa96c7c63da99617ad2fad07a9be7
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Sat May 16 11:17:43 2026 +0800
fix(query): dedup IndexMode measures by series ID across data nodes (#1132)
---
.../logical/measure/measure_plan_distributed.go | 39 ++++++++-
.../measure/measure_plan_distributed_test.go | 93 ++++++++++++++++++++++
pkg/query/logical/measure/schema.go | 6 ++
pkg/query/logical/measure/schema_test.go | 90 +++++++++++++++++++++
4 files changed, 225 insertions(+), 3 deletions(-)
diff --git a/pkg/query/logical/measure/measure_plan_distributed.go b/pkg/query/logical/measure/measure_plan_distributed.go
index 01d46ee1a..b88ef5d07 100644
--- a/pkg/query/logical/measure/measure_plan_distributed.go
+++ b/pkg/query/logical/measure/measure_plan_distributed.go
@@ -116,6 +116,7 @@ func newUnresolvedDistributed(query *measurev1.QueryRequest, pushDownAgg bool) l
}
func (ud *unresolvedDistributed) Analyze(s logical.Schema) (logical.Plan, error) {
+ indexMode := s.(*schema).measure.IndexMode
projectionTags := logical.ToTags(ud.originalQuery.GetTagProjection())
if len(projectionTags) > 0 {
var err error
@@ -178,6 +179,7 @@ func (ud *unresolvedDistributed) Analyze(s logical.Schema) (logical.Plan, error)
sortTagSpec: *sortTagSpec,
pushDownAgg: ud.pushDownAgg,
groupByTagsRefs: groupByTagsRefs,
+ indexMode: indexMode,
}
if ud.originalQuery.OrderBy != nil && ud.originalQuery.OrderBy.Sort == modelv1.Sort_SORT_DESC {
result.desc = true
@@ -191,6 +193,7 @@ func (ud *unresolvedDistributed) Analyze(s logical.Schema) (logical.Plan, error)
sortByTime: true,
pushDownAgg: ud.pushDownAgg,
groupByTagsRefs: groupByTagsRefs,
+ indexMode: indexMode,
}, nil
}
if ud.originalQuery.OrderBy.IndexRuleName == "" {
@@ -200,6 +203,7 @@ func (ud *unresolvedDistributed) Analyze(s logical.Schema) (logical.Plan, error)
sortByTime: true,
pushDownAgg: ud.pushDownAgg,
groupByTagsRefs: groupByTagsRefs,
+ indexMode: indexMode,
}
if ud.originalQuery.OrderBy.Sort == modelv1.Sort_SORT_DESC {
result.desc = true
@@ -224,6 +228,7 @@ func (ud *unresolvedDistributed) Analyze(s logical.Schema) (logical.Plan, error)
sortTagSpec: *sortTagSpec,
pushDownAgg: ud.pushDownAgg,
groupByTagsRefs: groupByTagsRefs,
+ indexMode: indexMode,
}
if ud.originalQuery.OrderBy.Sort == modelv1.Sort_SORT_DESC {
result.desc = true
@@ -240,6 +245,7 @@ type distributedPlan struct {
sortByTime bool
desc bool
pushDownAgg bool
+ indexMode bool
}
func (t *distributedPlan) Execute(ctx context.Context) (mi executor.MIterator, err error) {
@@ -311,7 +317,8 @@ func (t *distributedPlan) Execute(ctx context.Context) (mi executor.MIterator, e
return &pushedDownAggregatedIterator{dataPoints: deduplicatedDps}, err
}
smi := &sortedMIterator{
- Iterator: sort.NewItemIter(see, t.desc),
+ Iterator: sort.NewItemIter(see, t.desc),
+ indexMode: t.indexMode,
}
smi.init()
return smi, err
@@ -421,9 +428,11 @@ type sortedMIterator struct {
data *list.List
uniqueData map[uint64]*measurev1.InternalDataPoint
cur *measurev1.InternalDataPoint
+ seenSids map[uint64]struct{}
initialized bool
exhausted bool
closed bool
+ indexMode bool
}
func (s *sortedMIterator) init() {
@@ -437,6 +446,9 @@ func (s *sortedMIterator) init() {
}
s.data = list.New()
s.uniqueData = make(map[uint64]*measurev1.InternalDataPoint)
+ if s.indexMode {
+ s.seenSids = make(map[uint64]struct{})
+ }
s.loadDps()
}
@@ -456,10 +468,23 @@ func (s *sortedMIterator) Next() bool {
return true
}
+// loadDps consumes sort-field-equal groups from the underlying iterator and
+// pushes their deduplicated rows into s.data. When indexMode is true, rows
+// whose Sid was already emitted in a prior group are skipped — without this,
+// cross-node duplicates that carry different per-node "last-write" timestamps
+// would slip past hashDataPoint (which keys on Sid+timestamp) because they
+// land in different sort-field groups. When every row in a group is filtered,
+// the loop advances to the next group so callers see continued iteration.
func (s *sortedMIterator) loadDps() {
- if s.exhausted {
- return
+ for !s.exhausted {
+ s.loadOneGroup()
+ if s.data.Len() > 0 {
+ return
+ }
}
+}
+
+func (s *sortedMIterator) loadOneGroup() {
for k := range s.uniqueData {
delete(s.uniqueData, k)
}
@@ -485,6 +510,13 @@ func (s *sortedMIterator) loadDps() {
}
}
for _, v := range s.uniqueData {
+ if s.indexMode {
+ sid := v.GetDataPoint().GetSid()
+ if _, dup := s.seenSids[sid]; dup {
+ continue
+ }
+ s.seenSids[sid] = struct{}{}
+ }
s.data.PushBack(v)
}
}
@@ -501,6 +533,7 @@ func (s *sortedMIterator) Close() error {
s.exhausted = true
s.data = nil
s.uniqueData = nil
+ s.seenSids = nil
s.cur = nil
if s.Iterator == nil {
return nil
diff --git a/pkg/query/logical/measure/measure_plan_distributed_test.go b/pkg/query/logical/measure/measure_plan_distributed_test.go
index 7468b4fd1..80b44897b 100644
--- a/pkg/query/logical/measure/measure_plan_distributed_test.go
+++ b/pkg/query/logical/measure/measure_plan_distributed_test.go
@@ -62,6 +62,99 @@ func makeComparableDP(sid uint64, seconds, nanos int64, version int64, sortField
}
}
+func TestSortedMIterator_IndexModeDedup(t *testing.T) {
+ // Cross-node duplicates for an IndexMode measure carry the same Sid but
+ // different per-node "last-write" timestamps. They live in different
+ // sort-field groups when sorted by time, so hashDataPoint (which keys on
+ // Sid+timestamp) cannot collapse them — only Sid-based cross-group dedup
+ // can. See the package-level comment on sortedMIterator.loadDps.
+ testCases := []struct {
+ name string
+ data []*comparableDataPoint
+ wantSids []uint64
+ indexMode bool
+ }{
+ {
+ // Sort-by-time, two nodes return the same Sid at different ts.
+ // IndexMode=true → only first-seen Sid survives.
+ name: "indexMode collapses same Sid across sort-field groups",
+ data: []*comparableDataPoint{
+ makeComparableDP(1, 1, 0, 1, 1),
+ makeComparableDP(1, 2, 0, 1, 2),
+ },
+ indexMode: true,
+ wantSids: []uint64{1},
+ },
+ {
+ // Same input, IndexMode=false → existing behavior preserved.
+ name: "row-path keeps Sid duplicates with different timestamps",
+ data: []*comparableDataPoint{
+ makeComparableDP(1, 1, 0, 1, 1),
+ makeComparableDP(1, 2, 0, 1, 2),
+ },
+ indexMode: false,
+ wantSids: []uint64{1, 1},
+ },
+ {
+ // Sort-by-tag (same sort field) — both modes already dedup via
+ // hashDataPoint here. Verify IndexMode does not regress that.
+ name: "indexMode collapses same Sid within one sort-field group",
+ data: []*comparableDataPoint{
+ makeComparableDP(2, 1, 0, 1, 5),
+ makeComparableDP(2, 2, 0, 1, 5),
+ },
+ indexMode: true,
+ wantSids: []uint64{2},
+ },
+ {
+ // Three Sids interleaved across two sort-field groups.
+ // IndexMode=true keeps one per Sid (first-seen).
+ name: "indexMode preserves distinct Sids across groups",
+ data: []*comparableDataPoint{
+ makeComparableDP(1, 1, 0, 1, 1),
+ makeComparableDP(2, 1, 0, 1, 1),
+ makeComparableDP(1, 2, 0, 1, 2),
+ makeComparableDP(3, 2, 0, 1, 2),
+ },
+ indexMode: true,
+ wantSids: []uint64{1, 2, 3},
+ },
+ {
+ // Group at sf=2 contains only already-seen Sids — loadDps's outer
+ // loop must advance past the fully-filtered group to reach sf=3
+ // instead of stopping iteration. Without that, `Next` returns
+ // false after the first emission and Sid=2 is lost.
+ name: "indexMode advances past a fully-filtered intermediate group",
+ data: []*comparableDataPoint{
+ makeComparableDP(1, 1, 0, 1, 1),
+ makeComparableDP(1, 2, 0, 1, 2),
+ makeComparableDP(2, 3, 0, 1, 3),
+ },
+ indexMode: true,
+ wantSids: []uint64{1, 2},
+ },
+ }
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ iter := &sortedMIterator{
+ Iterator: &mockIterator{data: tc.data, idx: -1},
+ indexMode: tc.indexMode,
+ }
+ iter.init()
+ gotSids := make([]uint64, 0, len(tc.wantSids))
+ for iter.Next() {
+ gotSids = append(gotSids, iter.Current()[0].GetDataPoint().GetSid())
+ }
+ slices.Sort(gotSids)
+ wantSids := append([]uint64(nil), tc.wantSids...)
+ slices.Sort(wantSids)
+ if diff := cmp.Diff(wantSids, gotSids); diff != "" {
+ t.Errorf("Sids mismatch (-want +got):\n%s", diff)
+ }
+ })
+ }
+}
+
func TestSortedMIterator(t *testing.T) {
testCases := []struct {
name string
diff --git a/pkg/query/logical/measure/schema.go b/pkg/query/logical/measure/schema.go
index 3bdc81cad..3ce7b789f 100644
--- a/pkg/query/logical/measure/schema.go
+++ b/pkg/query/logical/measure/schema.go
@@ -136,6 +136,7 @@ func mergeSchema(schemas []logical.Schema) (logical.Schema, error) {
var commonSchemas []*logical.CommonSchema
var tagFamilies []*databasev1.TagFamilySpec
+ var canonicalMeasure *databasev1.Measure
fieldMap := make(map[string]*logical.FieldSpec)
for _, sm := range schemas {
@@ -149,6 +150,10 @@ func mergeSchema(schemas []logical.Schema) (logical.Schema, error) {
tagFamilies = logical.MergeTagFamilySpecs(tagFamilies, mSchema.measure.GetTagFamilies())
+ if canonicalMeasure == nil {
+ canonicalMeasure = mSchema.measure
+ }
+
commonSchemas = append(commonSchemas, mSchema.common)
for name, spec := range mSchema.fieldMap {
@@ -180,6 +185,7 @@ func mergeSchema(schemas []logical.Schema) (logical.Schema, error) {
}
ret := &schema{
+ measure: canonicalMeasure,
common: mergedCommon,
children: schemas,
fieldMap: fieldMap,
diff --git a/pkg/query/logical/measure/schema_test.go b/pkg/query/logical/measure/schema_test.go
new file mode 100644
index 000000000..8fc8fe173
--- /dev/null
+++ b/pkg/query/logical/measure/schema_test.go
@@ -0,0 +1,90 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions
+// and limitations under the License.
+
+package measure
+
+import (
+ "testing"
+
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ "github.com/apache/skywalking-banyandb/pkg/query/logical"
+)
+
+// mergeSchema is called from DistributedAnalyze whenever a measure query
+// spans more than one group. The merged schema must keep enough of the
+// underlying *databasev1.Measure that downstream consumers (notably the
+// IndexMode check in unresolvedDistributed.Analyze) do not nil-deref.
+func TestMergeSchema_PropagatesMeasure(t *testing.T) {
+ build := func(group string, indexMode bool) *schema {
+ md := &databasev1.Measure{
+ Metadata: &commonv1.Metadata{Name: "zipkin_service_traffic_minute", Group: group},
+ Entity: &databasev1.Entity{TagNames: []string{"id"}},
+ TagFamilies: []*databasev1.TagFamilySpec{
+ {Name: "storage-only", Tags: []*databasev1.TagSpec{{Name: "id", Type: databasev1.TagType_TAG_TYPE_STRING}}},
+ },
+ IndexMode: indexMode,
+ }
+ s, err := BuildSchema(md, nil)
+ if err != nil {
+ t.Fatalf("BuildSchema(%s) failed: %v", group, err)
+ }
+ return s.(*schema)
+ }
+
+ t.Run("multi-group merge carries IndexMode=true", func(t *testing.T) {
+ a := build("sw_metadata", true)
+ b := build("sw_metadata_replica", true)
+ merged, err := mergeSchema([]logical.Schema{a, b})
+ if err != nil {
+ t.Fatalf("mergeSchema: %v", err)
+ }
+ ms := merged.(*schema)
+ if ms.measure == nil {
+ t.Fatal("merged schema lost its *databasev1.Measure; downstream IndexMode checks will nil-deref")
+ }
+ if !ms.measure.IndexMode {
+ t.Fatalf("merged IndexMode = false, want true")
+ }
+ })
+
+ t.Run("multi-group merge carries IndexMode=false", func(t *testing.T) {
+ a := build("sw_metricsMinute", false)
+ b := build("sw_metricsMinute_replica", false)
+ merged, err := mergeSchema([]logical.Schema{a, b})
+ if err != nil {
+ t.Fatalf("mergeSchema: %v", err)
+ }
+ ms := merged.(*schema)
+ if ms.measure == nil {
+ t.Fatal("merged schema lost its *databasev1.Measure")
+ }
+ if ms.measure.IndexMode {
+ t.Fatalf("merged IndexMode = true, want false")
+ }
+ })
+
+ t.Run("single-group fast path returns input unchanged", func(t *testing.T) {
+ a := build("sw_metadata", true)
+ merged, err := mergeSchema([]logical.Schema{a})
+ if err != nil {
+ t.Fatalf("mergeSchema: %v", err)
+ }
+ if merged.(*schema) != a {
+ t.Fatal("single-group mergeSchema returned a new instance; expected pass-through")
+ }
+ })
+}
(skywalking-banyandb) 09/11: fix(trace): move storage metrics under trace.storage subscope (#1155)
Posted by bu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ButterBright pushed a commit to branch v0.10.x
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit 7b3227edf1658b63c94ad9f5cf5a6000a2be1721
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Thu Jun 4 20:14:03 2026 +0800
fix(trace): move storage metrics under trace.storage subscope (#1155)
The trace package's StorageMetricsFactory was using the root `trace`
scope, so per-segment inverted-index metrics from
`pkg/index/inverted` were emitted as `banyandb_trace_inverted_index_*`.
The dashboard in `docs/operation/grafana-cluster.json` (and the
sibling stream package) expect `banyandb_trace_storage_inverted_index_*`.
---
CHANGES.md | 1 +
banyand/trace/metadata.go | 2 +-
banyand/trace/metrics.go | 4 +--
banyand/trace/storage_scope_test.go | 54 +++++++++++++++++++++++++++++++++++++
4 files changed, 58 insertions(+), 3 deletions(-)
diff --git a/CHANGES.md b/CHANGES.md
index bd27150f3..c752a8c6b 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -8,6 +8,7 @@ Release Notes.
- Fix trace query identity-tag projection: when `trace_id`/`span_id` are explicitly projected, reconstruct them from span identity at response build time instead of requesting them as stored tags, and preserve tag order with null-filled per-span value alignment in the distributed trace result iterator.
- Fix FODC proxy corrupting Prometheus metric types. The agent dropped the `# TYPE` line while parsing banyandb `/metrics`, the `StreamMetrics` proto carried no type field, and the proxy guessed the type from a name-suffix heuristic — downgrading counters to gauge, mislabeling `_count`-suffixed counters as histograms, and splitting summaries into two conflicting `# TYPE` lines. Capture the type with the Prometheus `expfmt` parser, store it in the flight recorder, thread it through a new [...]
+- Trace storage metrics now expose the `storage` sub-scope, matching the `stream_storage_*` naming. The `StorageMetricsFactory` for trace switched from the root `trace` scope to `trace.storage`, so per-segment inverted-index metrics (`inverted_index_total_updates`, `inverted_index_total_doc_count`, `inverted_index_total_term_searchers_started`) are now emitted as `banyandb_trace_storage_*` instead of `banyandb_trace_*`, aligning the dashboard query names. Other trace metrics (`trace_tst_ [...]
## 0.10.2
diff --git a/banyand/trace/metadata.go b/banyand/trace/metadata.go
index 60bc4731d..9e2594cff 100644
--- a/banyand/trace/metadata.go
+++ b/banyand/trace/metadata.go
@@ -607,7 +607,7 @@ func (s *supplier) OpenDB(groupSchema *commonv1.Group) (resourceSchema.DB, error
Option: s.option,
SeriesIndexFlushTimeoutSeconds: s.option.flushTimeout.Nanoseconds() / int64(time.Second),
SeriesIndexCacheMaxBytes: int(s.option.seriesCacheMaxSize),
- StorageMetricsFactory: s.omr.With(traceScope.ConstLabels(meter.ToLabelPairs(common.DBLabelNames(), p.DBLabelValues()))),
+ StorageMetricsFactory: s.omr.With(storageScope.ConstLabels(meter.ToLabelPairs(common.DBLabelNames(), p.DBLabelValues()))),
SegmentIdleTimeout: segmentIdleTimeout,
DisableRetention: disableRetention,
DisableRotation: disableRotation,
diff --git a/banyand/trace/metrics.go b/banyand/trace/metrics.go
index d3c520afa..0c3594e2f 100644
--- a/banyand/trace/metrics.go
+++ b/banyand/trace/metrics.go
@@ -26,8 +26,8 @@ import (
)
var (
- streamScope = observability.RootScope.SubScope("trace")
- tbScope = streamScope.SubScope("tst")
+ tbScope = traceScope.SubScope("tst")
+ storageScope = traceScope.SubScope("storage")
)
type metrics struct {
diff --git a/banyand/trace/storage_scope_test.go b/banyand/trace/storage_scope_test.go
new file mode 100644
index 000000000..3d3e458cd
--- /dev/null
+++ b/banyand/trace/storage_scope_test.go
@@ -0,0 +1,54 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package trace
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "github.com/apache/skywalking-banyandb/pkg/meter"
+)
+
+// TestStorageScopeNamespace verifies that the new storageScope in this
+// package produces the metric-name namespace expected by the dashboard
+// and by the sibling stream package.
+//
+// Expected namespace chain (root separator is "_"):
+//
+// banyandb → banyandb_trace → banyandb_trace_tst (tbScope, unchanged)
+// banyandb_trace_storage (storageScope, NEW)
+func TestStorageScopeNamespace(t *testing.T) {
+ cases := []struct {
+ name string
+ scope meter.Scope
+ want string
+ }{
+ {"traceScope", traceScope, "banyandb_trace"},
+ {"tbScope", tbScope, "banyandb_trace_tst"},
+ {"storageScope", storageScope, "banyandb_trace_storage"},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ hs, ok := tc.scope.(*meter.HierarchicalScope)
+ require.Truef(t, ok, "%s must be a *meter.HierarchicalScope, got %T", tc.name, tc.scope)
+ assert.Equal(t, tc.want, hs.GetNamespace())
+ })
+ }
+}
(skywalking-banyandb) 05/11: fix: release bluge index writers on segment rotation (#13874) (#1128)
Posted by bu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ButterBright pushed a commit to branch v0.10.x
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit b2ced812aad0d15afc0e021a5d3e8535f4305566
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Thu May 21 14:41:49 2026 +0800
fix: release bluge index writers on segment rotation (#13874) (#1128)
---
CHANGES.md | 1 +
banyand/internal/storage/rotation.go | 11 +++++-
banyand/internal/storage/segment.go | 71 +++++++++++++++++++++++++++++------
banyand/internal/storage/tsdb_test.go | 26 +++++++------
banyand/measure/metadata.go | 4 +-
banyand/stream/metadata.go | 4 +-
banyand/trace/metadata.go | 4 +-
banyand/trace/metadata_test.go | 8 ++++
8 files changed, 101 insertions(+), 28 deletions(-)
diff --git a/CHANGES.md b/CHANGES.md
index fe42fa088..8a64b558d 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -40,6 +40,7 @@ Release Notes.
- Fix `CollectDataInfo` and `CollectLiaisonInfo` not handling `CATALOG_PROPERTY` groups.
- Fix lifecycle migration where the receiving node could create segments shorter than the configured `SegmentInterval`.
- Fail fast on incompatible storage version at boot. Previously the server would start in a degraded `SERVING` state with affected groups un-loaded because the property schema-registry retry loop swallowed the version-incompatibility panic. Compatible versions are listed in `banyand/internal/storage/versions.yml`.
+- Release bluge index writers on segment rotation so `analysisWorker` pools sized from `GOMAXPROCS` don't accumulate across rotations. Two layered defects kept the existing idle-segment reclaim path from running: `segmentIdleTimeout` defaulted to `0` (which disabled the 10-minute reclaim ticker), and `incRef` refreshed `lastAccessed` on every rotation tick so `closeIdleSegments` never observed an idle segment. Defaults to `time.Hour`, moves the `lastAccessed` bump to real read/write call [...]
### Chores
diff --git a/banyand/internal/storage/rotation.go b/banyand/internal/storage/rotation.go
index 3923cc99c..b97e80e00 100644
--- a/banyand/internal/storage/rotation.go
+++ b/banyand/internal/storage/rotation.go
@@ -55,15 +55,24 @@ func (d *database[T, O]) startRotationTask() error {
var idleCheckTicker *time.Ticker
var idleCheckC <-chan time.Time
- // Only create the ticker if idleTimeout is at least 1 second
+ // Only create the ticker if idleTimeout is at least 1 second.
+ // When disabled, idle segments are never reclaimed and their
+ // bluge writers (analysisWorker pools) accumulate across rotations.
if d.segmentController.idleTimeout >= time.Second {
idleCheckTicker = time.NewTicker(10 * time.Minute)
idleCheckC = idleCheckTicker.C
+ d.logger.Info().
+ Stringer("idle_timeout", d.segmentController.idleTimeout).
+ Msg("idle segment reclaimer enabled")
defer func() {
if idleCheckTicker != nil {
idleCheckTicker.Stop()
}
}()
+ } else {
+ d.logger.Warn().
+ Stringer("idle_timeout", d.segmentController.idleTimeout).
+ Msg("idle segment reclaimer disabled (idle_timeout < 1s); bluge writers will not be released on segment rotation")
}
for {
diff --git a/banyand/internal/storage/segment.go b/banyand/internal/storage/segment.go
index 75a2994b0..e0a9ab787 100644
--- a/banyand/internal/storage/segment.go
+++ b/banyand/internal/storage/segment.go
@@ -162,6 +162,14 @@ func (s *segment[T, O]) TablesWithShardIDs() (tt []T, shardIDs []common.ShardID,
// guaranteed that the segment is alive (s.index != nil, shards loaded) and
// will stay alive until the matching DecRef.
//
+// incRef intentionally does NOT touch lastAccessed: the idle-segment
+// reclaimer (closeIdleSegments) decides eligibility from that field, and
+// the rotation-tick scan iterates every segment via incRef on each tick.
+// Refreshing lastAccessed here would defeat the reclaimer for any segment
+// outside the active write window. Callers that represent a real read or
+// write touch must bump lastAccessed explicitly (see selectSegments and
+// createSegment); housekeeping iterators must not.
+//
// The CAS loop closes a TOCTOU race that the older "Load + AddInt32"
// shape suffered from: a goroutine could read refCount=1, race against a
// concurrent DecRef that drives it to 0 and triggers performCleanup
@@ -177,7 +185,6 @@ func (s *segment[T, O]) TablesWithShardIDs() (tt []T, shardIDs []common.ShardID,
// DecRef that drives refCount to 0 sees no concurrent +1 sneak in
// before performCleanup runs.
func (s *segment[T, O]) incRef(ctx context.Context) error {
- s.lastAccessed.Store(time.Now().UnixNano())
for {
current := atomic.LoadInt32(&s.refCount)
if current <= 0 {
@@ -424,6 +431,7 @@ func (sc *segmentController[T, O]) selectSegments(timeRange timestamp.TimeRange)
defer sc.RUnlock()
last := len(sc.lst) - 1
ctx := context.WithValue(context.Background(), logger.ContextKey, sc.l)
+ now := time.Now().UnixNano()
for i := range sc.lst {
s := sc.lst[last-i]
if s.GetTimeRange().End.Before(timeRange.Start) {
@@ -433,6 +441,7 @@ func (sc *segmentController[T, O]) selectSegments(timeRange timestamp.TimeRange)
if err = s.incRef(ctx); err != nil {
return nil, err
}
+ s.lastAccessed.Store(now)
tt = append(tt, s)
}
}
@@ -444,7 +453,11 @@ func (sc *segmentController[T, O]) createSegment(ts time.Time) (*segment[T, O],
if err != nil {
return nil, err
}
- return s, s.incRef(context.WithValue(context.Background(), logger.ContextKey, sc.l))
+ if err = s.incRef(context.WithValue(context.Background(), logger.ContextKey, sc.l)); err != nil {
+ return nil, err
+ }
+ s.lastAccessed.Store(time.Now().UnixNano())
+ return s, nil
}
func (sc *segmentController[T, O]) segments(reopenClosed bool) (ss []*segment[T, O], err error) {
@@ -473,20 +486,54 @@ func (sc *segmentController[T, O]) closeIdleSegments() int {
now := time.Now().UnixNano()
idleThreshold := now - maxIdleTime.Nanoseconds()
- segs, _ := sc.segments(false)
- closedCount := 0
+ // Take our own snapshot rather than going through segments(false): that
+ // helper's "Load>0 then Add" bump is non-atomic and can resurrect a
+ // segment that DecRef just cleaned up, and its caller has no way to tell
+ // which entries it actually bumped. closeIdleSegments must know, because
+ // it issues an unconditional DecRef per entry to release the bump; if a
+ // concurrent selectSegments reopens an entry we didn't bump, that DecRef
+ // would drop the query's only live ref and trigger performCleanup under
+ // active use. Iterate sc.lst under RLock and CAS-bump each open segment;
+ // closed segments are recorded as "not bumped" and skipped in the loop.
+ sc.RLock()
+ segs := make([]*segment[T, O], 0, len(sc.lst))
+ bumped := make([]bool, 0, len(sc.lst))
+ refAtBump := make([]int32, 0, len(sc.lst))
+ for _, s := range sc.lst {
+ didBump := false
+ var snapRef int32
+ for {
+ current := atomic.LoadInt32(&s.refCount)
+ if current <= 0 {
+ break
+ }
+ if atomic.CompareAndSwapInt32(&s.refCount, current, current+1) {
+ didBump = true
+ snapRef = current
+ break
+ }
+ }
+ segs = append(segs, s)
+ bumped = append(bumped, didBump)
+ refAtBump = append(refAtBump, snapRef)
+ }
+ sc.RUnlock()
- for _, seg := range segs {
- lastAccess := seg.lastAccessed.Load()
- // Only consider segments that have been idle for longer than the threshold
- // and have active references (are not already closed)
- if lastAccess < idleThreshold && atomic.LoadInt32(&seg.refCount) > 0 {
- seg.DecRef()
+ closedCount := 0
+ for i, seg := range segs {
+ if !bumped[i] {
+ continue
}
- seg.DecRef()
- if atomic.LoadInt32(&seg.refCount) == 0 {
+ // Only close when the snapshot proved refCount==1 (baseline only):
+ // a higher value means other callers hold active references, and
+ // decrementing past our bump would steal one of their refs,
+ // potentially triggering performCleanup under active use on a
+ // subsequent tick.
+ if refAtBump[i] == 1 && seg.lastAccessed.Load() < idleThreshold {
+ seg.DecRef()
closedCount++
}
+ seg.DecRef()
}
return closedCount
diff --git a/banyand/internal/storage/tsdb_test.go b/banyand/internal/storage/tsdb_test.go
index 4852b9bd6..9b97d4ca2 100644
--- a/banyand/internal/storage/tsdb_test.go
+++ b/banyand/internal/storage/tsdb_test.go
@@ -606,37 +606,39 @@ func TestCollectWithPartialClosedSegments(t *testing.T) {
var segments []Segment[*MockTSTable, any]
- // Create segments and keep track of them
+ // Create segments and keep track of them.
+ // openSegment.initialize sets refCount=1, then createSegment.incRef
+ // bumps it to 2. The DecRef below releases the createSegment ref,
+ // leaving refCount=1 (the initialize ref) so the idle reclaimer can
+ // CAS-bump and see refAtBump==1.
for _, date := range segmentDates {
mc.Set(date)
- seg, err := tsdb.CreateSegmentIfNotExist(date)
- require.NoError(t, err)
+ seg, segErr := tsdb.CreateSegmentIfNotExist(date)
+ require.NoError(t, segErr)
require.NotNil(t, seg)
segments = append(segments, seg)
- seg.DecRef() // Release reference
+ seg.DecRef() // Release createSegment's ref; refCount 2->1
}
// Set a specific tick time for testing
tickTime := int64(123456789)
db.latestTickTime.Store(tickTime)
- // Manually close some segments (first and third)
- segments[0].DecRef() // Close first segment
- segments[2].DecRef() // Close third segment
-
- // Manually set the closed segments to have idle time in the past
+ // Mark the first and third segments as idle by backdating their
+ // lastAccessed timestamp. Do NOT DecRef them — they must stay alive
+ // (refCount=1) so closeIdleSegments can CAS-bump (refAtBump==1) and
+ // close them.
sc := db.segmentController
ss, _ := sc.segments(false)
for _, s := range ss {
- // Find segments we want to mark as idle (first and third)
if s.Start.Equal(segmentDates[0]) || s.Start.Equal(segmentDates[2]) {
s.lastAccessed.Store(time.Now().Add(-2 * time.Hour).UnixNano())
- s.DecRef() // Force close
}
s.DecRef() // Release our reference from segments()
}
- // Call closeIdleSegments to ensure our target segments are closed
+ // closeIdleSegments should close the 2 idle segments (refAtBump==1 and
+ // lastAccessed older than idleThreshold) and leave the other 2 open.
closedCount := sc.closeIdleSegments()
require.Equal(t, 2, closedCount, "Should have closed 2 segments")
diff --git a/banyand/measure/metadata.go b/banyand/measure/metadata.go
index d206cd342..f8ea68692 100644
--- a/banyand/measure/metadata.go
+++ b/banyand/measure/metadata.go
@@ -680,7 +680,9 @@ func (s *supplier) OpenDB(groupSchema *commonv1.Group) (resourceSchema.DB, error
shardNum := ro.ShardNum
ttl := ro.Ttl
segInterval := ro.SegmentInterval
- segmentIdleTimeout := time.Duration(0)
+ // Non-zero default so the idle-segment reclaimer ticker actually starts
+ // (storage/rotation.go gates it on >=1s). Staged Close paths override below.
+ segmentIdleTimeout := time.Hour
disableRetention := false
disableRotation := false
if len(ro.Stages) > 0 && len(s.nodeLabels) > 0 {
diff --git a/banyand/stream/metadata.go b/banyand/stream/metadata.go
index dd91f1481..bc48c00c9 100644
--- a/banyand/stream/metadata.go
+++ b/banyand/stream/metadata.go
@@ -537,7 +537,9 @@ func (s *supplier) OpenDB(groupSchema *commonv1.Group) (resourceSchema.DB, error
shardNum := ro.ShardNum
ttl := ro.Ttl
segInterval := ro.SegmentInterval
- segmentIdleTimeout := time.Duration(0)
+ // Non-zero default so the idle-segment reclaimer ticker actually starts
+ // (storage/rotation.go gates it on >=1s). Staged Close paths override below.
+ segmentIdleTimeout := time.Hour
disableRetention := false
disableRotation := false
if len(ro.Stages) > 0 && len(s.nodeLabels) > 0 {
diff --git a/banyand/trace/metadata.go b/banyand/trace/metadata.go
index 3b02231b5..60bc4731d 100644
--- a/banyand/trace/metadata.go
+++ b/banyand/trace/metadata.go
@@ -560,7 +560,9 @@ func (s *supplier) OpenDB(groupSchema *commonv1.Group) (resourceSchema.DB, error
shardNum := ro.ShardNum
ttl := ro.Ttl
segInterval := ro.SegmentInterval
- segmentIdleTimeout := time.Duration(0)
+ // Non-zero default so the idle-segment reclaimer ticker actually starts
+ // (storage/rotation.go gates it on >=1s). Staged Close paths override below.
+ segmentIdleTimeout := time.Hour
disableRetention := false
disableRotation := false
if len(ro.Stages) > 0 && len(s.nodeLabels) > 0 {
diff --git a/banyand/trace/metadata_test.go b/banyand/trace/metadata_test.go
index 58994f525..44f18ac51 100644
--- a/banyand/trace/metadata_test.go
+++ b/banyand/trace/metadata_test.go
@@ -372,9 +372,17 @@ var _ = Describe("Metadata", func() {
Eventually(func() int64 {
return getFilePartCount(svcs, groupName)
}, flags.EventuallyTimeout).Should(BeNumerically(">=", 1))
+ filePartCountAfterFirstBatch := getFilePartCount(svcs, groupName)
changeTraceExtraTagType(svcs, traceName, groupName)
writeSchemaChangeTraceData(svcs, traceName, groupName, now.Add(-1*time.Hour), 3,
writeTraceDataOptions{extraTag: extraTagString, traceIDPrefix: "trace_new_"})
+ // Wait for the second batch to flush to disk, creating additional
+ // file parts that the merge loop can pick up. Without this gate the
+ // merge Eventually below also has to absorb flush latency, which can
+ // exceed 30 s on resource-constrained CI runners with -race.
+ Eventually(func() int64 {
+ return getFilePartCount(svcs, groupName)
+ }, flags.EventuallyTimeout).Should(BeNumerically(">", filePartCountAfterFirstBatch))
partCountBeforeMerge := getTotalPartCount(svcs, groupName)
Eventually(func() int64 {
return getTotalPartCount(svcs, groupName)
(skywalking-banyandb) 07/11: Fix trace identity-tag projection and distributed span tag alignment (#1147)
Posted by bu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ButterBright pushed a commit to branch v0.10.x
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit 7b2a552a7249efc8b63c195365955d0961c823e1
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Tue Jun 2 12:57:52 2026 +0800
Fix trace identity-tag projection and distributed span tag alignment (#1147)
---
CHANGES.md | 6 +
banyand/trace/block.go | 3 +
banyand/trace/metadata_test.go | 6 +-
banyand/trace/query.go | 26 ++++-
banyand/trace/query_test.go | 15 +++
pkg/query/logical/trace/trace_plan_distributed.go | 27 ++++-
.../logical/trace/trace_plan_distributed_test.go | 130 +++++++++++++++++++++
pkg/query/logical/trace/trace_plan_tag_filter.go | 15 ++-
8 files changed, 217 insertions(+), 11 deletions(-)
diff --git a/CHANGES.md b/CHANGES.md
index 0f007b0aa..c0928ad31 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -2,6 +2,12 @@
Release Notes.
+## 0.10.3
+
+### Bug Fixes
+
+- Fix trace query identity-tag projection: when `trace_id`/`span_id` are explicitly projected, reconstruct them from span identity at response build time instead of requesting them as stored tags, and preserve tag order with null-filled per-span value alignment in the distributed trace result iterator.
+
## 0.10.2
### Bug Fixes
diff --git a/banyand/trace/block.go b/banyand/trace/block.go
index 0db54fdd1..c950e852a 100644
--- a/banyand/trace/block.go
+++ b/banyand/trace/block.go
@@ -457,6 +457,9 @@ func (bc *blockCursor) copyAllTo(r *model.TraceResult) {
r.Spans = append(r.Spans, bc.spans...)
r.SpanIDs = append(r.SpanIDs, bc.spanIDs...)
+ if bc.tagProjection == nil || len(bc.tagProjection.Names) == 0 {
+ return
+ }
if len(r.Tags) != len(bc.tagProjection.Names) {
r.Tags = make([]model.Tag, len(bc.tagProjection.Names))
for i, name := range bc.tagProjection.Names {
diff --git a/banyand/trace/metadata_test.go b/banyand/trace/metadata_test.go
index 44f18ac51..19b479731 100644
--- a/banyand/trace/metadata_test.go
+++ b/banyand/trace/metadata_test.go
@@ -384,9 +384,13 @@ var _ = Describe("Metadata", func() {
return getFilePartCount(svcs, groupName)
}, flags.EventuallyTimeout).Should(BeNumerically(">", filePartCountAfterFirstBatch))
partCountBeforeMerge := getTotalPartCount(svcs, groupName)
+ // The background merge of the freshly-flushed parts runs asynchronously and,
+ // on resource-constrained CI runners (notably under -race), can take well
+ // beyond the default Eventually budget. Allow a generous, environment-scaled
+ // timeout so the wait reflects merge latency rather than a fixed ceiling.
Eventually(func() int64 {
return getTotalPartCount(svcs, groupName)
- }, flags.EventuallyTimeout).Should(BeNumerically("<", partCountBeforeMerge))
+ }, 3*flags.EventuallyTimeout).Should(BeNumerically("<", partCountBeforeMerge))
Eventually(func(innerGm Gomega) {
spans := querySchemaChangeTraceData(svcs, traceName, groupName,
diff --git a/banyand/trace/query.go b/banyand/trace/query.go
index 7845d0eb7..5d6906c08 100644
--- a/banyand/trace/query.go
+++ b/banyand/trace/query.go
@@ -82,10 +82,13 @@ func (t *trace) Query(ctx context.Context, tqo model.TraceQueryOptions) (model.T
}
}()
+ storageTagProjection := omitIdentityTagProjection(tqo.TagProjection, t.schema.GetTraceIdTagName(), t.schema.GetSpanIdTagName())
+ storageTQO := tqo
+ storageTQO.TagProjection = storageTagProjection
result := queryResult{
ctx: ctx,
segments: segments,
- tagProjection: tqo.TagProjection,
+ tagProjection: storageTagProjection,
}
segmentsNeedRelease = false
defer func() {
@@ -105,7 +108,7 @@ func (t *trace) Query(ctx context.Context, tqo model.TraceQueryOptions) (model.T
}
qo := queryOptions{
- TraceQueryOptions: tqo,
+ TraceQueryOptions: storageTQO,
traceIDs: tqo.TraceIDs,
schemaTagTypes: schemaTagTypes,
}
@@ -116,7 +119,7 @@ func (t *trace) Query(ctx context.Context, tqo model.TraceQueryOptions) (model.T
tables := collectTables(segments)
- sidxInstances, sidxQueryRequest, useSIDXStreaming := t.prepareSIDXStreaming(tqo, qo, tables)
+ sidxInstances, sidxQueryRequest, useSIDXStreaming := t.prepareSIDXStreaming(storageTQO, qo, tables)
if len(qo.traceIDs) == 0 && !useSIDXStreaming {
result.Release()
return nilResult, nil
@@ -149,6 +152,23 @@ func (t *trace) Query(ctx context.Context, tqo model.TraceQueryOptions) (model.T
return &result, nil
}
+func omitIdentityTagProjection(projection *model.TagProjection, traceIDTagName, spanIDTagName string) *model.TagProjection {
+ if projection == nil {
+ return nil
+ }
+ filtered := &model.TagProjection{
+ Family: projection.Family,
+ Names: make([]string, 0, len(projection.Names)),
+ }
+ for _, name := range projection.Names {
+ if name == traceIDTagName || name == spanIDTagName {
+ continue
+ }
+ filtered.Names = append(filtered.Names, name)
+ }
+ return filtered
+}
+
func validateTraceQueryOptions(tqo model.TraceQueryOptions) error {
if tqo.TimeRange == nil {
return errors.New("invalid query options: timeRange are required")
diff --git a/banyand/trace/query_test.go b/banyand/trace/query_test.go
index 54abde693..5dab69d51 100644
--- a/banyand/trace/query_test.go
+++ b/banyand/trace/query_test.go
@@ -225,6 +225,21 @@ func TestQueryResult(t *testing.T) {
}
}
+func TestOmitIdentityTagProjection(t *testing.T) {
+ projection := &model.TagProjection{
+ Family: "default",
+ Names: []string{"trace_id", "service_id", "span_id", "duration"},
+ }
+
+ got := omitIdentityTagProjection(projection, "trace_id", "span_id")
+
+ require.Equal(t, &model.TagProjection{
+ Family: "default",
+ Names: []string{"service_id", "duration"},
+ }, got)
+ require.Equal(t, []string{"trace_id", "service_id", "span_id", "duration"}, projection.Names)
+}
+
func TestQueryResultMultipleBatches(t *testing.T) {
tests := []struct {
name string
diff --git a/pkg/query/logical/trace/trace_plan_distributed.go b/pkg/query/logical/trace/trace_plan_distributed.go
index bedc26ae0..5b5116d39 100644
--- a/pkg/query/logical/trace/trace_plan_distributed.go
+++ b/pkg/query/logical/trace/trace_plan_distributed.go
@@ -33,6 +33,7 @@ import (
"github.com/apache/skywalking-banyandb/pkg/iter"
"github.com/apache/skywalking-banyandb/pkg/iter/sort"
"github.com/apache/skywalking-banyandb/pkg/logger"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
"github.com/apache/skywalking-banyandb/pkg/query"
"github.com/apache/skywalking-banyandb/pkg/query/executor"
"github.com/apache/skywalking-banyandb/pkg/query/logical"
@@ -377,21 +378,37 @@ func (t *distributedTraceResultIterator) Next() (model.TraceResult, bool) {
SpanIDs: make([]string, 0, len(trace.Spans)),
Tags: make([]model.Tag, 0, len(trace.Spans)),
}
- tagMap := make(map[string][]*modelv1.TagValue)
+ tagValuesByName := make(map[string][]*modelv1.TagValue)
+ var tagOrder []string
- for _, span := range trace.Spans {
+ for spanIdx, span := range trace.Spans {
// Add span data
result.Spans = append(result.Spans, span.Span)
// Extract tags from this span and aggregate by name
+ spanTags := make(map[string]*modelv1.TagValue, len(span.Tags))
for _, tag := range span.Tags {
- tagMap[tag.Key] = append(tagMap[tag.Key], tag.Value)
+ if _, exists := tagValuesByName[tag.Key]; !exists {
+ tagOrder = append(tagOrder, tag.Key)
+ tagValuesByName[tag.Key] = make([]*modelv1.TagValue, spanIdx)
+ for fillIdx := range tagValuesByName[tag.Key] {
+ tagValuesByName[tag.Key][fillIdx] = pbv1.NullTagValue
+ }
+ }
+ spanTags[tag.Key] = tag.Value
+ }
+ for _, tagName := range tagOrder {
+ tagValue, exists := spanTags[tagName]
+ if !exists {
+ tagValue = pbv1.NullTagValue
+ }
+ tagValuesByName[tagName] = append(tagValuesByName[tagName], tagValue)
}
result.SpanIDs = append(result.SpanIDs, span.SpanId)
}
- for tagName, values := range tagMap {
+ for _, tagName := range tagOrder {
result.Tags = append(result.Tags, model.Tag{
Name: tagName,
- Values: values,
+ Values: tagValuesByName[tagName],
})
}
return result, true
diff --git a/pkg/query/logical/trace/trace_plan_distributed_test.go b/pkg/query/logical/trace/trace_plan_distributed_test.go
new file mode 100644
index 000000000..9464627d2
--- /dev/null
+++ b/pkg/query/logical/trace/trace_plan_distributed_test.go
@@ -0,0 +1,130 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package trace
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+ tracev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/trace/v1"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
+)
+
+func TestDistributedTraceResultIteratorPreservesTagOrderAndSpanAlignment(t *testing.T) {
+ iterator := &distributedTraceResultIterator{
+ traces: []*tracev1.InternalTrace{
+ {
+ TraceId: "trace-1",
+ Spans: []*tracev1.Span{
+ {
+ Span: []byte("span-1"),
+ SpanId: "span-1",
+ Tags: []*modelv1.Tag{
+ {Key: "service_id", Value: strTagValueForDistributedTest("svc")},
+ {Key: "trace_id", Value: pbv1.NullTagValue},
+ {Key: "trace_id", Value: strTagValueForDistributedTest("trace-1")},
+ {Key: "span_id", Value: strTagValueForDistributedTest("span-1")},
+ },
+ },
+ {
+ Span: []byte("span-2"),
+ SpanId: "span-2",
+ Tags: []*modelv1.Tag{
+ {Key: "service_id", Value: strTagValueForDistributedTest("svc")},
+ {Key: "trace_id", Value: strTagValueForDistributedTest("trace-1")},
+ {Key: "span_id", Value: strTagValueForDistributedTest("span-2")},
+ },
+ },
+ },
+ },
+ },
+ }
+
+ got, ok := iterator.Next()
+
+ require.True(t, ok)
+ require.Len(t, got.Tags, 3)
+ require.Equal(t, "service_id", got.Tags[0].Name)
+ require.Equal(t, "trace_id", got.Tags[1].Name)
+ require.Equal(t, "span_id", got.Tags[2].Name)
+ require.Equal(t, "trace-1", got.Tags[1].Values[0].GetStr().GetValue())
+ require.Equal(t, "trace-1", got.Tags[1].Values[1].GetStr().GetValue())
+ require.Equal(t, "span-1", got.Tags[2].Values[0].GetStr().GetValue())
+ require.Equal(t, "span-2", got.Tags[2].Values[1].GetStr().GetValue())
+}
+
+func TestDistributedTraceResultIteratorNullFillsMissingSpanTags(t *testing.T) {
+ iterator := &distributedTraceResultIterator{
+ traces: []*tracev1.InternalTrace{
+ {
+ TraceId: "trace-1",
+ Spans: []*tracev1.Span{
+ {
+ Span: []byte("span-1"),
+ SpanId: "span-1",
+ Tags: []*modelv1.Tag{
+ {Key: "service_id", Value: strTagValueForDistributedTest("svc")},
+ {Key: "only_on_first", Value: strTagValueForDistributedTest("first")},
+ },
+ },
+ {
+ Span: []byte("span-2"),
+ SpanId: "span-2",
+ Tags: []*modelv1.Tag{
+ {Key: "service_id", Value: strTagValueForDistributedTest("svc")},
+ {Key: "only_on_second", Value: strTagValueForDistributedTest("second")},
+ },
+ },
+ },
+ },
+ },
+ }
+
+ got, ok := iterator.Next()
+
+ require.True(t, ok)
+ spanCount := len(got.Spans)
+ require.Equal(t, 2, spanCount)
+
+ values := make(map[string][]*modelv1.TagValue, len(got.Tags))
+ for _, tag := range got.Tags {
+ require.Lenf(t, tag.Values, spanCount, "tag %q must have one value per span", tag.Name)
+ values[tag.Name] = tag.Values
+ }
+
+ require.Equal(t, "svc", values["service_id"][0].GetStr().GetValue())
+ require.Equal(t, "svc", values["service_id"][1].GetStr().GetValue())
+
+ // only_on_first is present on span-1 and forward-filled with NULL on span-2.
+ require.Equal(t, "first", values["only_on_first"][0].GetStr().GetValue())
+ require.Same(t, pbv1.NullTagValue, values["only_on_first"][1])
+
+ // only_on_second is back-filled with NULL on span-1 and present on span-2.
+ require.Same(t, pbv1.NullTagValue, values["only_on_second"][0])
+ require.Equal(t, "second", values["only_on_second"][1].GetStr().GetValue())
+}
+
+func strTagValueForDistributedTest(value string) *modelv1.TagValue {
+ return &modelv1.TagValue{
+ Value: &modelv1.TagValue_Str{
+ Str: &modelv1.Str{Value: value},
+ },
+ }
+}
diff --git a/pkg/query/logical/trace/trace_plan_tag_filter.go b/pkg/query/logical/trace/trace_plan_tag_filter.go
index d407a7607..25b06f3fb 100644
--- a/pkg/query/logical/trace/trace_plan_tag_filter.go
+++ b/pkg/query/logical/trace/trace_plan_tag_filter.go
@@ -117,8 +117,8 @@ func (uis *unresolvedTraceTagFilter) Analyze(s logical.Schema) (logical.Plan, er
conditionSchema = s.ProjTags(conditionTagRefs...)
}
- // Deduplicate tag names
- ctx.projectionTags.Names = deduplicateStrings(ctx.projectionTags.Names)
+ // Deduplicate tag names and leave identity tags to response construction.
+ ctx.projectionTags.Names = omitIdentityProjectionTagNames(deduplicateStrings(ctx.projectionTags.Names), uis.traceIDTagName, uis.spanIDTagName)
// Create tag references if we have any projection tags
if len(ctx.projectionTags.Names) > 0 {
@@ -212,6 +212,17 @@ func deduplicateStrings(strings []string) []string {
return result
}
+func omitIdentityProjectionTagNames(tagNames []string, traceIDTagName, spanIDTagName string) []string {
+ result := make([]string, 0, len(tagNames))
+ for _, tagName := range tagNames {
+ if tagName == traceIDTagName || tagName == spanIDTagName {
+ continue
+ }
+ result = append(result, tagName)
+ }
+ return result
+}
+
// buildTraceFilter builds a filter for trace queries and returns both the filter and collected tag names.
// Unlike stream, trace only needs skipping filter.
// Returns min/max int64 values for the orderByTag if provided, otherwise returns math.MaxInt64, math.MinInt64.
(skywalking-banyandb) 10/11: Fix FODC agent leaving node_role unspecified on startup race (#1157)
Posted by bu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ButterBright pushed a commit to branch v0.10.x
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit 5fa118632e5da4debd2e3628db30f542ea4ea935
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Fri Jun 5 16:01:34 2026 +0800
Fix FODC agent leaving node_role unspecified on startup race (#1157)
* Fix FODC agent leaving node_role unspecified on startup race
The agent resolved the node role exactly once at startup via a single
GetCurrentNode poll whose endpoint retries spanned only ~1s. When the
sibling lifecycle/banyandb gRPC server was not yet listening
(connect: cannot assign requested address), the role fell back to
ROLE_UNSPECIFIED permanently, so 6 of 7 nodes reported the wrong type
instead of ROLE_DATA/ROLE_LIAISON.
---
CHANGES.md | 1 +
fodc/agent/internal/cluster/collector.go | 48 +++++++++++++++++++++++++
fodc/agent/internal/cluster/collector_test.go | 50 +++++++++++++++++++++++++++
fodc/agent/internal/cmd/root.go | 2 +-
4 files changed, 100 insertions(+), 1 deletion(-)
diff --git a/CHANGES.md b/CHANGES.md
index c752a8c6b..c6d75b767 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -9,6 +9,7 @@ Release Notes.
- Fix trace query identity-tag projection: when `trace_id`/`span_id` are explicitly projected, reconstruct them from span identity at response build time instead of requesting them as stored tags, and preserve tag order with null-filled per-span value alignment in the distributed trace result iterator.
- Fix FODC proxy corrupting Prometheus metric types. The agent dropped the `# TYPE` line while parsing banyandb `/metrics`, the `StreamMetrics` proto carried no type field, and the proxy guessed the type from a name-suffix heuristic — downgrading counters to gauge, mislabeling `_count`-suffixed counters as histograms, and splitting summaries into two conflicting `# TYPE` lines. Capture the type with the Prometheus `expfmt` parser, store it in the flight recorder, thread it through a new [...]
- Trace storage metrics now expose the `storage` sub-scope, matching the `stream_storage_*` naming. The `StorageMetricsFactory` for trace switched from the root `trace` scope to `trace.storage`, so per-segment inverted-index metrics (`inverted_index_total_updates`, `inverted_index_total_doc_count`, `inverted_index_total_term_searchers_started`) are now emitted as `banyandb_trace_storage_*` instead of `banyandb_trace_*`, aligning the dashboard query names. Other trace metrics (`trace_tst_ [...]
+- Fix FODC agent labeling metrics with `node_role="ROLE_UNSPECIFIED"`. The agent resolved the node role exactly once at startup via a single `GetCurrentNode` poll whose endpoint retries spanned only ~1s; when the sibling lifecycle/banyandb gRPC server was not yet listening (`connect: cannot assign requested address`) the role fell back to `ROLE_UNSPECIFIED` permanently, so most nodes never reported their real `ROLE_DATA`/`ROLE_LIAISON`. Retry the initial node-role resolution with exponen [...]
## 0.10.2
diff --git a/fodc/agent/internal/cluster/collector.go b/fodc/agent/internal/cluster/collector.go
index ba53c4e0d..42f60e3a0 100644
--- a/fodc/agent/internal/cluster/collector.go
+++ b/fodc/agent/internal/cluster/collector.go
@@ -39,6 +39,10 @@ const (
initialBackoff = 100 * time.Millisecond
maxBackoff = 5 * time.Second
nodeInfoFetchTimeout = 30 * time.Second
+ // nodeRoleResolveBudget bounds how long the initial node-role resolution retries.
+ // Keep it below nodeInfoFetchTimeout so the poll finishes before StartCollector's
+ // WaitForNodeFetched deadline.
+ nodeRoleResolveBudget = 25 * time.Second
)
// TopologyMap represents processed cluster data for a single endpoint.
@@ -66,6 +70,7 @@ type Collector struct {
podName string
addrs []string
interval time.Duration
+ resolveBudget time.Duration
mu sync.RWMutex
}
@@ -76,6 +81,7 @@ func NewCollector(log *logger.Logger, addrs []string, interval time.Duration, po
addrs: addrs,
interval: interval,
podName: podName,
+ resolveBudget: nodeRoleResolveBudget,
closer: run.NewCloser(0),
nodeFetchedCh: make(chan struct{}),
clients: make([]*endpointClient, 0, len(addrs)),
@@ -259,6 +265,48 @@ func (c *Collector) updateCurrentNodes(nodes map[string]*databasev1.Node) {
c.log.Info().Int("nodes_count", len(nodes)).Msg("Updated current nodes from all endpoints")
}
+// ResolveNodeRole polls the current node until a resolved (non-unspecified) role is
+// obtained or resolveBudget elapses, then returns the role string and labels. All
+// per-attempt fetches share a budget-scoped context, so the total runtime is bounded by
+// resolveBudget regardless of the per-attempt gRPC timeouts and backoffs. This guards the
+// startup race where the sibling lifecycle/banyandb gRPC server is not yet listening when
+// the agent first polls, which previously left node_role permanently ROLE_UNSPECIFIED.
+// It does not touch nodeFetchedCh, so the collector's readiness signal stays prompt.
+func (c *Collector) ResolveNodeRole(ctx context.Context) (nodeRole string, nodeLabels map[string]string) {
+ budgetCtx, cancel := context.WithTimeout(ctx, c.resolveBudget)
+ defer cancel()
+ backoff := initialBackoff
+ for !c.hasResolvedRole() {
+ c.pollCurrentNode(budgetCtx)
+ if c.hasResolvedRole() || budgetCtx.Err() != nil || c.closer.Closed() {
+ break
+ }
+ select {
+ case <-budgetCtx.Done():
+ case <-c.closer.CloseNotify():
+ case <-time.After(backoff):
+ }
+ if budgetCtx.Err() != nil || c.closer.Closed() {
+ break
+ }
+ backoff = min(backoff*2, maxBackoff)
+ }
+ return c.GetNodeInfo()
+}
+
+// hasResolvedRole reports whether any current node has a known (non-unspecified) role.
+func (c *Collector) hasResolvedRole() bool {
+ c.mu.RLock()
+ defer c.mu.RUnlock()
+ unspecified := databasev1.Role_name[int32(databasev1.Role_ROLE_UNSPECIFIED)]
+ for _, node := range c.currentNodes {
+ if node != nil && NodeRoleFromNode(node) != unspecified {
+ return true
+ }
+ }
+ return false
+}
+
func (c *Collector) updateClusterStates(states map[string]*databasev1.GetClusterStateResponse) {
if len(states) == 0 {
c.log.Warn().Msg("Received empty cluster states map")
diff --git a/fodc/agent/internal/cluster/collector_test.go b/fodc/agent/internal/cluster/collector_test.go
index 4b127ad56..71ab0850b 100644
--- a/fodc/agent/internal/cluster/collector_test.go
+++ b/fodc/agent/internal/cluster/collector_test.go
@@ -19,11 +19,13 @@ package cluster
import (
"context"
+ "fmt"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
+ "google.golang.org/grpc"
commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
@@ -766,6 +768,54 @@ func TestProcessClusterStates_EmptyPodName(t *testing.T) {
}
}
+type fakeNodeQueryClient struct {
+ node *databasev1.Node
+ failCount int
+ calls int
+}
+
+func (f *fakeNodeQueryClient) GetCurrentNode(_ context.Context, _ *databasev1.GetCurrentNodeRequest,
+ _ ...grpc.CallOption,
+) (*databasev1.GetCurrentNodeResponse, error) {
+ f.calls++
+ if f.calls <= f.failCount {
+ return nil, fmt.Errorf("simulated unavailable on call %d", f.calls)
+ }
+ return &databasev1.GetCurrentNodeResponse{Node: f.node}, nil
+}
+
+func TestCollector_ResolveNodeRole_ResolvesAfterTransientFailure(t *testing.T) {
+ log := initTestLogger(t)
+ c := NewCollector(log, []string{"test"}, time.Second, "test-pod")
+ c.resolveBudget = 5 * time.Second
+ node := &databasev1.Node{
+ Metadata: &commonv1.Metadata{Name: "data-0"},
+ Roles: []databasev1.Role{databasev1.Role_ROLE_DATA},
+ }
+ // fetchCurrentNodeFromEndpoint itself retries maxRetries(=3) times per poll, so
+ // failing the first 3 GetCurrentNode calls forces a second outer poll iteration.
+ fake := &fakeNodeQueryClient{node: node, failCount: maxRetries}
+ c.clients = []*endpointClient{{nodeQueryClient: fake, addr: "test"}}
+ role, _ := c.ResolveNodeRole(context.Background())
+ assert.Equal(t, databasev1.Role_name[int32(databasev1.Role_ROLE_DATA)], role)
+ assert.Greater(t, fake.calls, maxRetries)
+}
+
+func TestCollector_ResolveNodeRole_GivesUpWithinBudget(t *testing.T) {
+ log := initTestLogger(t)
+ c := NewCollector(log, []string{"test"}, time.Second, "test-pod")
+ c.resolveBudget = 500 * time.Millisecond
+ fake := &fakeNodeQueryClient{failCount: 1 << 30} // always fails
+ c.clients = []*endpointClient{{nodeQueryClient: fake, addr: "test"}}
+ start := time.Now()
+ role, _ := c.ResolveNodeRole(context.Background())
+ elapsed := time.Since(start)
+ assert.Equal(t, databasev1.Role_name[int32(databasev1.Role_ROLE_UNSPECIFIED)], role)
+ // Total runtime must stay bounded by the budget (plus slack for one in-flight gRPC
+ // attempt), proving the budget-scoped context actually caps the retry loop.
+ assert.Less(t, elapsed, c.resolveBudget+2*time.Second)
+}
+
func TestProcessClusterStates_ComplexTopology(t *testing.T) {
log := initTestLogger(t)
collector := NewCollector(log, []string{"localhost:17914", "localhost:17915", "localhost:17916"}, 10*time.Second, "test-pod")
diff --git a/fodc/agent/internal/cmd/root.go b/fodc/agent/internal/cmd/root.go
index d0ded635b..3070b9dc3 100644
--- a/fodc/agent/internal/cmd/root.go
+++ b/fodc/agent/internal/cmd/root.go
@@ -223,7 +223,7 @@ func runFODC(_ *cobra.Command, _ []string) error {
var nodeRole string
var nodeLabels map[string]string
if clusterCollector != nil {
- nodeRole, nodeLabels = clusterCollector.GetNodeInfo()
+ nodeRole, nodeLabels = clusterCollector.ResolveNodeRole(ctx)
log.Info().Str("node_role", nodeRole).Int("labels_count", len(nodeLabels)).Msg("Node info fetched")
}