You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ha...@apache.org on 2026/08/19 03:49:46 UTC
(skywalking-banyandb) 01/01: feat(trace): expose sampler decision and row metrics
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch hapi-trace-plugin-ob
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit 3e16eda519170dbe4531d006a6d9c3a78b8e084d
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Tue Aug 18 22:51:37 2026 +0000
feat(trace): expose sampler decision and row metrics
---
.../tracefixture/sampler_seed_bench_test.go | 92 +++++++-
banyand/trace/plugin_telemetry_test.go | 41 ++++
docs/operation/grafana-fodc-trace-plugin.json | 131 ++++++++++-
docs/operation/observability/metrics.md | 33 +++
plugins/README.md | 2 +-
.../internal/tracesampler/metrics_reasons_test.go | 220 +++++++++++++++++++
.../internal/tracesampler/metrics_test.go | 163 ++++++++++++++
.../skywalking/internal/tracesampler/sampler.go | 239 ++++++++++++++++++---
.../internal/tracesampler/sampler_bench_test.go | 132 +++++++++++-
.../internal/tracesampler/sampler_test.go | 2 +-
10 files changed, 1012 insertions(+), 43 deletions(-)
diff --git a/banyand/internal/benchmark/tracefixture/sampler_seed_bench_test.go b/banyand/internal/benchmark/tracefixture/sampler_seed_bench_test.go
index 205ca0158..adf08dcb2 100644
--- a/banyand/internal/benchmark/tracefixture/sampler_seed_bench_test.go
+++ b/banyand/internal/benchmark/tracefixture/sampler_seed_bench_test.go
@@ -25,6 +25,7 @@ import (
"os"
"sort"
"strings"
+ "sync/atomic"
"testing"
"github.com/stretchr/testify/require"
@@ -35,11 +36,12 @@ import (
)
const (
- seedSourceEnv = "TRACE_SAMPLER_BENCH_SOURCE"
- seedCatalogEnv = "TRACE_SAMPLER_BENCH_CATALOG"
- seedScheduleEnv = "TRACE_SAMPLER_BENCH_SCHEDULE"
- seedExpectedEnv = "TRACE_SAMPLER_BENCH_EXPECTED"
- seedPluginEnv = "TRACE_SAMPLER_BENCH_PLUGIN"
+ seedSourceEnv = "TRACE_SAMPLER_BENCH_SOURCE"
+ seedCatalogEnv = "TRACE_SAMPLER_BENCH_CATALOG"
+ seedScheduleEnv = "TRACE_SAMPLER_BENCH_SCHEDULE"
+ seedExpectedEnv = "TRACE_SAMPLER_BENCH_EXPECTED"
+ seedPluginEnv = "TRACE_SAMPLER_BENCH_PLUGIN"
+ seedDecisionMetricName = "trace_sampler_decisions_total"
)
type seedSamplerInputs struct {
@@ -53,8 +55,50 @@ type seedSamplerCase struct {
name string
config []byte
batchSize int
+ telemetry bool
}
+type seedBenchmarkCounter struct {
+ total atomic.Uint64
+}
+
+func (counter *seedBenchmarkCounter) Inc(delta float64, _ ...string) {
+ counter.total.Add(uint64(delta))
+}
+
+type seedBenchmarkMeter struct {
+ decisions *seedBenchmarkCounter
+ rows *seedBenchmarkCounter
+ droppedRows *seedBenchmarkCounter
+ unavailable *seedBenchmarkCounter
+}
+
+func (meter *seedBenchmarkMeter) Counter(name string, _ ...string) sdk.Counter {
+ switch name {
+ case seedDecisionMetricName:
+ return meter.decisions
+ case "trace_sampler_rows_total":
+ return meter.rows
+ case "trace_sampler_rows_dropped_total":
+ return meter.droppedRows
+ case "trace_sampler_row_count_unavailable_total":
+ return meter.unavailable
+ default:
+ return &seedBenchmarkCounter{}
+ }
+}
+func (meter *seedBenchmarkMeter) Gauge(_ string, _ ...string) sdk.Gauge { return nil }
+func (meter *seedBenchmarkMeter) Histogram(_ string, _ []float64, _ ...string) sdk.Histogram {
+ return nil
+}
+
+type seedBenchmarkHost struct {
+ meter *seedBenchmarkMeter
+}
+
+func (host *seedBenchmarkHost) Meter() sdk.Meter { return host.meter }
+func (host *seedBenchmarkHost) Logger() sdk.Logger { return nil }
+
type seedTestSampler struct {
decide func(*sdk.TraceBatch) (sdk.Verdict, error)
projection sdk.Projection
@@ -227,7 +271,7 @@ var (
)
func benchmarkSeedSamplerCase(b *testing.B, sampler sdk.Sampler, builder *SamplerBatchBuilder, instances []Instance, batchSize int,
- expectedDigest string,
+ expectedDigest string, telemetryMeter *seedBenchmarkMeter,
) {
b.Helper()
batches, buildErr := buildSeedBatches(context.Background(), builder, instances, sampler.Project(), batchSize)
@@ -241,6 +285,12 @@ func benchmarkSeedSamplerCase(b *testing.B, sampler sdk.Sampler, builder *Sample
if expectedDigest != "" && seedVerdictDigest(preflight) != expectedDigest {
b.Fatalf("verdict drift: got %s, want %s", seedVerdictDigest(preflight), expectedDigest)
}
+ if telemetryMeter != nil {
+ telemetryMeter.decisions.total.Store(0)
+ telemetryMeter.rows.total.Store(0)
+ telemetryMeter.droppedRows.total.Store(0)
+ telemetryMeter.unavailable.total.Store(0)
+ }
b.ReportAllocs()
b.ResetTimer()
for iteration := 0; iteration < b.N; iteration++ {
@@ -253,7 +303,20 @@ func benchmarkSeedSamplerCase(b *testing.B, sampler sdk.Sampler, builder *Sample
b.StopTimer()
traceOperations := float64(b.N * len(instances))
b.ReportMetric(float64(b.Elapsed().Nanoseconds())/traceOperations, "ns/trace")
+ b.ReportMetric(traceOperations/b.Elapsed().Seconds(), "traces/s")
b.ReportMetric(float64(len(instances)), "traces/op")
+ if telemetryMeter != nil {
+ expectedTotal := uint64(b.N * len(instances))
+ if actualTotal := telemetryMeter.decisions.total.Load(); actualTotal != expectedTotal {
+ b.Fatalf("telemetry counter total %d, want %d", actualTotal, expectedTotal)
+ }
+ if unavailable := telemetryMeter.unavailable.total.Load(); unavailable != 0 {
+ b.Fatalf("real seed has %d trace evaluations without row counts", unavailable)
+ }
+ if totalRows := telemetryMeter.rows.total.Load(); totalRows > 0 {
+ b.ReportMetric(100*float64(telemetryMeter.droppedRows.total.Load())/float64(totalRows), "row-drop-%")
+ }
+ }
}
func TestSeedSamplerArtifact(t *testing.T) {
@@ -342,6 +405,7 @@ func BenchmarkSeedSampler(b *testing.B) {
{name: "tag-regex", config: []byte(`{"keepTagRules":[{"tagKey":"db.type","regex":"Postgre.*"}]}`), batchSize: 512},
{name: "status-regex", config: []byte(`{"keepTagRules":[{"tagKey":"http.status_code","regex":"5\\d{2}"}]}`), batchSize: 512},
{name: "default", config: DefaultSkyWalkingSamplerConfig, batchSize: 512},
+ {name: "default-telemetry", config: DefaultSkyWalkingSamplerConfig, batchSize: 512, telemetry: true},
}
for _, batchSize := range []int{1, 16, 64, 256, 512, 4096, 8192, 16384} {
cases = append(cases,
@@ -362,7 +426,19 @@ func BenchmarkSeedSampler(b *testing.B) {
if strings.HasPrefix(benchmarkCase.name, "default") {
expectedDigest = inputs.expected.VerdictSHA256
}
- benchmarkSeedSamplerCase(b, sampler, builder, inputs.plan.Instances, benchmarkCase.batchSize, expectedDigest)
+ var telemetryMeter *seedBenchmarkMeter
+ if benchmarkCase.telemetry {
+ hostAwareSampler, hostAware := sampler.(sdk.HostAware)
+ if !hostAware {
+ b.Fatal("loaded sampler does not implement sdk.HostAware")
+ }
+ telemetryMeter = &seedBenchmarkMeter{
+ decisions: &seedBenchmarkCounter{}, rows: &seedBenchmarkCounter{},
+ droppedRows: &seedBenchmarkCounter{}, unavailable: &seedBenchmarkCounter{},
+ }
+ hostAwareSampler.UseHost(&seedBenchmarkHost{meter: telemetryMeter})
+ }
+ benchmarkSeedSamplerCase(b, sampler, builder, inputs.plan.Instances, benchmarkCase.batchSize, expectedDigest, telemetryMeter)
})
}
projectionCases := []struct {
@@ -398,7 +474,7 @@ func BenchmarkSeedSampler(b *testing.B) {
seedBenchmarkPayloadSink = payloadBytes
return sdk.Verdict{Keep: keep}, nil
}}
- benchmarkSeedSamplerCase(b, sampler, builder, inputs.plan.Instances, 512, "")
+ benchmarkSeedSamplerCase(b, sampler, builder, inputs.plan.Instances, 512, "", nil)
})
}
}
diff --git a/banyand/trace/plugin_telemetry_test.go b/banyand/trace/plugin_telemetry_test.go
index 72a20c148..5650b7f7d 100644
--- a/banyand/trace/plugin_telemetry_test.go
+++ b/banyand/trace/plugin_telemetry_test.go
@@ -61,6 +61,47 @@ func TestPluginTelemetry_MetricNamePrefix(t *testing.T) {
assert.True(t, hasGauge, "gauge name must be prefixed with plugin_")
}
+// TestPluginTelemetry_TraceSamplerDecisionContract verifies the real host
+// adapter supplies attribution labels, prefixes the first-party decision
+// metric, records its fixed labels, and removes the emitted series at teardown.
+func TestPluginTelemetry_TraceSamplerDecisionContract(t *testing.T) {
+ const (
+ group = "sw_trace"
+ pluginName = "sw-trace-sampler"
+ verdictKeep = "keep"
+ ruleTag31 = "tag_31"
+ )
+ sm, factory := newTestSamplerMetrics()
+ telemetry := newPluginTelemetry(factory, logger.GetLogger("trace"), sm, group, pluginName)
+ counter := telemetry.Meter().Counter("trace_sampler_decisions_total", "verdict", "rule")
+ counter.Inc(3, verdictKeep, ruleTag31)
+ rowCounter := telemetry.Meter().Counter("trace_sampler_rows_total")
+ rowCounter.Inc(12)
+ droppedRowCounter := telemetry.Meter().Counter("trace_sampler_rows_dropped_total", "rule")
+ droppedRowCounter.Inc(4, ruleTag31)
+ unavailableCounter := telemetry.Meter().Counter("trace_sampler_row_count_unavailable_total")
+ unavailableCounter.Inc(1)
+
+ underlying := factory.counter("plugin_trace_sampler_decisions_total")
+ require.NotNil(t, underlying)
+ assert.Equal(t, 1, underlying.callsWithLabels(group, pluginName, verdictKeep, ruleTag31))
+ underlyingRows := factory.counter("plugin_trace_sampler_rows_total")
+ require.NotNil(t, underlyingRows)
+ assert.Equal(t, 1, underlyingRows.callsWithLabels(group, pluginName))
+ underlyingDroppedRows := factory.counter("plugin_trace_sampler_rows_dropped_total")
+ require.NotNil(t, underlyingDroppedRows)
+ assert.Equal(t, 1, underlyingDroppedRows.callsWithLabels(group, pluginName, ruleTag31))
+ underlyingUnavailable := factory.counter("plugin_trace_sampler_row_count_unavailable_total")
+ require.NotNil(t, underlyingUnavailable)
+ assert.Equal(t, 1, underlyingUnavailable.callsWithLabels(group, pluginName))
+
+ telemetry.teardown()
+ assert.Equal(t, 1, underlying.deletesWithPrefix(group, pluginName, verdictKeep, ruleTag31))
+ assert.Equal(t, 1, underlyingRows.deletesWithPrefix(group, pluginName))
+ assert.Equal(t, 1, underlyingDroppedRows.deletesWithPrefix(group, pluginName, ruleTag31))
+ assert.Equal(t, 1, underlyingUnavailable.deletesWithPrefix(group, pluginName))
+}
+
// TestPluginTelemetry_MetricNameSanitization verifies that mixed-case and
// special-character names are lowercased and sanitized.
func TestPluginTelemetry_MetricNameSanitization(t *testing.T) {
diff --git a/docs/operation/grafana-fodc-trace-plugin.json b/docs/operation/grafana-fodc-trace-plugin.json
index f689f36df..6c0eb96ac 100644
--- a/docs/operation/grafana-fodc-trace-plugin.json
+++ b/docs/operation/grafana-fodc-trace-plugin.json
@@ -944,7 +944,7 @@
},
"gridPos": {
"h": 8,
- "w": 12,
+ "w": 8,
"x": 0,
"y": 24
},
@@ -1081,8 +1081,8 @@
},
"gridPos": {
"h": 8,
- "w": 12,
- "x": 12,
+ "w": 8,
+ "x": 8,
"y": 24
},
"id": 302,
@@ -1206,6 +1206,131 @@
"title": "Safety Events",
"type": "timeseries"
},
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "First-party sampler decisions and evaluated rows by the first matching fixed rule slot. Proposed drops are not committed storage deletions.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 16,
+ "y": 24
+ },
+ "id": 303,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull",
+ "max",
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true,
+ "sortBy": "Last *",
+ "sortDesc": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.2.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (group, plugin_name, verdict, rule) (rate(banyandb_trace_pipeline_plugin_trace_sampler_decisions_total{job=~\"$job\", container_name=~\"$role\", pod_name=~\"$pod\", group=~\"$group\"}[$__rate_interval]))",
+ "instant": false,
+ "legendFormat": "{{group}} / {{plugin_name}} / {{verdict}} / {{rule}}",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (group, plugin_name, rule) (rate(banyandb_trace_pipeline_plugin_trace_sampler_rows_dropped_total{job=~\"$job\", container_name=~\"$role\", pod_name=~\"$pod\", group=~\"$group\"}[$__rate_interval]))",
+ "instant": false,
+ "legendFormat": "{{group}} / {{plugin_name}} / {{rule}} dropped rows",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (group, plugin_name) (rate(banyandb_trace_pipeline_plugin_trace_sampler_rows_total{job=~\"$job\", container_name=~\"$role\", pod_name=~\"$pod\", group=~\"$group\"}[$__rate_interval]))",
+ "instant": false,
+ "legendFormat": "{{group}} / {{plugin_name}} total rows",
+ "range": true,
+ "refId": "C"
+ }
+ ],
+ "title": "Sampler Decisions & Rows by Rule",
+ "type": "timeseries"
+ },
{
"collapsed": false,
"gridPos": {
diff --git a/docs/operation/observability/metrics.md b/docs/operation/observability/metrics.md
index 2603b6e24..331f123d0 100644
--- a/docs/operation/observability/metrics.md
+++ b/docs/operation/observability/metrics.md
@@ -392,6 +392,39 @@ therefore cannot be enumerated in the host catalog. The host caps each plugin in
series and directs overflow to a bounded sentinel series. See
[Plugin Telemetry](../../design/post-trace-pipeline.md#27-plugin-telemetry-meter--logger) for the complete contract.
+The first-party `sw-trace-sampler` and `zipkin-trace-sampler` publish the following bounded decision metric:
+
+| Metric | Type | Labels | Meaning |
+| --- | --- | --- | --- |
+| `banyandb_trace_pipeline_plugin_trace_sampler_decisions_total` | Counter | `group`, `plugin_name`, `verdict`, `rule` | Trace evaluations attributed to the first matching sampler rule. |
+| `banyandb_trace_pipeline_plugin_trace_sampler_rows_total` | Counter | `group`, `plugin_name` | All span or segment rows carried by trace evaluations when a projected row-aligned column is available. |
+| `banyandb_trace_pipeline_plugin_trace_sampler_rows_dropped_total` | Counter | `group`, `plugin_name`, `rule` | Rows belonging to traces the sampler proposed dropping, attributed to the fixed drop rule. |
+| `banyandb_trace_pipeline_plugin_trace_sampler_row_count_unavailable_total` | Counter | `group`, `plugin_name` | Trace evaluations whose metadata-only projection exposed no row count; these traces are omitted from `rows_total`. |
+
+`verdict` is `keep` or `drop`. `rule` is one of the fixed built-in values `duration`, `error`, `healthy_sample`,
+`healthy_rejected`, `no_keep_rule`, `decode_failure_duration`, `decode_failure_error`, and `decode_failure_tags`, or a
+fixed positional tag-rule slot from `tag_00` through `tag_31`. Tag slots follow `keepTagRules` configuration order; tag
+keys and values are never copied into metric labels. Configurations with more than 32 tag rules are rejected.
+
+This counter reports sampler evaluations, not unique traces or committed storage deletions. A trace may be evaluated in
+multiple merge or finalization rounds, and BanyanDB safety guards may retain a trace for which a plugin returned a drop
+verdict. Compare requested drops by rule with the engine's committed drop rate rather than treating them as equal:
+
+`sum by (group, plugin_name, rule) (rate(banyandb_trace_pipeline_plugin_trace_sampler_decisions_total{verdict="drop"}[5m]))`
+
+`sum by (group) (rate(banyandb_trace_tst_pipeline_traces_dropped[5m]))`
+
+For the first-party default SkyWalking policy, projected duration and error columns make the row count available. The
+proposed dropped-row ratio is:
+
+`sum by (group, plugin_name) (rate(banyandb_trace_pipeline_plugin_trace_sampler_rows_dropped_total[5m]))`
+` / sum by (group, plugin_name) (rate(banyandb_trace_pipeline_plugin_trace_sampler_rows_total[5m]))`
+
+Break down the dropped rows by the responsible rule with
+`sum by (group, plugin_name, rule) (rate(banyandb_trace_pipeline_plugin_trace_sampler_rows_dropped_total[5m]))`.
+Alert on any increase of `trace_sampler_row_count_unavailable_total` before treating the ratio as complete. Like the
+decision counter, this is the plugin's proposed row-drop ratio, not the committed storage deletion ratio after guards.
+
## Data: Inverted Index
`Inverted Index` metrics monitor the series index of measures and streams, grouped by the `group` tag. Rapid growth or churn signals a cardinality problem that bloats the index and slows queries.
diff --git a/plugins/README.md b/plugins/README.md
index 9ec967299..d0514d485 100644
--- a/plugins/README.md
+++ b/plugins/README.md
@@ -80,7 +80,7 @@ overrides, and tuning guidance, see SkyWalking's
| `durationThresholdMs` | int | Keep a trace whose end-to-end envelope reaches this many **milliseconds**. `0` disables. |
| `keepErrors` | bool | Keep a trace carrying an error. Rejected on a schema that expresses error neither as a column nor as an array key. On the Zipkin schema see the truncation caveat below. |
| `errorTag` | string | Override the tag `keepErrors` reads. Ignored unless `keepErrors` is set. |
-| `keepTagRules` | list \| string | Sure-keep rules over the searchable tags: `{tagKey, exists\|equals\|in\|regex}`, or the compact `key=value,key=~regex,key` string form. |
+| `keepTagRules` | list \| string | Up to 32 sure-keep rules over the searchable tags: `{tagKey, exists\|equals\|in\|regex}`, or the compact `key=value,key=~regex,key` string form. |
| `healthySampleRate` | float | Fraction in `[0,1]` of the remaining traces to keep, by FNV-1a hash of the trace ID. Accepts a quoted number, since SkyWalking's config loader stringifies float placeholders. |
**Unknown keys are rejected.** Every option is a keep rule, so a key that
diff --git a/plugins/skywalking/internal/tracesampler/metrics_reasons_test.go b/plugins/skywalking/internal/tracesampler/metrics_reasons_test.go
new file mode 100644
index 000000000..1d88dfff2
--- /dev/null
+++ b/plugins/skywalking/internal/tracesampler/metrics_reasons_test.go
@@ -0,0 +1,220 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package tracesampler
+
+import (
+ "fmt"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "github.com/apache/skywalking-banyandb/pkg/pb/v1/valuetype"
+ "github.com/apache/skywalking-banyandb/pkg/pipeline/sdk"
+ "github.com/apache/skywalking-banyandb/pkg/pipeline/sdk/sdktest"
+)
+
+func TestDecideAggregatesMixedReasonsByBatch(t *testing.T) {
+ plugin, newErr := New([]byte(`{
+ "durationThresholdMs":500,"keepErrors":true,"healthySampleRate":0.5,
+ "keepTagRules":[
+ {"tagKey":"missing","exists":true},
+ {"tagKey":"db.type","equals":"PostgreSQL"},
+ {"tagKey":"http.method","in":["POST","PUT"]},
+ {"tagKey":"http.status_code","regex":"5\\d\\d"}
+ ]}`), segmentSchema)
+ require.NoError(t, newErr)
+ sampler := plugin.(*Sampler)
+ counter := &recordingCounter{}
+ sampler.UseHost(&testHost{meter: &testMeter{counter: counter}})
+ rowCounter := &recordingCounter{}
+ droppedRowCounter := &recordingCounter{}
+ unavailableCounter := &recordingCounter{}
+ sampler.rowsTotal = rowCounter
+ sampler.rowsDropped = droppedRowCounter
+ sampler.rowCountUnavailable = unavailableCounter
+
+ healthyID := traceIDForSampleResult(true, 0.5)
+ rejectedID := traceIDForSampleResult(false, 0.5)
+ batch := &sdk.TraceBatch{Traces: []sdk.TraceBlock{
+ metricSegmentBlock(t, "duration", 600, 0, nil),
+ metricSegmentBlock(t, "error", 1, 1, nil),
+ metricSegmentBlock(t, "tag-equals-1", 1, 0, []string{"db.type=PostgreSQL"}),
+ metricSegmentBlock(t, "tag-equals-2", 1, 0, []string{"db.type=PostgreSQL"}),
+ metricSegmentBlock(t, "tag-regex", 1, 0, []string{"http.status_code=503"}),
+ metricSegmentBlock(t, healthyID, 1, 0, nil),
+ metricSegmentBlock(t, rejectedID, 1, 0, nil),
+ }}
+ verdict, decideErr := sampler.Decide(batch)
+ require.NoError(t, decideErr)
+ assert.Equal(t, []bool{true, true, true, true, true, true, false}, verdict.Keep)
+
+ got := counterCallsByLabels(counter.calls)
+ assert.Equal(t, float64(1), got[decisionVerdictKeep+"/"+decisionReasonDurationLabel])
+ assert.Equal(t, float64(1), got[decisionVerdictKeep+"/"+decisionReasonErrorLabel])
+ assert.Equal(t, float64(2), got[decisionVerdictKeep+"/"+decisionReasonTag01Label])
+ assert.Equal(t, float64(1), got[decisionVerdictKeep+"/"+decisionReasonTag03Label])
+ assert.Equal(t, float64(1), got[decisionVerdictKeep+"/"+decisionReasonHealthySampleLabel])
+ assert.Equal(t, float64(1), got[decisionVerdictDrop+"/"+decisionReasonHealthyRejectedLabel])
+ assert.Equal(t, float64(len(batch.Traces)), sumCounterDeltas(counter.calls))
+ require.Len(t, rowCounter.calls, 1)
+ assert.Equal(t, float64(len(batch.Traces)), rowCounter.calls[0].delta)
+ assert.Empty(t, rowCounter.calls[0].labels)
+ require.Len(t, droppedRowCounter.calls, 1)
+ assert.Equal(t, float64(1), droppedRowCounter.calls[0].delta)
+ assert.Equal(t, []string{decisionReasonHealthyRejectedLabel}, droppedRowCounter.calls[0].labels)
+ assert.Empty(t, unavailableCounter.calls)
+}
+
+func TestDecideReportsUnavailableRowCount(t *testing.T) {
+ plugin, newErr := New([]byte(`{"healthySampleRate":1}`), segmentSchema)
+ require.NoError(t, newErr)
+ sampler := plugin.(*Sampler)
+ sampler.decisions = &recordingCounter{}
+ sampler.rowsTotal = &recordingCounter{}
+ sampler.rowsDropped = &recordingCounter{}
+ unavailableCounter := &recordingCounter{}
+ sampler.rowCountUnavailable = unavailableCounter
+
+ _, decideErr := sampler.Decide(&sdk.TraceBatch{Traces: []sdk.TraceBlock{{TraceID: "metadata-only"}}})
+ require.NoError(t, decideErr)
+ assert.Empty(t, sampler.rowsTotal.(*recordingCounter).calls)
+ assert.Empty(t, sampler.rowsDropped.(*recordingCounter).calls)
+ require.Len(t, unavailableCounter.calls, 1)
+ assert.Equal(t, float64(1), unavailableCounter.calls[0].delta)
+ assert.Empty(t, unavailableCounter.calls[0].labels)
+}
+
+func TestDecideTraceSuccessfulReasonsAndPrecedence(t *testing.T) {
+ allMatchBlock := metricSegmentBlock(t, "all", 600, 1, []string{"db.type=PostgreSQL"})
+ errorAndTagBlock := metricSegmentBlock(t, "error-tag", 1, 1, []string{"db.type=PostgreSQL"})
+ tests := []struct {
+ name string
+ config string
+ block sdk.TraceBlock
+ want traceDecision
+ }{
+ {name: "duration wins over error and tag", config: `{"durationThresholdMs":500,"keepErrors":true,"keepTagRules":[{"tagKey":"db.type","exists":true}]}`,
+ block: allMatchBlock, want: traceDecision{keep: true, reason: decisionReasonDuration}},
+ {name: "error wins over tag", config: `{"keepErrors":true,"keepTagRules":[{"tagKey":"db.type","exists":true}]}`,
+ block: errorAndTagBlock, want: traceDecision{keep: true, reason: decisionReasonError}},
+ {name: "tag decode failure", config: `{"keepTagRules":[{"tagKey":"db.type","exists":true}]}`,
+ block: malformedMetricTagBlock(), want: traceDecision{keep: true, reason: decisionReasonDecodeTags}},
+ }
+ for _, testCase := range tests {
+ t.Run(testCase.name, func(t *testing.T) {
+ plugin, newErr := New([]byte(testCase.config), segmentSchema)
+ require.NoError(t, newErr)
+ assert.Equal(t, testCase.want, plugin.(*Sampler).decideTrace(&testCase.block))
+ })
+ }
+}
+
+func TestDecideTraceEveryTagMatcherReason(t *testing.T) {
+ tests := []struct {
+ name string
+ rule string
+ entry string
+ }{
+ {name: "exists", rule: `{"tagKey":"debug","exists":true}`, entry: "debug"},
+ {name: "equals", rule: `{"tagKey":"db.type","equals":"PostgreSQL"}`, entry: "db.type=PostgreSQL"},
+ {name: "in", rule: `{"tagKey":"http.method","in":["POST","PUT"]}`, entry: "http.method=PUT"},
+ {name: "regex", rule: `{"tagKey":"http.status_code","regex":"5\\d\\d"}`, entry: "http.status_code=503"},
+ }
+ for _, testCase := range tests {
+ t.Run(testCase.name, func(t *testing.T) {
+ config := `{"keepTagRules":[{"tagKey":"never","exists":true},` + testCase.rule + `]}`
+ plugin, newErr := New([]byte(config), segmentSchema)
+ require.NoError(t, newErr)
+ block := metricSegmentBlock(t, testCase.name, 1, 0, []string{testCase.entry})
+ want := traceDecision{keep: true, reason: decisionReasonTagRule, tagRuleIndex: 1}
+ assert.Equal(t, want, plugin.(*Sampler).decideTrace(&block))
+ })
+ }
+}
+
+func TestDecideTraceZipkinErrorReason(t *testing.T) {
+ plugin, newErr := New([]byte(`{"keepErrors":true}`), zipkinSchema)
+ require.NoError(t, newErr)
+ block, buildErr := sdktest.NewTrace("zipkin-error").Tag("query", []string{"error", "error=boom"}).Build()
+ require.NoError(t, buildErr)
+ assert.Equal(t, traceDecision{keep: true, reason: decisionReasonError}, plugin.(*Sampler).decideTrace(&block))
+}
+
+func TestDecideEmitsHighestFixedTagSlot(t *testing.T) {
+ rules := make([]string, maxKeepTagRules)
+ for ruleIndex := range rules {
+ rules[ruleIndex] = fmt.Sprintf(`{"tagKey":"key%d","exists":true}`, ruleIndex)
+ }
+ plugin, newErr := New([]byte(`{"keepTagRules":[`+strings.Join(rules, ",")+`]}`), segmentSchema)
+ require.NoError(t, newErr)
+ sampler := plugin.(*Sampler)
+ counter := &recordingCounter{}
+ sampler.UseHost(&testHost{meter: &testMeter{counter: counter}})
+ block := metricSegmentBlock(t, "last-slot", 1, 0, []string{"key31=value"})
+ _, decideErr := sampler.Decide(&sdk.TraceBatch{Traces: []sdk.TraceBlock{block}})
+ require.NoError(t, decideErr)
+ require.Len(t, counter.calls, 1)
+ assert.Equal(t, []string{decisionVerdictKeep, decisionReasonTag31Label}, counter.calls[0].labels)
+}
+
+func metricSegmentBlock(t *testing.T, traceID string, durationMs, errorValue int64, entries []string) sdk.TraceBlock {
+ t.Helper()
+ builder := sdktest.NewTrace(traceID).
+ TagAs("start_time", valuetype.ValueTypeTimestamp, int64(0)).
+ Tag("latency", durationMs).
+ Tag("is_error", errorValue)
+ if entries != nil {
+ builder.Tag("tags", entries)
+ }
+ block, buildErr := builder.Build()
+ require.NoError(t, buildErr)
+ return block
+}
+
+func malformedMetricTagBlock() sdk.TraceBlock {
+ return sdk.TraceBlock{TraceID: "malformed", Tags: []sdk.TagColumn{{
+ Name: "tags", ValueType: valuetype.ValueTypeStrArr, Values: [][]byte{[]byte("unterminated")},
+ }}}
+}
+
+func traceIDForSampleResult(keep bool, rate float64) string {
+ for candidateIndex := 0; ; candidateIndex++ {
+ traceID := fmt.Sprintf("sample-%d", candidateIndex)
+ if (sampleFraction(traceID) < rate) == keep {
+ return traceID
+ }
+ }
+}
+
+func counterCallsByLabels(calls []counterCall) map[string]float64 {
+ result := make(map[string]float64, len(calls))
+ for _, call := range calls {
+ result[strings.Join(call.labels, "/")] += call.delta
+ }
+ return result
+}
+
+func sumCounterDeltas(calls []counterCall) float64 {
+ var total float64
+ for _, call := range calls {
+ total += call.delta
+ }
+ return total
+}
diff --git a/plugins/skywalking/internal/tracesampler/metrics_test.go b/plugins/skywalking/internal/tracesampler/metrics_test.go
new file mode 100644
index 000000000..ce6a2eee6
--- /dev/null
+++ b/plugins/skywalking/internal/tracesampler/metrics_test.go
@@ -0,0 +1,163 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package tracesampler
+
+import (
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "github.com/apache/skywalking-banyandb/pkg/encoding/vararray"
+ "github.com/apache/skywalking-banyandb/pkg/pb/v1/valuetype"
+ "github.com/apache/skywalking-banyandb/pkg/pipeline/sdk"
+)
+
+type counterCall struct {
+ labels []string
+ delta float64
+}
+
+type recordingCounter struct{ calls []counterCall }
+
+func (r *recordingCounter) Inc(delta float64, labels ...string) {
+ r.calls = append(r.calls, counterCall{delta: delta, labels: append([]string(nil), labels...)})
+}
+
+type testMeter struct {
+ counter *recordingCounter
+ registered map[string][]string
+}
+
+func (m *testMeter) Counter(name string, labelNames ...string) sdk.Counter {
+ if m.registered == nil {
+ m.registered = make(map[string][]string)
+ }
+ m.registered[name] = append([]string(nil), labelNames...)
+ if name == decisionMetricName {
+ return m.counter
+ }
+ return discardCounter{}
+}
+func (m *testMeter) Gauge(_ string, _ ...string) sdk.Gauge { return nil }
+func (m *testMeter) Histogram(_ string, _ []float64, _ ...string) sdk.Histogram {
+ return nil
+}
+
+type testHost struct{ meter *testMeter }
+
+type discardCounter struct{}
+
+func (discardCounter) Inc(_ float64, _ ...string) {}
+
+func (h *testHost) Meter() sdk.Meter { return h.meter }
+func (h *testHost) Logger() sdk.Logger { return nil }
+
+func TestSamplerImplementsHostAware(t *testing.T) {
+ var _ sdk.HostAware = (*Sampler)(nil)
+}
+
+func TestUseHostRegistersDecisionMetricContract(t *testing.T) {
+ plugin, newErr := New([]byte(`{"healthySampleRate":0}`), segmentSchema)
+ require.NoError(t, newErr)
+ meter := &testMeter{counter: &recordingCounter{}}
+ plugin.(*Sampler).UseHost(&testHost{meter: meter})
+ assert.Equal(t, []string{decisionLabelVerdict, decisionLabelRule}, meter.registered[decisionMetricName])
+ assert.Empty(t, meter.registered[rowMetricName])
+ assert.Equal(t, []string{decisionLabelRule}, meter.registered[rowDroppedMetricName])
+ assert.Empty(t, meter.registered[rowCountUnavailableMetricName])
+}
+
+func TestDecideAggregatesDecisionMetricsByBatch(t *testing.T) {
+ plugin, newErr := New([]byte(`{"healthySampleRate":0}`), segmentSchema)
+ require.NoError(t, newErr)
+ sampler := plugin.(*Sampler)
+ counter := &recordingCounter{}
+ sampler.UseHost(&testHost{meter: &testMeter{counter: counter}})
+
+ verdict, decideErr := sampler.Decide(&sdk.TraceBatch{Traces: []sdk.TraceBlock{{TraceID: "a"}, {TraceID: "b"}, {TraceID: "c"}}})
+ require.NoError(t, decideErr)
+ assert.Equal(t, []bool{false, false, false}, verdict.Keep)
+ require.Len(t, counter.calls, 1, "one metric write represents every trace with the same reason")
+ assert.Equal(t, float64(3), counter.calls[0].delta)
+ assert.Equal(t, []string{decisionVerdictDrop, decisionReasonNoKeepRuleLabel}, counter.calls[0].labels)
+}
+
+func TestDecideWithoutHostPreservesVerdict(t *testing.T) {
+ plugin, newErr := New([]byte(`{"healthySampleRate":1}`), segmentSchema)
+ require.NoError(t, newErr)
+ verdict, decideErr := plugin.Decide(&sdk.TraceBatch{Traces: []sdk.TraceBlock{{TraceID: "a"}}})
+ require.NoError(t, decideErr)
+ assert.Equal(t, []bool{true}, verdict.Keep)
+}
+
+func TestNewRejectsMoreThanFixedTagSlots(t *testing.T) {
+ rules := make([]string, maxKeepTagRules+1)
+ for ruleIndex := range rules {
+ rules[ruleIndex] = `{"tagKey":"key","exists":true}`
+ }
+ _, newErr := New([]byte(`{"keepTagRules":[`+strings.Join(rules, ",")+`]}`), segmentSchema)
+ require.Error(t, newErr)
+ assert.Contains(t, newErr.Error(), "maximum is 32")
+}
+
+func TestDecideTraceReasons(t *testing.T) {
+ encodedTag := vararray.MarshalVarArray(nil, []byte("db.type=PostgreSQL"))
+ tagBlock := sdk.TraceBlock{TraceID: "tag", Tags: []sdk.TagColumn{{
+ Name: "tags", ValueType: valuetype.ValueTypeStrArr, Values: [][]byte{encodedTag},
+ }}}
+ tests := []struct {
+ name string
+ config string
+ block sdk.TraceBlock
+ want traceDecision
+ }{
+ {
+ name: "tag rule", config: `{"keepTagRules":[{"tagKey":"missing","exists":true},{"tagKey":"db.type","equals":"PostgreSQL"}]}`,
+ block: tagBlock, want: traceDecision{keep: true, reason: decisionReasonTagRule, tagRuleIndex: 1},
+ },
+ {
+ name: "duration decode failure", config: `{"durationThresholdMs":1}`,
+ block: sdk.TraceBlock{}, want: traceDecision{keep: true, reason: decisionReasonDecodeDuration},
+ },
+ {
+ name: "error decode failure", config: `{"keepErrors":true}`,
+ block: sdk.TraceBlock{}, want: traceDecision{keep: true, reason: decisionReasonDecodeError},
+ },
+ {
+ name: "healthy sample", config: `{"healthySampleRate":1}`,
+ block: sdk.TraceBlock{TraceID: "sample"}, want: traceDecision{keep: true, reason: decisionReasonHealthySample},
+ },
+ {
+ name: "healthy rejected", config: `{"healthySampleRate":0.000000001}`,
+ block: sdk.TraceBlock{TraceID: "reject"}, want: traceDecision{reason: decisionReasonHealthyRejected},
+ },
+ {
+ name: "no keep rule", config: `{"healthySampleRate":0}`,
+ block: sdk.TraceBlock{TraceID: "drop"}, want: traceDecision{reason: decisionReasonNoKeepRule},
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ plugin, newErr := New([]byte(test.config), segmentSchema)
+ require.NoError(t, newErr)
+ assert.Equal(t, test.want, plugin.(*Sampler).decideTrace(&test.block))
+ })
+ }
+}
diff --git a/plugins/skywalking/internal/tracesampler/sampler.go b/plugins/skywalking/internal/tracesampler/sampler.go
index 9f6450e9e..c79e3c729 100644
--- a/plugins/skywalking/internal/tracesampler/sampler.go
+++ b/plugins/skywalking/internal/tracesampler/sampler.go
@@ -330,6 +330,107 @@ type config struct {
KeepErrors bool `json:"keepErrors"`
}
+const (
+ decisionMetricName = "trace_sampler_decisions_total"
+ rowMetricName = "trace_sampler_rows_total"
+ rowDroppedMetricName = "trace_sampler_rows_dropped_total"
+ rowCountUnavailableMetricName = "trace_sampler_row_count_unavailable_total"
+ decisionLabelVerdict = "verdict"
+ decisionLabelRule = "rule"
+ decisionVerdictKeep = "keep"
+ decisionVerdictDrop = "drop"
+ decisionReasonDurationLabel = "duration"
+ decisionReasonErrorLabel = "error"
+ decisionReasonHealthySampleLabel = "healthy_sample"
+ decisionReasonHealthyRejectedLabel = "healthy_rejected"
+ decisionReasonNoKeepRuleLabel = "no_keep_rule"
+ decisionReasonDecodeDurationLabel = "decode_failure_duration"
+ decisionReasonDecodeErrorLabel = "decode_failure_error"
+ decisionReasonDecodeTagsLabel = "decode_failure_tags"
+ decisionReasonTag00Label = "tag_00"
+ decisionReasonTag01Label = "tag_01"
+ decisionReasonTag02Label = "tag_02"
+ decisionReasonTag03Label = "tag_03"
+ decisionReasonTag04Label = "tag_04"
+ decisionReasonTag05Label = "tag_05"
+ decisionReasonTag06Label = "tag_06"
+ decisionReasonTag07Label = "tag_07"
+ decisionReasonTag08Label = "tag_08"
+ decisionReasonTag09Label = "tag_09"
+ decisionReasonTag10Label = "tag_10"
+ decisionReasonTag11Label = "tag_11"
+ decisionReasonTag12Label = "tag_12"
+ decisionReasonTag13Label = "tag_13"
+ decisionReasonTag14Label = "tag_14"
+ decisionReasonTag15Label = "tag_15"
+ decisionReasonTag16Label = "tag_16"
+ decisionReasonTag17Label = "tag_17"
+ decisionReasonTag18Label = "tag_18"
+ decisionReasonTag19Label = "tag_19"
+ decisionReasonTag20Label = "tag_20"
+ decisionReasonTag21Label = "tag_21"
+ decisionReasonTag22Label = "tag_22"
+ decisionReasonTag23Label = "tag_23"
+ decisionReasonTag24Label = "tag_24"
+ decisionReasonTag25Label = "tag_25"
+ decisionReasonTag26Label = "tag_26"
+ decisionReasonTag27Label = "tag_27"
+ decisionReasonTag28Label = "tag_28"
+ decisionReasonTag29Label = "tag_29"
+ decisionReasonTag30Label = "tag_30"
+ decisionReasonTag31Label = "tag_31"
+)
+
+var tagDecisionReasonLabels = [...]string{
+ decisionReasonTag00Label, decisionReasonTag01Label, decisionReasonTag02Label, decisionReasonTag03Label,
+ decisionReasonTag04Label, decisionReasonTag05Label, decisionReasonTag06Label, decisionReasonTag07Label,
+ decisionReasonTag08Label, decisionReasonTag09Label, decisionReasonTag10Label, decisionReasonTag11Label,
+ decisionReasonTag12Label, decisionReasonTag13Label, decisionReasonTag14Label, decisionReasonTag15Label,
+ decisionReasonTag16Label, decisionReasonTag17Label, decisionReasonTag18Label, decisionReasonTag19Label,
+ decisionReasonTag20Label, decisionReasonTag21Label, decisionReasonTag22Label, decisionReasonTag23Label,
+ decisionReasonTag24Label, decisionReasonTag25Label, decisionReasonTag26Label, decisionReasonTag27Label,
+ decisionReasonTag28Label, decisionReasonTag29Label, decisionReasonTag30Label, decisionReasonTag31Label,
+}
+
+const maxKeepTagRules = len(tagDecisionReasonLabels)
+
+type decisionReason uint8
+
+const (
+ decisionReasonDuration decisionReason = iota
+ decisionReasonError
+ decisionReasonHealthySample
+ decisionReasonHealthyRejected
+ decisionReasonNoKeepRule
+ decisionReasonDecodeDuration
+ decisionReasonDecodeError
+ decisionReasonDecodeTags
+ decisionReasonBuiltInCount
+ decisionReasonTagRule
+)
+
+type decisionMetricLabels struct {
+ verdict string
+ rule string
+}
+
+var builtInDecisionMetricLabels = [...]decisionMetricLabels{
+ {verdict: decisionVerdictKeep, rule: decisionReasonDurationLabel},
+ {verdict: decisionVerdictKeep, rule: decisionReasonErrorLabel},
+ {verdict: decisionVerdictKeep, rule: decisionReasonHealthySampleLabel},
+ {verdict: decisionVerdictDrop, rule: decisionReasonHealthyRejectedLabel},
+ {verdict: decisionVerdictDrop, rule: decisionReasonNoKeepRuleLabel},
+ {verdict: decisionVerdictKeep, rule: decisionReasonDecodeDurationLabel},
+ {verdict: decisionVerdictKeep, rule: decisionReasonDecodeErrorLabel},
+ {verdict: decisionVerdictKeep, rule: decisionReasonDecodeTagsLabel},
+}
+
+type traceDecision struct {
+ reason decisionReason
+ tagRuleIndex uint8
+ keep bool
+}
+
// Sampler keeps a trace when any sure-keep rule matches, and otherwise admits a
// deterministic fraction of the healthy remainder. It implements sdk.Sampler.
type Sampler struct {
@@ -339,6 +440,10 @@ type Sampler struct {
startTimeTag string
rules []rule
requiredTags []string
+ decisions sdk.Counter
+ rowsTotal sdk.Counter
+ rowsDropped sdk.Counter
+ rowCountUnavailable sdk.Counter
errorRule rule
durationThresholdMs int64
durationTagNanosPerUnit int64
@@ -388,6 +493,9 @@ func New(configJSON []byte, schema Schema) (sdk.Sampler, error) {
if c.HealthySampleRate < 0 || c.HealthySampleRate > 1 {
return nil, fmt.Errorf("tracesampler: healthySampleRate %v out of [0,1]", c.HealthySampleRate)
}
+ if len(c.KeepTagRules) > maxKeepTagRules {
+ return nil, fmt.Errorf("tracesampler: keepTagRules contains %d rules, maximum is %d", len(c.KeepTagRules), maxKeepTagRules)
+ }
s := &Sampler{
arrayColumn: schema.ArrayTagColumn,
rules: c.KeepTagRules,
@@ -528,18 +636,80 @@ func (s *Sampler) Project() sdk.Projection {
// Close releases resources; this sampler holds none.
func (s *Sampler) Close() error { return nil }
+// UseHost binds the sampler's decision counter to its group-scoped host.
+func (s *Sampler) UseHost(host sdk.Host) {
+ meter := host.Meter()
+ s.decisions = meter.Counter(decisionMetricName, decisionLabelVerdict, decisionLabelRule)
+ s.rowsTotal = meter.Counter(rowMetricName)
+ s.rowsDropped = meter.Counter(rowDroppedMetricName, decisionLabelRule)
+ s.rowCountUnavailable = meter.Counter(rowCountUnavailableMetricName)
+}
+
// Decide returns a keep-mask aligned to batch.Traces. The batch is read-only.
// It never returns an error: a per-row decode failure fails open for that trace
// (kept), so one malformed value can never make the sampler drop data.
func (s *Sampler) Decide(batch *sdk.TraceBatch) (sdk.Verdict, error) {
keep := make([]bool, len(batch.Traces))
- for i := range batch.Traces {
- keep[i] = s.keepTrace(&batch.Traces[i])
+ var reasonCounts [decisionReasonBuiltInCount]uint64
+ var droppedReasonRowCounts [decisionReasonBuiltInCount]uint64
+ var tagReasonCounts [maxKeepTagRules]uint64
+ var totalRows uint64
+ var unavailableRowCounts uint64
+ for traceIndex := range batch.Traces {
+ decision := s.decideTrace(&batch.Traces[traceIndex])
+ keep[traceIndex] = decision.keep
+ if s.decisions != nil {
+ rowCount := batch.Traces[traceIndex].Len()
+ if decision.reason == decisionReasonTagRule {
+ tagReasonCounts[decision.tagRuleIndex]++
+ } else {
+ reasonCounts[decision.reason]++
+ }
+ if rowCount > 0 {
+ totalRows += uint64(rowCount)
+ if !decision.keep {
+ droppedReasonRowCounts[decision.reason] += uint64(rowCount)
+ }
+ } else {
+ unavailableRowCounts++
+ }
+ }
+ }
+ if s.decisions != nil {
+ emitSamplerCounts(s.decisions, reasonCounts, tagReasonCounts)
+ if totalRows > 0 {
+ s.rowsTotal.Inc(float64(totalRows))
+ }
+ for reasonIndex, rowCount := range droppedReasonRowCounts {
+ if rowCount > 0 {
+ s.rowsDropped.Inc(float64(rowCount), builtInDecisionMetricLabels[reasonIndex].rule)
+ }
+ }
+ if unavailableRowCounts > 0 {
+ s.rowCountUnavailable.Inc(float64(unavailableRowCounts))
+ }
}
return sdk.Verdict{Keep: keep}, nil
}
-// keepTrace applies the sure-keep rules, then the deterministic healthy sample.
+func emitSamplerCounts(counter sdk.Counter, builtInCounts [decisionReasonBuiltInCount]uint64,
+ tagCounts [maxKeepTagRules]uint64,
+) {
+ for reasonIndex, count := range builtInCounts {
+ if count == 0 {
+ continue
+ }
+ labels := builtInDecisionMetricLabels[reasonIndex]
+ counter.Inc(float64(count), labels.verdict, labels.rule)
+ }
+ for ruleIndex, count := range tagCounts {
+ if count > 0 {
+ counter.Inc(float64(count), decisionVerdictKeep, tagDecisionReasonLabels[ruleIndex])
+ }
+ }
+}
+
+// decideTrace applies the sure-keep rules, then the deterministic healthy sample.
// Any decode error encountered while evaluating a sure-keep predicate keeps the
// trace (fail open), never drops it.
//
@@ -555,13 +725,16 @@ func (s *Sampler) Decide(batch *sdk.TraceBatch) (sdk.Verdict, error) {
// inlined lazy decode exists to avoid repeated decode work and the per-row
// allocations it triggers, and to keep failure semantics consistent across
// every rule in the trace.
-func (s *Sampler) keepTrace(b *sdk.TraceBlock) bool {
+func (s *Sampler) decideTrace(traceBlock *sdk.TraceBlock) traceDecision {
// Duration keep: the trace's end-to-end envelope reaches the threshold.
// The early return keeps entriesPtr nil, so no pool release is needed.
if s.durationThresholdMs > 0 {
- hit, durErr := s.hasSlowTrace(b)
- if durErr != nil || hit {
- return true
+ hit, durErr := s.hasSlowTrace(traceBlock)
+ if durErr != nil {
+ return traceDecision{keep: true, reason: decisionReasonDecodeDuration}
+ }
+ if hit {
+ return traceDecision{keep: true, reason: decisionReasonDuration}
}
}
// Lazy tag-array decode. Entries are computed on first call and reused for
@@ -600,37 +773,41 @@ func (s *Sampler) keepTrace(b *sdk.TraceBlock) bool {
entriesErr error
decoded bool
)
- keep := false
+ decision := traceDecision{reason: decisionReasonNoKeepRule}
// Error keep.
if s.keepErrors {
if s.errorTagInArray {
if !decoded {
decoded = true
- entriesPtr, stableBufPtr, entriesErr = arrayEntries(b.Tag(s.arrayColumn))
+ entriesPtr, stableBufPtr, entriesErr = arrayEntries(traceBlock.Tag(s.arrayColumn))
}
- if entriesErr != nil || matchEntries(*entriesPtr, &s.errorRule) {
- keep = true
+ if entriesErr != nil {
+ decision = traceDecision{keep: true, reason: decisionReasonDecodeError}
+ } else if matchEntries(*entriesPtr, &s.errorRule) {
+ decision = traceDecision{keep: true, reason: decisionReasonError}
}
} else {
- hit, errColErr := s.hasErrorColumn(b)
- if errColErr != nil || hit {
- keep = true
+ hit, errColErr := s.hasErrorColumn(traceBlock)
+ if errColErr != nil {
+ decision = traceDecision{keep: true, reason: decisionReasonDecodeError}
+ } else if hit {
+ decision = traceDecision{keep: true, reason: decisionReasonError}
}
}
}
// Sure-keep tag rules. The decode is hoisted out of the loop so the
// lazy fetch fires exactly once even when multiple rules are configured.
- if !keep && len(s.rules) > 0 {
+ if !decision.keep && len(s.rules) > 0 {
if !decoded {
decoded = true
- entriesPtr, stableBufPtr, entriesErr = arrayEntries(b.Tag(s.arrayColumn))
+ entriesPtr, stableBufPtr, entriesErr = arrayEntries(traceBlock.Tag(s.arrayColumn))
}
if entriesErr != nil {
- keep = true
+ decision = traceDecision{keep: true, reason: decisionReasonDecodeTags}
} else {
- for i := range s.rules {
- if matchEntries(*entriesPtr, &s.rules[i]) {
- keep = true
+ for ruleIndex := range s.rules {
+ if matchEntries(*entriesPtr, &s.rules[ruleIndex]) {
+ decision = traceDecision{keep: true, reason: decisionReasonTagRule, tagRuleIndex: uint8(ruleIndex)}
break
}
}
@@ -638,8 +815,12 @@ func (s *Sampler) keepTrace(b *sdk.TraceBlock) bool {
}
// Healthy remainder: deterministic hash(trace_id) < rate, stable across
// re-evaluation at merge and finalization.
- if !keep && s.healthySampleRate > 0 && sampleFraction(b.TraceID) < s.healthySampleRate {
- keep = true
+ if !decision.keep && s.healthySampleRate > 0 {
+ if sampleFraction(traceBlock.TraceID) < s.healthySampleRate {
+ decision = traceDecision{keep: true, reason: decisionReasonHealthySample}
+ } else {
+ decision.reason = decisionReasonHealthyRejected
+ }
}
// Release only what a decode acquired. Keying this off decoded rather than
// off the pointers is what keeps the flag honest: it is the same flag both
@@ -648,7 +829,7 @@ func (s *Sampler) keepTrace(b *sdk.TraceBlock) bool {
releaseEntries(entriesPtr)
releaseStableBuf(stableBufPtr)
}
- return keep
+ return decision
}
// nanosPerMillis converts the millisecond threshold to nanoseconds for the
@@ -656,7 +837,7 @@ func (s *Sampler) keepTrace(b *sdk.TraceBlock) bool {
const nanosPerMillis = int64(1_000_000)
// errNoDurationEnvelope reports that no row yielded a usable start/duration pair,
-// so the envelope could not be computed at all. keepTrace treats any error as a
+// so the envelope could not be computed at all. decideTrace treats any error as a
// keep, which is the intended reading: "can't tell", not "not slow". The columns
// are declared by the Schema, so their absence means the block was written under a
// different schema — typically the wrong plugin for the group — and answering
@@ -797,7 +978,7 @@ func (s *Sampler) hasSlowTrace(b *sdk.TraceBlock) (bool, error) {
// Bounded by aggregate capacity: each retained slice's cap is counted at 8 bytes
// per string header, so a 64 KiB cap supports ~8000 cap-1 slices or ~340 cap-24
// slices (the realistic 3-row × 8-entry trace). The pool is checked-out at
-// arrayEntries entry and released by keepTrace once matchEntries is done with
+// arrayEntries entry and released by decideTrace once matchEntries is done with
// it; the call site owns the slice for the lifetime of the rule evaluation.
//
// Bounded[T] requires T to be comparable, so the pool element is *[]string
@@ -907,7 +1088,7 @@ func releaseStableBuf(b *[]byte) {
// arrayEntries decodes every entry of the flattened tag array, flattened across
// rows. All tag predicates are existential over rows and entries, so collapsing
// the rows loses nothing — and decoding once is what keeps the in-place string
-// array decode from corrupting later reads (see keepTrace).
+// array decode from corrupting later reads (see decideTrace).
//
// The returned *[]string is a pool-managed slice. The pointer is what the
// caller stores; passing it by value (not by address) keeps the caller's
@@ -959,9 +1140,9 @@ func arrayEntries(col *sdk.TagColumn) (*[]string, *[]byte, error) {
// never leak into the new one (matchEntries iterates whatever the length says).
//
// Ownership note: arrayEntries returns the pool pointer to its caller
- // (keepTrace), which calls releaseEntries after matchEntries no longer
+ // (decideTrace), which calls releaseEntries after matchEntries no longer
// references it. The error path here releases locally because the slice
- // never escapes to keepTrace.
+ // never escapes to decideTrace.
out := acquireEntries()
// The per-row copy buffer is acquired LAZILY, on the first row that actually
// carries an escape. A column whose rows are all escape-free never needs one,
@@ -1082,7 +1263,7 @@ func arrayEntries(col *sdk.TagColumn) (*[]string, *[]byte, error) {
}
// errNoErrorColumn reports that the schema's error column is absent from the block.
-// keepTrace treats any error as a keep, which is the intended reading: the column is
+// decideTrace treats any error as a keep, which is the intended reading: the column is
// schema-declared, so its absence means the block was written under a different
// schema — typically the wrong plugin attached to the group — and answering "no
// error" there would drop every trace keepErrors was enabled to save. This mirrors
diff --git a/plugins/skywalking/internal/tracesampler/sampler_bench_test.go b/plugins/skywalking/internal/tracesampler/sampler_bench_test.go
index b99c352be..6bbcd9e36 100644
--- a/plugins/skywalking/internal/tracesampler/sampler_bench_test.go
+++ b/plugins/skywalking/internal/tracesampler/sampler_bench_test.go
@@ -35,6 +35,7 @@ package tracesampler
import (
"fmt"
+ "sync/atomic"
"testing"
"github.com/apache/skywalking-banyandb/pkg/pb/v1/valuetype"
@@ -168,7 +169,10 @@ func runDecide(b *testing.B, s sdk.Sampler, batch *sdk.TraceBatch) {
benchSink = v
}
b.StopTimer()
- b.ReportMetric(float64(b.Elapsed().Nanoseconds())/float64(b.N*len(batch.Traces)), "ns/trace")
+ elapsed := b.Elapsed()
+ traceCount := float64(b.N * len(batch.Traces))
+ b.ReportMetric(float64(elapsed.Nanoseconds())/traceCount, "ns/trace")
+ b.ReportMetric(traceCount/elapsed.Seconds(), "traces/s")
}
// BenchmarkDecide isolates what each keep rule costs, so a config change's price is
@@ -254,3 +258,129 @@ func BenchmarkDecide_BatchSize(b *testing.B) {
}
}
}
+
+type benchmarkCounter struct{ total atomic.Uint64 }
+
+func (c *benchmarkCounter) Inc(delta float64, _ ...string) { c.total.Add(uint64(delta)) }
+
+type benchmarkMeter struct {
+ decisions *benchmarkCounter
+ rows *benchmarkCounter
+ droppedRows *benchmarkCounter
+ unavailable *benchmarkCounter
+}
+
+func (m *benchmarkMeter) Counter(name string, _ ...string) sdk.Counter {
+ switch name {
+ case decisionMetricName:
+ return m.decisions
+ case rowMetricName:
+ return m.rows
+ case rowDroppedMetricName:
+ return m.droppedRows
+ case rowCountUnavailableMetricName:
+ return m.unavailable
+ default:
+ return &benchmarkCounter{}
+ }
+}
+func (m *benchmarkMeter) Gauge(_ string, _ ...string) sdk.Gauge { return nil }
+func (m *benchmarkMeter) Histogram(_ string, _ []float64, _ ...string) sdk.Histogram {
+ return nil
+}
+
+type benchmarkHost struct{ meter *benchmarkMeter }
+
+func (h *benchmarkHost) Meter() sdk.Meter { return h.meter }
+func (h *benchmarkHost) Logger() sdk.Logger { return nil }
+
+// BenchmarkDecide_TelemetryRegression compares the unchanged no-host path with
+// batch-aggregated decision telemetry over both shipped schemas. It is intended
+// for repeated benchstat runs when changing the sampler hot path.
+func BenchmarkDecide_TelemetryRegression(b *testing.B) {
+ for _, plugin := range benchPlugins() {
+ config, batch := benchmarkDecisionMetricBatch(b, plugin)
+ b.Run(plugin.name+"/disabled", func(b *testing.B) {
+ sampler := mustSampler(b, config, plugin.schema)
+ runDecide(b, sampler, batch)
+ })
+ b.Run(plugin.name+"/enabled", func(b *testing.B) {
+ sampler := mustSampler(b, config, plugin.schema).(*Sampler)
+ meter := &benchmarkMeter{
+ decisions: &benchmarkCounter{}, rows: &benchmarkCounter{}, droppedRows: &benchmarkCounter{}, unavailable: &benchmarkCounter{},
+ }
+ sampler.UseHost(&benchmarkHost{meter: meter})
+ runDecide(b, sampler, batch)
+ expectedTotal := uint64(b.N * len(batch.Traces))
+ if actualTotal := meter.decisions.total.Load(); actualTotal != expectedTotal {
+ b.Fatalf("telemetry counter total %d, want %d", actualTotal, expectedTotal)
+ }
+ if totalRows := meter.rows.total.Load(); totalRows > 0 {
+ b.ReportMetric(100*float64(meter.droppedRows.total.Load())/float64(totalRows), "row-drop-%")
+ }
+ })
+ }
+}
+
+// benchmarkDecisionMetricBatch exercises every decision family and both tag
+// decode paths in the telemetry regression benchmark.
+func benchmarkDecisionMetricBatch(b *testing.B, plugin benchPlugin) (string, *sdk.TraceBatch) {
+ b.Helper()
+ config := fmt.Sprintf(`{
+ "durationThresholdMs":%d,"keepErrors":true,"healthySampleRate":0.5,
+ "keepTagRules":[
+ {"tagKey":"missing","exists":true},
+ {"tagKey":"db.type","equals":"PostgreSQL"},
+ {"tagKey":"http.method","in":["POST","PUT"]}
+ ]}`, plugin.durationMs)
+ healthyID := benchmarkTraceIDForSampleResult(true, 0.5)
+ rejectedID := benchmarkTraceIDForSampleResult(false, 0.5)
+ durationHit := plugin.durationMs*nanosPerMillis/plugin.schema.DurationTagNanosPerUnit + 1
+ blocks := []sdk.TraceBlock{
+ benchmarkDecisionMetricBlock(b, plugin, "duration", durationHit, false, nil),
+ benchmarkDecisionMetricBlock(b, plugin, "error", 1, true, nil),
+ benchmarkDecisionMetricBlock(b, plugin, "tag-equals", 1, false, []string{"db.type=PostgreSQL"}),
+ benchmarkDecisionMetricBlock(b, plugin, "tag-in", 1, false, []string{"note=a|b", "http.method=PUT"}),
+ benchmarkDecisionMetricBlock(b, plugin, healthyID, 1, false, nil),
+ benchmarkDecisionMetricBlock(b, plugin, rejectedID, 1, false, nil),
+ }
+ const realisticBatchSize = 64
+ seedCount := len(blocks)
+ for len(blocks) < realisticBatchSize {
+ blocks = append(blocks, blocks[len(blocks)%seedCount])
+ }
+ return config, sdktest.Batch(blocks...)
+}
+
+func benchmarkDecisionMetricBlock(b *testing.B, plugin benchPlugin, traceID string, duration int64, hasError bool, entries []string) sdk.TraceBlock {
+ b.Helper()
+ builder := sdktest.NewTrace(traceID).
+ TagAs(plugin.startCol, valuetype.ValueTypeTimestamp, int64(0)).
+ Tag(plugin.durCol, duration)
+ if plugin.errCol != "" {
+ errorValue := int64(0)
+ if hasError {
+ errorValue = 1
+ }
+ builder.Tag(plugin.errCol, errorValue)
+ } else if hasError {
+ entries = append(entries, "error")
+ }
+ if entries != nil {
+ builder.Tag(plugin.arrayCol, entries)
+ }
+ block, buildErr := builder.Build()
+ if buildErr != nil {
+ b.Fatalf("build decision metric fixture: %v", buildErr)
+ }
+ return block
+}
+
+func benchmarkTraceIDForSampleResult(keep bool, rate float64) string {
+ for candidateIndex := 0; ; candidateIndex++ {
+ traceID := fmt.Sprintf("benchmark-sample-%d", candidateIndex)
+ if (sampleFraction(traceID) < rate) == keep {
+ return traceID
+ }
+ }
+}
diff --git a/plugins/skywalking/internal/tracesampler/sampler_test.go b/plugins/skywalking/internal/tracesampler/sampler_test.go
index 18ebd5671..4b1d1df05 100644
--- a/plugins/skywalking/internal/tracesampler/sampler_test.go
+++ b/plugins/skywalking/internal/tracesampler/sampler_test.go
@@ -893,7 +893,7 @@ func TestDecide_FailOpenOnDecodeError(t *testing.T) {
// TestDecide_MissingTagArrayColumn is a regression test for the nil-pool-element
// dereference. arrayEntries must return a non-nil *[]string (an empty slice) when
-// the array column is missing from the block, so keepTrace can dereference it
+// the array column is missing from the block, so decideTrace can dereference it
// without crashing. The pre-pool version of arrayEntries returned a nil slice and
// matchEntries(nil, ...) was safe by accident; the pool-typed version is not.
func TestDecide_MissingTagArrayColumn(t *testing.T) {