You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ha...@apache.org on 2026/10/23 09:07:00 UTC
(skywalking-banyandb) branch main updated: Add lifecycle trace sampling with durable crash recovery (#1421)
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
The following commit(s) were added to refs/heads/main by this push:
new 96c46b7df Add lifecycle trace sampling with durable crash recovery (#1421)
96c46b7df is described below
commit 96c46b7df0989a1397201d29aa28ca8b95bf3856
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Fri Oct 23 17:06:55 2026 +0800
Add lifecycle trace sampling with durable crash recovery (#1421)
---
CHANGES.md | 1 +
api/validate/validate.go | 44 +-
api/validate/validate_test.go | 98 ++-
banyand/Dockerfile | 7 +-
banyand/backup/lifecycle/migration_integration.go | 44 ++
banyand/backup/lifecycle/progress.go | 24 +-
banyand/backup/lifecycle/row_replay_pipeline.go | 66 +-
.../backup/lifecycle/sampling_benchmark_test.go | 86 +++
.../backup/lifecycle/sampling_destination_test.go | 255 ++++++++
banyand/backup/lifecycle/sampling_policy.go | 228 +++++++
banyand/backup/lifecycle/sampling_policy_test.go | 165 +++++
banyand/backup/lifecycle/sampling_prepare.go | 338 +++++++++++
banyand/backup/lifecycle/sampling_real_test.go | 379 ++++++++++++
banyand/backup/lifecycle/sampling_recovery.go | 262 ++++++++
banyand/backup/lifecycle/sampling_replay.go | 180 ++++++
.../backup/lifecycle/sampling_roundtrip_test.go | 298 +++++++++
banyand/backup/lifecycle/sampling_store.go | 366 +++++++++++
banyand/backup/lifecycle/sampling_store_test.go | 224 +++++++
banyand/backup/lifecycle/sampling_visitor.go | 157 +++++
banyand/backup/lifecycle/service.go | 82 +++
banyand/backup/lifecycle/steps.go | 14 +-
.../backup/lifecycle/trace_migration_visitor.go | 17 +
banyand/internal/dump/iterator.go | 28 +
banyand/internal/dump/iterator_seek_test.go | 38 ++
.../pipeline/loader.go} | 130 ++--
banyand/internal/pipeline/loader_test.go | 70 +++
banyand/internal/pipeline/session.go | 253 ++++++++
banyand/internal/pipeline/session_test.go | 147 +++++
banyand/metadata/schema/property/client.go | 8 +
.../schema/property/pipeline_single_trace_test.go | 44 ++
banyand/trace/merge_benchmark_observer_test.go | 3 +-
banyand/trace/pipeline_chain.go | 219 ++-----
banyand/trace/pipeline_loader.go | 169 +-----
banyand/trace/pipeline_loader_test.go | 4 +-
banyand/trace/snapshot_scanner.go | 668 +++++++++++++++++++++
banyand/trace/snapshot_scanner_test.go | 185 ++++++
docs/design/0.12.0/README.md | 1 +
docs/design/0.12.0/lifecycle-trace-sampling.md | 418 +++++++++++++
docs/design/README.md | 1 +
docs/interacting/data-lifecycle.md | 16 +
docs/operation/lifecycle.md | 170 ++++++
docs/operation/plugins-debugging.md | 46 +-
docs/operation/plugins.md | 75 ++-
pkg/compress/zstd/bounded_test.go | 54 ++
pkg/compress/zstd/zstd.go | 42 ++
pkg/encoding/bytes.go | 34 +-
pkg/encoding/bytes_limit_test.go | 53 ++
pkg/encoding/dictionary.go | 43 +-
test/cases/lifecycle/lifecycle.go | 3 +-
test/cases/lifecycle/orphan.go | 18 +
test/cases/lifecycle/sampling.go | 274 +++++++++
test/cases/lifecycle/sampling_process.go | 232 +++++++
test/cases/lifecycle/sampling_recovery.go | 276 +++++++++
.../lifecycle/testdata/sampling-plugin/main.go | 43 ++
test/cases/measure/ql_generated_reuse_schema.go | 29 +-
test/e2e-v2/cases/lifecycle/lifecycle-cases.yaml | 4 +
.../distributed/lifecycle_real/README.md | 30 +
.../distributed/lifecycle_real/real_test.go | 434 +++++++++++++
.../distributed/lifecycle_real/verify_test.go | 442 ++++++++++++++
59 files changed, 7554 insertions(+), 485 deletions(-)
diff --git a/CHANGES.md b/CHANGES.md
index 726f05e31..74b83cff5 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -6,6 +6,7 @@ Release Notes.
### Features
+- Sample traces with source-stage native plugins before lifecycle migration, with durable decisions, checkpointed replay, and crash recovery.
- Add typed raw measure QL with WHERE time, aliases and bounded compiled-plan reuse, executed over shared columnar native batches.
- Add a native ICE v3 encoder that writes committed index generations compatible with existing readers.
- Cut the Property database's index writer over to the native segment plugin, selected by both the `property` and `schema-server` roles. `banyand/property/db.OpenDB` now acquires its exclusive `<Location>/lock` before scanning shard directories or opening any writer, and releases it if a later startup step fails. A rollback (flipping the selection back and restarting) reopens the same on-disk shards with the retained legacy writer unchanged.
diff --git a/api/validate/validate.go b/api/validate/validate.go
index e01bc5a5a..58f22ae91 100644
--- a/api/validate/validate.go
+++ b/api/validate/validate.go
@@ -212,35 +212,41 @@ func validateTracePipelineConfig(cfg *commonv1.TracePipelineConfig) error {
return fmt.Errorf("proto validation failed: %w", validateErr)
}
for idx, plugin := range cfg.GetPlugins() {
- sp := plugin.GetSampler()
- if sp == nil {
- continue
- }
- if sp.GetPath() == "" {
- return fmt.Errorf("plugins[%d].sampler.path is empty", idx)
- }
- // Reject absolute paths and path segments that escape via "..".
- cleaned := filepath.Clean(sp.GetPath())
- if filepath.IsAbs(cleaned) {
- return fmt.Errorf("plugins[%d].sampler.path must be a relative path, got %q", idx, sp.GetPath())
- }
- if cleaned == ".." || strings.HasPrefix(cleaned, ".."+string(filepath.Separator)) {
- return fmt.Errorf("plugins[%d].sampler.path %q escapes the trusted directory", idx, sp.GetPath())
- }
- if sp.GetAbiVersion() != sdk.ABIVersion {
- return fmt.Errorf("plugins[%d].sampler.abi_version must be %d, got %d", idx, sdk.ABIVersion, sp.GetAbiVersion())
+ if err := validatePipelineSampler(plugin, fmt.Sprintf("plugins[%d]", idx)); err != nil {
+ return err
}
}
for idx, stage := range cfg.GetStages() {
for jdx, plugin := range stage.GetPlugins() {
- if plugin.GetSampler() != nil {
- return fmt.Errorf("stages[%d].plugins[%d]: stages[].plugins are not supported in v1; declare plugins at the top level", idx, jdx)
+ if err := validatePipelineSampler(plugin, fmt.Sprintf("stages[%d].plugins[%d]", idx, jdx)); err != nil {
+ return err
}
}
}
return nil
}
+func validatePipelineSampler(plugin *commonv1.Plugin, location string) error {
+ sampler := plugin.GetSampler()
+ if sampler == nil {
+ return nil
+ }
+ if sampler.GetPath() == "" {
+ return fmt.Errorf("%s.sampler.path is empty", location)
+ }
+ cleaned := filepath.Clean(sampler.GetPath())
+ if filepath.IsAbs(cleaned) {
+ return fmt.Errorf("%s.sampler.path must be a relative path, got %q", location, sampler.GetPath())
+ }
+ if cleaned == ".." || strings.HasPrefix(cleaned, ".."+string(filepath.Separator)) {
+ return fmt.Errorf("%s.sampler.path %q escapes the trusted directory", location, sampler.GetPath())
+ }
+ if sampler.GetAbiVersion() != sdk.ABIVersion {
+ return fmt.Errorf("%s.sampler.abi_version must be %d, got %d", location, sdk.ABIVersion, sampler.GetAbiVersion())
+ }
+ return nil
+}
+
// Stream validates the provided Stream object.
// It checks for nil values, empty strings, and unspecified enum values.
func Stream(stream *databasev1.Stream) error {
diff --git a/api/validate/validate_test.go b/api/validate/validate_test.go
index 61e125932..203f512aa 100644
--- a/api/validate/validate_test.go
+++ b/api/validate/validate_test.go
@@ -180,10 +180,7 @@ func TestMeasurePassesWithNonPrefixShardingKey(t *testing.T) {
assert.NoError(t, err, "Measure() must not reject a non-prefix sharding key")
}
-func TestValidateTracePipelineConfig_StagePluginsRejected(t *testing.T) {
- // A config with sampler plugins declared only under stages[] must be rejected
- // with a clear error — not silently accepted (which would load zero samplers
- // and retain all traces).
+func TestValidateTracePipelineConfig_StagePluginsAccepted(t *testing.T) {
stagePlugin := &commonv1.Plugin{
Name: "lss",
Kind: &commonv1.Plugin_Sampler{
@@ -201,8 +198,7 @@ func TestValidateTracePipelineConfig_StagePluginsRejected(t *testing.T) {
},
}
err := validateTracePipelineConfig(cfg)
- assert.Error(t, err, "stages[].plugins with a sampler must be rejected")
- assert.Contains(t, err.Error(), "stages[].plugins are not supported in v1")
+ assert.NoError(t, err, "stage sampler chains run during source-stage migration")
}
func TestCheckShardingKeySubset(t *testing.T) {
@@ -369,3 +365,93 @@ func TestMeasureAndStreamStillLoadDuplicateTagNames(t *testing.T) {
assert.NoError(t, Stream(stream))
assert.Error(t, UniqueTagNames(duplicated))
}
+
+func TestValidateTracePipelineConfigStageSamplerValidation(t *testing.T) {
+ for _, test := range []struct {
+ name, path, message, stageContext string
+ abi uint32
+ }{
+ {name: "absolute", path: "/sampler.so", abi: 1, message: "invalid SamplerPlugin.Path", stageContext: "StageRule.Plugins[0]"},
+ {name: "escape", path: "../sampler.so", abi: 1, message: "invalid SamplerPlugin.Path", stageContext: "StageRule.Plugins[0]"},
+ {name: "ABI", path: "sampler.so", abi: 2, message: "abi_version must be 1", stageContext: "stages[0].plugins[0]"},
+ } {
+ t.Run(test.name, func(t *testing.T) {
+ config := &commonv1.TracePipelineConfig{
+ Enabled: true,
+ Stages: []*commonv1.StageRule{{
+ Stage: "hot",
+ Plugins: []*commonv1.Plugin{{
+ Name: "sampler",
+ Kind: &commonv1.Plugin_Sampler{Sampler: &commonv1.SamplerPlugin{
+ Path: test.path,
+ AbiVersion: test.abi,
+ }},
+ }},
+ }},
+ }
+ err := validateTracePipelineConfig(config)
+ assert.ErrorContains(t, err, test.message)
+ assert.ErrorContains(t, err, test.stageContext)
+ })
+ }
+}
+
+func TestGroupAcceptsImplicitHotAndNonterminalStageSampling(t *testing.T) {
+ plugin := &commonv1.Plugin{Name: "sampler", Kind: &commonv1.Plugin_Sampler{Sampler: &commonv1.SamplerPlugin{Path: "sampler.so", AbiVersion: 1}}}
+ group := &commonv1.Group{
+ Metadata: &commonv1.Metadata{Name: "trace-group"}, Catalog: commonv1.Catalog_CATALOG_TRACE,
+ ResourceOpts: &commonv1.ResourceOpts{
+ ShardNum: 1,
+ SegmentInterval: &commonv1.IntervalRule{
+ Unit: commonv1.IntervalRule_UNIT_DAY,
+ Num: 1,
+ },
+ Ttl: &commonv1.IntervalRule{
+ Unit: commonv1.IntervalRule_UNIT_DAY,
+ Num: 1,
+ },
+ Stages: []*commonv1.LifecycleStage{
+ {
+ Name: "warm",
+ ShardNum: 1,
+ SegmentInterval: &commonv1.IntervalRule{
+ Unit: commonv1.IntervalRule_UNIT_DAY,
+ Num: 1,
+ },
+ Ttl: &commonv1.IntervalRule{
+ Unit: commonv1.IntervalRule_UNIT_DAY,
+ Num: 1,
+ },
+ },
+ {
+ Name: "cold",
+ ShardNum: 1,
+ SegmentInterval: &commonv1.IntervalRule{
+ Unit: commonv1.IntervalRule_UNIT_DAY,
+ Num: 1,
+ },
+ Ttl: &commonv1.IntervalRule{
+ Unit: commonv1.IntervalRule_UNIT_DAY,
+ Num: 1,
+ },
+ },
+ },
+ },
+
+ Pipeline: &commonv1.TracePipelineConfig{
+ Enabled: true,
+ Stages: []*commonv1.StageRule{
+ {
+ Stage: "hot",
+ Plugins: []*commonv1.Plugin{plugin},
+ },
+ {
+ Stage: "warm",
+ Plugins: []*commonv1.Plugin{plugin},
+ },
+ },
+ },
+ }
+ assert.NoError(t, Group(group))
+ assert.NoError(t, group.Validate())
+}
diff --git a/banyand/Dockerfile b/banyand/Dockerfile
index 8ff31289b..28db73619 100644
--- a/banyand/Dockerfile
+++ b/banyand/Dockerfile
@@ -115,6 +115,7 @@ COPY --from=reporoot . .
FROM build-plugins-base AS build-plugins-server
ARG GO_LINK_VERSION
RUN go build -trimpath -ldflags "${GO_LINK_VERSION}" -tags slim -o /out/banyand-server-plugins ./banyand/cmd/server
+RUN go build -trimpath -ldflags "${GO_LINK_VERSION}" -tags slim -o /out/banyand-lifecycle-plugins ./banyand/cmd/lifecycle
# build-plugins-so compiles every first-party plugin under plugins/<vendor>/<name>
# with the IDENTICAL toolchain/flags as build-plugins-server above (same base
@@ -151,8 +152,8 @@ RUN set -e; \
# It ships an EMPTY /plugins directory (the trusted-dir mount POINT) and NO .so
# files: plugins are delivered at deploy time by MOUNTING the carrier image
# (final-carrier below) at /plugins. Because /plugins is empty, a mount there
-# shadows nothing. banyand-backup/restore/lifecycle/migration do not host
-# plugins, so they are reused unmodified from the ordinary `make release`
+# shadows nothing. The lifecycle agent hosts the same plugins and is built
+# alongside the data node above. Backup/restore/migration are reused from `make release`
# static build output (this Dockerfile's own context, like the default
# `build-linux` stage above) — the plugin CI job runs `make release` first for
# exactly this.
@@ -169,7 +170,7 @@ WORKDIR /plugins
WORKDIR /
COPY build/bin/linux/${TARGETARCH}/banyand-backup-static /backup
COPY build/bin/linux/${TARGETARCH}/banyand-restore-static /restore
-COPY build/bin/linux/${TARGETARCH}/banyand-lifecycle-static /lifecycle
+COPY --from=build-plugins-server /out/banyand-lifecycle-plugins /lifecycle
COPY build/bin/linux/${TARGETARCH}/banyand-migration-static /migration
ENV GRPC_GO_LOG_SEVERITY_LEVEL=ERROR
diff --git a/banyand/backup/lifecycle/migration_integration.go b/banyand/backup/lifecycle/migration_integration.go
index f4f278bc8..af8bc56db 100644
--- a/banyand/backup/lifecycle/migration_integration.go
+++ b/banyand/backup/lifecycle/migration_integration.go
@@ -19,6 +19,8 @@ package lifecycle
import (
"context"
+ "fmt"
+ "path/filepath"
"strings"
"github.com/apache/skywalking-banyandb/api/common"
@@ -249,6 +251,39 @@ func (pcv *measurePartCountVisitor) VisitPart(_ *timestamp.TimeRange, _ common.S
func migrateTraceWithFileBasedAndProgress(ctx context.Context, tsdbRootPath string, timeRange timestamp.TimeRange, group *GroupConfig,
logger *logger.Logger, progress *Progress, chunkSize int, md metadata.Repo,
) ([]string, error) {
+ var policy *pinnedSamplingPolicy
+ var policyErr error
+ pendingSampling, discoveryErr := hasPreparedSampling(group.SamplingHost.ArtifactRoot, group.Metadata.Name)
+ if discoveryErr != nil {
+ return nil, discoveryErr
+ }
+ if config := group.GetPipeline(); !group.LegacyTraceReplay && config != nil && config.Enabled && len(config.Stages) > 0 && group.SourceStage == "" {
+ return nil, fmt.Errorf("sampling source stage identity is missing")
+ }
+ desired := false
+ if config := group.GetPipeline(); config != nil && config.Enabled {
+ for _, stage := range config.Stages {
+ desired = desired || stage.Stage == group.SourceStage && len(stage.Plugins) > 0
+ }
+ }
+ sampled, modeErr := selectSamplingMode(group.SamplingHost.ArtifactRoot, group.Metadata.Name,
+ filepath.Dir(tsdbRootPath), group.LegacyTraceReplay, desired || pendingSampling)
+ if modeErr != nil {
+ return nil, modeErr
+ }
+ if sampled && !pendingSampling {
+ policy, policyErr = resolveSamplingPolicy(ctx, group, md)
+ }
+ if sampled && policy == nil && policyErr == nil && !pendingSampling {
+ modePath := filepath.Join(group.SamplingHost.ArtifactRoot, group.Metadata.Name, "mode.json")
+ if modeErr = saveSamplingMode(modePath, samplingModeRecord{Snapshot: filepath.Dir(tsdbRootPath)}); modeErr != nil {
+ return nil, modeErr
+ }
+ }
+ if policyErr != nil {
+ return nil, policyErr
+ }
+ defer policy.close()
// Convert segment interval to IntervalRule using storage.MustToIntervalRule
segmentIntervalRule := storage.MustToIntervalRule(group.SegmentInterval)
@@ -273,6 +308,12 @@ func migrateTraceWithFileBasedAndProgress(ctx context.Context, tsdbRootPath stri
group.SourceStage, group.TargetStage, segmentIntervalRule,
)
defer visitor.Close()
+ visitor.samplingPolicy = policy
+ visitor.samplingGroup = group
+ visitor.snapshot = filepath.Dir(tsdbRootPath)
+ if policy != nil || pendingSampling {
+ visitor.samplingRoot = group.SamplingHost.ArtifactRoot
+ }
// Set the planned source-item counts for progress tracking.
if counter != nil {
@@ -285,6 +326,9 @@ func migrateTraceWithFileBasedAndProgress(ctx context.Context, tsdbRootPath stri
if err != nil {
return nil, err
}
+ if err = visitor.completeSampledSeries(); err != nil {
+ return nil, err
+ }
return segmentSuffixes, nil
}
diff --git a/banyand/backup/lifecycle/progress.go b/banyand/backup/lifecycle/progress.go
index 6b4c5c8f9..e98fd1467 100644
--- a/banyand/backup/lifecycle/progress.go
+++ b/banyand/backup/lifecycle/progress.go
@@ -29,6 +29,8 @@ import (
// Progress tracks the lifecycle migration progress to support resume after crash.
type Progress struct {
+ samplingOwnedGroups map[string]bool
+ TraceSamplingReports map[string]samplingReport `json:"trace_sampling_reports,omitempty"`
logger *logger.Logger `json:"-"`
CompletedGroups map[string]bool `json:"completed_groups"`
DeletedStreamGroups map[string]bool `json:"deleted_stream_groups"`
@@ -57,7 +59,6 @@ type Progress struct {
CompletedTraceSeries map[string]map[string]map[common.ShardID]bool `json:"completed_trace_series"`
TraceSeriesCounts map[string]int `json:"trace_series_counts"`
TraceSeriesProgress map[string]int `json:"trace_series_progress"`
-
// Per-source completion tracking (advances *Progress idempotently once per source item).
SourceCompletedStreamParts map[string]map[string]map[common.ShardID]map[uint64]bool `json:"source_completed_stream_parts"`
SourceCompletedStreamSeries map[string]map[string]map[common.ShardID]bool `json:"source_completed_stream_series"`
@@ -66,7 +67,6 @@ type Progress struct {
SourceCompletedMeasureSeries map[string]map[string]map[common.ShardID]bool `json:"source_completed_measure_series"`
SourceCompletedTraceShards map[string]map[string]map[common.ShardID]bool `json:"source_completed_trace_shards"`
SourceCompletedTraceSeries map[string]map[string]map[common.ShardID]bool `json:"source_completed_trace_series"`
-
// Sync-mode breakdown per group: how many parts (measure/stream) or shards
// (trace) were migrated via chunk-sync vs row-replay, plus the real number
// of rows replayed. Accumulated across resume cycles, fed into the report's
@@ -80,26 +80,24 @@ type Progress struct {
TraceChunkSyncShards map[string]uint64 `json:"trace_chunk_sync_shards"`
TraceRowReplayParts map[string]uint64 `json:"trace_row_replay_parts"`
TraceRowReplayRows map[string]uint64 `json:"trace_row_replay_rows"`
-
// OrphanRows reports, per catalog -> group -> deleted measure/stream subject,
// how many rows were archived or discarded because the schema was deleted from
// the registry. Accumulated across resume cycles (orphan parts are marked
// completed, so each row is counted once), fed into the report's orphans.
- OrphanRows map[string]map[string]map[string]uint64 `json:"orphan_rows"`
-
+ OrphanRows map[string]map[string]map[string]uint64 `json:"orphan_rows"`
+ progressFilePath string `json:"-"`
+ SnapshotStreamDir string `json:"snapshot_stream_dir"`
+ SnapshotMeasureDir string `json:"snapshot_measure_dir"`
+ SnapshotTraceDir string `json:"snapshot_trace_dir"`
// MigrationErrors is the single source of structured migration errors for both
// the report (stage/group/catalog/scope/segment/interval/location/message) and
// the per-resource error counts. Reset by ClearErrors at the start of each run
// so the report reflects the current attempt; persisted only for crash
// durability within a run.
- MigrationErrors []MigrationError `json:"migration_errors"`
-
- progressFilePath string `json:"-"`
- SnapshotStreamDir string `json:"snapshot_stream_dir"`
- SnapshotMeasureDir string `json:"snapshot_measure_dir"`
- SnapshotTraceDir string `json:"snapshot_trace_dir"`
- GroupsToProcess []string `json:"groups_to_process"`
- mu sync.Mutex `json:"-"`
+ MigrationErrors []MigrationError `json:"migration_errors"`
+ GroupsToProcess []string `json:"groups_to_process"`
+ mu sync.Mutex `json:"-"`
+ legacyTraceSnapshot bool
}
// SetGroupsToProcess records the set of groups picked up by this migration cycle.
diff --git a/banyand/backup/lifecycle/row_replay_pipeline.go b/banyand/backup/lifecycle/row_replay_pipeline.go
index 7ad9afd4d..d59809e9a 100644
--- a/banyand/backup/lifecycle/row_replay_pipeline.go
+++ b/banyand/backup/lifecycle/row_replay_pipeline.go
@@ -115,8 +115,10 @@ func recordReplayNodeErrors(progress *Progress, group, sourceStage, targetStage,
// confirmations are observed in send order. It is driven by a single goroutine
// (the part's replay loop) and needs no internal locking.
type confirmPipeline struct {
- inflight []chan error
- depth int
+ inflight []chan error
+ onConfirmed []func() error
+ checkpointFailed bool
+ depth int
// waited accumulates, over the pipeline's lifetime, the wall time the driving
// goroutine spent blocked on confirmations (the in-flight bound in add plus
// drain). Since that goroutine is either building or blocked here, it measures
@@ -133,14 +135,34 @@ func newConfirmPipeline() *confirmPipeline {
// so no more than depth batches are ever outstanding, returning that awaited
// error (a non-nil error means an earlier batch must abort the part).
func (p *confirmPipeline) add(ch chan error) error {
+ return p.addConfirmed(ch, nil)
+}
+
+func (p *confirmPipeline) observe(index int, result error) error {
+ if result != nil {
+ p.checkpointFailed = true
+ return result
+ }
+ if !p.checkpointFailed && p.onConfirmed[index] != nil {
+ if err := p.onConfirmed[index](); err != nil {
+ p.checkpointFailed = true
+ return err
+ }
+ }
+ return nil
+}
+
+func (p *confirmPipeline) addConfirmed(ch chan error, confirmed func() error) error {
var awaited error
if len(p.inflight) >= p.depth {
start := time.Now()
- awaited = <-p.inflight[0]
+ awaited = p.observe(0, <-p.inflight[0])
p.waited += time.Since(start)
p.inflight = p.inflight[1:]
+ p.onConfirmed = p.onConfirmed[1:]
}
p.inflight = append(p.inflight, ch)
+ p.onConfirmed = append(p.onConfirmed, confirmed)
return awaited
}
@@ -149,15 +171,16 @@ func (p *confirmPipeline) add(ch chan error) error {
// a drained pipeline is empty.
func (p *confirmPipeline) drain() error {
var first error
- for _, ch := range p.inflight {
+ for index, ch := range p.inflight {
start := time.Now()
- out := <-ch
+ out := p.observe(index, <-ch)
p.waited += time.Since(start)
if first == nil && out != nil {
first = out
}
}
p.inflight = nil
+ p.onConfirmed = nil
return first
}
@@ -172,6 +195,9 @@ func (p *confirmPipeline) drain() error {
// it touches the buffer, and each background confirmation touches just its own
// captured publisher and channel, so no buffer locking is needed.
type batchSender struct {
+ // beforeFlush captures the ending source cursor before automatic flushing.
+ beforeFlush func() (func() error, error)
+
client queue.Client
publisher queue.BatchPublisher
pipeline *confirmPipeline
@@ -191,6 +217,7 @@ type batchSender struct {
maxRows int
maxBytes int
inflightBytes int
+ queuedBytes uint64
// Flush-reason tallies: which cap forced each batch out. A flush triggered by
// both caps at once is attributed to bytes (the memory-protecting cap that
// fires early in the big-body cluster). tailFlushes counts the trailing
@@ -321,6 +348,7 @@ func (s *batchSender) routeAndEnqueue(ctx context.Context, l *logger.Logger, sel
func (s *batchSender) enqueue(ctx context.Context, size int, msg bus.Message) error {
s.batch = append(s.batch, msg)
s.inflightBytes += size
+ s.queuedBytes += uint64(size)
byteCap := s.inflightBytes >= s.maxBytes
rowCap := len(s.batch) >= s.maxRows
if rowCap || byteCap {
@@ -341,6 +369,14 @@ func (s *batchSender) enqueue(ctx context.Context, size int, msg bus.Message) er
// the in-flight bound made it wait — an earlier batch's surfaced result (which
// takes priority, preserving send order). It is a no-op when the buffer is empty.
func (s *batchSender) flush(ctx context.Context) error {
+ var confirmed func() error
+ if len(s.batch) > 0 && s.beforeFlush != nil {
+ var prepareErr error
+ confirmed, prepareErr = s.beforeFlush()
+ if prepareErr != nil {
+ return prepareErr
+ }
+ }
pending := s.take()
if len(pending) == 0 {
return nil
@@ -378,7 +414,7 @@ func (s *batchSender) flush(ctx context.Context) error {
}()
// Abort promptly: an earlier in-flight batch's failure (via the bound) wins,
// otherwise this batch's own synchronous send error.
- if awaited := s.pipeline.add(ch); awaited != nil {
+ if awaited := s.pipeline.addConfirmed(ch, confirmed); awaited != nil {
return awaited
}
if pubErr != nil {
@@ -546,3 +582,21 @@ func (s *batchSender) discard() {
s.lent = s.lent[:0]
s.inflightBytes = 0
}
+
+// poll observes completed batches without waiting, always in send order.
+func (p *confirmPipeline) poll() error {
+ for len(p.inflight) > 0 {
+ select {
+ case result := <-p.inflight[0]:
+ err := p.observe(0, result)
+ p.inflight = p.inflight[1:]
+ p.onConfirmed = p.onConfirmed[1:]
+ if err != nil {
+ return err
+ }
+ default:
+ return nil
+ }
+ }
+ return nil
+}
diff --git a/banyand/backup/lifecycle/sampling_benchmark_test.go b/banyand/backup/lifecycle/sampling_benchmark_test.go
new file mode 100644
index 000000000..0c8fdc7e8
--- /dev/null
+++ b/banyand/backup/lifecycle/sampling_benchmark_test.go
@@ -0,0 +1,86 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package lifecycle
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ "github.com/apache/skywalking-banyandb/banyand/internal/pipeline"
+ "github.com/apache/skywalking-banyandb/banyand/trace"
+ "github.com/apache/skywalking-banyandb/pkg/fs"
+ "github.com/apache/skywalking-banyandb/pkg/pipeline/sdk"
+ "github.com/apache/skywalking-banyandb/pkg/timestamp"
+)
+
+func BenchmarkSamplingPreparation(b *testing.B) {
+ for _, partCount := range []int{1, 100} {
+ for _, spanBytes := range []int{1024, 1 << 20} {
+ for _, mode := range []string{"mixed", "keep-all", "drop-all"} {
+ b.Run(fmt.Sprintf("parts=%d/body=%d/%s", partCount, spanBytes, mode), func(b *testing.B) {
+ shard := filepath.Join(b.TempDir(), "00000000")
+ now := time.Now()
+ for partID := 1; partID <= partCount; partID++ {
+ rows := []trace.PartEncoderRow{
+ {TraceID: "a-drop", SpanID: "span", Span: make([]byte, spanBytes), Timestamp: now.UnixNano()},
+ {TraceID: "b-keep", SpanID: "span", Span: make([]byte, spanBytes), Timestamp: now.UnixNano()},
+ }
+ _, release := trace.EncodePart(shard, fs.NewLocalFileSystem(), uint64(partID), rows)
+ release()
+ }
+ group := &GroupConfig{Group: &commonv1.Group{Metadata: &commonv1.Metadata{Name: "benchmark"}}, SourceStage: "hot", TargetStage: "warm", TargetShardNum: 1}
+ schema := &databasev1.Trace{Metadata: &commonv1.Metadata{Name: "trace", Group: "benchmark"}}
+ segment := ×tamp.TimeRange{Start: now.Add(-24 * time.Hour), End: now.Add(24 * time.Hour)}
+ artifactRoot := b.TempDir()
+ b.ReportAllocs()
+ b.SetBytes(int64(2 * partCount * spanBytes))
+ b.ResetTimer()
+ for iteration := 0; iteration < b.N; iteration++ {
+ sampler := &migrationTestSampler{mode: mode}
+ session := pipeline.NewSession([]sdk.Sampler{sampler}, 3)
+ policy := &pinnedSamplingPolicy{
+ schema: schema, config: &commonv1.TracePipelineConfig{Enabled: true},
+ session: session, samplers: []sdk.Sampler{sampler}, host: samplingHostConfig{Timeout: time.Second},
+ }
+ directory := filepath.Join(artifactRoot, fmt.Sprintf("attempt-%d", iteration))
+ prepared, err := prepareTraceMigration(context.Background(), directory, shard, filepath.Dir(shard), segment, 0, group, policy)
+ if err != nil {
+ b.Fatal(err)
+ }
+ info, statErr := os.Stat(filepath.Join(directory, "decisions"))
+ if statErr != nil {
+ b.Fatal(statErr)
+ }
+ b.ReportMetric(float64(info.Size()), "decision-bytes")
+ b.ReportMetric(float64(prepared.manifest.KeptSpans), "retained-spans")
+ if err = prepared.Close(); err != nil {
+ b.Fatal(err)
+ }
+ policy.close()
+ }
+ })
+ }
+ }
+ }
+}
diff --git a/banyand/backup/lifecycle/sampling_destination_test.go b/banyand/backup/lifecycle/sampling_destination_test.go
new file mode 100644
index 000000000..2f36eb819
--- /dev/null
+++ b/banyand/backup/lifecycle/sampling_destination_test.go
@@ -0,0 +1,255 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package lifecycle
+
+import (
+ "context"
+ "fmt"
+ "math"
+ "os"
+ "path/filepath"
+ "sort"
+ "testing"
+ "time"
+
+ "github.com/onsi/gomega"
+ "github.com/stretchr/testify/require"
+ "go.uber.org/mock/gomock"
+ "google.golang.org/protobuf/types/known/timestamppb"
+
+ "github.com/apache/skywalking-banyandb/api/common"
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+ "github.com/apache/skywalking-banyandb/banyand/internal/pipeline"
+ "github.com/apache/skywalking-banyandb/banyand/internal/storage"
+ metadataservice "github.com/apache/skywalking-banyandb/banyand/metadata/service"
+ obsservice "github.com/apache/skywalking-banyandb/banyand/observability/services"
+ "github.com/apache/skywalking-banyandb/banyand/protector"
+ "github.com/apache/skywalking-banyandb/banyand/queue"
+ "github.com/apache/skywalking-banyandb/banyand/trace"
+ "github.com/apache/skywalking-banyandb/pkg/fs"
+ "github.com/apache/skywalking-banyandb/pkg/index"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+ "github.com/apache/skywalking-banyandb/pkg/pipeline/sdk"
+ "github.com/apache/skywalking-banyandb/pkg/query/model"
+ "github.com/apache/skywalking-banyandb/pkg/test"
+ "github.com/apache/skywalking-banyandb/pkg/timestamp"
+)
+
+func TestSamplingDestinationRebuildsIndexesAcrossRoutingChanges(t *testing.T) {
+ require.NoError(t, logger.Init(logger.Logging{Env: "dev", Level: "warn"}))
+ gomega.RegisterFailHandler(func(message string, _ ...int) { panic(message) })
+ cases := []struct {
+ name string
+ storageUnit storage.IntervalUnit
+ unit commonv1.IntervalRule_Unit
+ shards uint32
+ }{
+ {name: "same-interval", unit: commonv1.IntervalRule_UNIT_DAY, storageUnit: storage.DAY, shards: 2},
+ {name: "split-interval", unit: commonv1.IntervalRule_UNIT_HOUR, storageUnit: storage.HOUR, shards: 2},
+ {name: "changed-shard-count", unit: commonv1.IntervalRule_UNIT_DAY, storageUnit: storage.DAY, shards: 3},
+ }
+ for _, scenario := range cases {
+ t.Run(scenario.name, func(t *testing.T) {
+ localQueue := queue.Local()
+ metadataService, err := metadataservice.NewService()
+ require.NoError(t, err)
+ metrics := obsservice.NewMetricService(metadataService, localQueue, "sampling-destination", nil)
+ traceService, err := trace.NewService(metadataService, localQueue, metrics, protector.NewMemory(metrics))
+ require.NoError(t, err)
+ ports, err := test.AllocateFreePorts(1)
+ require.NoError(t, err)
+ targetRoot := t.TempDir()
+ stop := test.SetupModules([]string{
+ "--schema-server-root-path=" + t.TempDir(), fmt.Sprintf("--schema-server-grpc-port=%d", ports[0]),
+ "--schema-server-grpc-host=127.0.0.1", "--trace-root-path=" + targetRoot, "--trace-flush-timeout=200ms",
+ }, localQueue, metadataService, traceService)
+ defer stop()
+ targetInterval := &commonv1.IntervalRule{Unit: scenario.unit, Num: 1}
+ sourceInterval := &commonv1.IntervalRule{Unit: commonv1.IntervalRule_UNIT_DAY, Num: 1}
+ targetGroup := &commonv1.Group{
+ Metadata: &commonv1.Metadata{Name: roundtripTraceGroup}, Catalog: commonv1.Catalog_CATALOG_TRACE,
+ ResourceOpts: &commonv1.ResourceOpts{
+ ShardNum: scenario.shards,
+ SegmentInterval: targetInterval,
+ Ttl: &commonv1.IntervalRule{
+ Unit: commonv1.IntervalRule_UNIT_DAY,
+ Num: 7,
+ },
+ },
+ }
+ _, err = metadataService.SchemaRegistry().CreateGroup(context.Background(), targetGroup)
+ require.NoError(t, err)
+ traceSchema := &databasev1.Trace{Metadata: &commonv1.Metadata{Name: roundtripTraceName, Group: roundtripTraceGroup}, Tags: []*databasev1.TraceTagSpec{
+ {Name: "trace_id", Type: databasev1.TagType_TAG_TYPE_STRING},
+ {Name: "span_id", Type: databasev1.TagType_TAG_TYPE_STRING},
+ {Name: "timestamp", Type: databasev1.TagType_TAG_TYPE_TIMESTAMP},
+ }, TraceIdTagName: "trace_id", SpanIdTagName: "span_id", TimestampTagName: "timestamp"}
+ _, err = metadataService.SchemaRegistry().CreateTrace(context.Background(), traceSchema)
+ require.NoError(t, err)
+
+ timestampIndex := &databasev1.IndexRule{
+ Metadata: &commonv1.Metadata{Name: "migration-timestamp", Group: roundtripTraceGroup},
+ Tags: []string{"timestamp"}, Type: databasev1.IndexRule_TYPE_TREE,
+ }
+ _, err = metadataService.SchemaRegistry().CreateIndexRule(context.Background(), timestampIndex)
+ require.NoError(t, err)
+ _, err = metadataService.SchemaRegistry().CreateIndexRuleBinding(context.Background(), &databasev1.IndexRuleBinding{
+ Metadata: &commonv1.Metadata{Name: "migration-timestamp-binding", Group: roundtripTraceGroup},
+ Subject: &databasev1.Subject{Name: roundtripTraceName, Catalog: commonv1.Catalog_CATALOG_TRACE},
+ Rules: []string{timestampIndex.Metadata.Name},
+ BeginAt: timestamppb.New(time.Now().Add(-7 * 24 * time.Hour)),
+ ExpireAt: timestamppb.New(time.Now().Add(365 * 24 * time.Hour)),
+ })
+ require.NoError(t, err)
+ var destination trace.Trace
+ require.Eventually(t, func() bool {
+ destination, err = traceService.Trace(traceSchema.Metadata)
+ return err == nil && len(destination.GetIndexRules()) == 1
+ }, 30*time.Second, 100*time.Millisecond, "destination index binding must be active before replay")
+ // The group watcher and write subscriber start independently of schema lookup.
+ require.Eventually(t, func() bool { _, ok := traceService.LoadGroup(roundtripTraceGroup); return ok }, 30*time.Second, 100*time.Millisecond)
+ sourceRoot := t.TempDir()
+ sourceShard := filepath.Join(sourceRoot, "00000000")
+ start := time.Now().UTC().Truncate(24 * time.Hour).Add(-24 * time.Hour)
+ segment := ×tamp.TimeRange{Start: start, End: start.Add(24 * time.Hour), IncludeStart: true}
+ wanted := []string{"b-keep-1", "b-keep-2", "b-keep-3"}
+ for partID := uint64(1); partID <= 3; partID++ {
+ ts := start.Add(time.Duration(3+partID*2) * time.Hour)
+ var rows []trace.PartEncoderRow
+ for _, traceID := range []string{"a-drop", "b-keep"} {
+ rows = append(rows,
+ trace.PartEncoderRow{
+ TraceID: traceID,
+ SpanID: fmt.Sprintf("%s-%d",
+ traceID,
+ partID),
+ Span: []byte("payload"),
+ Timestamp: ts.UnixNano(),
+ Tags: []trace.PartEncoderTag{{
+ Name: "timestamp",
+ Value: trace.EncodedTimestamp(ts.UnixNano()),
+ }},
+ })
+ }
+ _, release := trace.EncodePart(sourceShard, fs.NewLocalFileSystem(), partID, rows)
+ release()
+ }
+ // Invalid original index bytes prove sampled visitation never ships or opens them.
+ seriesPath := filepath.Join(sourceRoot, "series")
+ require.NoError(t, os.MkdirAll(seriesPath, 0o700))
+ require.NoError(t, os.WriteFile(filepath.Join(seriesPath, "0000000000000000.seg"), []byte("unshippable source series"), 0o600))
+ require.NoError(t, os.MkdirAll(filepath.Join(sourceShard, "sidx"), 0o700))
+ require.NoError(t, os.WriteFile(filepath.Join(sourceShard, "sidx", "invalid"), []byte("unshippable source sidx"), 0o600))
+ sampler := &migrationTestSampler{mode: "mixed"}
+ session := pipeline.NewSession([]sdk.Sampler{sampler}, 3)
+ policy := &pinnedSamplingPolicy{
+ schema: traceSchema,
+ config: &commonv1.TracePipelineConfig{Enabled: true},
+ session: session,
+ host: samplingHostConfig{Timeout: time.Second},
+ digest: "destination-test",
+ }
+ sourceGroup := &GroupConfig{
+ Group: &commonv1.Group{
+ Metadata: targetGroup.Metadata,
+ Catalog: commonv1.Catalog_CATALOG_TRACE,
+ },
+ SourceStage: "hot",
+ TargetStage: "warm",
+ SegmentInterval: sourceInterval,
+ TargetSegmentInterval: targetInterval,
+ TargetShardNum: scenario.shards,
+ TargetReplicas: 1,
+ }
+ // Only batch publication is allowed; chunk/file transfer would fail this mock.
+ client := queue.NewMockClient(gomock.NewController(t))
+ client.EXPECT().NewBatchPublisher(gomock.Any()).DoAndReturn(func(timeout time.Duration) queue.BatchPublisher {
+ return localQueue.NewBatchPublisher(timeout)
+ }).AnyTimes()
+ log := logger.GetLogger("sampling-destination")
+ progress := NewProgress(filepath.Join(t.TempDir(), "progress.json"), log)
+ visitor := newTraceMigrationVisitor(context.Background(), sourceGroup.Group, scenario.shards, 1, singleNodeSelector(t), client, log, progress, 1<<20,
+ storage.IntervalRule{Unit: scenario.storageUnit, Num: 1}, metadataService, "hot", "warm", storage.IntervalRule{Unit: storage.DAY, Num: 1})
+ visitor.samplingPolicy = policy
+ visitor.samplingGroup = sourceGroup
+ visitor.samplingRoot = t.TempDir()
+ visitor.snapshot = sourceRoot
+ sampled, modeErr := selectSamplingMode(visitor.samplingRoot, roundtripTraceGroup, sourceRoot, false, true)
+ require.NoError(t, modeErr)
+ require.True(t, sampled)
+ selected, modeErr := selectSamplingMode(visitor.samplingRoot, sourceGroup.Metadata.Name, sourceRoot, false, true)
+ require.NoError(t, modeErr)
+ require.True(t, selected)
+ defer visitor.Close()
+ require.NoError(t, visitor.VisitSeries(segment, seriesPath, []common.ShardID{0}))
+ require.Empty(t, progress.SourceCompletedTraceSeries)
+ require.NoError(t, visitor.VisitShard(segment, 0, sourceShard))
+ require.NoError(t, visitor.completeSampledSeries())
+ chunked, _ := visitor.CounterSnapshot()
+ require.Zero(t, chunked)
+ require.Equal(t, []int{3, 3}, sampler.sizes)
+ require.True(t, progress.IsTraceShardCompleted(roundtripTraceGroup, segment.String(), 0))
+ queryRange := ×tamp.TimeRange{Start: start, End: start.Add(24 * time.Hour), IncludeStart: true, IncludeEnd: true}
+ query := func(indexed bool) ([]string, []string, error) {
+ options := model.TraceQueryOptions{
+ TimeRange: queryRange,
+ MaxTraceSize: 100,
+ TagProjection: &model.TagProjection{Names: []string{"timestamp"}},
+ MinVal: math.MinInt64,
+ MaxVal: math.MaxInt64,
+ }
+ if indexed {
+ options.Order = &index.OrderBy{Sort: modelv1.Sort_SORT_ASC, Index: timestampIndex}
+ options.Name = roundtripTraceName
+ options.Entities = [][]*modelv1.TagValue{{}}
+ } else {
+ options.TraceIDs = []string{"a-drop", "b-keep"}
+ }
+ results, queryErr := destination.Query(context.Background(), options)
+ if queryErr != nil {
+ return nil, nil, queryErr
+ }
+ if results == nil {
+ return nil, nil, nil
+ }
+ defer results.Release()
+ var ids, spanIDs []string
+ for result := results.Pull(); result != nil; result = results.Pull() {
+ if result.Error != nil {
+ return nil, nil, result.Error
+ }
+ ids = append(ids, result.TID)
+ spanIDs = append(spanIDs, result.SpanIDs...)
+ }
+ sort.Strings(spanIDs)
+ return ids, spanIDs, nil
+ }
+ require.Eventually(t, func() bool {
+ ids, spans, queryErr := query(false)
+ return queryErr == nil && len(ids) == 1 && ids[0] == "b-keep" && fmt.Sprint(spans) == fmt.Sprint(wanted)
+ }, 30*time.Second, 100*time.Millisecond, "retained trace must have every span and dropped trace must be absent")
+ require.Eventually(t, func() bool {
+ ids, spans, queryErr := query(true)
+ return queryErr == nil && len(ids) == 1 && ids[0] == "b-keep" && fmt.Sprint(spans) == fmt.Sprint(wanted)
+ }, 30*time.Second, 100*time.Millisecond, "rebuilt destination series and SIDX must find only the whole retained trace")
+ require.Eventually(t, func() bool { return len(findRoundtripPartDirs(targetRoot)) > 0 }, 30*time.Second, 100*time.Millisecond)
+ })
+ }
+}
diff --git a/banyand/backup/lifecycle/sampling_policy.go b/banyand/backup/lifecycle/sampling_policy.go
new file mode 100644
index 000000000..8995b4660
--- /dev/null
+++ b/banyand/backup/lifecycle/sampling_policy.go
@@ -0,0 +1,228 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package lifecycle
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "fmt"
+ "path/filepath"
+ "regexp"
+ "sync"
+ "time"
+
+ "google.golang.org/protobuf/proto"
+
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ "github.com/apache/skywalking-banyandb/banyand/internal/pipeline"
+ "github.com/apache/skywalking-banyandb/banyand/metadata"
+ "github.com/apache/skywalking-banyandb/banyand/metadata/schema"
+ "github.com/apache/skywalking-banyandb/pkg/pipeline/sdk"
+)
+
+type samplingHostConfig struct {
+ TrustedDirectory string
+ ArtifactRoot string
+ Timeout time.Duration
+ CircuitBreakN int
+ Enabled bool
+}
+
+type pinnedSamplingPolicy struct {
+ schema *databasev1.Trace
+ config *commonv1.TracePipelineConfig
+ session *pipeline.Session
+ digest string
+ plugins []*commonv1.Plugin
+ binaryDigests []string
+ samplers []sdk.Sampler
+ host samplingHostConfig
+ closeOnce sync.Once
+}
+
+func sourceSamplingChain(config *commonv1.TracePipelineConfig, source, name string) ([]*commonv1.Plugin, error) {
+ if config == nil || !config.Enabled {
+ return nil, nil
+ }
+ if len(config.SchemaNames) > 0 || config.SchemaNameRegex != "" {
+ matches := false
+ for _, selected := range config.SchemaNames {
+ matches = matches || selected == name
+ }
+ if config.SchemaNameRegex != "" {
+ expression, err := regexp.Compile(config.SchemaNameRegex)
+ if err != nil {
+ return nil, fmt.Errorf("invalid sampling schema selector: %w", err)
+ }
+ matches = matches || expression.MatchString(name)
+ }
+ if !matches {
+ return nil, nil
+ }
+ }
+ if source == "" {
+ return nil, fmt.Errorf("sampling source stage identity is missing")
+ }
+ var result []*commonv1.Plugin
+ found := false
+ for _, stage := range config.Stages {
+ if stage.Stage != source {
+ continue
+ }
+ if found {
+ return nil, fmt.Errorf("multiple sampling rules for source stage %q", source)
+ }
+ found = true
+ result = stage.Plugins
+ }
+ return result, nil
+}
+
+func resolveSamplingPolicy(ctx context.Context, group *GroupConfig, md metadata.Repo) (*pinnedSamplingPolicy, error) {
+ config := group.GetPipeline()
+ if config == nil || !config.Enabled {
+ return nil, nil
+ }
+ // Resolve only when a nonempty source chain exists, keeping legacy migrations untouched.
+ applicable := false
+ for _, stage := range config.Stages {
+ applicable = applicable || stage.Stage == group.SourceStage && len(stage.Plugins) > 0
+ }
+ if group.SourceStage == "" && len(config.Stages) > 0 {
+ return nil, fmt.Errorf("sampling source stage identity is missing")
+ }
+ if !applicable {
+ return nil, nil
+ }
+ traces, err := md.TraceRegistry().ListTrace(ctx, schema.ListOpt{Group: group.Metadata.Name})
+ if err != nil {
+ return nil, fmt.Errorf("resolve sampling trace schema: %w", err)
+ }
+ if len(traces) != 1 {
+ return nil, fmt.Errorf("sampling group %s has %d trace schemas; exactly one is required", group.Metadata.Name, len(traces))
+ }
+ plugins, err := sourceSamplingChain(config, group.SourceStage, traces[0].Metadata.Name)
+ if err != nil || len(plugins) == 0 {
+ return nil, err
+ }
+ if !group.SamplingHost.Enabled {
+ return nil, fmt.Errorf("source-stage sampling configured but lifecycle plugin host is disabled")
+ }
+ policy := &pinnedSamplingPolicy{
+ schema: proto.Clone(traces[0]).(*databasev1.Trace),
+ config: proto.Clone(config).(*commonv1.TracePipelineConfig), host: group.SamplingHost,
+ }
+ if err = config.ValidateAll(); err != nil {
+ return nil, fmt.Errorf("invalid migration sampling chain: %w", err)
+ }
+ for _, plugin := range plugins {
+ samplerConfig := plugin.GetSampler()
+ if samplerConfig == nil {
+ policy.close()
+ return nil, fmt.Errorf("unsupported migration plugin %s", plugin.Name)
+ }
+ digest, digestErr := samplingFileDigest(filepath.Join(policy.host.TrustedDirectory, samplerConfig.Path))
+ if digestErr != nil {
+ policy.close()
+ return nil, fmt.Errorf("digest plugin %s: %w", plugin.Name, digestErr)
+ }
+ sampler, loadErr := pipeline.NewLoader().LoadSamplerPlugin(samplerConfig, policy.host.TrustedDirectory, group.Metadata.Name, nil)
+ if loadErr != nil {
+ policy.close()
+ return nil, fmt.Errorf("load migration plugin %s: %w", plugin.Name, loadErr)
+ }
+ loadedDigest, loadedDigestErr := samplingFileDigest(filepath.Join(policy.host.TrustedDirectory, samplerConfig.Path))
+ if loadedDigestErr != nil || loadedDigest != digest {
+ _ = sampler.Close()
+ policy.close()
+ return nil, fmt.Errorf("plugin binary changed during migration preflight: %s", plugin.Name)
+ }
+ policy.samplers = append(policy.samplers, sampler)
+ policy.plugins = append(policy.plugins, proto.Clone(plugin).(*commonv1.Plugin))
+ policy.binaryDigests = append(policy.binaryDigests, digest)
+ }
+ encoded, err := proto.MarshalOptions{Deterministic: true}.Marshal(policy.config)
+ if err != nil {
+ policy.close()
+ return nil, err
+ }
+ digest := sha256.Sum256(encoded)
+ policy.digest = hex.EncodeToString(digest[:])
+ policy.session, err = pipeline.NewSessionChecked(policy.samplers, policy.host.CircuitBreakN)
+ if err != nil {
+ policy.close()
+ return nil, err
+ }
+ return policy, nil
+}
+
+func (p *pinnedSamplingPolicy) close() {
+ if p == nil {
+ return
+ }
+ p.closeOnce.Do(func() {
+ if p.session != nil {
+ p.session.ClosePluginsWhenIdle()
+ return
+ }
+ for _, sampler := range p.samplers {
+ _ = sampler.Close()
+ }
+ })
+}
+
+// newShardSession starts native state at the preparation restart boundary.
+func (p *pinnedSamplingPolicy) newShardSession(group string) (*pinnedSamplingPolicy, error) {
+ return p.newShardSessionWithLoader(group, pipeline.NewLoader)
+}
+
+func (p *pinnedSamplingPolicy) newShardSessionWithLoader(group string, newLoader func() *pipeline.Loader) (*pinnedSamplingPolicy, error) {
+ child := &pinnedSamplingPolicy{schema: p.schema, config: p.config, plugins: p.plugins, host: p.host, digest: p.digest, binaryDigests: p.binaryDigests}
+ for index, plugin := range p.plugins {
+ samplerConfig := plugin.GetSampler()
+ digest, err := samplingFileDigest(filepath.Join(p.host.TrustedDirectory, samplerConfig.Path))
+ if err != nil {
+ child.close()
+ return nil, err
+ }
+ if digest != p.binaryDigests[index] {
+ child.close()
+ return nil, fmt.Errorf("pinned sampling plugin binary changed: %s", plugin.Name)
+ }
+ sampler, err := newLoader().LoadSamplerPlugin(samplerConfig, p.host.TrustedDirectory, group, nil)
+ if err != nil {
+ child.close()
+ return nil, err
+ }
+ child.samplers = append(child.samplers, sampler)
+ loadedDigest, err := samplingFileDigest(filepath.Join(p.host.TrustedDirectory, samplerConfig.Path))
+ if err != nil || loadedDigest != digest {
+ child.close()
+ return nil, fmt.Errorf("pinned sampling plugin changed during load: %s", plugin.Name)
+ }
+ }
+ var err error
+ child.session, err = pipeline.NewSessionChecked(child.samplers, p.host.CircuitBreakN)
+ if err != nil {
+ child.close()
+ return nil, err
+ }
+ return child, nil
+}
diff --git a/banyand/backup/lifecycle/sampling_policy_test.go b/banyand/backup/lifecycle/sampling_policy_test.go
new file mode 100644
index 000000000..3caddd9aa
--- /dev/null
+++ b/banyand/backup/lifecycle/sampling_policy_test.go
@@ -0,0 +1,165 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package lifecycle
+
+import (
+ "os"
+ "path/filepath"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ "github.com/apache/skywalking-banyandb/banyand/internal/pipeline"
+ "github.com/apache/skywalking-banyandb/pkg/pipeline/sdk"
+)
+
+func TestSourceSamplingChainSelection(t *testing.T) {
+ hot := &commonv1.Plugin{Name: "hot"}
+ warm := &commonv1.Plugin{Name: "warm"}
+ config := &commonv1.TracePipelineConfig{
+ Enabled: true,
+ Plugins: []*commonv1.Plugin{{Name: "top-level"}},
+ SchemaNames: []string{"selected"},
+ Stages: []*commonv1.StageRule{
+ {
+ Stage: "hot",
+ Plugins: []*commonv1.Plugin{hot},
+ },
+ {
+ Stage: "warm",
+ Plugins: []*commonv1.Plugin{warm},
+ },
+ },
+ }
+ selected, err := sourceSamplingChain(config, "hot", "selected")
+ require.NoError(t, err)
+ require.Equal(t, []*commonv1.Plugin{hot}, selected, "migration must select source chain even without MERGE/FINALIZE enablement")
+ selected, err = sourceSamplingChain(config, "warm", "selected")
+ require.NoError(t, err)
+ require.Equal(t, []*commonv1.Plugin{warm}, selected)
+ for _, test := range []struct{ source, name string }{{"hot", "unmatched"}, {"cold", "selected"}} {
+ selected, err = sourceSamplingChain(config, test.source, test.name)
+ require.NoError(t, err)
+ require.Empty(t, selected)
+ }
+ config.Enabled = false
+ selected, err = sourceSamplingChain(config, "", "selected")
+ require.NoError(t, err)
+ require.Empty(t, selected)
+}
+
+func TestSourceSamplingChainRejectsAmbiguityAndInvalidSelector(t *testing.T) {
+ config := &commonv1.TracePipelineConfig{Enabled: true, Stages: []*commonv1.StageRule{{Stage: "hot"}, {Stage: "hot"}}}
+ _, err := sourceSamplingChain(config, "hot", "trace")
+ require.ErrorContains(t, err, "multiple sampling rules")
+ _, err = sourceSamplingChain(config, "", "trace")
+ require.ErrorContains(t, err, "identity is missing")
+ config.SchemaNameRegex = "["
+ _, err = sourceSamplingChain(config, "hot", "trace")
+ require.ErrorContains(t, err, "invalid sampling schema selector")
+ config.SchemaNameRegex = "^trace$"
+ config.Stages = config.Stages[:1]
+ selected, err := sourceSamplingChain(config, "hot", "trace")
+ require.NoError(t, err)
+ require.Empty(t, selected, "empty stage chain preserves legacy migration")
+}
+
+func TestSamplingHostFlagsValidate(t *testing.T) {
+ service := &lifecycleService{orphanPolicyStr: "archive", samplingHost: samplingHostConfig{Enabled: true, Timeout: time.Second, CircuitBreakN: 3}}
+ require.ErrorContains(t, service.Validate(), "trusted-dir is required")
+ service.samplingHost.TrustedDirectory = t.TempDir()
+ require.NoError(t, service.Validate())
+ service.samplingHost.Timeout = 0
+ require.ErrorContains(t, service.Validate(), "timeout must be positive")
+ service.samplingHost.Timeout = time.Second
+ service.samplingHost.CircuitBreakN = 0
+ require.ErrorContains(t, service.Validate(), "circuit-break must be positive")
+}
+
+type statefulShardSampler struct {
+ closed chan struct{}
+ calls int
+ closes atomic.Int64
+}
+
+func (*statefulShardSampler) Kind() sdk.Kind { return sdk.KindSampler }
+func (*statefulShardSampler) Project() sdk.Projection { return sdk.Projection{} }
+func (s *statefulShardSampler) Decide(*sdk.TraceBatch) (sdk.Verdict, error) {
+ s.calls++
+ return sdk.Verdict{Keep: []bool{s.calls%2 == 1}}, nil
+}
+func (s *statefulShardSampler) Close() error { s.closes.Add(1); close(s.closed); return nil }
+
+func TestSamplingShardSessionsRestartNativeState(t *testing.T) {
+ directory := t.TempDir()
+ path := filepath.Join(directory, "stateful.so")
+ require.NoError(t, os.WriteFile(path, []byte("stateful-native-binary"), 0o600))
+ digest, err := samplingFileDigest(path)
+ require.NoError(t, err)
+ preflight := &statefulShardSampler{closed: make(chan struct{})}
+ policy := &pinnedSamplingPolicy{
+ plugins: []*commonv1.Plugin{{
+ Name: "stateful",
+ Kind: &commonv1.Plugin_Sampler{Sampler: &commonv1.SamplerPlugin{
+ Path: "stateful.so",
+ AbiVersion: uint32(sdk.ABIVersion),
+ }},
+ }},
+
+ binaryDigests: []string{digest}, samplers: []sdk.Sampler{preflight}, host: samplingHostConfig{TrustedDirectory: directory, CircuitBreakN: 3},
+ }
+ var constructed []*statefulShardSampler
+ newLoader := func() *pipeline.Loader {
+ loader := pipeline.NewLoader()
+ loader.OpenSampler = func(string, string, []byte) (sdk.Sampler, error) {
+ sampler := &statefulShardSampler{closed: make(chan struct{})}
+ constructed = append(constructed, sampler)
+ return sampler, nil
+ }
+ return loader
+ }
+ batch := &sdk.TraceBatch{Traces: []sdk.TraceBlock{{TraceID: "trace"}}}
+ for shard := 0; shard < 3; shard++ {
+ child, loadErr := policy.newShardSessionWithLoader("group", newLoader)
+ require.NoError(t, loadErr)
+ first, executeErr := child.session.Execute(batch, time.Second)
+ require.NoError(t, executeErr)
+ require.Equal(t, []bool{true}, first.Keep, "new shard and restarted shard must begin at identical native state")
+ second, executeErr := child.session.Execute(batch, time.Second)
+ require.NoError(t, executeErr)
+ require.Equal(t, []bool{false}, second.Keep, "sampler state must persist within one shard")
+ child.close()
+ child.close()
+ select {
+ case <-constructed[shard].closed:
+ case <-time.After(time.Second):
+ t.Fatal("idle shard sampler was not closed")
+ }
+ require.Equal(t, int64(1), constructed[shard].closes.Load())
+ }
+ require.Len(t, constructed, 3)
+ require.NotSame(t, constructed[0], constructed[1])
+ require.Zero(t, preflight.calls, "preflight instance must never influence shard decision state")
+ require.Zero(t, preflight.closes.Load())
+ policy.close()
+ policy.close()
+ require.Equal(t, int64(1), preflight.closes.Load())
+}
diff --git a/banyand/backup/lifecycle/sampling_prepare.go b/banyand/backup/lifecycle/sampling_prepare.go
new file mode 100644
index 000000000..d44d602a1
--- /dev/null
+++ b/banyand/backup/lifecycle/sampling_prepare.go
@@ -0,0 +1,338 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package lifecycle
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+ "sort"
+ "strconv"
+ "time"
+
+ "google.golang.org/protobuf/proto"
+
+ "github.com/apache/skywalking-banyandb/banyand/trace"
+ "github.com/apache/skywalking-banyandb/pkg/pipeline/sdk"
+ "github.com/apache/skywalking-banyandb/pkg/timestamp"
+)
+
+type preparedTraceMigration struct {
+ table *samplingDecisionTable
+ directory string
+ checkpoint samplingCheckpoint
+ manifest samplingManifest
+ replayedBytes uint64
+}
+
+func samplingArtifactPath(root, group string, segment *timestamp.TimeRange, shard uint32) string {
+ return filepath.Join(root, group, strconv.FormatInt(segment.Start.UnixNano(), 10), strconv.FormatUint(uint64(shard), 10))
+}
+
+func samplingCoreParts(shardPath string) ([]string, error) {
+ entries, err := os.ReadDir(shardPath)
+ if err != nil {
+ return nil, err
+ }
+ var parts []string
+ for _, entry := range entries {
+ if !entry.IsDir() || len(entry.Name()) != 16 {
+ continue
+ }
+ if _, err = strconv.ParseUint(entry.Name(), 16, 64); err != nil {
+ continue
+ }
+ parts = append(parts, filepath.Join(shardPath, entry.Name()))
+ }
+ sort.Strings(parts)
+ return parts, nil
+}
+
+func samplingInventory(parts []string) ([]samplingInventoryFile, error) {
+ var result []samplingInventoryFile
+ for _, part := range parts {
+ err := filepath.WalkDir(part, func(path string, entry os.DirEntry, walkErr error) error {
+ if walkErr != nil {
+ return walkErr
+ }
+ if entry.IsDir() {
+ return nil
+ }
+ if entry.Type()&os.ModeSymlink != 0 {
+ return fmt.Errorf("snapshot inventory contains symlink %s", path)
+ }
+ info, err := entry.Info()
+ if err != nil {
+ return err
+ }
+ digest, err := samplingFileDigest(path)
+ if err != nil {
+ return err
+ }
+ result = append(result, samplingInventoryFile{Path: path, Size: info.Size(), Digest: digest})
+ return nil
+ })
+ if err != nil {
+ return nil, err
+ }
+ }
+ return result, nil
+}
+
+func openPreparedTraceMigration(directory string) (*preparedTraceMigration, error) {
+ payload, err := os.ReadFile(filepath.Join(directory, "manifest.json"))
+ if err != nil {
+ return nil, err
+ }
+ prepared := &preparedTraceMigration{directory: directory}
+ if err = json.Unmarshal(payload, &prepared.manifest); err != nil {
+ return nil, fmt.Errorf("decode sampling manifest: %w", err)
+ }
+ if err = verifySamplingManifest(&prepared.manifest); err != nil {
+ return nil, err
+ }
+ if prepared.manifest.Version != samplingArtifactVersion {
+ return nil, fmt.Errorf("unsupported sampling manifest version %d", prepared.manifest.Version)
+ }
+ for _, file := range prepared.manifest.Inventory {
+ info, statErr := os.Stat(file.Path)
+ if statErr != nil {
+ return nil, fmt.Errorf("required sampling snapshot: %w", statErr)
+ }
+ digest, digestErr := samplingFileDigest(file.Path)
+ if digestErr != nil {
+ return nil, digestErr
+ }
+ if info.Size() != file.Size || digest != file.Digest {
+ return nil, fmt.Errorf("sampling snapshot inventory mismatch: %s", file.Path)
+ }
+ }
+ prepared.table, err = openSamplingDecisionTable(directory, &prepared.manifest)
+ if err != nil {
+ return nil, err
+ }
+ payload, err = os.ReadFile(filepath.Join(directory, "checkpoint.json"))
+ if err != nil {
+ prepared.Close()
+ return nil, err
+ }
+ if prepared.checkpoint, err = readSamplingCheckpoint(payload); err != nil {
+ prepared.Close()
+ return nil, err
+ }
+ if prepared.checkpoint.ManifestDigest != prepared.manifest.ManifestDigest || prepared.checkpoint.Sequence > prepared.manifest.KeptSpans {
+ prepared.Close()
+ return nil, fmt.Errorf("sampling checkpoint does not belong to manifest")
+ }
+ if err = syncSamplingDirectory(directory); err != nil {
+ prepared.Close()
+ return nil, err
+ }
+ if err = syncSamplingDirectory(filepath.Dir(directory)); err != nil {
+ prepared.Close()
+ return nil, err
+ }
+ return prepared, nil
+}
+
+func prepareTraceMigration(ctx context.Context, directory, shardPath, snapshot string, segment *timestamp.TimeRange,
+ shard uint32, group *GroupConfig, policy *pinnedSamplingPolicy,
+) (*preparedTraceMigration, error) {
+ // A published manifest is authoritative even when policy or schema changed.
+ if _, err := os.Stat(directory); err == nil {
+ return openPreparedTraceMigration(directory)
+ } else if !errors.Is(err, os.ErrNotExist) {
+ return nil, err
+ }
+ start := time.Now()
+ parent := filepath.Dir(directory)
+ if err := makeSamplingDirectory(parent); err != nil {
+ return nil, err
+ }
+ temporary, err := os.MkdirTemp(parent, ".preparing-")
+ if err != nil {
+ return nil, err
+ }
+ defer os.RemoveAll(temporary)
+ parts, err := samplingCoreParts(shardPath)
+ if err != nil {
+ return nil, err
+ }
+ inventory, err := samplingInventory(parts)
+ if err != nil {
+ return nil, err
+ }
+ manifest := samplingManifest{
+ Version: samplingArtifactVersion, Snapshot: snapshot, Group: group.Metadata.Name,
+ SourceStage: group.SourceStage, TargetStage: group.TargetStage, Shard: shard, SegmentStart: segment.Start.UnixNano(), SegmentEnd: segment.End.UnixNano(),
+ TargetShardNum: group.TargetShardNum, Replicas: group.TargetReplicas, Inventory: inventory, Parts: parts,
+ PolicyDigest: policy.digest, PluginDigests: policy.binaryDigests, FailOpen: make(map[string]uint64),
+ }
+ manifest.SourceInterval, err = proto.MarshalOptions{Deterministic: true}.Marshal(group.SegmentInterval)
+ if err != nil {
+ return nil, err
+ }
+ manifest.TargetInterval, err = proto.MarshalOptions{Deterministic: true}.Marshal(group.TargetSegmentInterval)
+ if err != nil {
+ return nil, err
+ }
+ manifest.Schema, err = proto.MarshalOptions{Deterministic: true}.Marshal(policy.schema)
+ if err != nil {
+ return nil, err
+ }
+ manifest.GroupConfig, err = proto.MarshalOptions{Deterministic: true}.Marshal(group.Group)
+ if err != nil {
+ return nil, err
+ }
+ manifest.Policy, err = proto.MarshalOptions{Deterministic: true}.Marshal(policy.config)
+ if err != nil {
+ return nil, err
+ }
+ writer, err := newSamplingDecisionWriter(temporary)
+ if err != nil {
+ return nil, err
+ }
+ closed := false
+ defer func() {
+ if !closed {
+ _ = writer.close()
+ }
+ }()
+ scanner, err := trace.NewProjectedSnapshotScanner(ctx, parts, policy.session.Projection(), trace.SnapshotScanOptions{
+ MaxTraceBytes: 32 << 20, MaxMetadataBytes: 8 << 20, MaxOpenReaders: 16, TempDir: temporary,
+ })
+ if err != nil {
+ return nil, err
+ }
+ defer scanner.Close()
+ if err = prepareSamplingDecisions(ctx, scanner, writer, &manifest, policy, segment); err != nil {
+ return nil, err
+ }
+ if err = scanner.Close(); err != nil {
+ return nil, err
+ }
+ if err = writer.close(); err != nil {
+ return nil, err
+ }
+ closed = true
+ manifest.TableDigest, err = samplingFileDigest(filepath.Join(temporary, "decisions"))
+ if err != nil {
+ return nil, err
+ }
+ manifest.IndexDigest, err = samplingFileDigest(filepath.Join(temporary, "offsets"))
+ if err != nil {
+ return nil, err
+ }
+ manifest.PreparationNanos = int64(time.Since(start))
+ if err = sealSamplingManifest(&manifest); err != nil {
+ return nil, err
+ }
+ if err = atomicSamplingJSON(filepath.Join(temporary, "manifest.json"), &manifest); err != nil {
+ return nil, err
+ }
+ if err = saveSamplingCheckpoint(filepath.Join(temporary, "checkpoint.json"), samplingCheckpoint{ManifestDigest: manifest.ManifestDigest}); err != nil {
+ return nil, err
+ }
+ if err = syncSamplingDirectory(temporary); err != nil {
+ return nil, err
+ }
+ if err = os.Rename(temporary, directory); err != nil {
+ return nil, err
+ }
+ if err = syncSamplingDirectory(parent); err != nil {
+ return nil, err
+ }
+ return openPreparedTraceMigration(directory)
+}
+
+func (p *preparedTraceMigration) Close() error {
+ if p.table != nil {
+ return p.table.close()
+ }
+ return nil
+}
+
+func prepareSamplingDecisions(ctx context.Context, scanner *trace.ProjectedSnapshotScanner, writer *samplingDecisionWriter,
+ manifest *samplingManifest, policy *pinnedSamplingPolicy, segment *timestamp.TimeRange,
+) error {
+ for {
+ assembled, scanErr := scanner.Next(ctx)
+ if errors.Is(scanErr, io.EOF) {
+ break
+ }
+ if scanErr != nil {
+ return scanErr
+ }
+ keep := true
+ reason := ""
+ switch {
+ case assembled.InputConflict:
+ reason = "input_conflict"
+ case assembled.Oversized:
+ reason = "oversized_trace"
+ default:
+ bypasses := make(chan string, max(1, len(policy.samplers)))
+ verdict, _, executionErr := policy.session.ExecuteInto(&sdk.TraceBatch{Traces: []sdk.TraceBlock{assembled.Trace}}, policy.host.Timeout, nil,
+ func(samplers []sdk.Sampler, batch *sdk.TraceBatch, mask []bool) sdk.Verdict {
+ return sdk.EvaluateChainInto(samplers, batch, mask, func(_ int, info sdk.BypassInfo) { bypasses <- info.Reason })
+ })
+ for len(bypasses) > 0 {
+ manifest.FailOpen[<-bypasses]++
+ }
+ if open, _ := policy.session.State(); open && executionErr == nil {
+ reason = "circuit_open"
+ }
+ switch {
+ case executionErr != nil:
+ reason = executionErr.Error()
+ case len(verdict.Keep) != 1:
+ reason = "invalid_verdict"
+ default:
+ keep = verdict.Keep[0]
+ }
+ }
+ // Apply the same temporal assumptions and edge protection as data-node sampling.
+ grace := 2 * time.Hour
+ if policy.config != nil && policy.config.MergeGrace != nil {
+ grace = policy.config.MergeGrace.AsDuration()
+ }
+ guardKeep, guardReason := trace.GuardSnapshotTrace(ctx, *assembled, segment.Start.UnixNano(), segment.End.UnixNano(), grace, keep)
+ if !keep && guardKeep {
+ reason = guardReason
+ }
+ keep = guardKeep
+ if reason != "" {
+ manifest.FailOpen[reason]++
+ }
+ manifest.Traces++
+ manifest.Spans += assembled.SpanCount
+ if keep {
+ manifest.KeptTraces++
+ manifest.KeptSpans += assembled.SpanCount
+ }
+ if err := writer.append(assembled.Trace.TraceID, keep); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
diff --git a/banyand/backup/lifecycle/sampling_real_test.go b/banyand/backup/lifecycle/sampling_real_test.go
new file mode 100644
index 000000000..f69ee5e70
--- /dev/null
+++ b/banyand/backup/lifecycle/sampling_real_test.go
@@ -0,0 +1,379 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package lifecycle
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "math"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "runtime/debug"
+ "strconv"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/onsi/gomega"
+ "github.com/stretchr/testify/require"
+ "go.uber.org/mock/gomock"
+ "google.golang.org/protobuf/encoding/protojson"
+ "google.golang.org/protobuf/proto"
+ "google.golang.org/protobuf/types/known/structpb"
+ "google.golang.org/protobuf/types/known/timestamppb"
+
+ "github.com/apache/skywalking-banyandb/api/common"
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+ "github.com/apache/skywalking-banyandb/banyand/internal/storage"
+ metadataservice "github.com/apache/skywalking-banyandb/banyand/metadata/service"
+ obsservice "github.com/apache/skywalking-banyandb/banyand/observability/services"
+ "github.com/apache/skywalking-banyandb/banyand/protector"
+ "github.com/apache/skywalking-banyandb/banyand/queue"
+ "github.com/apache/skywalking-banyandb/banyand/trace"
+ "github.com/apache/skywalking-banyandb/pkg/index"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+ "github.com/apache/skywalking-banyandb/pkg/pipeline/sdk"
+ "github.com/apache/skywalking-banyandb/pkg/query/model"
+ "github.com/apache/skywalking-banyandb/pkg/test"
+ "github.com/apache/skywalking-banyandb/pkg/timestamp"
+)
+
+// TestSamplingRealSnapshot is opt-in; downloaded input and output stay outside the checkout.
+// REAL_TRACE_DIR contains metadata/{group,trace}.json and source/sw_trace/seg-20261012/shard-1.
+// REAL_TRACE_SHARD, REAL_TRACE_BEGIN, REAL_TRACE_END override the shard and its exact source bounds.
+func TestSamplingRealSnapshot(t *testing.T) {
+ root := os.Getenv("REAL_TRACE_DIR")
+ if root == "" {
+ t.Skip("set REAL_TRACE_DIR to an independently downloaded native trace snapshot")
+ }
+ root, err := filepath.Abs(root)
+ require.NoError(t, err)
+ originalGroup := &commonv1.Group{}
+ schema := &databasev1.Trace{}
+ readRealTraceMetadata(t, filepath.Join(root, "metadata", "group.json"), "group", originalGroup)
+ readRealTraceMetadata(t, filepath.Join(root, "metadata", "trace.json"), "trace", schema)
+ shard := os.Getenv("REAL_TRACE_SHARD")
+ if shard == "" {
+ shard = filepath.Join(root, "source", originalGroup.Metadata.Name, "seg-20261012", "shard-1")
+ }
+ shardName := filepath.Base(shard)
+ shardNumber, shardErr := strconv.ParseUint(strings.TrimPrefix(shardName, "shard-"), 10, 32)
+ require.NoError(t, shardErr, "source shard name must identify its actual shard ID")
+ begin := realTraceTime(t, "REAL_TRACE_BEGIN", "2026-10-12T00:00:00Z")
+ end := realTraceTime(t, "REAL_TRACE_END", "2026-10-13T00:00:00Z")
+ segment := ×tamp.TimeRange{Start: begin, End: end, IncludeStart: true}
+ output, outputErr := os.MkdirTemp(root, "simulation-")
+ require.NoError(t, outputErr)
+ pluginDirectory := filepath.Join(output, "plugins")
+ require.NoError(t, makeSamplingDirectory(pluginDirectory))
+ buildRealTraceSampler(t, pluginDirectory)
+ require.NoError(t, logger.Init(logger.Logging{Env: "dev", Level: "warn"}))
+ gomega.RegisterFailHandler(func(message string, _ ...int) { panic(message) })
+ localQueue := queue.Local()
+ metadataService, err := metadataservice.NewService()
+ require.NoError(t, err)
+ metrics := obsservice.NewMetricService(metadataService, localQueue, "real-trace-simulation", nil)
+ destinationService, err := trace.NewService(metadataService, localQueue, metrics, protector.NewMemory(metrics))
+ require.NoError(t, err)
+ ports, err := test.AllocateFreePorts(1)
+ require.NoError(t, err)
+ destinationRoot := filepath.Join(output, "destination")
+ stop := test.SetupModules([]string{
+ "--schema-server-root-path=" + filepath.Join(output, "metadata-store"),
+ fmt.Sprintf("--schema-server-grpc-port=%d", ports[0]), "--schema-server-grpc-host=127.0.0.1",
+ "--trace-root-path=" + destinationRoot, "--trace-flush-timeout=200ms",
+ }, localQueue, metadataService, destinationService)
+ defer stop()
+ targetGroup := proto.Clone(originalGroup).(*commonv1.Group)
+ targetGroup.Pipeline = nil
+ targetGroup.ResourceOpts.Stages = nil
+ targetGroup.ResourceOpts.DefaultStages = nil
+ targetGroup.ResourceOpts.Ttl = &commonv1.IntervalRule{Unit: commonv1.IntervalRule_UNIT_DAY, Num: 3650}
+ targetGroup.Metadata.ModRevision = 0
+ _, err = metadataService.SchemaRegistry().CreateGroup(context.Background(), targetGroup)
+ require.NoError(t, err)
+ localSchema := proto.Clone(schema).(*databasev1.Trace)
+ localSchema.Metadata.ModRevision = 0
+ _, err = metadataService.SchemaRegistry().CreateTrace(context.Background(), localSchema)
+ require.NoError(t, err)
+ timestampIndex := &databasev1.IndexRule{
+ Metadata: &commonv1.Metadata{Name: "simulation-timestamp", Group: originalGroup.Metadata.Name},
+ Tags: []string{schema.TimestampTagName}, Type: databasev1.IndexRule_TYPE_TREE,
+ }
+ _, err = metadataService.SchemaRegistry().CreateIndexRule(context.Background(), timestampIndex)
+ require.NoError(t, err)
+ _, err = metadataService.SchemaRegistry().CreateIndexRuleBinding(context.Background(), &databasev1.IndexRuleBinding{
+ Metadata: &commonv1.Metadata{Name: "simulation-timestamp-binding", Group: originalGroup.Metadata.Name},
+ Subject: &databasev1.Subject{Name: schema.Metadata.Name, Catalog: commonv1.Catalog_CATALOG_TRACE}, Rules: []string{timestampIndex.Metadata.Name},
+ BeginAt: timestamppb.New(time.Unix(0, 0)), ExpireAt: timestamppb.New(time.Now().AddDate(100, 0, 0)),
+ })
+ require.NoError(t, err)
+ var destination trace.Trace
+ require.Eventually(t, func() bool {
+ destination, err = destinationService.Trace(localSchema.Metadata)
+ return err == nil && len(destination.GetIndexRules()) == 1
+ }, 30*time.Second, 100*time.Millisecond)
+ require.Eventually(t, func() bool { _, loaded := destinationService.LoadGroup(originalGroup.Metadata.Name); return loaded }, 30*time.Second, 100*time.Millisecond)
+ config, err := structpb.NewStruct(map[string]any{"durationThresholdMs": 500, "keepErrors": true, "healthySampleRate": 0.01})
+ require.NoError(t, err)
+ sourceGroup := proto.Clone(originalGroup).(*commonv1.Group)
+ sourceGroup.Pipeline = &commonv1.TracePipelineConfig{
+ Enabled: true, SchemaNames: []string{schema.Metadata.Name},
+ MergeGrace: originalGroup.GetPipeline().GetMergeGrace(), Stages: []*commonv1.StageRule{{Stage: "hot", Plugins: []*commonv1.Plugin{
+ {Name: "real-skywalking", Kind: &commonv1.Plugin_Sampler{Sampler: &commonv1.SamplerPlugin{Path: "sw-trace-sampler.so", AbiVersion: sdk.ABIVersion, Config: config}}},
+ }}},
+ }
+ group := &GroupConfig{
+ Group: sourceGroup, SourceStage: "hot", TargetStage: "warm", TargetShardNum: targetGroup.ResourceOpts.ShardNum,
+ SegmentInterval: sourceGroup.ResourceOpts.SegmentInterval, TargetSegmentInterval: targetGroup.ResourceOpts.SegmentInterval,
+ SamplingHost: samplingHostConfig{
+ Enabled: true, TrustedDirectory: pluginDirectory, Timeout: time.Second, CircuitBreakN: 3, ArtifactRoot: filepath.Join(output, "artifacts"),
+ },
+ }
+ policy, err := resolveSamplingPolicy(context.Background(), group, metadataService)
+ require.NoError(t, err)
+ defer policy.close()
+ // Bind preparation to the downloaded schema revision, rather than the local registry's revision.
+ policy.schema = schema
+ selector := singleNodeSelector(t)
+ client := queue.NewMockClient(gomock.NewController(t))
+ client.EXPECT().NewBatchPublisher(gomock.Any()).DoAndReturn(func(timeout time.Duration) queue.BatchPublisher { return localQueue.NewBatchPublisher(timeout) }).AnyTimes()
+ progress := NewProgress(filepath.Join(output, "progress.json"), logger.GetLogger("real-trace-simulation"))
+ visitor := newTraceMigrationVisitor(context.Background(), sourceGroup, group.TargetShardNum, 0, selector, client,
+ logger.GetLogger("real-trace-simulation"), progress, 1<<20, storage.MustToIntervalRule(group.TargetSegmentInterval), metadataService,
+ "hot", "warm", storage.MustToIntervalRule(group.SegmentInterval))
+ visitor.samplingPolicy = policy
+ visitor.samplingGroup = group
+ visitor.samplingRoot = group.SamplingHost.ArtifactRoot
+ visitor.snapshot = root
+ defer visitor.Close()
+ selected, err := selectSamplingMode(visitor.samplingRoot, sourceGroup.Metadata.Name, root, false, true)
+ require.NoError(t, err)
+ require.True(t, selected)
+ start := time.Now()
+ require.NoError(t, visitor.VisitShard(segment, common.ShardID(shardNumber), shard))
+ directory := samplingArtifactPath(visitor.samplingRoot, sourceGroup.Metadata.Name, segment, uint32(shardNumber))
+ prepared, err := openPreparedTraceMigration(directory)
+ require.NoError(t, err)
+ defer prepared.Close()
+ verified, signature := verifyRealTraceReplay(t, prepared, destination, segment)
+ require.Eventually(t, func() bool {
+ indexCount, indexSignature := queryRealTraceIndex(t, destination, schema, timestampIndex, segment)
+ return indexCount == verified && signature == indexSignature
+ }, 30*time.Second, 100*time.Millisecond, "rebuilt timestamp index must match every retained source body")
+ summary := map[string]any{
+ "source": shard, "policy": "SkyWalking duration500ms/errors/healthy0.01", "manifest": prepared.manifest,
+ "verified_unique_spans": verified, "payload_signature": hex.EncodeToString(signature[:]), "elapsed_seconds": time.Since(start).Seconds(),
+ }
+ require.NoError(t, atomicSamplingJSON(filepath.Join(output, "result.json"), summary))
+ t.Logf("real simulation: traces=%d kept=%d dropped=%d spans=%d kept_spans=%d verified=%d elapsed=%s output=%s",
+ prepared.manifest.Traces, prepared.manifest.KeptTraces, prepared.manifest.Traces-prepared.manifest.KeptTraces,
+ prepared.manifest.Spans, prepared.manifest.KeptSpans, verified, time.Since(start), output)
+}
+
+func readRealTraceMetadata(t *testing.T, path, key string, message proto.Message) {
+ t.Helper()
+ payload, err := os.ReadFile(path)
+ require.NoError(t, err)
+ var wrapper map[string]json.RawMessage
+ require.NoError(t, json.Unmarshal(payload, &wrapper))
+ if wrapped := wrapper[key]; wrapped != nil {
+ payload = wrapped
+ }
+ require.NoError(t, protojson.Unmarshal(payload, message))
+}
+
+func realTraceTime(t *testing.T, key, fallback string) time.Time {
+ t.Helper()
+ value := os.Getenv(key)
+ if value == "" {
+ value = fallback
+ }
+ parsed, err := time.Parse(time.RFC3339, value)
+ require.NoError(t, err)
+ return parsed
+}
+
+func buildRealTraceSampler(t *testing.T, directory string) {
+ t.Helper()
+ root, err := os.Getwd()
+ require.NoError(t, err)
+ for {
+ if _, err = os.Stat(filepath.Join(root, "go.mod")); err == nil {
+ break
+ }
+ parent := filepath.Dir(root)
+ require.NotEqual(t, root, parent)
+ root = parent
+ }
+ args := []string{"build", "-buildmode=plugin", "-o", filepath.Join(directory, "sw-trace-sampler.so")}
+ if info, ok := debug.ReadBuildInfo(); ok {
+ for _, setting := range info.Settings {
+ switch setting.Key {
+ case "-race", "-trimpath":
+ if setting.Value == "true" {
+ args = append(args, setting.Key)
+ }
+ case "-tags":
+ if setting.Value != "" {
+ args = append(args, "-tags", setting.Value)
+ }
+ }
+ }
+ }
+ args = append(args, "./plugins/skywalking/sw-trace-sampler")
+ command := exec.Command("go", args...)
+ command.Dir = root
+ command.Env = append(os.Environ(), "CGO_ENABLED=1")
+ output, err := command.CombinedOutput()
+ require.NoError(t, err, "native sampler build: %s", output)
+}
+
+func verifyRealTraceReplay(t *testing.T, prepared *preparedTraceMigration, destination trace.Trace, segment *timestamp.TimeRange) (uint64, [32]byte) {
+ t.Helper()
+ scanner, err := trace.NewProjectedSnapshotScanner(context.Background(), prepared.manifest.Parts, sdk.Projection{SpanIDs: true, Spans: true},
+ trace.SnapshotScanOptions{MaxTraceBytes: 32 << 20, MaxMetadataBytes: 8 << 20, MaxOpenReaders: 16, TempDir: prepared.directory})
+ require.NoError(t, err)
+ defer scanner.Close()
+ var count uint64
+ var signature [32]byte
+ batch := make(map[string]map[string][32]byte)
+ flush := func() {
+ if len(batch) == 0 {
+ return
+ }
+ ids := make([]string, 0, len(batch))
+ for id := range batch {
+ ids = append(ids, id)
+ }
+ got := make(map[string]map[string][32]byte)
+ require.Eventually(t, func() bool {
+ got = make(map[string]map[string][32]byte)
+ results, queryErr := destination.Query(context.Background(), model.TraceQueryOptions{TimeRange: segment, TraceIDs: ids, MaxTraceSize: 100000})
+ if queryErr != nil {
+ return false
+ }
+ if results != nil {
+ defer results.Release()
+ for row := results.Pull(); row != nil; row = results.Pull() {
+ if row.Error != nil {
+ return false
+ }
+ if got[row.TID] == nil {
+ got[row.TID] = make(map[string][32]byte)
+ }
+ for index, id := range row.SpanIDs {
+ got[row.TID][id] = sha256.Sum256(row.Spans[index])
+ }
+ }
+ }
+ for id, wanted := range batch {
+ if len(wanted) == 0 {
+ if len(got[id]) != 0 {
+ return false
+ }
+ continue
+ }
+ if len(got[id]) != len(wanted) {
+ return false
+ }
+ for spanID, digest := range wanted {
+ if got[id][spanID] != digest {
+ return false
+ }
+ }
+ }
+ return true
+ }, 30*time.Second, 100*time.Millisecond, "bounded destination query must match retained source bodies and exclude drops")
+ for id, spans := range got {
+ for spanID, digest := range spans {
+ count++
+ addRealTraceSignature(&signature, id, spanID, digest)
+ }
+ }
+ clear(batch)
+ }
+ for {
+ assembled, scanErr := scanner.Next(context.Background())
+ if errors.Is(scanErr, io.EOF) {
+ break
+ }
+ require.NoError(t, scanErr)
+ require.False(t, assembled.Oversized, "real-data verification limits each trace to32MiB")
+ keep, lookupErr := prepared.table.lookup(assembled.Trace.TraceID)
+ require.NoError(t, lookupErr)
+ spans := make(map[string][32]byte)
+ if keep {
+ for index, id := range assembled.Trace.SpanIDs {
+ spans[id] = sha256.Sum256(assembled.Trace.Spans[index])
+ }
+ }
+ batch[assembled.Trace.TraceID] = spans
+ if len(batch) >= 128 {
+ flush()
+ }
+ }
+ flush()
+ return count, signature
+}
+
+func addRealTraceSignature(signature *[32]byte, id, spanID string, body [32]byte) {
+ digest := sha256.New()
+ _, _ = io.WriteString(digest, id)
+ _, _ = digest.Write([]byte{0})
+ _, _ = io.WriteString(digest, spanID)
+ _, _ = digest.Write(body[:])
+ value := digest.Sum(nil)
+ for index := range signature {
+ signature[index] ^= value[index]
+ }
+}
+
+func queryRealTraceIndex(t *testing.T, destination trace.Trace, schema *databasev1.Trace, rule *databasev1.IndexRule,
+ segment *timestamp.TimeRange,
+) (uint64, [32]byte) {
+ t.Helper()
+ var count uint64
+ var signature [32]byte
+ results, err := destination.Query(context.Background(), model.TraceQueryOptions{
+ TimeRange: segment, Name: schema.Metadata.Name,
+ Entities: [][]*modelv1.TagValue{{}}, Order: &index.OrderBy{Index: rule, Sort: modelv1.Sort_SORT_ASC},
+ MinVal: math.MinInt64, MaxVal: math.MaxInt64, MaxTraceSize: 100000,
+ })
+ require.NoError(t, err)
+ if results == nil {
+ return 0, signature
+ }
+ defer results.Release()
+ for row := results.Pull(); row != nil; row = results.Pull() {
+ require.NoError(t, row.Error)
+ for index, spanID := range row.SpanIDs {
+ count++
+ addRealTraceSignature(&signature, row.TID, spanID, sha256.Sum256(row.Spans[index]))
+ }
+ }
+ return count, signature
+}
diff --git a/banyand/backup/lifecycle/sampling_recovery.go b/banyand/backup/lifecycle/sampling_recovery.go
new file mode 100644
index 000000000..378b41db1
--- /dev/null
+++ b/banyand/backup/lifecycle/sampling_recovery.go
@@ -0,0 +1,262 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package lifecycle
+
+import (
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "google.golang.org/protobuf/proto"
+
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+)
+
+// recoverSamplingAttempts runs before legacy progress can hide a prepared shard.
+func recoverSamplingAttempts(root string, progress *Progress) (map[string]*commonv1.Group, error) {
+ groups := make(map[string]*commonv1.Group)
+ err := filepath.WalkDir(root, func(path string, entry os.DirEntry, walkErr error) error {
+ if errors.Is(walkErr, os.ErrNotExist) && path == root {
+ return nil
+ }
+ if walkErr != nil {
+ return walkErr
+ }
+ if entry.IsDir() && strings.HasPrefix(entry.Name(), ".preparing-") {
+ return filepath.SkipDir
+ }
+ if !entry.IsDir() && entry.Name() == "mode.json" {
+ return recoverSamplingMode(path, progress, groups)
+ }
+ if entry.IsDir() || entry.Name() != "manifest.json" {
+ return nil
+ }
+ return recoverSamplingManifest(path, progress, groups)
+ })
+ return groups, err
+}
+
+type samplingModeRecord struct {
+ Prepared map[string]string `json:"prepared,omitempty"`
+ Digest string `json:"sha256"`
+ Snapshot string `json:"snapshot"`
+ GroupConfig []byte `json:"group_config,omitempty"`
+ Sampled bool `json:"sampled"`
+}
+
+func selectSamplingMode(root, group, snapshot string, legacy bool, desired bool) (bool, error) {
+ if root == "" {
+ return desired && !legacy, nil
+ }
+ path := filepath.Join(root, group, "mode.json")
+ payload, err := os.ReadFile(path)
+ if err == nil {
+ var mode samplingModeRecord
+ if err = json.Unmarshal(payload, &mode); err != nil {
+ return false, err
+ }
+ if mode.Snapshot != snapshot {
+ return false, fmt.Errorf("sampling mode snapshot mismatch")
+ }
+ if err = validateSamplingMode(filepath.Dir(path), &mode); err != nil {
+ return false, err
+ }
+ return mode.Sampled, nil
+ }
+ if !errors.Is(err, os.ErrNotExist) {
+ return false, err
+ }
+ if err = makeSamplingDirectory(filepath.Dir(path)); err != nil {
+ return false, err
+ }
+ mode := samplingModeRecord{Snapshot: snapshot, Sampled: desired && !legacy}
+ if err = saveSamplingMode(path, mode); err != nil {
+ return false, err
+ }
+ return mode.Sampled, nil
+}
+
+func hasPreparedSampling(root, group string) (bool, error) {
+ found := false
+ err := filepath.WalkDir(filepath.Join(root, group), func(_ string, entry os.DirEntry, walkErr error) error {
+ if errors.Is(walkErr, os.ErrNotExist) {
+ return nil
+ }
+ if walkErr != nil {
+ return walkErr
+ }
+ if entry.IsDir() && strings.HasPrefix(entry.Name(), ".preparing-") {
+ return filepath.SkipDir
+ }
+ if !entry.IsDir() && entry.Name() == "manifest.json" {
+ found = true
+ }
+ return nil
+ })
+ return found, err
+}
+
+func validateSamplingMode(directory string, mode *samplingModeRecord) error {
+ copyMode := *mode
+ copyMode.Digest = ""
+ encoded, err := json.Marshal(copyMode)
+ if err != nil {
+ return err
+ }
+ digest := sha256.Sum256(encoded)
+ if mode.Digest != hex.EncodeToString(digest[:]) {
+ return fmt.Errorf("sampling mode checksum mismatch")
+ }
+ for relative, digest := range mode.Prepared {
+ path := filepath.Join(directory, relative)
+ if !strings.HasPrefix(path, directory+string(filepath.Separator)) {
+ return fmt.Errorf("prepared sampling path escapes mode directory")
+ }
+ payload, err := os.ReadFile(filepath.Join(path, "manifest.json"))
+ if err != nil {
+ return fmt.Errorf("required prepared sampling artifact missing: %w", err)
+ }
+ var manifest samplingManifest
+ if err = json.Unmarshal(payload, &manifest); err != nil {
+ return err
+ }
+ if err = verifySamplingManifest(&manifest); err != nil {
+ return err
+ }
+ if manifest.ManifestDigest != digest {
+ return fmt.Errorf("prepared sampling artifact identity mismatch")
+ }
+ }
+ return nil
+}
+
+func registerPreparedSampling(root string, prepared *preparedTraceMigration) error {
+ directory := filepath.Join(root, prepared.manifest.Group)
+ path := filepath.Join(directory, "mode.json")
+ payload, err := os.ReadFile(path)
+ if err != nil {
+ return err
+ }
+ var mode samplingModeRecord
+ if err = json.Unmarshal(payload, &mode); err != nil {
+ return err
+ }
+ if err = validateSamplingMode(directory, &mode); err != nil {
+ return err
+ }
+ if !mode.Sampled || mode.Snapshot != prepared.manifest.Snapshot {
+ return fmt.Errorf("prepared sampling mode ownership mismatch")
+ }
+ if mode.Prepared == nil {
+ mode.Prepared = make(map[string]string)
+ }
+ relative, err := filepath.Rel(directory, prepared.directory)
+ if err != nil {
+ return err
+ }
+ mode.Prepared[relative] = prepared.manifest.ManifestDigest
+ mode.GroupConfig = prepared.manifest.GroupConfig
+ return saveSamplingMode(path, mode)
+}
+
+func saveSamplingMode(path string, mode samplingModeRecord) error {
+ mode.Digest = ""
+ encoded, err := json.Marshal(mode)
+ if err != nil {
+ return err
+ }
+ digest := sha256.Sum256(encoded)
+ mode.Digest = hex.EncodeToString(digest[:])
+ return atomicSamplingJSON(path, mode)
+}
+
+func recoverSamplingMode(path string, progress *Progress, groups map[string]*commonv1.Group) error {
+ payload, err := os.ReadFile(path)
+ if err != nil {
+ return err
+ }
+ var mode samplingModeRecord
+ if err = json.Unmarshal(payload, &mode); err != nil {
+ return err
+ }
+ if mode.Sampled {
+ if err = validateSamplingMode(filepath.Dir(path), &mode); err != nil {
+ return err
+ }
+ if progress.SnapshotTraceDir != "" && progress.SnapshotTraceDir != mode.Snapshot {
+ return fmt.Errorf("conflicting sampled mode snapshots")
+ }
+ progress.SnapshotTraceDir = mode.Snapshot
+ if len(mode.GroupConfig) > 0 {
+ group := &commonv1.Group{}
+ if err = proto.Unmarshal(mode.GroupConfig, group); err != nil {
+ return err
+ }
+ groups[group.Metadata.Name] = group
+ }
+
+ if progress.samplingOwnedGroups == nil {
+ progress.samplingOwnedGroups = make(map[string]bool)
+ }
+ progress.samplingOwnedGroups[filepath.Base(filepath.Dir(path))] = true
+ }
+ return nil
+}
+
+func recoverSamplingManifest(path string, progress *Progress, groups map[string]*commonv1.Group) error {
+ payload, err := os.ReadFile(path)
+ if err != nil {
+ return err
+ }
+ manifest := &samplingManifest{}
+ if err = json.Unmarshal(payload, manifest); err != nil {
+ return fmt.Errorf("recover sampling manifest %s: %w", path, err)
+ }
+ if err = verifySamplingManifest(manifest); err != nil {
+ return err
+ }
+ if manifest.Version != samplingArtifactVersion {
+ return fmt.Errorf("unsupported pending sampling manifest %s", path)
+ }
+ group := &commonv1.Group{}
+ if err = proto.Unmarshal(manifest.GroupConfig, group); err != nil {
+ return err
+ }
+ if group.Metadata == nil || group.Metadata.Name != manifest.Group {
+ return fmt.Errorf("sampling manifest group mismatch %s", path)
+ }
+ if existing := groups[manifest.Group]; existing != nil && !proto.Equal(existing, group) {
+ return fmt.Errorf("conflicting pinned sampling groups %s", manifest.Group)
+ }
+ if _, snapshotErr := os.Stat(manifest.Snapshot); snapshotErr != nil {
+ return fmt.Errorf("required sampling snapshot unavailable: %w", snapshotErr)
+ }
+ groups[manifest.Group] = group
+ if progress.SnapshotTraceDir != "" && progress.SnapshotTraceDir != manifest.Snapshot {
+ return fmt.Errorf("conflicting sampling snapshot ownership")
+ }
+ progress.SnapshotTraceDir = manifest.Snapshot
+ delete(progress.CompletedGroups, manifest.Group)
+ delete(progress.DeletedTraceGroups, manifest.Group)
+ return nil
+}
diff --git a/banyand/backup/lifecycle/sampling_replay.go b/banyand/backup/lifecycle/sampling_replay.go
new file mode 100644
index 000000000..2f5c0866d
--- /dev/null
+++ b/banyand/backup/lifecycle/sampling_replay.go
@@ -0,0 +1,180 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package lifecycle
+
+import (
+ "context"
+ "fmt"
+ "path/filepath"
+ "time"
+
+ "google.golang.org/protobuf/proto"
+
+ databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ "github.com/apache/skywalking-banyandb/banyand/internal/dump"
+ dumptrace "github.com/apache/skywalking-banyandb/banyand/internal/dump/trace"
+)
+
+// Replay uses a continuous bounded sender session across source part boundaries.
+func (p *preparedTraceMigration) Replay(ctx context.Context, r *traceRowReplayer, shardPath string) (int, error) {
+ if p.checkpoint.Cursor.Part != "" {
+ found := false
+ for _, part := range p.manifest.Parts {
+ found = found || filepath.Base(part) == p.checkpoint.Cursor.Part
+ }
+ if !found || p.checkpoint.Cursor.Block < 0 || p.checkpoint.Cursor.Row < 0 {
+ return 0, fmt.Errorf("sampling checkpoint outside pinned inventory")
+ }
+ }
+ queuedBefore := r.sender.queuedBytes
+ defer func() { p.replayedBytes += r.sender.queuedBytes - queuedBefore }()
+ if p.checkpoint.Delivered {
+ return 0, nil
+ }
+ pinnedSchema := &databasev1.Trace{}
+ if err := proto.Unmarshal(p.manifest.Schema, pinnedSchema); err != nil {
+ return 0, err
+ }
+ r.schema = pinnedSchema
+ r.traceName = pinnedSchema.Metadata.Name
+ r.targetShardNum = p.manifest.TargetShardNum
+ parts := p.manifest.Parts
+ for _, part := range parts {
+ if filepath.Dir(part) != shardPath {
+ return 0, fmt.Errorf("sampling source part outside pinned shard")
+ }
+ }
+ cursor := p.checkpoint.Cursor
+ sequence := p.checkpoint.Sequence
+ r.sender.beforeFlush = func() (func() error, error) {
+ sequence++
+ checkpoint := samplingCheckpoint{ManifestDigest: p.manifest.ManifestDigest, Sequence: sequence, Cursor: cursor}
+ return func() error {
+ if err := saveSamplingCheckpoint(filepath.Join(p.directory, "checkpoint.json"), checkpoint); err != nil {
+ return fmt.Errorf("sampling delivery checkpoint: %w", err)
+ }
+ p.checkpoint = checkpoint
+ return nil
+ }, nil
+ }
+ defer func() { r.sender.beforeFlush = nil }()
+ rows := 0
+ lastFlush := time.Now()
+ var failure error
+ for _, partPath := range parts {
+ part := filepath.Base(partPath)
+ if p.checkpoint.Cursor.Part != "" && part < p.checkpoint.Cursor.Part {
+ continue
+ }
+ partRows, partErr := p.replaySourcePart(ctx, r, partPath, &cursor, &lastFlush)
+ rows += partRows
+ if partErr != nil {
+ failure = partErr
+ break
+ }
+ }
+ if failure == nil {
+ failure = r.sender.flush(ctx)
+ }
+ if drainErr := r.sender.drain(); failure == nil {
+ failure = drainErr
+ }
+ if failure != nil {
+ r.sender.discard()
+ return rows, failure
+ }
+ completed := p.checkpoint
+ completed.Cursor = cursor
+ completed.Delivered = true
+ if checkpointErr := saveSamplingCheckpoint(filepath.Join(p.directory, "checkpoint.json"), completed); checkpointErr != nil {
+ return rows, checkpointErr
+ }
+ p.checkpoint = completed
+ return rows, nil
+}
+
+func (p *preparedTraceMigration) replaySourcePart(ctx context.Context, r *traceRowReplayer, partPath string,
+ cursor *samplingCursor, lastFlush *time.Time,
+) (int, error) {
+ part := filepath.Base(partPath)
+ rows := 0
+ var failure error
+ partID, sourcePath, _, parseErr := parseReplayPartPath(partPath)
+ if parseErr != nil {
+ return 0, parseErr
+ }
+ reader, openErr := dumptrace.OpenPart(partID, sourcePath, r.fs)
+ if openErr != nil {
+ return 0, openErr
+ }
+ iterator := reader.Iterator()
+ if part == p.checkpoint.Cursor.Part {
+ if seekErr := iterator.SeekAfter(dump.Position{BlockIdx: p.checkpoint.Cursor.Block, RowIdx: p.checkpoint.Cursor.Row}); seekErr != nil {
+ iterator.Close()
+ reader.Close()
+ return 0, seekErr
+ }
+ }
+ for iterator.Next() {
+ if pollErr := r.sender.pipeline.poll(); pollErr != nil {
+ failure = pollErr
+ break
+ }
+ if ctx.Err() != nil {
+ failure = ctx.Err()
+ break
+ }
+ row := iterator.Row()
+ keep, lookupErr := p.table.lookup(row.TraceID)
+ if lookupErr != nil {
+ failure = lookupErr
+ break
+ }
+ if time.Since(*lastFlush) >= time.Second {
+ if flushErr := r.sender.flush(ctx); flushErr != nil {
+ failure = flushErr
+ break
+ }
+ *lastFlush = time.Now()
+ }
+ position := iterator.Position()
+ *cursor = samplingCursor{Part: part, Block: position.BlockIdx, Row: position.RowIdx}
+
+ if !keep {
+ continue
+ }
+ if publishErr := r.publishRow(ctx, row); publishErr != nil {
+ failure = publishErr
+ break
+ }
+ rows++
+ if time.Since(*lastFlush) >= time.Second {
+ if flushErr := r.sender.flush(ctx); flushErr != nil {
+ failure = flushErr
+ break
+ }
+ *lastFlush = time.Now()
+ }
+ }
+ if failure == nil {
+ failure = iterator.Err()
+ }
+ iterator.Close()
+ reader.Close()
+ return rows, failure
+}
diff --git a/banyand/backup/lifecycle/sampling_roundtrip_test.go b/banyand/backup/lifecycle/sampling_roundtrip_test.go
new file mode 100644
index 000000000..1f6b02a6b
--- /dev/null
+++ b/banyand/backup/lifecycle/sampling_roundtrip_test.go
@@ -0,0 +1,298 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package lifecycle
+
+import (
+ "context"
+ "fmt"
+ "path/filepath"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+ "go.uber.org/mock/gomock"
+ "google.golang.org/protobuf/proto"
+
+ "github.com/apache/skywalking-banyandb/api/common"
+ "github.com/apache/skywalking-banyandb/api/data"
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ tracev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/trace/v1"
+ "github.com/apache/skywalking-banyandb/banyand/internal/pipeline"
+ "github.com/apache/skywalking-banyandb/banyand/queue"
+ "github.com/apache/skywalking-banyandb/banyand/trace"
+ "github.com/apache/skywalking-banyandb/pkg/bus"
+ "github.com/apache/skywalking-banyandb/pkg/fs"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+ "github.com/apache/skywalking-banyandb/pkg/node"
+ "github.com/apache/skywalking-banyandb/pkg/pipeline/sdk"
+ "github.com/apache/skywalking-banyandb/pkg/timestamp"
+)
+
+const samplingPartialMode = "partial"
+
+type migrationTestSampler struct {
+ mode string
+ sizes []int
+ calls int
+}
+
+func (*migrationTestSampler) Kind() sdk.Kind { return sdk.KindSampler }
+func (*migrationTestSampler) Project() sdk.Projection { return sdk.Projection{Spans: true} }
+func (*migrationTestSampler) Close() error { return nil }
+func (s *migrationTestSampler) Decide(batch *sdk.TraceBatch) (sdk.Verdict, error) {
+ s.calls++
+ keep := make([]bool, len(batch.Traces))
+ for index, traceBlock := range batch.Traces {
+ s.sizes = append(s.sizes, len(traceBlock.Spans))
+ keep[index] = s.mode == "keep-all" || ((s.mode == "mixed" || s.mode == samplingPartialMode) && traceBlock.TraceID == "b-keep")
+ }
+ return sdk.Verdict{Keep: keep}, nil
+}
+
+type samplingCapturePublisher struct {
+ writes []*tracev1.InternalWriteRequest
+ mu sync.Mutex
+ batches int
+ failAfter int
+}
+
+func (p *samplingCapturePublisher) Publish(_ context.Context, _ bus.Topic, messages ...bus.Message) (bus.Future, error) {
+ p.mu.Lock()
+ defer p.mu.Unlock()
+ p.batches++
+ for _, message := range messages {
+ switch payload := message.Data().(type) {
+ case *tracev1.InternalWriteRequest:
+ p.writes = append(p.writes, proto.Clone(payload).(*tracev1.InternalWriteRequest))
+ case []byte:
+ write := &tracev1.InternalWriteRequest{}
+ if err := proto.Unmarshal(payload, write); err != nil {
+ return nil, err
+ }
+ p.writes = append(p.writes, write)
+ default:
+ return nil, fmt.Errorf("unexpected sampling write %T", payload)
+ }
+ }
+ return nil, nil
+}
+func (*samplingCapturePublisher) Close() (map[string]*common.Error, error) { return nil, nil }
+
+type samplingCaptureStream struct {
+ capture *samplingCapturePublisher
+ number int
+}
+
+func (p *samplingCaptureStream) Publish(ctx context.Context, topic bus.Topic, messages ...bus.Message) (bus.Future, error) {
+ return p.capture.Publish(ctx, topic, messages...)
+}
+
+func (p *samplingCaptureStream) Close() (map[string]*common.Error, error) {
+ if p.capture.failAfter > 0 && p.number == 2 {
+ return nil, fmt.Errorf("injected transport failure")
+ }
+ return nil, nil
+}
+
+func TestSamplingMultipartPreparationReplayAndResume(t *testing.T) {
+ for _, mode := range []string{"mixed", "keep-all", "drop-all", samplingPartialMode} {
+ t.Run(mode, func(t *testing.T) {
+ now := time.Now().Truncate(time.Second)
+ shard := filepath.Join(t.TempDir(), "00000000")
+ for partID := uint64(1); partID <= 3; partID++ {
+ rows := make([]trace.PartEncoderRow, 0, 2)
+ for _, id := range []string{"a-drop", "b-keep"} {
+ rows = append(rows,
+ trace.PartEncoderRow{
+ TraceID: id,
+ SpanID: fmt.Sprintf("%s-%d",
+ id,
+ partID),
+ Span: []byte("body"),
+ Timestamp: now.UnixNano(),
+ Tags: []trace.PartEncoderTag{{
+ Name: "timestamp",
+ Value: trace.EncodedTimestamp(now.UnixNano()),
+ }},
+ })
+ }
+ _, release := trace.EncodePart(shard, fs.NewLocalFileSystem(), partID, rows)
+ release()
+ }
+ schema := &databasev1.Trace{
+ Metadata: &commonv1.Metadata{
+ Name: "trace",
+ Group: "group",
+ },
+ Tags: []*databasev1.TraceTagSpec{
+ {
+ Name: "trace_id",
+ Type: databasev1.TagType_TAG_TYPE_STRING,
+ },
+ {
+ Name: "span_id",
+ Type: databasev1.TagType_TAG_TYPE_STRING,
+ },
+ {
+ Name: "timestamp",
+ Type: databasev1.TagType_TAG_TYPE_TIMESTAMP,
+ },
+ },
+ TraceIdTagName: "trace_id",
+ SpanIdTagName: "span_id",
+ TimestampTagName: "timestamp",
+ }
+ group := &GroupConfig{
+ Group: &commonv1.Group{
+ Metadata: &commonv1.Metadata{Name: "group"},
+ Catalog: commonv1.Catalog_CATALOG_TRACE,
+ },
+ SourceStage: "hot",
+ TargetStage: "warm",
+ TargetShardNum: 2,
+ TargetReplicas: 1,
+ }
+ sampler := &migrationTestSampler{mode: mode}
+ session := pipeline.NewSession([]sdk.Sampler{sampler}, 3)
+ defer session.Close()
+ policy := &pinnedSamplingPolicy{
+ schema: schema,
+ config: &commonv1.TracePipelineConfig{Enabled: true},
+ session: session,
+ host: samplingHostConfig{Timeout: time.Second},
+ digest: "original-policy",
+ }
+ segment := ×tamp.TimeRange{Start: now.Add(-24 * time.Hour), End: now.Add(24 * time.Hour)}
+ artifactRoot := t.TempDir()
+ directory := samplingArtifactPath(artifactRoot, "group", segment, 0)
+ selectedMode, modeErr := selectSamplingMode(artifactRoot, "group", filepath.Dir(shard), false, true)
+ require.NoError(t, modeErr)
+ require.True(t, selectedMode)
+ prepared, err := prepareTraceMigration(context.Background(), directory, shard, filepath.Dir(shard), segment, 0, group, policy)
+ require.NoError(t, err)
+ require.Equal(t, []int{3, 3}, sampler.sizes, "all source parts must be assembled before Decide")
+ require.Equal(t, uint64(2), prepared.manifest.Traces)
+ require.Equal(t, uint64(6), prepared.manifest.Spans)
+ expectedRows := map[string]int{"mixed": 3, "keep-all": 6, "drop-all": 0, samplingPartialMode: 3}[mode]
+ require.Equal(t, uint64(expectedRows), prepared.manifest.KeptSpans)
+ selector, err := node.NewPickFirstSelector()
+ require.NoError(t, err)
+ selector.AddNode(&databasev1.Node{Metadata: &commonv1.Metadata{Name: "target"}})
+ client := queue.NewMockClient(gomock.NewController(t))
+ capture := &samplingCapturePublisher{}
+ streamNumber := 0
+ client.EXPECT().NewBatchPublisher(gomock.Any()).DoAndReturn(func(time.Duration) queue.BatchPublisher {
+ streamNumber++
+ return &samplingCaptureStream{capture: capture, number: streamNumber}
+ }).AnyTimes()
+ replayer := &traceRowReplayer{
+ group: "group",
+ selector: selector,
+ fs: fs.NewLocalFileSystem(),
+ logger: logger.GetLogger("sampling-test"),
+ sender: newBatchSender(client,
+ data.TopicTraceWrite,
+ 100,
+ 1<<20,
+ time.Second),
+ }
+
+ if mode == samplingPartialMode {
+ replayer.sender.maxRows = 1
+ capture.failAfter = 1
+ _, partialErr := prepared.Replay(context.Background(), replayer, shard)
+ require.ErrorContains(t, partialErr, "injected transport failure")
+ require.False(t, prepared.checkpoint.Delivered)
+ require.Equal(t, uint64(1), prepared.checkpoint.Sequence)
+ require.GreaterOrEqual(t, len(capture.writes), 2, "the failed acknowledgement may have reached the receiver")
+ require.LessOrEqual(t, len(capture.writes), 3, "in-flight delivery remains bounded")
+ require.Equal(t, "b-keep-1", capture.writes[0].Request.Tags[1].GetStr().Value)
+ capture.writes = nil
+ require.NoError(t, prepared.Close())
+ priorCalls := sampler.calls
+ prepared, err = openPreparedTraceMigration(directory)
+ require.NoError(t, err)
+ require.Equal(t, priorCalls, sampler.calls)
+ _, closeErr := replayer.Close()
+ require.NoError(t, closeErr)
+ capture.failAfter = 0
+ capture.batches = 0
+ replayer.sender = newBatchSender(client, data.TopicTraceWrite, 100, 1<<20, time.Second)
+ }
+ replayed, err := prepared.Replay(context.Background(), replayer, shard)
+ require.NoError(t, err)
+ if mode == samplingPartialMode {
+ require.Equal(t, expectedRows-1, replayed)
+ } else {
+ require.Equal(t, expectedRows, replayed)
+ }
+ if mode == samplingPartialMode {
+ require.Len(t, capture.writes, expectedRows-1)
+ require.Equal(t, "b-keep-2", capture.writes[0].Request.Tags[1].GetStr().Value)
+ } else {
+ require.Len(t, capture.writes, expectedRows)
+ }
+ expectedBatches := 0
+ if expectedRows > 0 {
+ expectedBatches = 1
+ }
+ require.Equal(t, expectedBatches, capture.batches, "part boundaries must not flush replay batches")
+ for _, write := range capture.writes {
+ if mode == "mixed" || mode == samplingPartialMode {
+ require.Equal(t, "b-keep", write.Request.Tags[0].GetStr().Value)
+ }
+ }
+ require.True(t, prepared.checkpoint.Delivered)
+ require.NoError(t, prepared.Close())
+ progress := NewProgress("", logger.GetLogger("sampling-test"))
+ visitor := &traceMigrationVisitor{
+ ctx: context.Background(),
+ group: "group",
+ samplingRoot: artifactRoot,
+ samplingGroup: group,
+ sourceStage: "hot",
+ targetStage: "warm",
+ targetShardNum: 2,
+ replicas: 1,
+ lfs: fs.NewLocalFileSystem(),
+ replayer: replayer,
+ progress: progress,
+ logger: logger.GetLogger("sampling-test"),
+ }
+ require.NoError(t, visitor.VisitSeries(segment, "missing-source-index", []common.ShardID{0}))
+ require.Len(t, visitor.sampledSeries, 1, "source index transfer is deferred in sampled mode")
+ require.Empty(t, progress.SourceCompletedTraceSeries, "series cannot complete before retained-row delivery")
+ require.NoError(t, visitor.VisitShard(segment, 0, shard), "sampled path must precede ordinary chunk transfer even when no live policy remains")
+ require.True(t, progress.IsTraceShardCompleted("group", segment.String(), 0))
+
+ calls := sampler.calls
+ restored, err := prepareTraceMigration(context.Background(), directory, "missing-new-snapshot", "changed", segment, 0, group, nil)
+ require.NoError(t, err)
+ defer restored.Close()
+ require.Equal(t, "original-policy", restored.manifest.PolicyDigest)
+ replayed, err = restored.Replay(context.Background(), replayer, shard)
+ require.NoError(t, err)
+ require.Zero(t, replayed)
+ require.Equal(t, calls, sampler.calls, "delivery resume must not invoke sampler")
+ _, err = replayer.Close()
+ require.NoError(t, err)
+ })
+ }
+}
diff --git a/banyand/backup/lifecycle/sampling_store.go b/banyand/backup/lifecycle/sampling_store.go
new file mode 100644
index 000000000..f437ece27
--- /dev/null
+++ b/banyand/backup/lifecycle/sampling_store.go
@@ -0,0 +1,366 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package lifecycle
+
+import (
+ "crypto/sha256"
+ "encoding/binary"
+ "encoding/hex"
+ "encoding/json"
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+)
+
+const samplingArtifactVersion = 1
+
+type samplingCursor struct {
+ Part string `json:"part"`
+ Block int `json:"block"`
+ Row int `json:"row"`
+}
+
+type samplingCheckpoint struct {
+ ManifestDigest string `json:"manifest_sha256"`
+ Cursor samplingCursor `json:"cursor"`
+ Sequence uint64 `json:"sequence"`
+ Delivered bool `json:"delivered"`
+}
+
+type samplingInventoryFile struct {
+ Path string `json:"path"`
+ Digest string `json:"sha256"`
+ Size int64 `json:"size"`
+}
+
+type samplingManifest struct {
+ FailOpen map[string]uint64 `json:"fail_open"`
+ PolicyDigest string `json:"policy_digest"`
+ IndexDigest string `json:"index_digest"`
+ ManifestDigest string `json:"manifest_sha256"`
+ Snapshot string `json:"snapshot"`
+ Group string `json:"group"`
+ SourceStage string `json:"source_stage"`
+ TargetStage string `json:"target_stage"`
+ TableDigest string `json:"table_digest"`
+ SourceInterval []byte `json:"source_interval"`
+ Parts []string `json:"parts"`
+ Schema []byte `json:"schema"`
+ PluginDigests []string `json:"plugin_digests"`
+ TargetInterval []byte `json:"target_interval"`
+ Policy []byte `json:"policy"`
+ GroupConfig []byte `json:"group_config"`
+ Inventory []samplingInventoryFile `json:"inventory"`
+ KeptSpans uint64 `json:"kept_spans"`
+ SegmentEnd int64 `json:"segment_end"`
+ SegmentStart int64 `json:"segment_start"`
+ Traces uint64 `json:"traces"`
+ KeptTraces uint64 `json:"kept_traces"`
+ Spans uint64 `json:"spans"`
+ Version int `json:"version"`
+ PreparationNanos int64 `json:"preparation_nanos"`
+ Replicas uint32 `json:"replicas"`
+ TargetShardNum uint32 `json:"target_shards"`
+ Shard uint32 `json:"shard"`
+}
+
+// atomicSamplingJSON makes durability errors visible to the migration caller.
+func atomicSamplingJSON(path string, value any) error {
+ payload, err := json.Marshal(value)
+ if err != nil {
+ return err
+ }
+ file, err := os.CreateTemp(filepath.Dir(path), ".commit-*")
+ if err != nil {
+ return err
+ }
+ temporary := file.Name()
+ defer os.Remove(temporary)
+ if _, err = file.Write(payload); err != nil {
+ file.Close()
+ return err
+ }
+ if err = file.Sync(); err != nil {
+ file.Close()
+ return err
+ }
+ if err = file.Close(); err != nil {
+ return err
+ }
+ if err = os.Rename(temporary, path); err != nil {
+ return err
+ }
+ return syncSamplingDirectory(filepath.Dir(path))
+}
+
+func syncSamplingDirectory(path string) error {
+ directory, err := os.Open(path)
+ if err != nil {
+ return err
+ }
+ err = directory.Sync()
+ closeErr := directory.Close()
+ if err != nil {
+ return err
+ }
+ return closeErr
+}
+
+func samplingFileDigest(path string) (string, error) {
+ file, err := os.Open(path)
+ if err != nil {
+ return "", err
+ }
+ defer file.Close()
+ digest := sha256.New()
+ if _, err = io.Copy(digest, file); err != nil {
+ return "", err
+ }
+ return hex.EncodeToString(digest.Sum(nil)), nil
+}
+
+type samplingDecisionWriter struct {
+ table *os.File
+ index *os.File
+ previous string
+ count uint64
+}
+
+func newSamplingDecisionWriter(dir string) (*samplingDecisionWriter, error) {
+ table, err := os.Create(filepath.Join(dir, "decisions"))
+ if err != nil {
+ return nil, err
+ }
+ index, err := os.Create(filepath.Join(dir, "offsets"))
+ if err != nil {
+ table.Close()
+ return nil, err
+ }
+ return &samplingDecisionWriter{table: table, index: index}, nil
+}
+
+func (w *samplingDecisionWriter) append(id string, keep bool) error {
+ if w.count > 0 && id <= w.previous {
+ return fmt.Errorf("sampling decisions are not strictly ordered: %q", id)
+ }
+ if len(id) > 1<<20 {
+ return fmt.Errorf("trace ID exceeds decision record limit")
+ }
+ offset, err := w.table.Seek(0, io.SeekCurrent)
+ if err != nil {
+ return err
+ }
+ var index [8]byte
+ binary.LittleEndian.PutUint64(index[:], uint64(offset))
+ if _, err = w.index.Write(index[:]); err != nil {
+ return err
+ }
+ var header [5]byte
+ binary.LittleEndian.PutUint32(header[:4], uint32(len(id)))
+ if keep {
+ header[4] = 1
+ }
+ if _, err = w.table.Write(header[:]); err != nil {
+ return err
+ }
+ if _, err = io.WriteString(w.table, id); err != nil {
+ return err
+ }
+ w.previous = id
+ w.count++
+ return nil
+}
+
+func (w *samplingDecisionWriter) close() error {
+ var first error
+ for _, file := range []*os.File{w.table, w.index} {
+ if err := file.Sync(); first == nil {
+ first = err
+ }
+ if err := file.Close(); first == nil {
+ first = err
+ }
+ }
+ return first
+}
+
+type samplingDecisionTable struct {
+ table *os.File
+ index *os.File
+ count uint64
+}
+
+func openSamplingDecisionTable(dir string, manifest *samplingManifest) (*samplingDecisionTable, error) {
+ for name, expected := range map[string]string{"decisions": manifest.TableDigest, "offsets": manifest.IndexDigest} {
+ digest, err := samplingFileDigest(filepath.Join(dir, name))
+ if err != nil {
+ return nil, err
+ }
+ if digest != expected {
+ return nil, fmt.Errorf("sampling %s checksum mismatch", name)
+ }
+ }
+ table, err := os.Open(filepath.Join(dir, "decisions"))
+ if err != nil {
+ return nil, err
+ }
+ index, err := os.Open(filepath.Join(dir, "offsets"))
+ if err != nil {
+ table.Close()
+ return nil, err
+ }
+ info, err := index.Stat()
+ if err != nil {
+ table.Close()
+ index.Close()
+ return nil, fmt.Errorf("stat sampling decision index: %w", err)
+ }
+ if info.Size() != int64(manifest.Traces)*8 {
+ table.Close()
+ index.Close()
+ return nil, fmt.Errorf("sampling index count mismatch")
+ }
+ return &samplingDecisionTable{table: table, index: index, count: manifest.Traces}, nil
+}
+
+func (t *samplingDecisionTable) lookup(id string) (bool, error) {
+ low, high := uint64(0), t.count
+ for low < high {
+ mid := low + (high-low)/2
+ var offset [8]byte
+ if _, err := t.index.ReadAt(offset[:], int64(mid)*8); err != nil {
+ return false, err
+ }
+ position := int64(binary.LittleEndian.Uint64(offset[:]))
+ var header [5]byte
+ if _, err := t.table.ReadAt(header[:], position); err != nil {
+ return false, err
+ }
+ length := binary.LittleEndian.Uint32(header[:4])
+ if length > 1<<20 || header[4] > 1 {
+ return false, fmt.Errorf("invalid sampling decision record")
+ }
+ value := make([]byte, length)
+ if _, err := t.table.ReadAt(value, position+5); err != nil {
+ return false, err
+ }
+ key := string(value)
+ if key == id {
+ return header[4] == 1, nil
+ }
+ if key < id {
+ low = mid + 1
+ } else {
+ high = mid
+ }
+ }
+ return false, fmt.Errorf("trace %q missing from authoritative sampling decisions", id)
+}
+
+func (t *samplingDecisionTable) close() error {
+ first := t.table.Close()
+ second := t.index.Close()
+ if first != nil {
+ return first
+ }
+ return second
+}
+
+func makeSamplingDirectory(path string) error {
+ if info, err := os.Stat(path); err == nil {
+ if !info.IsDir() {
+ return fmt.Errorf("sampling directory is not a directory: %s", path)
+ }
+ if err = syncSamplingDirectory(path); err != nil {
+ return err
+ }
+ return syncSamplingDirectory(filepath.Dir(path))
+ } else if !os.IsNotExist(err) {
+ return err
+ }
+ parent := filepath.Dir(path)
+ if parent == path {
+ return fmt.Errorf("cannot create sampling root %s", path)
+ }
+ if err := makeSamplingDirectory(parent); err != nil {
+ return err
+ }
+ if err := os.Mkdir(path, 0o700); err != nil && !os.IsExist(err) {
+ return err
+ }
+ if err := syncSamplingDirectory(path); err != nil {
+ return err
+ }
+ return syncSamplingDirectory(parent)
+}
+
+type samplingCheckpointRecord struct {
+ Digest string `json:"sha256"`
+ Checkpoint samplingCheckpoint `json:"checkpoint"`
+}
+
+func saveSamplingCheckpoint(path string, checkpoint samplingCheckpoint) error {
+ payload, err := json.Marshal(checkpoint)
+ if err != nil {
+ return err
+ }
+ digest := sha256.Sum256(payload)
+ return atomicSamplingJSON(path, samplingCheckpointRecord{Checkpoint: checkpoint, Digest: hex.EncodeToString(digest[:])})
+}
+
+func readSamplingCheckpoint(payload []byte) (samplingCheckpoint, error) {
+ var record samplingCheckpointRecord
+ if err := json.Unmarshal(payload, &record); err != nil {
+ return samplingCheckpoint{}, err
+ }
+ encoded, err := json.Marshal(record.Checkpoint)
+ if err != nil {
+ return samplingCheckpoint{}, err
+ }
+ digest := sha256.Sum256(encoded)
+ if record.Digest != hex.EncodeToString(digest[:]) {
+ return samplingCheckpoint{}, fmt.Errorf("sampling checkpoint checksum mismatch")
+ }
+ if record.Checkpoint.Sequence == 0 && record.Checkpoint.Cursor.Part != "" && !record.Checkpoint.Delivered {
+ return samplingCheckpoint{}, fmt.Errorf("sampling checkpoint cursor without confirmed sequence")
+ }
+ return record.Checkpoint, nil
+}
+
+func sealSamplingManifest(manifest *samplingManifest) error {
+ manifest.ManifestDigest = ""
+ payload, err := json.Marshal(manifest)
+ if err != nil {
+ return err
+ }
+ digest := sha256.Sum256(payload)
+ manifest.ManifestDigest = hex.EncodeToString(digest[:])
+ return nil
+}
+
+func verifySamplingManifest(manifest *samplingManifest) error {
+ copyManifest := *manifest
+ if err := sealSamplingManifest(©Manifest); err != nil {
+ return err
+ }
+ if manifest.ManifestDigest != copyManifest.ManifestDigest {
+ return fmt.Errorf("sampling manifest checksum mismatch")
+ }
+ return nil
+}
diff --git a/banyand/backup/lifecycle/sampling_store_test.go b/banyand/backup/lifecycle/sampling_store_test.go
new file mode 100644
index 000000000..747909115
--- /dev/null
+++ b/banyand/backup/lifecycle/sampling_store_test.go
@@ -0,0 +1,224 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package lifecycle
+
+import (
+ "encoding/json"
+ "errors"
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+ "google.golang.org/protobuf/proto"
+
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+)
+
+func TestSamplingDecisionTableIsAuthoritative(t *testing.T) {
+ directory := t.TempDir()
+ writer, err := newSamplingDecisionWriter(directory)
+ require.NoError(t, err)
+ require.NoError(t, writer.append("a", false))
+ require.NoError(t, writer.append("b", true))
+ require.NoError(t, writer.append("c", false))
+ require.ErrorContains(t, writer.append("b", true), "not strictly ordered")
+ require.NoError(t, writer.close())
+ tableDigest, err := samplingFileDigest(filepath.Join(directory, "decisions"))
+ require.NoError(t, err)
+ indexDigest, err := samplingFileDigest(filepath.Join(directory, "offsets"))
+ require.NoError(t, err)
+ manifest := &samplingManifest{TableDigest: tableDigest, IndexDigest: indexDigest, Traces: 3}
+ table, err := openSamplingDecisionTable(directory, manifest)
+ require.NoError(t, err)
+ for id, expected := range map[string]bool{"a": false, "b": true, "c": false} {
+ keep, lookupErr := table.lookup(id)
+ require.NoError(t, lookupErr)
+ require.Equal(t, expected, keep)
+ }
+ _, err = table.lookup("missing")
+ require.ErrorContains(t, err, "missing from authoritative")
+ require.NoError(t, table.close())
+ require.NoError(t, os.WriteFile(filepath.Join(directory, "decisions"), []byte("corruption"), 0o600))
+ _, err = openSamplingDecisionTable(directory, manifest)
+ require.ErrorContains(t, err, "checksum mismatch")
+ require.NoError(t, os.Remove(filepath.Join(directory, "decisions")))
+ _, err = openSamplingDecisionTable(directory, manifest)
+ require.Error(t, err, "missing state must not create replacement verdicts")
+}
+
+func TestSamplingAtomicCheckpointPublication(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "checkpoint.json")
+ initial := samplingCheckpoint{Sequence: 1, Cursor: samplingCursor{Part: "part-1", Block: 2, Row: 3}}
+ require.NoError(t, atomicSamplingJSON(path, initial))
+ advanced := samplingCheckpoint{Sequence: 2, Cursor: samplingCursor{Part: "part-2", Block: 0, Row: 1}, Delivered: true}
+ require.NoError(t, atomicSamplingJSON(path, advanced))
+ payload, err := os.ReadFile(path)
+ require.NoError(t, err)
+ var restored samplingCheckpoint
+ require.NoError(t, json.Unmarshal(payload, &restored))
+ require.Equal(t, advanced, restored)
+ entries, err := os.ReadDir(filepath.Dir(path))
+ require.NoError(t, err)
+ require.Len(t, entries, 1, "publication should leave no temporary commit artifacts")
+ require.Error(t, atomicSamplingJSON(filepath.Join(path, "invalid-parent"), advanced), "durability failures must propagate")
+ require.Error(t, atomicSamplingJSON(filepath.Dir(path), advanced), "rename over directory must fail")
+ payload, err = os.ReadFile(path)
+ require.NoError(t, err)
+ require.NoError(t, json.Unmarshal(payload, &restored))
+ require.Equal(t, advanced, restored, "failed publication must preserve last durable cursor")
+}
+
+func TestSamplingCheckpointsWaitForContiguousConfirmations(t *testing.T) {
+ pipeline := newConfirmPipeline()
+ first, second := make(chan error, 1), make(chan error, 1)
+ checkpoints := make(chan int, 2)
+ require.NoError(t, pipeline.addConfirmed(first, func() error { checkpoints <- 1; return nil }))
+ require.NoError(t, pipeline.addConfirmed(second, func() error { checkpoints <- 2; return nil }))
+ second <- nil
+ finished := make(chan error, 1)
+ go func() { finished <- pipeline.drain() }()
+ select {
+ case sequence := <-checkpoints:
+ t.Fatalf("checkpoint %d passed an unconfirmed predecessor", sequence)
+ case <-time.After(10 * time.Millisecond):
+ }
+ first <- nil
+ require.NoError(t, <-finished)
+ require.Equal(t, 1, <-checkpoints)
+ require.Equal(t, 2, <-checkpoints)
+}
+
+func TestSamplingCheckpointFailureStopsAdvanceAndDrains(t *testing.T) {
+ pipeline := newConfirmPipeline()
+ first, second := make(chan error, 1), make(chan error, 1)
+ advanced := 0
+ failure := errors.New("manifest filesystem full")
+ require.NoError(t, pipeline.addConfirmed(first, func() error { return failure }))
+ require.NoError(t, pipeline.addConfirmed(second, func() error { advanced++; return nil }))
+ first <- nil
+ second <- nil
+ require.ErrorIs(t, pipeline.drain(), failure)
+ require.Zero(t, advanced)
+ require.Empty(t, pipeline.inflight, "failure must still drain outstanding confirmations")
+ require.True(t, pipeline.checkpointFailed)
+ require.NoError(t, pipeline.drain())
+}
+
+func TestSamplingAuthoritativeStateRejectsTampering(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "checkpoint.json")
+ checkpoint := samplingCheckpoint{Sequence: 7, Cursor: samplingCursor{Part: "part", Block: 2, Row: 4}}
+ require.NoError(t, saveSamplingCheckpoint(path, checkpoint))
+ payload, err := os.ReadFile(path)
+ require.NoError(t, err)
+ restored, err := readSamplingCheckpoint(payload)
+ require.NoError(t, err)
+ require.Equal(t, checkpoint, restored)
+ var record samplingCheckpointRecord
+ require.NoError(t, json.Unmarshal(payload, &record))
+ record.Checkpoint.Sequence++
+ tampered, err := json.Marshal(record)
+ require.NoError(t, err)
+ _, err = readSamplingCheckpoint(tampered)
+ require.ErrorContains(t, err, "checksum mismatch")
+ manifest := &samplingManifest{Version: samplingArtifactVersion, Group: "group", PolicyDigest: "policy"}
+ require.NoError(t, sealSamplingManifest(manifest))
+ require.NoError(t, verifySamplingManifest(manifest))
+ manifest.TargetShardNum++
+ require.ErrorContains(t, verifySamplingManifest(manifest), "checksum mismatch")
+}
+
+func TestSamplingRecoveryRestoresPinnedGroupWithoutLegacyProgress(t *testing.T) {
+ root := t.TempDir()
+ incomplete := filepath.Join(root, ".preparing-interrupted")
+ require.NoError(t, os.Mkdir(incomplete, 0o700))
+ require.NoError(t, os.WriteFile(filepath.Join(incomplete, "manifest.json"), []byte("unfinished"), 0o600))
+ progress := &Progress{CompletedGroups: map[string]bool{"group": true}, DeletedTraceGroups: map[string]bool{"group": true}}
+ groups, err := recoverSamplingAttempts(root, progress)
+ require.NoError(t, err)
+ require.Empty(t, groups)
+ group := &commonv1.Group{Metadata: &commonv1.Metadata{Name: "group"}, Catalog: commonv1.Catalog_CATALOG_TRACE, ResourceOpts: &commonv1.ResourceOpts{ShardNum: 3}}
+ encoded, err := proto.Marshal(group)
+ require.NoError(t, err)
+ manifest := &samplingManifest{Version: samplingArtifactVersion, Group: "group", Snapshot: root, GroupConfig: encoded}
+ require.NoError(t, sealSamplingManifest(manifest))
+ prepared := filepath.Join(root, "published")
+ require.NoError(t, os.Mkdir(prepared, 0o700))
+ require.NoError(t, atomicSamplingJSON(filepath.Join(prepared, "manifest.json"), manifest))
+ groups, err = recoverSamplingAttempts(root, progress)
+ require.NoError(t, err)
+ require.True(t, proto.Equal(group, groups["group"]))
+ require.Equal(t, root, progress.SnapshotTraceDir)
+ require.False(t, progress.CompletedGroups["group"])
+ require.False(t, progress.DeletedTraceGroups["group"])
+}
+
+func TestSamplingModePinsLegacyAndSampledWork(t *testing.T) {
+ root, snapshot := t.TempDir(), t.TempDir()
+ selected, err := selectSamplingMode(root, "legacy", snapshot, true, true)
+ require.NoError(t, err)
+ require.False(t, selected, "new stage rules cannot retroactively sample legacy in-flight work")
+ selected, err = selectSamplingMode(root, "legacy", snapshot, false, true)
+ require.NoError(t, err)
+ require.False(t, selected, "restart must preserve original transport mode")
+ selected, err = selectSamplingMode(root, "sampled", snapshot, false, true)
+ require.NoError(t, err)
+ require.True(t, selected)
+ selected, err = selectSamplingMode(root, "sampled", snapshot, false, false)
+ require.NoError(t, err)
+ require.True(t, selected, "disabled current policy must not remove prepared-mode ownership")
+}
+
+func TestSamplingRecoveryRejectsMissingRegisteredArtifact(t *testing.T) {
+ root, snapshot := t.TempDir(), t.TempDir()
+ selected, err := selectSamplingMode(root, "group", snapshot, false, true)
+ require.NoError(t, err)
+ require.True(t, selected)
+ preparing := filepath.Join(root, "group", ".preparing-interrupted")
+ require.NoError(t, os.Mkdir(preparing, 0o700))
+ require.NoError(t, os.WriteFile(filepath.Join(preparing, "manifest.json"), []byte("incomplete"), 0o600))
+ found, err := hasPreparedSampling(root, "group")
+ require.NoError(t, err)
+ require.False(t, found, "incomplete preparation cannot select prepared replay")
+ group := &commonv1.Group{Metadata: &commonv1.Metadata{Name: "group"}, Catalog: commonv1.Catalog_CATALOG_TRACE}
+ encoded, err := proto.Marshal(group)
+ require.NoError(t, err)
+ manifest := samplingManifest{Version: samplingArtifactVersion, Snapshot: snapshot, Group: "group", GroupConfig: encoded}
+ require.NoError(t, sealSamplingManifest(&manifest))
+ directory := filepath.Join(root, "group", "published")
+ require.NoError(t, os.Mkdir(directory, 0o700))
+ require.NoError(t, atomicSamplingJSON(filepath.Join(directory, "manifest.json"), &manifest))
+ require.NoError(t, registerPreparedSampling(root, &preparedTraceMigration{directory: directory, manifest: manifest}))
+ require.NoError(t, os.RemoveAll(directory))
+ _, err = recoverSamplingAttempts(root, &Progress{})
+ require.ErrorContains(t, err, "required prepared sampling artifact missing", "registered prepared work cannot silently become a new sampling attempt")
+}
+
+func TestSamplingModeOnlyRecoveryPreservesSnapshotWithoutPinningConfiguration(t *testing.T) {
+ root, snapshot := t.TempDir(), t.TempDir()
+ selected, err := selectSamplingMode(root, "group", snapshot, false, true)
+ require.NoError(t, err)
+ require.True(t, selected)
+ progress := &Progress{}
+ groups, err := recoverSamplingAttempts(root, progress)
+ require.NoError(t, err)
+ require.Empty(t, groups, "preparation failures must allow correction of invalid current policy")
+ require.Equal(t, snapshot, progress.SnapshotTraceDir)
+ require.True(t, progress.samplingOwnedGroups["group"])
+}
diff --git a/banyand/backup/lifecycle/sampling_visitor.go b/banyand/backup/lifecycle/sampling_visitor.go
new file mode 100644
index 000000000..dbf2ea06e
--- /dev/null
+++ b/banyand/backup/lifecycle/sampling_visitor.go
@@ -0,0 +1,157 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package lifecycle
+
+import (
+ "bytes"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strconv"
+ "strings"
+
+ "google.golang.org/protobuf/proto"
+
+ "github.com/apache/skywalking-banyandb/api/common"
+ "github.com/apache/skywalking-banyandb/api/data"
+ databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ "github.com/apache/skywalking-banyandb/pkg/timestamp"
+)
+
+type sampledSeriesWork struct {
+ path string
+ shards []common.ShardID
+}
+
+func (mv *traceMigrationVisitor) visitSampledShard(segment *timestamp.TimeRange, shard common.ShardID, shardPath, directory string) error {
+ var prepared *preparedTraceMigration
+ var err error
+ if mv.lfs.IsExist(directory) {
+ prepared, err = openPreparedTraceMigration(directory)
+ } else {
+ if mv.samplingPolicy == nil {
+ mv.samplingPolicy, err = resolveSamplingPolicy(mv.ctx, mv.samplingGroup, mv.metadata)
+ if err != nil {
+ return err
+ }
+ if mv.samplingPolicy == nil {
+ return fmt.Errorf("pending sampled migration has no pinned source policy")
+ }
+ }
+ shardPolicy := mv.samplingPolicy
+ if len(shardPolicy.plugins) > 0 {
+ shardPolicy, err = mv.samplingPolicy.newShardSession(mv.group)
+ if err != nil {
+ return err
+ }
+ defer shardPolicy.close()
+ }
+ prepared, err = prepareTraceMigration(mv.ctx, directory, shardPath, mv.snapshot, segment, uint32(shard), mv.samplingGroup, shardPolicy)
+ }
+ if err != nil {
+ mv.recordError(scopeShard, segment, shard, err.Error())
+ return err
+ }
+ defer prepared.Close()
+ if err = registerPreparedSampling(mv.samplingRoot, prepared); err != nil {
+ return err
+ }
+ sourceInterval, intervalErr := proto.MarshalOptions{Deterministic: true}.Marshal(mv.samplingGroup.SegmentInterval)
+ if intervalErr != nil {
+ return intervalErr
+ }
+ targetInterval, intervalErr := proto.MarshalOptions{Deterministic: true}.Marshal(mv.samplingGroup.TargetSegmentInterval)
+ if intervalErr != nil {
+ return intervalErr
+ }
+ if prepared.manifest.SourceStage != mv.sourceStage || prepared.manifest.TargetStage != mv.targetStage ||
+ prepared.manifest.TargetShardNum != mv.targetShardNum || prepared.manifest.Replicas != mv.replicas ||
+ !bytes.Equal(sourceInterval, prepared.manifest.SourceInterval) || !bytes.Equal(targetInterval, prepared.manifest.TargetInterval) {
+ return fmt.Errorf("prepared sampling routing differs from resolved stages; retaining source")
+ }
+ if mv.replayer == nil {
+ pinnedSchema := &databasev1.Trace{}
+ if err = proto.Unmarshal(prepared.manifest.Schema, pinnedSchema); err != nil {
+ return err
+ }
+ mv.replayer = &traceRowReplayer{
+ group: mv.group, targetShardNum: prepared.manifest.TargetShardNum, selector: mv.selector,
+ sender: newBatchSender(mv.client, data.TopicTraceWrite, rowReplayMaxBatchRows, int(rowReplayMaxBatchBytes), traceReplayBatchTimeout),
+ fs: mv.lfs, logger: mv.logger, schema: pinnedSchema, traceName: pinnedSchema.Metadata.Name, counter: &mv.partsReplayedRowLevel,
+ }
+ }
+ replayer := mv.replayer
+ resumed := prepared.checkpoint.Sequence > 0
+ rows, err := prepared.Replay(mv.ctx, replayer, shardPath)
+ info, statErr := os.Stat(filepath.Join(directory, "manifest.json"))
+ if statErr != nil {
+ return statErr
+ }
+ mv.progress.mu.Lock()
+ if mv.progress.TraceSamplingReports == nil {
+ mv.progress.TraceSamplingReports = make(map[string]samplingReport)
+ }
+ report := mv.progress.TraceSamplingReports[directory]
+ report.Manifest = prepared.manifest
+ report.Artifact = directory
+ report.ManifestBytes = info.Size()
+ report.ReplayedBytes += prepared.replayedBytes
+ if resumed {
+ report.ResumeCount++
+ }
+ mv.progress.TraceSamplingReports[directory] = report
+ mv.progress.mu.Unlock()
+ if err != nil {
+ mv.recordError(scopeShard, segment, shard, err.Error())
+ return err
+ }
+ mv.progress.MarkTraceShardCompleted(mv.group, segment.String(), shard)
+ mv.progress.MarkSourceTraceShardCompleted(mv.group, segment.String(), shard)
+ mv.progress.AddTraceRowReplay(mv.group, len(prepared.manifest.Parts), rows)
+ mv.logger.Info().Str("group", mv.group).Str("source_stage", prepared.manifest.SourceStage).
+ Str("target_stage", prepared.manifest.TargetStage).Str("policy_digest", prepared.manifest.PolicyDigest).
+ Str("artifact", directory).Uint64("input_traces", prepared.manifest.Traces).Uint64("kept_traces", prepared.manifest.KeptTraces).
+ Uint64("dropped_traces", prepared.manifest.Traces-prepared.manifest.KeptTraces).Uint64("input_spans", prepared.manifest.Spans).
+ Uint64("kept_spans", prepared.manifest.KeptSpans).Interface("fail_open", prepared.manifest.FailOpen).
+ Int64("preparation_ns", prepared.manifest.PreparationNanos).Int("replayed_rows", rows).Msg("sampled trace shard delivered")
+ return nil
+}
+
+func (mv *traceMigrationVisitor) completeSampledSeries() error {
+ for _, work := range mv.sampledSeries {
+ for _, entry := range mv.lfs.ReadDir(work.path) {
+ if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".seg") {
+ continue
+ }
+ value, err := strconv.ParseUint(strings.TrimSuffix(entry.Name(), ".seg"), 16, 64)
+ if err != nil {
+ return fmt.Errorf("sampled source series %s: %w", filepath.Join(work.path, entry.Name()), err)
+ }
+ mv.progress.MarkSourceTraceSeriesCompleted(mv.group, work.path, common.ShardID(value))
+ }
+ }
+ return nil
+}
+
+type samplingReport struct {
+ Artifact string `json:"artifact"`
+ Manifest samplingManifest `json:"manifest"`
+ ManifestBytes int64 `json:"manifest_bytes"`
+ ReplayedBytes uint64 `json:"replayed_bytes"`
+ ResumeCount uint64 `json:"resume_count"`
+}
diff --git a/banyand/backup/lifecycle/service.go b/banyand/backup/lifecycle/service.go
index 450dd184f..52a59e23f 100644
--- a/banyand/backup/lifecycle/service.go
+++ b/banyand/backup/lifecycle/service.go
@@ -131,6 +131,7 @@ type lifecycleService struct {
orphanArchiveSubdir string
orphanCfg orphanConfig
localNodeMD schema.Metadata
+ samplingHost samplingHostConfig
maxExecutionTimes int
chunkSize run.Bytes
lifecycleGRPCPort uint32
@@ -215,6 +216,11 @@ func (l *lifecycleService) FlagSet() *run.FlagSet {
flagS.DurationVar(&lifecycleSendRetryTimeout, "lifecycle-send-retry-timeout", defaultLifecycleSendRetryTimeout,
"Max total time to retry streaming a migration part to a target node on transient failures (target restarting, disconnect, receiver busy)")
+ flagS.BoolVar(&l.samplingHost.Enabled, "trace-sampling-enabled", false, "Enable native sampler hosting for source-stage trace migration policies")
+ flagS.StringVar(&l.samplingHost.TrustedDirectory, "trace-sampling-trusted-dir", "", "Trusted directory containing migration sampler plugins")
+ flagS.DurationVar(&l.samplingHost.Timeout, "trace-sampling-timeout", time.Second, "Maximum execution time for a migration sampling batch")
+ flagS.IntVar(&l.samplingHost.CircuitBreakN, "trace-sampling-circuit-break", 3, "Consecutive sampling timeouts before retaining all subsequent batches")
+
// Lifecycle server flags
flagS.BoolVar(&l.lifecycleTLS, "lifecycle-tls", false, "connection uses TLS if true, else plain TCP")
flagS.StringVar(&l.lifecycleCertFile, "lifecycle-cert-file", "", "the TLS cert file")
@@ -227,6 +233,25 @@ func (l *lifecycleService) FlagSet() *run.FlagSet {
}
func (l *lifecycleService) Validate() error {
+ if l.samplingHost.Enabled {
+ if l.samplingHost.TrustedDirectory == "" {
+ return errors.New("trace-sampling-trusted-dir is required when trace sampling is enabled")
+ }
+ directory, directoryErr := os.Stat(l.samplingHost.TrustedDirectory)
+ if directoryErr != nil {
+ return fmt.Errorf("trace sampling trusted directory: %w", directoryErr)
+ }
+ if !directory.IsDir() {
+ return errors.New("trace-sampling-trusted-dir must be a directory")
+ }
+ if l.samplingHost.Timeout <= 0 {
+ return errors.New("trace-sampling-timeout must be positive")
+ }
+ if l.samplingHost.CircuitBreakN <= 0 {
+ return errors.New("trace-sampling-circuit-break must be positive")
+ }
+ }
+
if l.schedule != "" {
l.lifecycleGRPCAddr = net.JoinHostPort(l.lifecycleHost, strconv.FormatUint(uint64(l.lifecycleGRPCPort), 10))
if l.lifecycleGRPCAddr == ":" {
@@ -615,7 +640,31 @@ func (l *lifecycleService) action(ctx context.Context) (err error) {
defer func() {
l.recordLastRun(runStart, err)
}()
+ // Discover durable sampled work before interpreting advisory legacy progress.
+ recoveredProgress := NewProgress(l.progressFilePath, l.l)
+ recoveredGroups, recoveryErr := recoverSamplingAttempts(l.progressFilePath+".sampling", recoveredProgress)
+ if recoveryErr != nil {
+ return recoveryErr
+ }
progress := LoadProgress(l.progressFilePath, l.l)
+ progress.legacyTraceSnapshot = progress.SnapshotTraceDir != ""
+ progress.samplingOwnedGroups = recoveredProgress.samplingOwnedGroups
+ if progress.samplingOwnedGroups == nil {
+ progress.samplingOwnedGroups = make(map[string]bool)
+ }
+ for groupName := range recoveredGroups {
+ progress.samplingOwnedGroups[groupName] = true
+ }
+ if recoveredProgress.SnapshotTraceDir != "" {
+ if progress.SnapshotTraceDir != "" && progress.SnapshotTraceDir != recoveredProgress.SnapshotTraceDir {
+ return fmt.Errorf("legacy progress conflicts with sampling snapshot")
+ }
+ progress.SnapshotTraceDir = recoveredProgress.SnapshotTraceDir
+ for groupName := range progress.samplingOwnedGroups {
+ delete(progress.CompletedGroups, groupName)
+ delete(progress.DeletedTraceGroups, groupName)
+ }
+ }
progress.ClearErrors()
groups, err := l.getGroupsToProcess(ctx, progress)
@@ -624,6 +673,15 @@ func (l *lifecycleService) action(ctx context.Context) (err error) {
return err
}
+ for index, group := range groups {
+ if pinned := recoveredGroups[group.Metadata.Name]; pinned != nil {
+ groups[index] = pinned
+ delete(recoveredGroups, group.Metadata.Name)
+ }
+ }
+ for _, pinned := range recoveredGroups {
+ groups = append(groups, pinned)
+ }
l.l.Info().Msgf("starting migration for %d groups: %v", len(groups), getGroupNames(groups))
if len(groups) == 0 {
@@ -893,6 +951,7 @@ func (l *lifecycleService) buildMigrationReport(p *Progress) map[string]interfac
"summary": l.buildSummaryStats(p),
"errors": l.buildErrorSummary(p),
"orphans": orphans,
+ "trace_sampling": p.TraceSamplingReports,
"snapshot_info": map[string]interface{}{
"stream_dir": p.SnapshotStreamDir,
"measure_dir": p.SnapshotMeasureDir,
@@ -1432,6 +1491,10 @@ func (l *lifecycleService) processTraceGroup(ctx context.Context, g *commonv1.Gr
l.l.Error().Err(err).Msgf("failed to parse group %s", g.Metadata.Name)
return
}
+ group.LegacyTraceReplay = progress.legacyTraceSnapshot && !progress.samplingOwnedGroups[g.Metadata.Name]
+ group.SamplingHost = l.samplingHost
+ group.SamplingHost.ArtifactRoot = l.progressFilePath + ".sampling"
+
l.recordCycleGroup(g.Metadata.Name, senderNode, senderRole, senderTier)
defer group.Close()
@@ -1452,6 +1515,22 @@ func (l *lifecycleService) processTraceGroup(ctx context.Context, g *commonv1.Gr
l.l.Info().Msgf("deleting expired trace segments for group: %s", g.Metadata.Name)
l.deleteExpiredTraceSegments(ctx, g, segmentSuffixes, progress)
+ if group.SamplingHost.ArtifactRoot != "" {
+ artifactGroup := filepath.Join(group.SamplingHost.ArtifactRoot, g.Metadata.Name)
+ if _, artifactErr := os.Stat(artifactGroup); artifactErr == nil {
+ if !progress.IsTraceGroupDeleted(g.Metadata.Name) {
+ return
+ }
+ if cleanupErr := os.RemoveAll(artifactGroup); cleanupErr != nil {
+ l.l.Error().Err(cleanupErr).Msg("sampling artifact cleanup failed")
+ return
+ }
+ if cleanupErr := syncSamplingDirectory(group.SamplingHost.ArtifactRoot); cleanupErr != nil {
+ l.l.Error().Err(cleanupErr).Msg("sampling cleanup durability failed")
+ return
+ }
+ }
+ }
progress.MarkGroupCompleted(g.Metadata.Name)
progress.Save(l.progressFilePath, l.l)
}
@@ -1470,6 +1549,9 @@ func (l *lifecycleService) processTraceGroupFileBased(ctx context.Context, g *Gr
// skip the counting if the tsdb root path does not exist
// may no data found in the snapshot
if _, err := os.Stat(rootDir); err != nil && errors.Is(err, os.ErrNotExist) {
+ if _, artifactErr := os.Stat(filepath.Join(g.SamplingHost.ArtifactRoot, g.Metadata.Name)); artifactErr == nil {
+ return nil, fmt.Errorf("required prepared sampling snapshot group %s is missing", g.Metadata.Name)
+ }
l.l.Info().Msgf("skipping file-based trace migration for group because is empty in the snapshot dir: %s", g.Metadata.Name)
return nil, nil
}
diff --git a/banyand/backup/lifecycle/steps.go b/banyand/backup/lifecycle/steps.go
index e0d7540ee..a3c68f72b 100644
--- a/banyand/backup/lifecycle/steps.go
+++ b/banyand/backup/lifecycle/steps.go
@@ -246,9 +246,9 @@ func isLoopbackHost(host string) bool {
// GroupConfig encapsulates the parsed lifecycle configuration for a Group.
// It contains all necessary information for migration and deletion operations.
type GroupConfig struct {
+ NodeSelector node.Selector
+ QueueClient queue.Client
*commonv1.Group
- NodeSelector node.Selector
- QueueClient queue.Client
AccumulatedTTL *commonv1.IntervalRule
// SegmentInterval is the current stage's segment interval, used to read
// source segments on this node.
@@ -259,10 +259,12 @@ type GroupConfig struct {
TargetSegmentInterval *commonv1.IntervalRule
// SourceStage and TargetStage are the migration's source and target stage
// names (e.g. "hot" -> "warm"), surfaced in the migration report's errors.
- SourceStage string
- TargetStage string
- TargetShardNum uint32
- TargetReplicas uint32
+ SourceStage string
+ TargetStage string
+ SamplingHost samplingHostConfig
+ TargetShardNum uint32
+ TargetReplicas uint32
+ LegacyTraceReplay bool
}
// Close releases resources held by the GroupConfig.
diff --git a/banyand/backup/lifecycle/trace_migration_visitor.go b/banyand/backup/lifecycle/trace_migration_visitor.go
index 2b5e2501e..cb3e7ad24 100644
--- a/banyand/backup/lifecycle/trace_migration_visitor.go
+++ b/banyand/backup/lifecycle/trace_migration_visitor.go
@@ -52,9 +52,14 @@ type traceMigrationVisitor struct {
logger *logger.Logger
progress *Progress
replayer *traceRowReplayer
+ samplingPolicy *pinnedSamplingPolicy
+ samplingGroup *GroupConfig
+ snapshot string
+ samplingRoot string
group string
sourceStage string
targetStage string
+ sampledSeries []sampledSeriesWork
targetStageInterval storage.IntervalRule
sourceSegmentInterval storage.IntervalRule
chunkSize int
@@ -126,6 +131,10 @@ func (mv *traceMigrationVisitor) ensureReplayer(ctx context.Context) (*traceRowR
// VisitSeries implements trace.Visitor.
func (mv *traceMigrationVisitor) VisitSeries(segmentTR *timestamp.TimeRange, seriesIndexPath string, shardIDs []common.ShardID) error {
+ if mv.samplingPolicy != nil || mv.samplingRoot != "" {
+ mv.sampledSeries = append(mv.sampledSeries, sampledSeriesWork{path: seriesIndexPath, shards: shardIDs})
+ return nil
+ }
mv.logger.Info().
Str("path", seriesIndexPath).
Int64("min_timestamp", segmentTR.Start.UnixNano()).
@@ -293,6 +302,10 @@ func (mv *traceMigrationVisitor) VisitSeries(segmentTR *timestamp.TimeRange, ser
// VisitShard implements trace.Visitor - core and sidx migration logic.
func (mv *traceMigrationVisitor) VisitShard(timestampTR *timestamp.TimeRange, sourceShardID common.ShardID, shardPath string) error {
segmentIDStr := timestampTR.String()
+ if mv.samplingRoot != "" {
+ directory := samplingArtifactPath(mv.samplingRoot, mv.group, timestampTR, uint32(sourceShardID))
+ return mv.visitSampledShard(timestampTR, sourceShardID, shardPath, directory)
+ }
// Match the key used by Mark below; using shardPath here would always
// miss on resume and re-run the migration even though the shard had
// completed, wasting bandwidth and writing duplicates to the target.
@@ -720,6 +733,10 @@ func (mv *traceMigrationVisitor) streamPartToNode(nodeID string, targetShardID u
// Close cleans up all chunked sync clients and the trace row-replayer.
func (mv *traceMigrationVisitor) Close() error {
+ if mv.samplingPolicy != nil {
+ mv.samplingPolicy.close()
+ mv.samplingPolicy = nil
+ }
if mv.replayer != nil {
cee, replayCloseErr := mv.replayer.Close()
recordNodeErrors(mv.progress, mv.group, mv.sourceStage, mv.targetStage, catalogTrace, cee)
diff --git a/banyand/internal/dump/iterator.go b/banyand/internal/dump/iterator.go
index 7e3c83189..a262a3373 100644
--- a/banyand/internal/dump/iterator.go
+++ b/banyand/internal/dump/iterator.go
@@ -20,6 +20,8 @@
// specifics: the per-format block layout lives in the sub-packages.
package dump
+import "fmt"
+
// Position is the (block, row) position of an iterator, or where Err() was
// raised when the iterator is in terminal state. BlockIdx is 0-based, numbered
// globally across all primary blocks. RowIdx is 0-based within the current
@@ -110,3 +112,29 @@ func (it *Iterator[Row]) Close() error {
it.rowBuf = nil
return nil
}
+
+// SeekAfter resumes directly after position, decoding only its containing block.
+// The caller must bind the position to the same immutable source part.
+func (it *Iterator[Row]) SeekAfter(position Position) error {
+ if it.closed {
+ return fmt.Errorf("iterator is closed")
+ }
+ if it.err != nil {
+ return it.err
+ }
+ if position.BlockIdx < 0 || position.BlockIdx >= it.numBlocks || position.RowIdx < 0 {
+ return fmt.Errorf("invalid iterator position: %+v", position)
+ }
+ rows, err := it.decode(position.BlockIdx)
+ if err != nil {
+ it.err = err
+ return err
+ }
+ if position.RowIdx >= len(rows) {
+ return fmt.Errorf("row %d outside block %d", position.RowIdx, position.BlockIdx)
+ }
+ it.blockCursor = position.BlockIdx
+ it.rowIdx = position.RowIdx
+ it.rowBuf = rows
+ return nil
+}
diff --git a/banyand/internal/dump/iterator_seek_test.go b/banyand/internal/dump/iterator_seek_test.go
new file mode 100644
index 000000000..1868d2bf7
--- /dev/null
+++ b/banyand/internal/dump/iterator_seek_test.go
@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with this work for
+// additional information regarding copyright ownership. The ASF licenses this
+// file to you under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package dump
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestSnapshotIteratorSeekAfterDecodesOnlyCursorBlock(t *testing.T) {
+ var decoded []int
+ iterator := NewIterator(100, func(index int) ([]int, error) {
+ decoded = append(decoded, index)
+ return []int{index * 2, index*2 + 1}, nil
+ })
+ require.NoError(t, iterator.SeekAfter(Position{BlockIdx: 80, RowIdx: 0}))
+ require.Equal(t, []int{80}, decoded)
+ require.True(t, iterator.Next())
+ require.Equal(t, 161, iterator.Row())
+ require.True(t, iterator.Next())
+ require.Equal(t, 162, iterator.Row())
+ require.Equal(t, []int{80, 81}, decoded)
+ require.Error(t, iterator.SeekAfter(Position{BlockIdx: 80, RowIdx: 5}))
+}
diff --git a/banyand/trace/pipeline_loader.go b/banyand/internal/pipeline/loader.go
similarity index 63%
copy from banyand/trace/pipeline_loader.go
copy to banyand/internal/pipeline/loader.go
index fff378fc0..bb01e5723 100644
--- a/banyand/trace/pipeline_loader.go
+++ b/banyand/internal/pipeline/loader.go
@@ -15,12 +15,15 @@
// specific language governing permissions and limitations
// under the License.
-package trace
+// Package pipeline hosts native trace samplers for ingestion and lifecycle migration.
+package pipeline
import (
"crypto/sha256"
"encoding/hex"
"fmt"
+ "io"
+ "os"
"path/filepath"
"strings"
"sync"
@@ -31,11 +34,8 @@ import (
"github.com/apache/skywalking-banyandb/pkg/pipeline/sdk"
)
-// computeConfigHash returns the sha256 hex of the protojson-marshaled
-// SamplerPlugin.config (or of "{}" when config is nil). It mirrors the hash
-// used inside loadSamplerPlugin for the pluginCacheKey so callers can build
-// namedSampler entries without re-computing the hash.
-func computeConfigHash(sp *commonv1.SamplerPlugin) string {
+// ConfigHash returns the SHA-256 digest of the canonical plugin configuration.
+func ConfigHash(sp *commonv1.SamplerPlugin) string {
var cfgJSON []byte
if cfg := sp.GetConfig(); cfg != nil {
var marshalErr error
@@ -62,12 +62,43 @@ func pathWithin(base, target string) bool {
return rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator))
}
-// pluginCache caches loaded .so plugins by (path, symbol, configHash) to avoid
-// re-opening the same .so with the same config. Go plugins cannot be unloaded;
-// we re-use cached instances. A different config yields a different key and
-// triggers a fresh ABI-checked construction.
-var pluginCache = &loadedPluginCache{
- m: make(map[pluginCacheKey]sdk.Sampler),
+// Go's plugin host pins mapped code for a process lifetime. A replacement at
+// the same path must never be represented as new code while Open reuses it.
+var pluginBinaryIdentities = struct {
+ digests map[string]string
+ sync.Mutex
+}{digests: make(map[string]string)}
+
+func binaryDigest(path string) (string, error) {
+ file, err := os.Open(path)
+ if err != nil {
+ return "", err
+ }
+ defer file.Close()
+ digest := sha256.New()
+ if _, err = io.Copy(digest, file); err != nil {
+ return "", err
+ }
+ return hex.EncodeToString(digest.Sum(nil)), nil
+}
+
+// Loader caches data-node sampler instances. Sessions use a fresh Loader.
+type Loader struct {
+ m map[pluginCacheKey]sdk.Sampler
+ OpenSampler func(string, string, []byte) (sdk.Sampler, error)
+ mu sync.Mutex
+}
+
+// NewLoader constructs an independent plugin cache.
+func NewLoader() *Loader {
+ return &Loader{m: make(map[pluginCacheKey]sdk.Sampler), OpenSampler: sdk.OpenSampler}
+}
+
+// Reset clears cached instances; callers must ensure no executions are active.
+func (l *Loader) Reset() {
+ l.mu.Lock()
+ defer l.mu.Unlock()
+ l.m = make(map[pluginCacheKey]sdk.Sampler)
}
type pluginCacheKey struct {
@@ -77,19 +108,14 @@ type pluginCacheKey struct {
group string // "" for a shared (non-HostAware) instance; group name for a per-group HostAware instance
}
-type loadedPluginCache struct {
- m map[pluginCacheKey]sdk.Sampler
- mu sync.Mutex
-}
-
-// loadSamplerPlugin loads (or returns cached) a sampler plugin from the given SamplerPlugin config.
+// LoadSamplerPlugin loads (or returns cached) a sampler plugin from the given SamplerPlugin config.
// trustedDir is the only directory .so paths may resolve within; empty trustedDir disables loading.
// group scopes a HostAware instance to one group so its injected telemetry is per-group; a
// non-HostAware sampler is cached under the shared key (group=="") and re-used across groups.
// bindHost, when non-nil, is invoked EXACTLY ONCE on a freshly constructed HostAware sampler,
-// under pluginCache.mu and BEFORE the instance is stored, so no concurrent cache-hit can hand a
+// under l.mu and BEFORE the instance is stored, so no concurrent cache-hit can hand a
// not-yet-hosted instance to a merge. It is never called on a cache hit. bindHost may be nil.
-func loadSamplerPlugin(sp *commonv1.SamplerPlugin, trustedDir, group string, bindHost func(sdk.Sampler)) (_ sdk.Sampler, retErr error) {
+func (l *Loader) LoadSamplerPlugin(sp *commonv1.SamplerPlugin, trustedDir, group string, bindHost func(sdk.Sampler)) (_ sdk.Sampler, retErr error) {
if trustedDir == "" {
return nil, fmt.Errorf("trusted plugin dir not configured")
}
@@ -136,35 +162,65 @@ func loadSamplerPlugin(sp *commonv1.SamplerPlugin, trustedDir, group string, bin
hashBytes := sha256.Sum256(cfgJSON)
configHash := hex.EncodeToString(hashBytes[:])
+ if sp.GetAbiVersion() != uint32(sdk.ABIVersion) {
+ return nil, fmt.Errorf("plugin ABI version %d does not match engine ABI version %d",
+ sp.GetAbiVersion(), sdk.ABIVersion)
+ }
+
+ canonicalPath, canonicalErr := filepath.Abs(resolvedPath)
+ if canonicalErr != nil {
+ return nil, canonicalErr
+ }
+ if realPath, pathErr := filepath.EvalSymlinks(canonicalPath); pathErr == nil {
+ canonicalPath = realPath
+ }
+ beforeDigest, digestErr := binaryDigest(canonicalPath)
+ if digestErr != nil {
+ return nil, fmt.Errorf("read plugin binary identity: %w", digestErr)
+ }
+ pluginBinaryIdentities.Lock()
+ defer pluginBinaryIdentities.Unlock()
+ if previous, ok := pluginBinaryIdentities.digests[canonicalPath]; ok && previous != beforeDigest {
+ return nil, fmt.Errorf("plugin binary changed at previously loaded path %q; restart plugin host", canonicalPath)
+ }
+
key := pluginCacheKey{path: resolvedPath, symbol: symbol, configHash: configHash}
- pluginCache.mu.Lock()
- defer pluginCache.mu.Unlock()
- // 3-step lookup, all under pluginCache.mu:
+ l.mu.Lock()
+ defer l.mu.Unlock()
+ // 3-step lookup, all under l.mu:
// 1. shared key (group=="") — a non-HostAware instance re-used across groups.
// 2. group key — an already-hosted HostAware instance; return WITHOUT calling bindHost.
// 3. miss — fall through to construct (ABI check, plugin.Open, ctor) under the lock.
- if cached, ok := pluginCache.m[key]; ok {
+ if cached, ok := l.m[key]; ok {
return cached, nil
}
groupKey := key
groupKey.group = group
- if cached, ok := pluginCache.m[groupKey]; ok {
+ if cached, ok := l.m[groupKey]; ok {
return cached, nil
}
- if sp.GetAbiVersion() != uint32(sdk.ABIVersion) {
- return nil, fmt.Errorf("plugin ABI version %d does not match engine ABI version %d",
- sp.GetAbiVersion(), sdk.ABIVersion)
- }
-
- sampler, err := newSamplerFromPlugin(resolvedPath, symbol, cfgJSON)
+ // OpenSampler may map native code before an ABI or constructor failure.
+ // Pin the attempted identity so a later retry cannot attribute mapped code
+ // to a replacement binary at the same path.
+ pluginBinaryIdentities.digests[canonicalPath] = beforeDigest
+ sampler, err := l.OpenSampler(resolvedPath, symbol, cfgJSON)
if err != nil {
return nil, err
}
+ afterDigest, afterErr := binaryDigest(canonicalPath)
+ if afterErr != nil || afterDigest != beforeDigest {
+ _ = sampler.Close()
+ if afterErr != nil {
+ return nil, fmt.Errorf("verify loaded plugin binary identity: %w", afterErr)
+ }
+ return nil, fmt.Errorf("plugin binary changed while loading %q", canonicalPath)
+ }
+
// Classify the freshly constructed instance and key it accordingly. For a
// HostAware sampler, inject the host via bindHost while still holding
- // pluginCache.mu and BEFORE storing, so no concurrent cache-hit can observe a
+ // l.mu and BEFORE storing, so no concurrent cache-hit can observe a
// not-yet-hosted instance. UseHost/telemetry setup thus runs under the same
// lock that already guards plugin.Open and the constructor; keep it fast.
if _, isHostAware := sampler.(sdk.HostAware); isHostAware {
@@ -173,14 +229,6 @@ func loadSamplerPlugin(sp *commonv1.SamplerPlugin, trustedDir, group string, bin
bindHost(sampler)
}
}
- pluginCache.m[key] = sampler
+ l.m[key] = sampler
return sampler, nil
}
-
-// newSamplerFromPlugin opens the .so, verifies its ABIVersion symbol, and calls
-// the constructor. It is a package var so tests can substitute a Go sampler and
-// exercise the cache/keying/bindHost logic under -race without a real .so. The
-// default implementation delegates to sdk.OpenSampler — the single shared
-// loader-contract implementation also used by sdktest.LoadSO — so the host and
-// the offline dev toolkit never drift.
-var newSamplerFromPlugin = sdk.OpenSampler
diff --git a/banyand/internal/pipeline/loader_test.go b/banyand/internal/pipeline/loader_test.go
new file mode 100644
index 000000000..1dc5ba91f
--- /dev/null
+++ b/banyand/internal/pipeline/loader_test.go
@@ -0,0 +1,70 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package pipeline
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ "github.com/apache/skywalking-banyandb/pkg/pipeline/sdk"
+)
+
+func TestLoaderRejectsChangedMappedBinary(t *testing.T) {
+ directory := t.TempDir()
+ path := filepath.Join(directory, "sampler.so")
+ require.NoError(t, os.WriteFile(path, []byte("binary-one"), 0o600))
+ loader := NewLoader()
+ calls := 0
+ loader.OpenSampler = func(string, string, []byte) (sdk.Sampler, error) { calls++; return &testSampler{}, nil }
+ config := &commonv1.SamplerPlugin{Path: "sampler.so", AbiVersion: uint32(sdk.ABIVersion)}
+ _, err := loader.LoadSamplerPlugin(config, directory, "group", nil)
+ require.NoError(t, err)
+ require.NoError(t, os.WriteFile(path, []byte("binary-two"), 0o600))
+ _, err = loader.LoadSamplerPlugin(config, directory, "group", nil)
+ require.ErrorContains(t, err, "changed at previously loaded path")
+ require.Equal(t, 1, calls, "a cache hit cannot mask replacement of mapped code")
+ fresh := NewLoader()
+ fresh.OpenSampler = loader.OpenSampler
+ _, err = fresh.LoadSamplerPlugin(config, directory, "group", nil)
+ require.ErrorContains(t, err, "changed at previously loaded path", "new policy sessions share process-wide mapped identities")
+}
+
+func TestLoaderRejectsReplacementDuringConstruction(t *testing.T) {
+ directory := t.TempDir()
+ path := filepath.Join(directory, "sampler.so")
+ require.NoError(t, os.WriteFile(path, []byte("binary-one"), 0o600))
+ sampler := &testSampler{closed: make(chan struct{})}
+ loader := NewLoader()
+ loader.OpenSampler = func(string, string, []byte) (sdk.Sampler, error) {
+ if err := os.WriteFile(path, []byte("binary-two"), 0o600); err != nil {
+ return nil, err
+ }
+ return sampler, nil
+ }
+ _, err := loader.LoadSamplerPlugin(&commonv1.SamplerPlugin{Path: "sampler.so", AbiVersion: uint32(sdk.ABIVersion)}, directory, "group", nil)
+ require.ErrorContains(t, err, "changed while loading")
+ select {
+ case <-sampler.closed:
+ default:
+ t.Fatal("discarded instance was not closed")
+ }
+}
diff --git a/banyand/internal/pipeline/session.go b/banyand/internal/pipeline/session.go
new file mode 100644
index 000000000..59fa3527d
--- /dev/null
+++ b/banyand/internal/pipeline/session.go
@@ -0,0 +1,253 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package pipeline
+
+import (
+ "fmt"
+ "reflect"
+ "sync"
+ "time"
+
+ "github.com/apache/skywalking-banyandb/pkg/cgroups"
+ "github.com/apache/skywalking-banyandb/pkg/pipeline/sdk"
+)
+
+var executionSlots = make(chan struct{}, cgroups.CPUs())
+
+// Session serializes a pinned chain and bounds calls which outlive their deadline.
+// Batches remain owned by the caller until reusable is true or Wait returns.
+type Session struct {
+ timer *time.Timer
+ done chan struct{}
+ slots chan struct{}
+ stop chan struct{}
+ results chan sdk.Verdict
+ requests chan sessionRequest
+ samplers []sdk.Sampler
+ projection sdk.Projection
+ workers sync.WaitGroup
+ timeouts int
+ threshold int
+ closePluginsOnce sync.Once
+ mu sync.Mutex
+ busy bool
+ closed bool
+ open bool
+}
+
+type sessionRequest struct {
+ batch *sdk.TraceBatch
+ evaluate func([]sdk.Sampler, *sdk.TraceBatch, []bool) sdk.Verdict
+ mask []bool
+}
+
+// NewSession builds a session over sampler instances owned by the caller.
+func NewSession(samplers []sdk.Sampler, circuitBreakN int) *Session {
+ s := &Session{threshold: circuitBreakN, slots: executionSlots}
+ seen := make(map[string]bool)
+ for _, sampler := range samplers {
+ if sampler == nil {
+ continue
+ }
+ s.samplers = append(s.samplers, sampler)
+ projection := sampler.Project()
+ for _, tag := range projection.Tags {
+ if !seen[tag] {
+ seen[tag] = true
+ s.projection.Tags = append(s.projection.Tags, tag)
+ }
+ }
+ s.projection.Spans = s.projection.Spans || projection.Spans
+ s.projection.SpanIDs = s.projection.SpanIDs || projection.SpanIDs
+ }
+ return s
+}
+
+// NewSessionChecked validates plugin projections without letting a native panic
+// terminate the host during policy preflight.
+func NewSessionChecked(samplers []sdk.Sampler, circuitBreakN int) (session *Session, err error) {
+ defer func() {
+ if recovered := recover(); recovered != nil {
+ session = nil
+ err = fmt.Errorf("sampler projection panic: %v", recovered)
+ }
+ }()
+ return NewSession(samplers, circuitBreakN), nil
+}
+
+// Projection returns the union of the chain's requested columns.
+func (s *Session) Projection() sdk.Projection { return s.projection }
+
+// SetExecutionSlots overrides the shared bound before the session is first used.
+func (s *Session) SetExecutionSlots(slots chan struct{}) {
+ if s.slots != slots {
+ s.slots = slots
+ }
+}
+
+// Execute evaluates the chain, retaining all traces on a deadline or open circuit.
+func (s *Session) Execute(batch *sdk.TraceBatch, timeout time.Duration) (sdk.Verdict, error) {
+ verdict, _, err := s.ExecuteInto(batch, timeout, nil, nil)
+ return verdict, err
+}
+
+// ExecuteInto reports whether the caller may recycle batch and mask immediately.
+// A false result means an outstanding native call still owns their buffers.
+func (s *Session) ExecuteInto(batch *sdk.TraceBatch, timeout time.Duration, mask []bool,
+ evaluate func([]sdk.Sampler, *sdk.TraceBatch, []bool) sdk.Verdict,
+) (sdk.Verdict, bool, error) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ if len(s.samplers) == 0 || s.open || s.closed {
+ return retainAll(len(batch.Traces), mask), true, nil
+ }
+ if s.busy {
+ select {
+ case <-s.done:
+ s.busy = false
+ s.requests = nil
+ s.results = nil
+ default:
+ return s.timeout(batch, mask, true)
+ }
+ }
+ if s.timer == nil {
+ s.timer = time.NewTimer(timeout)
+ } else {
+ s.stopTimer()
+ s.timer.Reset(timeout)
+ }
+ defer s.stopTimer()
+ select {
+ case s.slots <- struct{}{}:
+ case <-s.timer.C:
+ return s.timeout(batch, mask, true)
+ }
+ if s.requests == nil {
+ s.requests = make(chan sessionRequest)
+ s.results = make(chan sdk.Verdict)
+ s.stop = make(chan struct{})
+ s.done = make(chan struct{})
+ s.workers.Add(1)
+ go s.run()
+ }
+ s.requests <- sessionRequest{batch: batch, mask: mask, evaluate: evaluate}
+ select {
+ case verdict := <-s.results:
+ s.timeouts = 0
+ return verdict, true, nil
+ case <-s.timer.C:
+ s.busy = true
+ close(s.stop)
+ return s.timeout(batch, nil, false)
+ }
+}
+
+func (s *Session) run() {
+ defer s.workers.Done()
+ defer close(s.done)
+ for {
+ select {
+ case request := <-s.requests:
+ var verdict sdk.Verdict
+ if request.evaluate == nil {
+ verdict = sdk.EvaluateChainInto(s.samplers, request.batch, request.mask, nil)
+ } else {
+ verdict = request.evaluate(s.samplers, request.batch, request.mask)
+ }
+ <-s.slots
+ select {
+ case s.results <- verdict:
+ case <-s.stop:
+ return
+ }
+ case <-s.stop:
+ return
+ }
+ }
+}
+
+func (s *Session) stopTimer() {
+ if s.timer == nil || s.timer.Stop() {
+ return
+ }
+ select {
+ case <-s.timer.C:
+ default:
+ }
+}
+
+func (s *Session) timeout(batch *sdk.TraceBatch, mask []bool, reusable bool) (sdk.Verdict, bool, error) {
+ s.timeouts++
+ reason := "timeout"
+ if s.threshold > 0 && s.timeouts >= s.threshold {
+ s.open = true
+ reason = "circuit_open"
+ }
+ return retainAll(len(batch.Traces), mask), reusable, fmt.Errorf("%s", reason)
+}
+
+func retainAll(n int, mask []bool) sdk.Verdict {
+ if cap(mask) < n {
+ mask = make([]bool, n)
+ } else {
+ mask = mask[:n]
+ }
+ for i := range mask {
+ mask[i] = true
+ }
+ return sdk.Verdict{Keep: mask}
+}
+
+// State returns the circuit state and consecutive deadline count.
+func (s *Session) State() (bool, int) { s.mu.Lock(); defer s.mu.Unlock(); return s.open, s.timeouts }
+
+// Close prevents new work. It does not close instances while native calls run.
+func (s *Session) Close() {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ if !s.closed && s.stop != nil && !s.busy {
+ close(s.stop)
+ }
+ s.closed = true
+}
+
+// Wait waits for native calls before sampler instances or their input are released.
+func (s *Session) Wait() { s.workers.Wait() }
+
+// ClosePluginsWhenIdle stops new execution and asynchronously closes the owned
+// instances after native workers finish. An uncooperative plugin cannot stall
+// its caller or be closed while still accessing its input.
+func (s *Session) ClosePluginsWhenIdle() {
+ s.closePluginsOnce.Do(func() {
+ s.Close()
+ go func() {
+ s.Wait()
+ seen := make(map[sdk.Sampler]bool)
+ for _, sampler := range s.samplers {
+ if reflect.TypeOf(sampler).Comparable() {
+ if seen[sampler] {
+ continue
+ }
+ seen[sampler] = true
+ }
+ _ = sampler.Close()
+ }
+ }()
+ })
+}
diff --git a/banyand/internal/pipeline/session_test.go b/banyand/internal/pipeline/session_test.go
new file mode 100644
index 000000000..8882a672a
--- /dev/null
+++ b/banyand/internal/pipeline/session_test.go
@@ -0,0 +1,147 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package pipeline
+
+import (
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/apache/skywalking-banyandb/pkg/pipeline/sdk"
+)
+
+type testSampler struct {
+ decide func(*sdk.TraceBatch) (sdk.Verdict, error)
+ closed chan struct{}
+ projection sdk.Projection
+}
+
+func (s *testSampler) Kind() sdk.Kind { return sdk.KindSampler }
+func (s *testSampler) Project() sdk.Projection { return s.projection }
+func (s *testSampler) Decide(batch *sdk.TraceBatch) (sdk.Verdict, error) { return s.decide(batch) }
+func (s *testSampler) Close() error {
+ if s.closed != nil {
+ close(s.closed)
+ }
+ return nil
+}
+
+func TestSessionChainAndProjection(t *testing.T) {
+ first := &testSampler{
+ projection: sdk.Projection{
+ Tags: []string{"a"},
+ SpanIDs: true,
+ },
+ decide: func(*sdk.TraceBatch) (sdk.Verdict,
+ error,
+ ) {
+ return sdk.Verdict{Keep: []bool{
+ false,
+ true,
+ }},
+ nil
+ },
+ }
+ failed := &testSampler{projection: sdk.Projection{Tags: []string{"a", "b"}, Spans: true}, decide: func(*sdk.TraceBatch) (sdk.Verdict, error) { panic("broken link") }}
+ session := NewSession([]sdk.Sampler{first, failed}, 2)
+ defer session.Close()
+ require.Equal(t, sdk.Projection{Tags: []string{"a", "b"}, SpanIDs: true, Spans: true}, session.Projection())
+ verdict, err := session.Execute(&sdk.TraceBatch{Traces: make([]sdk.TraceBlock, 2)}, time.Second)
+ require.NoError(t, err)
+ require.Equal(t, []bool{false, true}, verdict.Keep)
+}
+
+func TestSessionTimeoutOwnsBuffersAndDefersClose(t *testing.T) {
+ release := make(chan struct{})
+ entered := make(chan struct{})
+ sampler := &testSampler{closed: make(chan struct{}), decide: func(batch *sdk.TraceBatch) (sdk.Verdict, error) {
+ close(entered)
+ <-release
+ return sdk.Verdict{Keep: make([]bool, len(batch.Traces))}, nil
+ }}
+ session := NewSession([]sdk.Sampler{sampler}, 2)
+ session.SetExecutionSlots(make(chan struct{}, 1))
+ verdict, reusable, err := session.ExecuteInto(&sdk.TraceBatch{Traces: make([]sdk.TraceBlock, 1)}, 10*time.Millisecond, nil, nil)
+ require.EqualError(t, err, "timeout")
+ require.False(t, reusable)
+ require.Equal(t, []bool{true}, verdict.Keep)
+ <-entered
+ _, reusable, err = session.ExecuteInto(&sdk.TraceBatch{Traces: make([]sdk.TraceBlock, 1)}, time.Second, nil, nil)
+ require.EqualError(t, err, "circuit_open")
+ require.True(t, reusable)
+ session.ClosePluginsWhenIdle()
+ select {
+ case <-sampler.closed:
+ t.Fatal("closed a running native instance")
+ default:
+ }
+ close(release)
+ select {
+ case <-sampler.closed:
+ case <-time.After(time.Second):
+ t.Fatal("instance never closed")
+ }
+}
+
+func TestSessionRecoversAfterTimedOutWorkerReturns(t *testing.T) {
+ release := make(chan struct{})
+ calls := 0
+ sampler := &testSampler{decide: func(*sdk.TraceBatch) (sdk.Verdict, error) {
+ calls++
+ if calls == 1 {
+ <-release
+ }
+ return sdk.Verdict{Keep: []bool{false}}, nil
+ }}
+ session := NewSession([]sdk.Sampler{sampler}, 3)
+ defer session.Close()
+ batch := &sdk.TraceBatch{Traces: make([]sdk.TraceBlock, 1)}
+ _, reusable, err := session.ExecuteInto(batch, 10*time.Millisecond, nil, nil)
+ require.EqualError(t, err, "timeout")
+ require.False(t, reusable)
+ close(release)
+ <-session.done
+ verdict, err := session.Execute(batch, time.Second)
+ require.NoError(t, err)
+ require.Equal(t, []bool{false}, verdict.Keep)
+ open, timeouts := session.State()
+ require.False(t, open)
+ require.Zero(t, timeouts)
+}
+
+func TestSessionClosesRepeatedInstanceOnce(t *testing.T) {
+ sampler := &testSampler{closed: make(chan struct{}), decide: func(*sdk.TraceBatch) (sdk.Verdict, error) { return sdk.Verdict{}, nil }}
+ session := NewSession([]sdk.Sampler{sampler, sampler}, 3)
+ session.ClosePluginsWhenIdle()
+ session.ClosePluginsWhenIdle()
+ select {
+ case <-sampler.closed:
+ case <-time.After(time.Second):
+ t.Fatal("owned instance not closed")
+ }
+}
+
+type projectionPanicSampler struct{ testSampler }
+
+func (*projectionPanicSampler) Project() sdk.Projection { panic("invalid projection") }
+func TestSessionCheckedRejectsProjectionPanic(t *testing.T) {
+ session, err := NewSessionChecked([]sdk.Sampler{&projectionPanicSampler{}}, 3)
+ require.Nil(t, session)
+ require.ErrorContains(t, err, "sampler projection panic")
+}
diff --git a/banyand/metadata/schema/property/client.go b/banyand/metadata/schema/property/client.go
index 2f16ccf7f..ffef2883d 100644
--- a/banyand/metadata/schema/property/client.go
+++ b/banyand/metadata/schema/property/client.go
@@ -870,6 +870,14 @@ func pipelineEnabledWithSampler(cfg *commonv1.TracePipelineConfig) bool {
return true
}
}
+ for _, stage := range cfg.GetStages() {
+ for _, plugin := range stage.GetPlugins() {
+ if plugin.GetSampler() != nil {
+ return true
+ }
+ }
+ }
+
return false
}
diff --git a/banyand/metadata/schema/property/pipeline_single_trace_test.go b/banyand/metadata/schema/property/pipeline_single_trace_test.go
index 91cee5390..159a03cbf 100644
--- a/banyand/metadata/schema/property/pipeline_single_trace_test.go
+++ b/banyand/metadata/schema/property/pipeline_single_trace_test.go
@@ -201,3 +201,47 @@ func TestUpdateGroup_MissingGroupReturnsNotFound(t *testing.T) {
require.NotEqual(t, codes.FailedPrecondition, status.Code(updateErr))
require.Contains(t, updateErr.Error(), "not exist")
}
+
+func stageSamplerGroup() *commonv1.Group {
+ config := samplerPipeline(true)
+ config.Stages = []*commonv1.StageRule{{Stage: "hot", Plugins: config.Plugins}}
+ config.Plugins = nil
+ config.EnabledEvents = nil
+ group := traceGroup(config)
+ group.ResourceOpts.Stages = []*commonv1.LifecycleStage{{
+ Name: "warm",
+ ShardNum: 1,
+ SegmentInterval: &commonv1.IntervalRule{
+ Unit: commonv1.IntervalRule_UNIT_DAY,
+ Num: 1,
+ },
+ Ttl: &commonv1.IntervalRule{
+ Unit: commonv1.IntervalRule_UNIT_DAY,
+ Num: 7,
+ },
+ NodeSelector: "type=warm",
+ }}
+ return group
+}
+
+func TestCreateTraceStageSamplerRejectsSecondTrace(t *testing.T) {
+ registry, ctx := newTraceRegistry(t)
+ _, err := registry.CreateGroup(ctx, stageSamplerGroup())
+ require.NoError(t, err)
+ _, err = registry.CreateTrace(ctx, traceNamed("trace-a"))
+ require.NoError(t, err)
+ _, err = registry.CreateTrace(ctx, traceNamed("trace-b"))
+ requireFailedPrecondition(t, err)
+}
+
+func TestUpdateGroupStageSamplerRejectsMultipleExistingTraces(t *testing.T) {
+ registry, ctx := newTraceRegistry(t)
+ _, err := registry.CreateGroup(ctx, traceGroup(nil))
+ require.NoError(t, err)
+ _, err = registry.CreateTrace(ctx, traceNamed("trace-a"))
+ require.NoError(t, err)
+ _, err = registry.CreateTrace(ctx, traceNamed("trace-b"))
+ require.NoError(t, err)
+ _, err = registry.UpdateGroup(ctx, stageSamplerGroup())
+ requireFailedPrecondition(t, err)
+}
diff --git a/banyand/trace/merge_benchmark_observer_test.go b/banyand/trace/merge_benchmark_observer_test.go
index 003b25f68..58b3a8459 100644
--- a/banyand/trace/merge_benchmark_observer_test.go
+++ b/banyand/trace/merge_benchmark_observer_test.go
@@ -261,7 +261,7 @@ func TestMergeBenchmarkObserverMaturesProductionSelectedHotOutputs(t *testing.T)
segmentRange := timestamp.NewInclusiveTimeRange(base.Add(-24*time.Hour), base.Add(24*time.Hour))
table, tableErr := newTSTable(fileSystem, root, common.Position{Database: group}, logger.GetLogger(group), segmentRange, option{
flushTimeout: 0, mergePolicy: newDefaultMergePolicyForTesting(), protector: protector.Nop{}, nativePipelineEnabled: true,
- mergeGraceDefault: 2 * time.Hour,
+ mergeGraceDefault: 2 * time.Hour, decideTimeout: time.Second,
}, nil)
require.NoError(t, tableErr)
t.Cleanup(func() { require.NoError(t, table.Close()) })
@@ -363,6 +363,7 @@ func runObservedBenchmarkMerge(t *testing.T, mature bool) (mergeBenchmarkEvent,
mergePolicy: newDefaultMergePolicyForTesting(),
protector: protector.Nop{},
nativePipelineEnabled: true,
+ decideTimeout: time.Second,
mergeGraceDefault: 2 * time.Hour,
}, nil)
require.NoError(t, tableErr)
diff --git a/banyand/trace/pipeline_chain.go b/banyand/trace/pipeline_chain.go
index 7feef2c52..8559285c9 100644
--- a/banyand/trace/pipeline_chain.go
+++ b/banyand/trace/pipeline_chain.go
@@ -23,19 +23,14 @@ import (
"time"
internalencoding "github.com/apache/skywalking-banyandb/banyand/internal/encoding"
+ runtime "github.com/apache/skywalking-banyandb/banyand/internal/pipeline"
pkgbytes "github.com/apache/skywalking-banyandb/pkg/bytes"
- "github.com/apache/skywalking-banyandb/pkg/cgroups"
"github.com/apache/skywalking-banyandb/pkg/convert"
"github.com/apache/skywalking-banyandb/pkg/logger"
"github.com/apache/skywalking-banyandb/pkg/pipeline/sdk"
)
-var (
- chainLog = logger.GetLogger("trace").Named("pipeline-chain")
- // samplerExecutionSlots bounds batches retained by sampler calls that ignore
- // the execution timeout. A slot is released only after Decide returns.
- samplerExecutionSlots = make(chan struct{}, cgroups.CPUs())
-)
+var chainLog = logger.GetLogger("trace").Named("pipeline-chain")
const (
pluginExecutionResultSuccess = "success"
@@ -66,8 +61,8 @@ type pluginLinkExecutionObservation struct {
// consecutive-timeout circuit breaker disables the chain after circuitBreakN
// timeouts.
type mergeChain struct {
+ session *runtime.Session
observeExecution func(pluginExecutionObservation)
- timer *time.Timer
worker *mergeDecisionWorker
executionSlots chan struct{}
observeLinkExecution func(pluginLinkExecutionObservation)
@@ -77,25 +72,15 @@ type mergeChain struct {
samplerNames []string
projection sdk.Projection
consecutiveTOs int
- circuitBreakN int
executionMu sync.Mutex
mu sync.Mutex
circuitOpen bool
}
-type mergeDecisionRequest struct {
- batch *sdk.TraceBatch
- observation *mergeEvaluationObservation
- decisionMask []bool
-}
-
type mergeDecisionWorker struct {
observation *mergeEvaluationObservation
- requests chan mergeDecisionRequest
- results chan sdk.Verdict
stopCh chan struct{}
samplers []sdk.Sampler
- stopOnce sync.Once
}
type observedSampler struct {
@@ -162,44 +147,28 @@ func defaultSamplerMetricName(idx int) string {
}
func newNamedMergeChain(group, schema string, samplers []namedSampler, circuitBreakN int) *mergeChain {
- var union sdk.Projection
- seen := make(map[string]struct{})
activeSamplers := make([]sdk.Sampler, 0, len(samplers))
activeNames := make([]string, 0, len(samplers))
for _, named := range samplers {
if named.sampler == nil {
continue
}
- sampler := named.sampler
- activeSamplers = append(activeSamplers, sampler)
+ activeSamplers = append(activeSamplers, named.sampler)
name := named.name
if name == "" {
name = defaultSamplerMetricName(len(activeNames))
}
activeNames = append(activeNames, name)
- proj := sampler.Project()
- for _, name := range proj.Tags {
- if _, ok := seen[name]; ok {
- continue
- }
- seen[name] = struct{}{}
- union.Tags = append(union.Tags, name)
- }
- if proj.SpanIDs {
- union.SpanIDs = true
- }
- if proj.Spans {
- union.Spans = true
- }
}
+ session := runtime.NewSession(activeSamplers, circuitBreakN)
return &mergeChain{
- projection: union,
+ projection: session.Projection(),
+ session: session,
samplers: activeSamplers,
samplerNames: activeNames,
- executionSlots: samplerExecutionSlots,
+ executionSlots: nil,
group: group,
schema: schema,
- circuitBreakN: circuitBreakN,
}
}
@@ -223,81 +192,49 @@ func (mc *mergeChain) executeObservedInto(batch *sdk.TraceBatch, timeout time.Du
) (sdk.Verdict, bool, error) {
mc.executionMu.Lock()
defer mc.executionMu.Unlock()
-
if len(mc.samplers) == 0 {
return retainAllVerdict(len(batch.Traces), decisionMask), true, nil
}
-
mc.mu.Lock()
- if mc.circuitOpen {
- mc.mu.Unlock()
- mc.observe(pluginExecutionObservation{
- result: pluginExecutionResultCircuitOpen, batchTraces: len(batch.Traces),
- }, observation)
- return retainAllVerdict(len(batch.Traces), decisionMask), true, nil
- }
+ circuitOpen := mc.circuitOpen
mc.mu.Unlock()
-
- if !mc.acquireExecutionSlot(timeout) {
- return mc.handleTimeout(batch, observation, decisionMask, nil, true)
- }
-
- worker := mc.acquireWorker()
- worker.requests <- mergeDecisionRequest{batch: batch, observation: observation, decisionMask: decisionMask}
- timeoutCh := mc.resetTimer(timeout)
-
- select {
- case verdict := <-worker.results:
- mc.stopTimer()
- mc.mu.Lock()
- mc.consecutiveTOs = 0
- mc.mu.Unlock()
- mc.observe(pluginExecutionObservation{
- result: pluginExecutionResultSuccess, batchTraces: len(batch.Traces),
- }, observation)
- return verdict, true, nil
- case <-timeoutCh:
- return mc.handleTimeout(batch, observation, nil, worker, false)
+ if circuitOpen {
+ mc.observe(pluginExecutionObservation{result: pluginExecutionResultCircuitOpen, batchTraces: len(batch.Traces)}, observation)
+ return retainAllVerdict(len(batch.Traces), decisionMask), true, nil
}
-}
-
-func (mc *mergeChain) acquireExecutionSlot(timeout time.Duration) bool {
- select {
- case mc.executionSlots <- struct{}{}:
- return true
- default:
+ if mc.executionSlots != nil {
+ mc.session.SetExecutionSlots(mc.executionSlots)
}
- timeoutCh := mc.resetTimer(timeout)
- select {
- case mc.executionSlots <- struct{}{}:
- mc.stopTimer()
- return true
- case <-timeoutCh:
- return false
+ worker := mc.worker
+ if worker == nil {
+ worker = &mergeDecisionWorker{stopCh: make(chan struct{})}
+ worker.samplers = make([]sdk.Sampler, len(mc.samplers))
+ for idx, sampler := range mc.samplers {
+ worker.samplers[idx] = &observedSampler{Sampler: sampler, chain: mc, worker: worker, pluginName: mc.samplerNames[idx]}
+ }
+ if len(mc.samplers) > 0 {
+ mc.worker = worker
+ }
}
-}
-
-func (mc *mergeChain) handleTimeout(batch *sdk.TraceBatch, observation *mergeEvaluationObservation, decisionMask []bool,
- worker *mergeDecisionWorker, reusable bool,
-) (sdk.Verdict, bool, error) {
- if worker != nil {
- mc.abandonWorker(worker)
+ worker.observation = observation
+ verdict, reusable, err := mc.session.ExecuteInto(batch, timeout, decisionMask,
+ func(_ []sdk.Sampler, batch *sdk.TraceBatch, mask []bool) sdk.Verdict {
+ return mc.runChainWithSamplers(worker.samplers, batch, observation, mask)
+ })
+ result := pluginExecutionResultSuccess
+ if err != nil {
+ close(worker.stopCh)
+ mc.worker = nil
+ result = pluginExecutionResultTimeout
}
mc.mu.Lock()
- mc.consecutiveTOs++
- opened := false
- if mc.circuitBreakN > 0 && mc.consecutiveTOs >= mc.circuitBreakN {
- mc.circuitOpen = true
- opened = true
+ mc.circuitOpen, mc.consecutiveTOs = mc.session.State()
+ if err == nil && mc.circuitOpen {
+ result = pluginExecutionResultCircuitOpen
}
mc.mu.Unlock()
- mc.observe(pluginExecutionObservation{
- result: pluginExecutionResultTimeout, batchTraces: len(batch.Traces),
- }, observation)
- if opened {
- return retainAllVerdict(len(batch.Traces), decisionMask), reusable, fmt.Errorf("circuit_open")
- }
- return retainAllVerdict(len(batch.Traces), decisionMask), reusable, fmt.Errorf("timeout")
+ mc.observe(pluginExecutionObservation{result: result, batchTraces: len(batch.Traces)}, observation)
+ return verdict, reusable, err
}
func (mc *mergeChain) observe(observation pluginExecutionObservation, evaluation *mergeEvaluationObservation) {
@@ -322,86 +259,14 @@ func retainAllVerdict(traceCount int, mask []bool) sdk.Verdict {
return sdk.Verdict{Keep: mask}
}
-func (mc *mergeChain) acquireWorker() *mergeDecisionWorker {
- if mc.worker != nil {
- return mc.worker
- }
- worker := &mergeDecisionWorker{
- requests: make(chan mergeDecisionRequest),
- results: make(chan sdk.Verdict),
- stopCh: make(chan struct{}),
- }
- worker.samplers = make([]sdk.Sampler, len(mc.samplers))
- for idx, sampler := range mc.samplers {
- worker.samplers[idx] = &observedSampler{
- Sampler: sampler, chain: mc, worker: worker, pluginName: mc.samplerNames[idx],
- }
- }
- mc.worker = worker
- go worker.run(mc)
- return worker
-}
-
-func (mdw *mergeDecisionWorker) run(chain *mergeChain) {
- for {
- select {
- case request := <-mdw.requests:
- if !mdw.execute(chain, request) {
- return
- }
- case <-mdw.stopCh:
- return
- }
- }
-}
-
-func (mdw *mergeDecisionWorker) execute(chain *mergeChain, request mergeDecisionRequest) (keepRunning bool) {
- defer func() { <-chain.executionSlots }()
- mdw.observation = request.observation
- result := chain.runChainWithSamplers(mdw.samplers, request.batch, request.observation, request.decisionMask)
- mdw.observation = nil
- select {
- case mdw.results <- result:
- return true
- case <-mdw.stopCh:
- return false
- }
-}
-
-func (mc *mergeChain) abandonWorker(worker *mergeDecisionWorker) {
- worker.stopOnce.Do(func() { close(worker.stopCh) })
- if mc.worker == worker {
- mc.worker = nil
- }
-}
-
-func (mc *mergeChain) resetTimer(timeout time.Duration) <-chan time.Time {
- if mc.timer == nil {
- mc.timer = time.NewTimer(timeout)
- return mc.timer.C
- }
- mc.stopTimer()
- mc.timer.Reset(timeout)
- return mc.timer.C
-}
-
-func (mc *mergeChain) stopTimer() {
- if mc.timer == nil || mc.timer.Stop() {
- return
- }
- select {
- case <-mc.timer.C:
- default:
- }
-}
-
func (mc *mergeChain) close() {
mc.executionMu.Lock()
defer mc.executionMu.Unlock()
- mc.stopTimer()
if mc.worker != nil {
- mc.abandonWorker(mc.worker)
+ close(mc.worker.stopCh)
+ mc.worker = nil
}
+ mc.session.Close()
}
// runChain evaluates the chain via the shared sdk.EvaluateChain — the same
diff --git a/banyand/trace/pipeline_loader.go b/banyand/trace/pipeline_loader.go
index fff378fc0..05fa7ab95 100644
--- a/banyand/trace/pipeline_loader.go
+++ b/banyand/trace/pipeline_loader.go
@@ -18,169 +18,22 @@
package trace
import (
- "crypto/sha256"
- "encoding/hex"
- "fmt"
- "path/filepath"
- "strings"
- "sync"
-
- "google.golang.org/protobuf/encoding/protojson"
-
commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ runtime "github.com/apache/skywalking-banyandb/banyand/internal/pipeline"
"github.com/apache/skywalking-banyandb/pkg/pipeline/sdk"
)
-// computeConfigHash returns the sha256 hex of the protojson-marshaled
-// SamplerPlugin.config (or of "{}" when config is nil). It mirrors the hash
-// used inside loadSamplerPlugin for the pluginCacheKey so callers can build
-// namedSampler entries without re-computing the hash.
-func computeConfigHash(sp *commonv1.SamplerPlugin) string {
- var cfgJSON []byte
- if cfg := sp.GetConfig(); cfg != nil {
- var marshalErr error
- cfgJSON, marshalErr = protojson.Marshal(cfg)
- if marshalErr != nil {
- cfgJSON = []byte("{}")
- }
- } else {
- cfgJSON = []byte("{}")
- }
- hashBytes := sha256.Sum256(cfgJSON)
- return hex.EncodeToString(hashBytes[:])
-}
-
-// pathWithin reports whether target is base itself or a descendant of base.
-// It treats only a bare ".." or a ".."+separator prefix as an escape, so a
-// legitimate child whose first segment merely starts with ".." (e.g. "..foo")
-// is not falsely rejected.
-func pathWithin(base, target string) bool {
- rel, err := filepath.Rel(base, target)
- if err != nil {
- return false
- }
- return rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator))
-}
-
-// pluginCache caches loaded .so plugins by (path, symbol, configHash) to avoid
-// re-opening the same .so with the same config. Go plugins cannot be unloaded;
-// we re-use cached instances. A different config yields a different key and
-// triggers a fresh ABI-checked construction.
-var pluginCache = &loadedPluginCache{
- m: make(map[pluginCacheKey]sdk.Sampler),
-}
-
-type pluginCacheKey struct {
- path string
- symbol string
- configHash string // sha256 hex of the protojson-marshaled config (or of "{}" when nil)
- group string // "" for a shared (non-HostAware) instance; group name for a per-group HostAware instance
-}
-
-type loadedPluginCache struct {
- m map[pluginCacheKey]sdk.Sampler
- mu sync.Mutex
-}
-
-// loadSamplerPlugin loads (or returns cached) a sampler plugin from the given SamplerPlugin config.
-// trustedDir is the only directory .so paths may resolve within; empty trustedDir disables loading.
-// group scopes a HostAware instance to one group so its injected telemetry is per-group; a
-// non-HostAware sampler is cached under the shared key (group=="") and re-used across groups.
-// bindHost, when non-nil, is invoked EXACTLY ONCE on a freshly constructed HostAware sampler,
-// under pluginCache.mu and BEFORE the instance is stored, so no concurrent cache-hit can hand a
-// not-yet-hosted instance to a merge. It is never called on a cache hit. bindHost may be nil.
-func loadSamplerPlugin(sp *commonv1.SamplerPlugin, trustedDir, group string, bindHost func(sdk.Sampler)) (_ sdk.Sampler, retErr error) {
- if trustedDir == "" {
- return nil, fmt.Errorf("trusted plugin dir not configured")
- }
- rawPath := sp.GetPath()
- if rawPath == "" {
- return nil, fmt.Errorf("plugin path is empty")
- }
- resolvedPath := filepath.Clean(filepath.Join(trustedDir, rawPath))
- cleanTrusted := filepath.Clean(trustedDir)
- if !pathWithin(cleanTrusted, resolvedPath) {
- return nil, fmt.Errorf("plugin path %q escapes trusted directory %q", rawPath, trustedDir)
- }
- // Defense-in-depth: the lexical check above cannot see through symlinks, so a
- // symlink inside the trusted dir could otherwise redirect the load outside it.
- // Resolve symlinks and re-check containment. EvalSymlinks requires the targets
- // to exist; if either cannot be resolved we keep the lexical result and let the
- // subsequent plugin.Open surface any real error.
- if realTrusted, trustedErr := filepath.EvalSymlinks(cleanTrusted); trustedErr == nil {
- if realPath, pathErr := filepath.EvalSymlinks(resolvedPath); pathErr == nil && !pathWithin(realTrusted, realPath) {
- return nil, fmt.Errorf("plugin path %q escapes trusted directory %q after symlink resolution", rawPath, trustedDir)
- }
- }
-
- symbol := sp.GetSymbol()
- if symbol == "" {
- symbol = "NewSampler"
- }
-
- // Marshal config before the cache lookup so the hash is part of the key.
- // protojson map-key ordering is deterministic for google.protobuf.Struct
- // (keys are sorted lexicographically by the protojson encoder), so hashing
- // the raw output is stable across calls with the same logical config.
- // A nil config hashes the canonical empty JSON object "{}".
- var cfgJSON []byte
- if cfg := sp.GetConfig(); cfg != nil {
- var marshalErr error
- cfgJSON, marshalErr = protojson.Marshal(cfg)
- if marshalErr != nil {
- return nil, fmt.Errorf("cannot marshal plugin config: %w", marshalErr)
- }
- } else {
- cfgJSON = []byte("{}")
- }
- hashBytes := sha256.Sum256(cfgJSON)
- configHash := hex.EncodeToString(hashBytes[:])
-
- key := pluginCacheKey{path: resolvedPath, symbol: symbol, configHash: configHash}
- pluginCache.mu.Lock()
- defer pluginCache.mu.Unlock()
- // 3-step lookup, all under pluginCache.mu:
- // 1. shared key (group=="") — a non-HostAware instance re-used across groups.
- // 2. group key — an already-hosted HostAware instance; return WITHOUT calling bindHost.
- // 3. miss — fall through to construct (ABI check, plugin.Open, ctor) under the lock.
- if cached, ok := pluginCache.m[key]; ok {
- return cached, nil
- }
- groupKey := key
- groupKey.group = group
- if cached, ok := pluginCache.m[groupKey]; ok {
- return cached, nil
- }
-
- if sp.GetAbiVersion() != uint32(sdk.ABIVersion) {
- return nil, fmt.Errorf("plugin ABI version %d does not match engine ABI version %d",
- sp.GetAbiVersion(), sdk.ABIVersion)
+var pluginCache = func() *runtime.Loader {
+ loader := runtime.NewLoader()
+ loader.OpenSampler = func(path, symbol string, config []byte) (sdk.Sampler, error) {
+ return newSamplerFromPlugin(path, symbol, config)
}
+ return loader
+}()
+var newSamplerFromPlugin = sdk.OpenSampler
- sampler, err := newSamplerFromPlugin(resolvedPath, symbol, cfgJSON)
- if err != nil {
- return nil, err
- }
+func computeConfigHash(sp *commonv1.SamplerPlugin) string { return runtime.ConfigHash(sp) }
- // Classify the freshly constructed instance and key it accordingly. For a
- // HostAware sampler, inject the host via bindHost while still holding
- // pluginCache.mu and BEFORE storing, so no concurrent cache-hit can observe a
- // not-yet-hosted instance. UseHost/telemetry setup thus runs under the same
- // lock that already guards plugin.Open and the constructor; keep it fast.
- if _, isHostAware := sampler.(sdk.HostAware); isHostAware {
- key.group = group
- if bindHost != nil {
- bindHost(sampler)
- }
- }
- pluginCache.m[key] = sampler
- return sampler, nil
+func loadSamplerPlugin(sp *commonv1.SamplerPlugin, trustedDir, group string, bindHost func(sdk.Sampler)) (sdk.Sampler, error) {
+ return pluginCache.LoadSamplerPlugin(sp, trustedDir, group, bindHost)
}
-
-// newSamplerFromPlugin opens the .so, verifies its ABIVersion symbol, and calls
-// the constructor. It is a package var so tests can substitute a Go sampler and
-// exercise the cache/keying/bindHost logic under -race without a real .so. The
-// default implementation delegates to sdk.OpenSampler — the single shared
-// loader-contract implementation also used by sdktest.LoadSO — so the host and
-// the offline dev toolkit never drift.
-var newSamplerFromPlugin = sdk.OpenSampler
diff --git a/banyand/trace/pipeline_loader_test.go b/banyand/trace/pipeline_loader_test.go
index f17b76c05..742a30a37 100644
--- a/banyand/trace/pipeline_loader_test.go
+++ b/banyand/trace/pipeline_loader_test.go
@@ -104,9 +104,7 @@ func buildTestPlugin(t *testing.T, dir string) string {
// permanent for the process lifetime, which is intentional for the wrong-symbol
// guard (it needs the .so already open so Lookup can be reached).
func resetPluginCache() {
- pluginCache.mu.Lock()
- pluginCache.m = make(map[pluginCacheKey]sdk.Sampler)
- pluginCache.mu.Unlock()
+ pluginCache.Reset()
}
// TestLoadSamplerPlugin exercises loadSamplerPlugin's happy path and every
diff --git a/banyand/trace/snapshot_scanner.go b/banyand/trace/snapshot_scanner.go
new file mode 100644
index 000000000..04e4512fc
--- /dev/null
+++ b/banyand/trace/snapshot_scanner.go
@@ -0,0 +1,668 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with this work for
+// additional information regarding copyright ownership. The ASF licenses this
+// file to you under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package trace
+
+import (
+ "context"
+ "encoding/gob"
+ "errors"
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+ "sort"
+ "time"
+
+ internalencoding "github.com/apache/skywalking-banyandb/banyand/internal/encoding"
+ pkgbytes "github.com/apache/skywalking-banyandb/pkg/bytes"
+ "github.com/apache/skywalking-banyandb/pkg/compress/zstd"
+ "github.com/apache/skywalking-banyandb/pkg/encoding"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
+ "github.com/apache/skywalking-banyandb/pkg/pipeline/sdk"
+)
+
+// SnapshotScanOptions bounds scratch metadata, merge fan-in and assembled trace bytes.
+type SnapshotScanOptions struct {
+ TempDir string
+ MaxMetadataBytes int64
+ MaxTraceBytes int64
+ MaxOpenReaders int
+}
+
+// SnapshotTrace is one whole local trace. Oversized traces have no projected payload.
+// The returned columns are independently owned and survive subsequent scanner calls.
+type SnapshotTrace struct {
+ Trace sdk.TraceBlock
+ SpanCount uint64
+ InputConflict bool
+ Oversized bool
+ CoverageKnown bool
+}
+
+// ProjectedSnapshotScanner assembles immutable source parts in trace-ID order.
+// It owns temporary merge runs but does not own or remove the source snapshot.
+type ProjectedSnapshotScanner struct {
+ input *os.File
+ decoder *gob.Decoder
+ pending *snapshotMetadata
+ scratch string
+ projection sdk.Projection
+ options SnapshotScanOptions
+}
+
+var errSnapshotTraceOversized = errors.New("projected trace exceeds budget")
+
+type snapshotMetadata struct {
+ TagTypes map[string]pbv1.ValueType
+ TraceID string
+ PartPath string
+ Metadata []byte
+}
+
+// NewProjectedSnapshotScanner prepares bounded external metadata merge runs.
+func NewProjectedSnapshotScanner(ctx context.Context, partPaths []string, projection sdk.Projection,
+ options SnapshotScanOptions,
+) (scanner *ProjectedSnapshotScanner, err error) {
+ if options.MaxMetadataBytes <= 0 {
+ options.MaxMetadataBytes = 8 << 20
+ }
+ if options.MaxTraceBytes <= 0 {
+ options.MaxTraceBytes = 16 << 20
+ }
+ if options.MaxOpenReaders <= 0 {
+ options.MaxOpenReaders = 16
+ }
+ if options.MaxOpenReaders < 2 {
+ return nil, errors.New("snapshot merge requires at least two readers")
+ }
+ scratch, err := os.MkdirTemp(options.TempDir, "trace-snapshot-scan-")
+ if err != nil {
+ return nil, err
+ }
+ scanner = &ProjectedSnapshotScanner{projection: projection, options: options, scratch: scratch}
+ cleanupScanner := scanner
+ defer func() {
+ if err != nil {
+ _ = cleanupScanner.Close()
+ }
+ }()
+ var runs []string
+ for _, partPath := range partPaths {
+ run, runErr := scanner.buildRun(ctx, partPath)
+ if runErr != nil {
+ return nil, fmt.Errorf("snapshot part %s: %w", partPath, runErr)
+ }
+ runs = append(runs, run)
+ }
+ for len(runs) > 1 {
+ var next []string
+ for offset := 0; offset < len(runs); offset += options.MaxOpenReaders {
+ end := min(offset+options.MaxOpenReaders, len(runs))
+ run, mergeErr := scanner.mergeRuns(ctx, runs[offset:end])
+ if mergeErr != nil {
+ return nil, mergeErr
+ }
+ next = append(next, run)
+ for _, old := range runs[offset:end] {
+ if removeErr := os.Remove(old); removeErr != nil {
+ return nil, removeErr
+ }
+ }
+ }
+ runs = next
+ }
+ if len(runs) == 0 {
+ return scanner, nil
+ }
+ scanner.input, err = os.Open(runs[0])
+ if err != nil {
+ return nil, err
+ }
+ scanner.decoder = gob.NewDecoder(scanner.input)
+ return scanner, nil
+}
+
+func (s *ProjectedSnapshotScanner) newRun() (*os.File, error) {
+ return os.CreateTemp(s.scratch, "run-")
+}
+
+func readSnapshotBytes(path string, limit int64) ([]byte, error) {
+ file, err := os.Open(path)
+ if err != nil {
+ return nil, err
+ }
+ defer file.Close()
+ data, err := io.ReadAll(io.LimitReader(file, limit+1))
+ if err != nil {
+ return nil, err
+ }
+ if int64(len(data)) > limit {
+ return nil, fmt.Errorf("metadata exceeds %d-byte budget", limit)
+ }
+ return data, nil
+}
+
+func (s *ProjectedSnapshotScanner) decompressMetadata(data []byte) ([]byte, error) {
+ return zstd.DecompressBounded(data, int(s.options.MaxMetadataBytes))
+}
+
+func (s *ProjectedSnapshotScanner) buildRun(ctx context.Context, path string) (run string, err error) {
+ defer func() {
+ if recovered := recover(); recovered != nil {
+ err = fmt.Errorf("invalid snapshot metadata: %v", recovered)
+ }
+ }()
+ typesData, err := readSnapshotBytes(filepath.Join(path, tagTypeFilename), s.options.MaxMetadataBytes)
+ if err != nil && !errors.Is(err, os.ErrNotExist) {
+ return "", err
+ }
+ types := make(tagType)
+ if len(typesData) > 0 {
+ if err = types.unmarshal(typesData); err != nil {
+ return "", err
+ }
+ }
+ meta, err := readSnapshotBytes(filepath.Join(path, metaFilename), s.options.MaxMetadataBytes)
+ if err != nil {
+ return "", err
+ }
+ meta, err = s.decompressMetadata(meta)
+ if err != nil {
+ return "", err
+ }
+ index, err := unmarshalPrimaryBlockMetadata(nil, meta)
+ if err != nil {
+ return "", err
+ }
+ primary, err := os.Open(filepath.Join(path, primaryFilename))
+ if err != nil {
+ return "", err
+ }
+ defer primary.Close()
+ output, err := s.newRun()
+ if err != nil {
+ return "", err
+ }
+ defer output.Close()
+ encoder := gob.NewEncoder(output)
+ previous := ""
+ for _, entry := range index {
+ if err = ctx.Err(); err != nil {
+ return "", err
+ }
+ if entry.size > uint64(s.options.MaxMetadataBytes) {
+ return "", errors.New("primary metadata exceeds budget")
+ }
+ compressed := make([]byte, int(entry.size))
+ if _, err = primary.ReadAt(compressed, int64(entry.offset)); err != nil {
+ return "", err
+ }
+ data, decodeErr := s.decompressMetadata(compressed)
+ if decodeErr != nil {
+ return "", decodeErr
+ }
+ for len(data) > 0 {
+ var metadata blockMetadata
+ tail, parseErr := metadata.unmarshal(data, types)
+ if parseErr != nil {
+ return "", parseErr
+ }
+ if metadata.traceID < previous {
+ return "", errors.New("source blocks are not trace-ID ordered")
+ }
+ previous = metadata.traceID
+ record := snapshotMetadata{TraceID: metadata.traceID, PartPath: path, Metadata: metadata.marshal(nil), TagTypes: types}
+ if err = encoder.Encode(record); err != nil {
+ return "", err
+ }
+ data = tail
+ }
+ }
+ if err = output.Close(); err != nil {
+ return "", err
+ }
+ return output.Name(), nil
+}
+
+func (s *ProjectedSnapshotScanner) mergeRuns(ctx context.Context, paths []string) (string, error) {
+ readers := make([]*os.File, 0, len(paths))
+ defer func() {
+ for _, reader := range readers {
+ _ = reader.Close()
+ }
+ }()
+ decoders := make([]*gob.Decoder, len(paths))
+ heads := make([]*snapshotMetadata, len(paths))
+ for index, path := range paths {
+ reader, err := os.Open(path)
+ if err != nil {
+ return "", err
+ }
+ readers = append(readers, reader)
+ decoders[index] = gob.NewDecoder(reader)
+ heads[index] = &snapshotMetadata{}
+ if err = decoders[index].Decode(heads[index]); errors.Is(err, io.EOF) {
+ heads[index] = nil
+ } else if err != nil {
+ return "", err
+ }
+ }
+ output, err := s.newRun()
+ if err != nil {
+ return "", err
+ }
+ defer output.Close()
+ encoder := gob.NewEncoder(output)
+ for {
+ if err = ctx.Err(); err != nil {
+ return "", err
+ }
+ best := -1
+ for index, head := range heads {
+ if head != nil && (best < 0 || head.TraceID < heads[best].TraceID) {
+ best = index
+ }
+ }
+ if best < 0 {
+ break
+ }
+ if err = encoder.Encode(heads[best]); err != nil {
+ return "", err
+ }
+ *heads[best] = snapshotMetadata{}
+ if err = decoders[best].Decode(heads[best]); errors.Is(err, io.EOF) {
+ heads[best] = nil
+ } else if err != nil {
+ return "", err
+ }
+ }
+ if err = output.Close(); err != nil {
+ return "", err
+ }
+ return output.Name(), nil
+}
+
+func (s *ProjectedSnapshotScanner) nextMetadata() (*snapshotMetadata, error) {
+ if s.pending != nil {
+ metadata := s.pending
+ s.pending = nil
+ return metadata, nil
+ }
+ if s.decoder == nil {
+ return nil, io.EOF
+ }
+ var metadata snapshotMetadata
+ if err := s.decoder.Decode(&metadata); err != nil {
+ return nil, err
+ }
+ return &metadata, nil
+}
+
+// Next returns the next whole trace, or io.EOF after the immutable inventory ends.
+func (s *ProjectedSnapshotScanner) Next(ctx context.Context) (result *SnapshotTrace, err error) {
+ defer func() {
+ if recovered := recover(); recovered != nil {
+ result = nil
+ err = fmt.Errorf("invalid snapshot payload: %v", recovered)
+ }
+ }()
+ first, err := s.nextMetadata()
+ if err != nil {
+ return nil, err
+ }
+ result = &SnapshotTrace{Trace: sdk.TraceBlock{TraceID: first.TraceID}, CoverageKnown: true}
+ used := int64(len(first.TraceID))
+ result.Oversized = used > s.options.MaxTraceBytes
+ current := first
+ for {
+ if err = ctx.Err(); err != nil {
+ return nil, err
+ }
+ var metadata blockMetadata
+ if _, err = metadata.unmarshal(current.Metadata, current.TagTypes); err != nil {
+ return nil, err
+ }
+ oldRows := int(result.SpanCount)
+ if metadata.count > ^uint64(0)-result.SpanCount {
+ return nil, errors.New("snapshot span count overflows")
+ }
+ result.SpanCount += metadata.count
+ if !metadata.timestamps.known {
+ result.CoverageKnown = false
+ }
+ if current == first {
+ result.Trace.MinTS = metadata.timestamps.min
+ result.Trace.MaxTS = metadata.timestamps.max
+ } else {
+ result.Trace.MinTS = min(result.Trace.MinTS, metadata.timestamps.min)
+ result.Trace.MaxTS = max(result.Trace.MaxTS, metadata.timestamps.max)
+ }
+ if !result.Oversized {
+ perRow := uint64(32 * (len(s.projection.Tags) + 1))
+ estimate := uint64(s.options.MaxTraceBytes) + 1
+ if result.SpanCount <= uint64(s.options.MaxTraceBytes)/perRow {
+ estimate = metadata.count * perRow
+ }
+ if s.projection.Spans || s.projection.SpanIDs {
+ if metadata.uncompressedSpanSizeBytes > uint64(s.options.MaxTraceBytes) || estimate > uint64(s.options.MaxTraceBytes)-metadata.uncompressedSpanSizeBytes {
+ estimate = uint64(s.options.MaxTraceBytes) + 1
+ } else {
+ estimate += metadata.uncompressedSpanSizeBytes
+ }
+ }
+ if estimate > uint64(s.options.MaxTraceBytes-used) {
+ result.Oversized = true
+ } else {
+ remaining := s.options.MaxTraceBytes - used
+ padding := snapshotProjectionPadding(result.Trace.Tags, metadata.tags, s.projection, oldRows)
+ var payload sdk.TraceBlock
+ var payloadErr error
+ if padding > remaining {
+ payloadErr = errSnapshotTraceOversized
+ } else {
+ payload, payloadErr = s.decodeProjection(current, &metadata, remaining-padding)
+ }
+ if errors.Is(payloadErr, errSnapshotTraceOversized) || errors.Is(payloadErr, zstd.ErrDecompressionLimit) {
+ result.Oversized = true
+ } else if payloadErr != nil {
+ return nil, payloadErr
+ }
+ for _, column := range payload.Tags {
+ for _, previousColumn := range result.Trace.Tags {
+ if column.Name == previousColumn.Name && column.ValueType != previousColumn.ValueType {
+ result.InputConflict = true
+ result.CoverageKnown = false
+ }
+ }
+ }
+ if !result.Oversized {
+ used += appendSnapshotProjection(&result.Trace, payload, oldRows, int(metadata.count), s.projection)
+ }
+ if used > s.options.MaxTraceBytes {
+ result.Oversized = true
+ }
+ }
+ if result.Oversized {
+ result.Trace.Tags = nil
+ result.Trace.SpanIDs = nil
+ result.Trace.Spans = nil
+ }
+ }
+ current, err = s.nextMetadata()
+ if errors.Is(err, io.EOF) {
+ return result, nil
+ }
+ if err != nil {
+ return nil, err
+ }
+ if current.TraceID != first.TraceID {
+ s.pending = current
+ return result, nil
+ }
+ }
+}
+
+func (s *ProjectedSnapshotScanner) decodeProjection(record *snapshotMetadata, metadata *blockMetadata, remaining int64) (sdk.TraceBlock, error) {
+ result := sdk.TraceBlock{}
+ decoder := &encoding.BytesBlockDecoder{}
+ decoder.SetMaxBytes(int(remaining))
+ readRange := func(name string, data *dataBlock) ([]byte, error) {
+ if data.size > uint64(remaining) {
+ return nil, errSnapshotTraceOversized
+ }
+ file, err := os.Open(filepath.Join(record.PartPath, name))
+ if err != nil {
+ return nil, err
+ }
+ defer file.Close()
+ buf := make([]byte, int(data.size))
+ _, err = file.ReadAt(buf, int64(data.offset))
+ return buf, err
+ }
+ if s.projection.Spans || s.projection.SpanIDs {
+ data, err := readRange(spansFilename, metadata.spans)
+ if err != nil {
+ return result, err
+ }
+ result, err = decodeSnapshotSpans(data, metadata.count, s.projection, &remaining)
+ if err != nil {
+ return result, err
+ }
+ }
+ names := make([]string, 0, len(metadata.tags))
+ for name := range metadata.tags {
+ names = append(names, name)
+ }
+ sort.Strings(names)
+ for _, projected := range s.projection.Tags {
+ for _, name := range names {
+ if decodeTypedTag(name) != projected {
+ continue
+ }
+ tagMeta, err := readRange(name+tagsMetadataFilenameExt, metadata.tags[name])
+ if err != nil {
+ return result, err
+ }
+ var tm tagMetadata
+ if err = tm.unmarshal(tagMeta); err != nil {
+ return result, err
+ }
+ if tm.size > uint64(remaining) {
+ return result, errSnapshotTraceOversized
+ }
+ valuesData, err := readRange(name+tagsFilenameExt, &tm.dataBlock)
+ if err != nil {
+ return result, err
+ }
+ if remaining <= 0 {
+ return result, errSnapshotTraceOversized
+ }
+ decoder.Reset()
+ decoder.SetMaxBytes(int(remaining))
+ var values [][]byte
+ valueType := record.TagTypes[name]
+ switch {
+ case len(valuesData) > 0 && encoding.EncodeType(valuesData[0]) == encoding.EncodeTypePlain:
+ values, err = decoder.Decode(nil, valuesData[1:], metadata.count)
+ case valueType == pbv1.ValueTypeInt64 || valueType == pbv1.ValueTypeFloat64:
+ values, err = internalencoding.DecodeTagValues(nil, decoder, &pkgbytes.Buffer{Buf: valuesData}, valueType, int(metadata.count))
+ case len(valuesData) > 0 && encoding.EncodeType(valuesData[0]) == encoding.EncodeTypeDictionary:
+ values, err = encoding.NewDictionary().DecodeWithLimit(nil, valuesData[1:], metadata.count, int(remaining))
+ case len(valuesData) > 0:
+ values, err = decoder.Decode(nil, valuesData[1:], metadata.count)
+ }
+ if err != nil {
+ return result, err
+ }
+ if uint64(len(values)) != metadata.count {
+ return result, fmt.Errorf("projected tag %s has %d values for %d rows", projected, len(values), metadata.count)
+ }
+ if err = consumeSnapshotValues(values, 24, &remaining); err != nil {
+ return result, err
+ }
+ column := sdk.TagColumn{Name: projected, ValueType: record.TagTypes[name]}
+ for _, value := range values {
+ var copied []byte
+ if value != nil {
+ copied = make([]byte, len(value))
+ copy(copied, value)
+ }
+ column.Values = append(column.Values, copied)
+ }
+ result.Tags = append(result.Tags, column)
+ break
+ }
+ }
+ return result, nil
+}
+
+func appendSnapshotProjection(target *sdk.TraceBlock, source sdk.TraceBlock, oldRows, rows int, projection sdk.Projection) int64 {
+ var used int64
+ target.SpanIDs = append(target.SpanIDs, source.SpanIDs...)
+ target.Spans = append(target.Spans, source.Spans...)
+ for _, id := range source.SpanIDs {
+ used += int64(len(id) + 16)
+ }
+ for _, span := range source.Spans {
+ used += int64(len(span) + 24)
+ }
+ for _, name := range projection.Tags {
+ index := -1
+ for columnIdx := range target.Tags {
+ if target.Tags[columnIdx].Name == name {
+ index = columnIdx
+ break
+ }
+ }
+ var column *sdk.TagColumn
+ for sourceIdx := range source.Tags {
+ if source.Tags[sourceIdx].Name == name {
+ column = &source.Tags[sourceIdx]
+ break
+ }
+ }
+ if index < 0 && column != nil {
+ target.Tags = append(target.Tags, sdk.TagColumn{Name: name, ValueType: column.ValueType, Values: make([][]byte, oldRows)})
+ index = len(target.Tags) - 1
+ used += int64(oldRows * 24)
+ }
+ if index < 0 {
+ continue
+ }
+ if column == nil {
+ target.Tags[index].Values = append(target.Tags[index].Values, make([][]byte, rows)...)
+ used += int64(rows * 24)
+ continue
+ }
+ target.Tags[index].Values = append(target.Tags[index].Values, column.Values...)
+ for _, value := range column.Values {
+ used += int64(len(value) + 24)
+ }
+ }
+ return used
+}
+
+// Close releases scan handles and temporary runs. Source parts are preserved.
+func (s *ProjectedSnapshotScanner) Close() error {
+ var closeErr error
+ if s.input != nil {
+ closeErr = s.input.Close()
+ s.input = nil
+ }
+ s.decoder = nil
+ s.pending = nil
+ if s.scratch != "" {
+ closeErr = errors.Join(closeErr, os.RemoveAll(s.scratch))
+ s.scratch = ""
+ }
+ return closeErr
+}
+
+// GuardSnapshotTrace applies the existing fragment guard to a proposed snapshot drop.
+// It inherits the storage runtime's grace and completeness semantics for the pinned inventory.
+func GuardSnapshotTrace(ctx context.Context, trace SnapshotTrace, sourceMin, sourceMax int64, grace time.Duration, proposedKeep bool) (bool, string) {
+ catalog := traceFragmentGuardCatalog{
+ Complete: true, CoverageKnown: true,
+ CoverageMinTimestamp: sourceMin, CoverageMaxTimestamp: sourceMax,
+ TemporalSafety: traceFragmentTemporalSafetyMaxGapEnforced, EnforcedMaxFragmentGap: grace, Pin: snapshotGuardPin{},
+ }
+ guard := newTraceFragmentGuard(traceFragmentGuardConfig{Grace: grace}, catalog)
+ defer guard.Close()
+ action := traceFragmentSamplerActionDrop
+ if proposedKeep {
+ action = traceFragmentSamplerActionKeep
+ }
+ decision := guard.Resolve(ctx, traceFragmentGuardTrace{
+ TraceID: trace.Trace.TraceID,
+ Complete: !trace.Oversized && trace.CoverageKnown,
+ Blocks: []traceFragmentGuardBlock{{MinTimestamp: trace.Trace.MinTS, MaxTimestamp: trace.Trace.MaxTS, BoundsKnown: trace.CoverageKnown}},
+ }, action)
+ return decision.Action != traceFragmentGuardActionDrop, string(decision.Reason)
+}
+
+type snapshotGuardPin struct{}
+
+func (snapshotGuardPin) Release() {}
+
+func consumeSnapshotValues(values [][]byte, referenceBytes int64, remaining *int64) error {
+ for _, value := range values {
+ size := int64(len(value)) + referenceBytes
+ if size > *remaining {
+ return errSnapshotTraceOversized
+ }
+ *remaining -= size
+ }
+ return nil
+}
+
+func snapshotProjectionPadding(columns []sdk.TagColumn, tags map[string]*dataBlock, projection sdk.Projection, oldRows int) int64 {
+ var padding int64
+ for _, projected := range projection.Tags {
+ exists := false
+ for _, column := range columns {
+ if column.Name == projected {
+ exists = true
+ break
+ }
+ }
+ if exists {
+ continue
+ }
+ for encoded := range tags {
+ if decodeTypedTag(encoded) == projected {
+ padding += int64(oldRows) * 24
+ break
+ }
+ }
+ }
+ return padding
+}
+
+func decodeSnapshotSpans(data []byte, count uint64, projection sdk.Projection, remaining *int64) (sdk.TraceBlock, error) {
+ result := sdk.TraceBlock{}
+ decoder := &encoding.BytesBlockDecoder{}
+ decoder.SetMaxBytes(int(*remaining))
+ ids, tail, err := decoder.DecodeWithTail(nil, data, count)
+ if err != nil {
+ return result, err
+ }
+ if projection.SpanIDs {
+ if err = consumeSnapshotValues(ids, 16, remaining); err != nil {
+ return result, err
+ }
+ for _, id := range ids {
+ result.SpanIDs = append(result.SpanIDs, string(id))
+ }
+ }
+ if projection.Spans {
+ if *remaining <= 0 {
+ return result, errSnapshotTraceOversized
+ }
+ decoder.Reset()
+ decoder.SetMaxBytes(int(*remaining))
+ spans, decodeErr := decoder.Decode(nil, tail, count)
+ if decodeErr != nil {
+ return result, decodeErr
+ }
+ if decodeErr = consumeSnapshotValues(spans, 24, remaining); decodeErr != nil {
+ return result, decodeErr
+ }
+ for _, span := range spans {
+ result.Spans = append(result.Spans, append([]byte(nil), span...))
+ }
+ }
+ return result, nil
+}
diff --git a/banyand/trace/snapshot_scanner_test.go b/banyand/trace/snapshot_scanner_test.go
new file mode 100644
index 000000000..8d65f4c0a
--- /dev/null
+++ b/banyand/trace/snapshot_scanner_test.go
@@ -0,0 +1,185 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with this work for
+// additional information regarding copyright ownership. The ASF licenses this
+// file to you under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package trace
+
+import (
+ "context"
+ "io"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/apache/skywalking-banyandb/pkg/fs"
+ "github.com/apache/skywalking-banyandb/pkg/pipeline/sdk"
+)
+
+func snapshotTestPart(t *testing.T, root string, id uint64, ids ...string) string {
+ t.Helper()
+ rows := make([]PartEncoderRow, 0, len(ids))
+ for index, traceID := range ids {
+ rows = append(rows, PartEncoderRow{
+ TraceID: traceID, SpanID: traceID + "-span", Span: []byte("payload"),
+ Timestamp: int64(index + 1), Tags: []PartEncoderTag{{Name: "status", Value: "error"}},
+ })
+ }
+ path, release := EncodePart(root, fs.NewLocalFileSystem(), id, rows)
+ release()
+ return path
+}
+
+func TestSnapshotScannerWholeTraceExternalMerge(t *testing.T) {
+ root := t.TempDir()
+ var paths []string
+ for index := 0; index < 9; index++ {
+ paths = append(paths, snapshotTestPart(t, root, uint64(index+1), "b", "a"))
+ }
+ scanner, err := NewProjectedSnapshotScanner(context.Background(), paths, sdk.Projection{Tags: []string{"status"}, SpanIDs: true, Spans: true},
+ SnapshotScanOptions{MaxOpenReaders: 2, TempDir: root})
+ require.NoError(t, err)
+ defer scanner.Close()
+ first, err := scanner.Next(context.Background())
+ require.NoError(t, err)
+ require.Equal(t, "a", first.Trace.TraceID)
+ require.Equal(t, uint64(9), first.SpanCount)
+ require.Len(t, first.Trace.Spans, 9)
+ require.Len(t, first.Trace.Tags, 1)
+ require.Len(t, first.Trace.Tags[0].Values, 9)
+ require.Equal(t, []byte("error"), first.Trace.Tags[0].Values[0])
+ second, err := scanner.Next(context.Background())
+ require.NoError(t, err)
+ require.Equal(t, "b", second.Trace.TraceID)
+ require.Equal(t, uint64(9), second.SpanCount)
+ require.Equal(t, []byte("payload"), first.Trace.Spans[0], "previous projected storage survives Next")
+ _, err = scanner.Next(context.Background())
+ require.ErrorIs(t, err, io.EOF)
+ scratch := scanner.scratch
+ require.NoError(t, scanner.Close())
+ _, err = os.Stat(scratch)
+ require.ErrorIs(t, err, os.ErrNotExist)
+}
+
+func TestSnapshotScannerMetadataOnlyDoesNotReadSpans(t *testing.T) {
+ root := t.TempDir()
+ part := snapshotTestPart(t, root, 1, "a")
+ require.NoError(t, os.Remove(filepath.Join(part, spansFilename)))
+ scanner, err := NewProjectedSnapshotScanner(context.Background(), []string{part}, sdk.Projection{}, SnapshotScanOptions{})
+ require.NoError(t, err)
+ defer scanner.Close()
+ trace, err := scanner.Next(context.Background())
+ require.NoError(t, err)
+ require.Equal(t, uint64(1), trace.SpanCount)
+ require.Nil(t, trace.Trace.Spans)
+ require.Nil(t, trace.Trace.SpanIDs)
+}
+
+func TestSnapshotScannerOversizedSkipsPayloadAndKeepsWholeTrace(t *testing.T) {
+ root := t.TempDir()
+ part := snapshotTestPart(t, root, 1, "a", "a", "b")
+ require.NoError(t, os.Remove(filepath.Join(part, spansFilename)))
+ scanner, err := NewProjectedSnapshotScanner(context.Background(), []string{part}, sdk.Projection{Spans: true}, SnapshotScanOptions{MaxTraceBytes: 2})
+ require.NoError(t, err)
+ defer scanner.Close()
+ trace, err := scanner.Next(context.Background())
+ require.NoError(t, err)
+ require.True(t, trace.Oversized)
+ require.Equal(t, uint64(2), trace.SpanCount)
+ require.Nil(t, trace.Trace.Spans)
+ next, err := scanner.Next(context.Background())
+ require.NoError(t, err)
+ require.Equal(t, "b", next.Trace.TraceID)
+}
+
+func TestSnapshotScannerRejectsCorruptMetadataAndCleansScratch(t *testing.T) {
+ root := t.TempDir()
+ part := snapshotTestPart(t, root, 1, "a")
+ require.NoError(t, os.WriteFile(filepath.Join(part, primaryFilename), []byte("broken"), 0o600))
+ _, err := NewProjectedSnapshotScanner(context.Background(), []string{part}, sdk.Projection{}, SnapshotScanOptions{TempDir: root})
+ require.Error(t, err)
+ scratch, err := filepath.Glob(filepath.Join(root, "trace-snapshot-scan-*"))
+ require.NoError(t, err)
+ require.Empty(t, scratch)
+}
+
+func TestSnapshotGuardPreservesRuntimeCompleteness(t *testing.T) {
+ trace := SnapshotTrace{Trace: sdk.TraceBlock{TraceID: "a", MinTS: 100, MaxTS: 200}, CoverageKnown: true}
+ keep, reason := GuardSnapshotTrace(context.Background(), trace, 0, 1000, time.Nanosecond, false)
+ require.False(t, keep)
+ require.Equal(t, string(traceFragmentGuardReasonNoCandidate), reason)
+ keep, reason = GuardSnapshotTrace(context.Background(), trace, 100, 1000, time.Nanosecond, false)
+ require.True(t, keep)
+ require.Equal(t, string(traceFragmentGuardReasonSegmentBoundary), reason)
+ keep, reason = GuardSnapshotTrace(context.Background(), trace, 0, 1000, time.Nanosecond, true)
+ require.True(t, keep)
+ require.Equal(t, string(traceFragmentGuardReasonSamplerKeep), reason)
+}
+
+func TestSnapshotScannerConflictingProjectedTypesFailOpen(t *testing.T) {
+ root := t.TempDir()
+ first := snapshotTestPart(t, root, 1, "a")
+ second, release := EncodePart(root, fs.NewLocalFileSystem(), 2, []PartEncoderRow{{
+ TraceID: "a", SpanID: "other", Span: []byte("payload"), Timestamp: 2,
+ Tags: []PartEncoderTag{{Name: "status", Value: int64(500)}},
+ }})
+ release()
+ scanner, err := NewProjectedSnapshotScanner(context.Background(), []string{first, second}, sdk.Projection{Tags: []string{"status"}}, SnapshotScanOptions{})
+ require.NoError(t, err)
+ defer scanner.Close()
+ result, err := scanner.Next(context.Background())
+ require.NoError(t, err)
+ require.True(t, result.InputConflict)
+ require.False(t, result.CoverageKnown)
+ keep, reason := GuardSnapshotTrace(context.Background(), *result, 0, 1000, time.Nanosecond, false)
+ require.True(t, keep)
+ require.Equal(t, string(traceFragmentGuardReasonTraceIncomplete), reason)
+}
+
+func TestSnapshotScannerCompressedProjectedTagOverBudgetRetains(t *testing.T) {
+ root := t.TempDir()
+ part, release := EncodePart(root, fs.NewLocalFileSystem(), 1, []PartEncoderRow{{
+ TraceID: "a", SpanID: "one", Span: []byte("body"), Timestamp: 1,
+ Tags: []PartEncoderTag{{Name: "status", Value: strings.Repeat("x", 1<<20)}},
+ }})
+ release()
+ scanner, err := NewProjectedSnapshotScanner(context.Background(), []string{part}, sdk.Projection{Tags: []string{"status"}}, SnapshotScanOptions{MaxTraceBytes: 1024})
+ require.NoError(t, err)
+ defer scanner.Close()
+ result, err := scanner.Next(context.Background())
+ require.NoError(t, err)
+ require.True(t, result.Oversized)
+ require.Nil(t, result.Trace.Tags)
+ require.Equal(t, uint64(1), result.SpanCount)
+}
+
+func TestSnapshotScannerAggregateProjectionBudgetRetainsWholeTrace(t *testing.T) {
+ root := t.TempDir()
+ part, release := EncodePart(root, fs.NewLocalFileSystem(), 1, []PartEncoderRow{{
+ TraceID: "a", SpanID: "one", Span: []byte("body"), Timestamp: 1,
+ Tags: []PartEncoderTag{{Name: "one", Value: strings.Repeat("x", 700)}, {Name: "two", Value: strings.Repeat("y", 700)}},
+ }})
+ release()
+ scanner, err := NewProjectedSnapshotScanner(context.Background(), []string{part}, sdk.Projection{Tags: []string{"one", "two"}}, SnapshotScanOptions{MaxTraceBytes: 1024})
+ require.NoError(t, err)
+ defer scanner.Close()
+ result, err := scanner.Next(context.Background())
+ require.NoError(t, err)
+ require.True(t, result.Oversized)
+ require.Nil(t, result.Trace.Tags)
+ require.Equal(t, uint64(1), result.SpanCount)
+}
diff --git a/docs/design/0.12.0/README.md b/docs/design/0.12.0/README.md
index 1ec90a391..2d182e0e5 100644
--- a/docs/design/0.12.0/README.md
+++ b/docs/design/0.12.0/README.md
@@ -1,5 +1,6 @@
# 0.12.0 Design Documents
+- [Trace sampling before lifecycle migration](lifecycle-trace-sampling.md)
- [Measure QL Catalog B: core SQL implementation plan](measure-ql-core-sql/README.md)
- [PR 1 scope and isolated verification](measure-ql-core-sql/pr1-verification.md)
- [Compiled measure QL plan reuse](measure-ql-core-sql/plan-reuse.md)
diff --git a/docs/design/0.12.0/lifecycle-trace-sampling.md b/docs/design/0.12.0/lifecycle-trace-sampling.md
new file mode 100644
index 000000000..08f4c1c94
--- /dev/null
+++ b/docs/design/0.12.0/lifecycle-trace-sampling.md
@@ -0,0 +1,418 @@
+# Trace sampling before lifecycle migration
+
+Status: implemented. This document specifies agent-side migration sampling and
+its durable preparation and replay protocol. For deployment, group API examples,
+and restart instructions, see [trace retention sampling](../../operation/lifecycle.md#trace-retention-sampling).
+
+## Scope decisions
+
+The migration-related questions are closed for this proposal. Post-sampling
+inherits the lifecycle agent's existing snapshot scope, late-write handling,
+source deletion, destination acknowledgement and at-least-once retry contracts.
+It does not require new trace sealing, source-node coordination, receiver
+deduplication or stronger delivery guarantees. These are scope decisions, not
+claims that the existing contracts have been strengthened or newly verified.
+
+Reuse the existing sampling completeness protections; uncertain decisions retain
+data. Row replay is the selected transport for sampled migrations. Native-part
+transfer and broader migration performance redesign are outside this scope;
+benchmarking is useful validation, not an unresolved architecture decision or a
+prerequisite for designing post-sampling.
+
+The new work is policy selection, trace assembly, sampling, durable decisions,
+delivery checkpoints, batching across source parts and focused verification of
+those changes. There are no remaining open design questions in this proposal.
+
+## Decision
+
+The lifecycle agent evaluates the **source stage's retention chain before sending
+trace data to the next stage**. Sampling applies to the immutable migration
+snapshot. It does not rewrite the live source segment.
+
+For the first implementation, sampled migrations use row replay for every
+retained span, including migrations that would otherwise use file transfer.
+The destination's normal write path rebuilds its indexes from those retained
+spans. Unsampled migrations keep their existing transport selection.
+
+Persist a complete sampling decision manifest for each source segment/shard
+before its first write to the destination. Retries replay the same snapshot
+with the same manifest, without rerunning the sampler.
+
+```mermaid
+flowchart TD
+ A[Immutable lifecycle snapshot] --> B{Source stage has a matching retention chain?}
+ B -->|No| C[Existing file transfer or row replay]
+ B -->|Yes| D[Assemble traces across all source shard parts]
+ D --> E[Evaluate stage sampler chain and completeness checks]
+ E --> F[Durably publish decision manifest]
+ F --> G[Replay retained spans through bounded sender]
+ G --> H[Destination rebuilds trace data and indexes]
+ H --> I[Drain confirmations and record completion]
+ I --> J[Existing source deletion after successful migration]
+```
+
+## Current implementation and placement
+
+- [`migrateTraceWithFileBasedAndProgress`](../../../banyand/backup/lifecycle/migration_integration.go)
+ traverses a filesystem snapshot and constructs a migration visitor.
+- [`traceMigrationVisitor.VisitShard`](../../../banyand/backup/lifecycle/trace_migration_visitor.go)
+ transfers core and SIDX files when the source fits one target segment, and
+ otherwise replays rows. Sampled migration runs **before** this choice.
+- `VisitSeries` can send the original series index before shard visitation.
+ Sampling mode is selected before traversal and suppresses that transfer. Series
+ work completes after retained rows have been delivered and their indexes rebuilt.
+- [`traceRowReplayer.replayPart`](../../../banyand/backup/lifecycle/row_replay_trace.go)
+ currently sends rows immediately, one part at a time. A predicate inside this
+ loop is insufficient: one trace may have blocks in several parts.
+- `traceRowReplayer.publishRow` already reconstructs writes and routes each trace
+ using the target shard count. Reuse it and the bounded sender after decisions
+ are fixed. Preserve its confirmation/error handling.
+
+This proposal supersedes the migration mechanism in
+[the earlier post-trace design, section 7.2](../archive/0.11.0/post-trace-pipeline.md#72-hot-to-warm-tier-migration-pre-migration-filter-rewrite):
+the agent performs a decision pass followed by filtered replay, rather than
+building reduced native parts for byte-copy transfer. The source-stage policy
+semantics remain the same. In particular, source TTL does not prove that every
+trace is complete or that late writes have stopped.
+
+## Policy selection
+
+Use the existing `TracePipelineConfig.stages` and `StageRule.plugins` fields;
+no new pipeline event or sampler ABI is needed.
+
+| Condition | Migration behavior |
+| --- | --- |
+| Pipeline absent or disabled | Existing lossless migration |
+| Schema not targeted by the pipeline selectors | Existing lossless migration |
+| No rule for `GroupConfig.SourceStage`, or empty chain | Existing lossless migration |
+| Matching source-stage chain | Prepare decisions, then replay retained rows |
+| Matching chain cannot be loaded or validated | Fail migration before sending; report configuration error |
+
+`MERGE` and `FINALIZE` in `enabled_events` control the data-node passes. They do
+not gate migration sampling. Do not rerun the top-level `pipeline.plugins`
+chain during migration, and do not apply the destination stage's chain early.
+All links see the same batch and their keep masks are AND-ed, using the existing
+SDK chain semantics.
+
+For example, `hot -> warm` uses the `hot` rule; `warm -> cold` uses the `warm`
+rule. If the initial hot tier is implicit in `resource_opts`, use the source
+stage identity already resolved by `parseGroup` from the node's `type` label.
+An ambiguous or missing source identity must be reported, not guessed. A stage
+without a successor has no migration pass.
+
+Resolve the trace schema before traversal. The current replay path supports
+exactly one trace schema per group; preserve that explicit restriction in v1.
+Pin the schema, ordered plugin configuration, source and destination stage
+configuration, and routing parameters for the migration attempt. Configuration
+updates take effect for new migration work, not midway through a prepared shard.
+
+## Sampling scope and completeness
+
+The decision unit is `(snapshot, group, source segment, source shard, trace ID)`.
+Assemble every block for that trace across **all core parts in the unit** before
+calling `Decide`. Make one decision for all its spans, before splitting retained
+rows between target segments or target shards. Never independently sample each
+part, row, or target segment.
+
+Use a trace-ID-ordered merge of part/block cursors. Decode the union of plugin
+projections for the decision pass; avoid decoding span bodies unless requested.
+Storage readers and column conversion stay in the trace storage module. The
+existing dump iterator decodes full blocks and eagerly gathers block metadata;
+it is not by itself a bounded, projected multi-part scanner. Implement the
+scanner explicitly and verify ordering instead of collecting the entire shard
+in memory. Bound open readers, metadata buffers, batch bytes and per-trace bytes;
+use bounded external merge runs when the part fan-in exceeds the reader budget.
+
+A trace exceeding the decision budget is retained in its entirety. Once that
+decision is made, skip remaining projected payloads for that trace while
+continuing the ordered scan. A partial trace must never be presented as complete
+to a sampler. Decoder-owned buffers must be copied or kept pinned until the
+sampler finishes, including a worker that outlives a timeout.
+
+The snapshot only establishes a stable local input set. It does not establish a
+globally complete trace across segments, nodes, or future arrivals. Apply the
+existing fragment-guard policy through a snapshot reader adapter: retain traces
+near source-segment edges, with unavailable/incomplete input coverage, or with
+possible fragments outside the decision unit. The snapshot adapter uses the
+same grace-window and temporal assumptions as the existing data-node sampling
+guard; establishing a new global fragment-gap contract is outside scope. If
+input bounds or coverage required by that guard are unavailable, convert the
+proposed drop to keep and report that reason. Tests must verify both eligible
+interior drops and conservative keeps under the inherited guard semantics.
+
+Migration sampling is therefore conservative and scoped to the captured input.
+It adds no global trace-sealing guarantee. Writes arriving after snapshot capture
+remain subject to the existing migration/deletion contract, which is inherited
+unchanged and is not a blocker for post-sampling. This proposal must not
+treat sampling completion as new evidence that live source data is sealed.
+
+## Module interface and ownership
+
+Keep the lifecycle-facing interface small. Package-private operations:
+
+```go
+prepareTraceMigration(ctx, sourceSnapshotShard, pinnedPolicy) (*preparedTraceMigration, error)
+prepared.Replay(ctx, traceRowReplayer) error
+prepared.Close() error
+```
+
+These are conceptual signatures, not new SDK declarations. Preparation owns
+assembly, plugin execution, conservative keep decisions and manifest durability.
+Replay owns manifest verification, filtering and acknowledgement draining.
+Close releases handles; it does not remove resumable state. Artifact reclamation
+belongs to successful migration cleanup.
+
+The storage module exposes a projected snapshot scanner, hiding native formats.
+The lifecycle module owns stage selection, durable state and sending. Factor the
+reusable plugin loader and bounded execution logic out of data-node-private code
+into an internal pipeline runtime, used by both callers. Keep `pkg/pipeline/sdk`
+and its existing `EvaluateChain` contract unchanged. Do not copy the loader,
+timeout, projection-union or panic-handling implementations into lifecycle.
+
+Plugin instances belong to a pinned policy execution session. Serialize calls to
+an instance unless concurrency is explicitly supported. Close instances only
+after workers finish; a timeout cannot kill a native Go call. Use bounded worker
+slots and the existing circuit-breaker behavior to prevent accumulation of
+timed-out workers and retained batches.
+
+## Merging and small outputs
+
+Source part boundaries are read boundaries, not output boundaries. The ordered
+decision pass logically merges blocks across parts to assemble complete local
+traces, but does not create a reduced file for each source part. Concurrent live
+source compaction cannot change the pinned filesystem snapshot's inputs; keep
+those snapshot files alive until the attempt no longer needs them.
+
+Sampled replay must feed retained rows across all source parts into a continuous,
+bounded batching session. Reuse `publishRow` and the sender's enqueue/flush/drain
+primitives, **not** the current `replayPart` driver unchanged: that driver flushes
+the trailing batch and drains confirmations at every part boundary. Flush on
+batch byte/row limits, a bounded linger deadline, or the end of the prepared
+shard. Check in-flight errors continuously and drain all confirmations before
+recording shard delivery. Crossing a source-part boundary alone causes no flush.
+
+Destination buffers and parts remain scoped to `(group, target segment, target
+shard)`. Route retained rows together for the same destination where practical;
+bound the number and combined memory of any destination-specific buffers. Never
+merge different target segments to reach a size goal. Transport batch size is
+not a guarantee of physical part size: destination flush timing and memory
+pressure can still produce small parts.
+
+Use the destination's existing memory-part merging and file compaction to combine
+these outputs. The current trace flusher already has a memory-part pileup/merge
+path (`banyand/trace/flusher.go`). This does not guarantee a minimum size or an
+immediate merge, particularly for a cold segment with little remaining traffic.
+A small final remainder is valid; do not retain extra traces, wait indefinitely,
+or force full-segment compaction merely to fill a part. Delivery completion does
+not wait for background compaction.
+
+For example, if 100 source parts each retain about 100 KiB and share one target
+segment/shard, replay treats the approximately 10 MiB of survivors as one bounded
+stream rather than forcing 100 part-ending batches. Actual destination part count
+depends on its write/flush/merge scheduling. If a later optimization writes native
+reduced parts in the agent, its writer must likewise span source parts and rotate
+based on retained output size, allowing a small terminal part and no empty parts.
+
+Verify this behavior with many parts and a low keep rate: assert that source part
+boundaries do not force replay flushes, memory stays bounded, final residual rows
+are confirmed, and all-drop inputs create no output. Benchmark destination part
+count, size distribution and compaction backlog as well as transfer bytes.
+
+## Manifest and retry protocol
+
+Use a versioned, disk-backed decision table sorted by trace ID, with bounded
+lookup buffers during replay. Record an explicit keep/drop result for every
+trace; an absent entry during replay is an integrity error. A table plus a sparse
+on-disk index avoids an unbounded in-memory trace-ID map. No span bodies need be
+duplicated in this artifact because replay reads the pinned source snapshot.
+
+The manifest binds the table to:
+
+- Snapshot identity and source part inventory, including integrity information.
+- Group, schema revision, source segment/shard and source/target stages.
+- Pinned destination interval, shard count and replication configuration.
+- Canonical policy digest, ordered plugin identities, ABI and plugin binary
+ digests. A filename alone does not identify plugin code.
+- Decision-table format, checksum, counts and preparation outcome.
+
+The durable state transitions are:
+
+```text
+PREPARING -> PREPARED -> DELIVERING -> DELIVERED -> cleanup after source removal
+```
+
+1. Write the table and manifest under a temporary artifact directory. Flush
+ files, publish by atomic rename on the same filesystem, and sync the parent
+ directory. Every durability failure returns an error. No destination write
+ occurs before `PREPARED` is durable.
+2. Replay only retained rows. Always skip source core/SIDX/series file transfer
+ in this mode, even if the manifest retains every trace. The destination write
+ path rebuilds all indexes from the retained rows.
+ Give delivery batches monotonically increasing sequence numbers and record
+ their ending source cursor (part ID, block position and row offset) against
+ the immutable input inventory. A batch may cross source-part boundaries.
+ Advance a durable delivery checkpoint only through the contiguous sequence
+ of batches whose required destination confirmations have all succeeded.
+ Later successful batches cannot advance past an earlier unconfirmed batch.
+ Persist checkpoints with the same error-returning atomic protocol as the
+ manifest; checkpoint failure stops further sending after safely draining
+ outstanding work. Keep in-flight batches and uncheckpointed acknowledged
+ batches bounded so crash replay remains bounded.
+3. Drain all required confirmations before durably recording `DELIVERED`.
+ A shard with zero retained rows is a successful, explicit all-drop result;
+ persist it and advance progress without sending an empty write batch.
+4. Project durable delivery status into the existing progress/report structures.
+ Only successful migration of all required work allows existing source
+ deletion. Persisted all-drop results count as completed work, not missing data.
+5. On restart, discard incomplete preparation only if no send was possible.
+ Resume a prepared or partially delivered shard using its original manifest
+ and snapshot, starting after the last durably confirmed delivery cursor.
+ Never resample it under new policy, new plugin code or new schema.
+ If the artifact or required snapshot is missing/corrupt, stop that migration
+ and retain available source state; do not silently generate different verdicts.
+
+[`Progress.Save`](../../../banyand/backup/lifecycle/progress.go) currently logs
+errors and uses `os.WriteFile` without an atomic durability protocol. It cannot
+serve as the sampling commit record. The new artifact store must return errors
+and be authoritative for sampled replay and completion. Discover its pending
+attempts before interpreting legacy progress, including when the progress file
+is missing or damaged.
+
+Delivery retains the current replay path's at-least-once behavior. A crash after
+receiver acknowledgement but before delivery checkpointing may resend retained
+rows. The manifest guarantees stable selection, not exactly-once delivery.
+Delivery checkpoints are independent of physical part creation and do not force
+the receiver to compact or create a part. Efficient cursor seeking needs an
+explicit reader operation; reopening an iterator and scanning to its cursor is
+correct but does not deliver the intended restart performance.
+
+Preparation has a different restart unit: the source segment/shard. A crash before
+its complete decision manifest is published can require resampling that shard;
+already prepared shards are reused. V1 does not promise mid-shard sampling resume.
+The general sampler ABI permits state across calls and offers no state checkpoint
+operation, so skipping earlier calls in a newly constructed plugin is not a
+transparent optimization. Finer sampling checkpoints would require a documented
+stateless/trace-local plugin contract or plugin state restoration. This restriction
+does not affect delivery resume, which never calls the sampler.
+Retain snapshot and artifact ownership until source deletion and cleanup are
+confirmed; filesystem paths alone must not permit another cleanup pass to remove
+an active attempt's inputs.
+
+## Failure behavior
+
+| Failure | Required result |
+| --- | --- |
+| Plugin panic, error, invalid verdict length | Bypass that link; other successful links still apply |
+| Execution timeout or open circuit | Retain affected batch |
+| Rule input cannot be evaluated | Preserve sampler's fail-open semantics |
+| Oversized trace or uncertain fragment coverage | Retain whole trace |
+| Missing plugin, incompatible ABI, invalid configured chain | Fail preflight; no migration send |
+| Source read/corruption error | Fail preparation/replay; never reinterpret missing input as a drop |
+| Manifest disk full, checksum mismatch or durability failure | Fail migration; no completion or source deletion |
+| Cancellation or send/confirmation failure | Stop, drain/close safely, preserve prepared state for retry |
+
+Runtime fail-open decisions are persisted like ordinary keeps. A retry must not
+turn an earlier timeout keep into a successful drop after some spans were sent.
+An unreadable source differs from a sampler failure: retaining data requires
+being able to read and deliver it.
+
+## Deployment, observability and compatibility
+
+The lifecycle process becomes a native plugin host. Add explicit enablement,
+trusted-directory, timeout and circuit-breaker configuration to its command and
+deployment. Mount matching plugin binaries into the lifecycle container and
+build it with the same Go/toolchain/CGO/SDK compatibility rules as the data-node
+plugin host. Loading a pipeline on a data node does not load it in the agent.
+If an applicable stage chain is configured but the agent cannot host it, fail
+preflight instead of silently migrating unfiltered data.
+
+Report per group and transition: input/kept/dropped traces and spans, fail-open
+reasons, preparation time, manifest bytes, replayed bytes, and resume counts.
+Keep intentional drops separate from missing/corrupt-source and transport errors.
+Record policy/artifact identities in structured reports rather than metric
+labels. The artifact's logical counts are recorded once; transport attempts may
+be counted repeatedly.
+
+Group protobuf and destination write protocol remain unchanged. New durable
+state belongs to the lifecycle agent. Deploy capable agents before activating
+stage rules; an old agent may ignore those rules. Do not downgrade an agent with
+pending sampled attempts to a binary that does not understand their manifests.
+Unsampled in-flight legacy migrations finish without retroactive filtering.
+
+## Alternatives and cost
+
+Filtering individual replay rows is simpler but cannot evaluate error/duration
+rules over a trace spanning parts. Destination-side filtering wastes bandwidth
+and cannot satisfy the agent-before-send requirement.
+
+A reduced native-part rewrite can preserve efficient chunk transfer, as proposed
+in the earlier design. It requires coordinated rewriting of core metadata,
+Bloom filters, SIDX and series indexes, plus a second path for target segment or
+shard repartitioning. Filtered replay reuses the existing destination write path
+and gives both transfer geometries the same sampling semantics. Its cost is a
+decision pass, local decision-table I/O, and replay decode/re-encode CPU. These
+costs are accepted for v1; measurements can inform subsequent tuning without
+reopening the transport decision. Reduced-part transfer is outside this proposal.
+
+## Verification requirements
+
+1. Add source-stage selection/preflight and shared runtime extraction. Test
+ disabled/unmatched/empty policy, implicit hot identity, schema selection,
+ chain semantics and independence from MERGE/FINALIZE enablement.
+2. Add bounded projected assembly and manifest storage. Test traces split across
+ parts, buffer lifetime, oversized traces, source-edge fragments, unavailable
+ coverage, malformed input and failure at every publication step.
+3. Add sampled replay before transport selection and suppress original index
+ shipping. Test same-interval, split-interval and changed-shard-count migrations;
+ query destination data and indexes to prove dropped traces are absent and all
+ spans of retained traces arrive. Include keep-all, drop-all and empty sources.
+4. Inject restart before send, during partial delivery and after acknowledgement.
+ Confirm out-of-order acknowledgements cannot advance the durable cursor past
+ an unconfirmed batch, checkpoint failure halts further sending, and restart
+ replays only the bounded uncheckpointed suffix without calling the sampler.
+ Change policy/schema/plugin code between attempts and prove the old decisions
+ remain authoritative. Verify failed delivery never marks work complete or
+ requests source deletion, including an all-drop manifest persistence failure.
+5. Run targeted lifecycle, pipeline-runtime, SDK and plugin tests, then the trace
+ migration roundtrip suite. Benchmark peak memory, local disk usage, transfer
+ bytes and elapsed time against unsampled replay and chunk transfer, varying
+ retention rate, part count and large-trace size.
+
+The acceptance criterion is observable at the sender: no span of a trace marked
+drop enters the destination queue, and no retained trace is partly omitted because
+of part layout, target segmentation, retry or sampler failure.
+
+
+## Implemented components and regression coverage
+
+- [Policy selection](../../../banyand/backup/lifecycle/sampling_policy.go) resolves
+ the source-stage chain and pins plugin identities; the shared
+ [pipeline runtime](../../../banyand/internal/pipeline) hosts native samplers.
+- The [snapshot scanner](../../../banyand/trace/snapshot_scanner.go) assembles
+ projected trace blocks across source parts with bounded metadata merging.
+- [Preparation](../../../banyand/backup/lifecycle/sampling_prepare.go), the
+ [artifact store](../../../banyand/backup/lifecycle/sampling_store.go), and
+ [recovery](../../../banyand/backup/lifecycle/sampling_recovery.go) publish and
+ validate decisions and recover pending attempts independently of legacy progress.
+- [Sampled replay](../../../banyand/backup/lifecycle/sampling_replay.go) filters
+ spans and persists confirmed source cursors while reusing the existing sender.
+- The regular lifecycle E2E suite includes
+ [normal sampling](../../../test/cases/lifecycle/sampling.go) and
+ [crash recovery](../../../test/cases/lifecycle/sampling_recovery.go). The recovery
+ case kills a separate lifecycle process after an incomplete durable delivery
+ checkpoint, removes access to its sampler binary, and restarts against the same
+ snapshots and artifacts. It verifies checkpoint advancement, unchanged committed
+ decisions, retained payloads and indexes, dropped-trace absence, source removal,
+ and artifact cleanup. Exact duplicate deliveries are permitted by the inherited
+ at-least-once contract. The process-kill case requires Linux.
+
+Run the regular suite from the repository root with the repository's Go toolchain
+and native plugin build prerequisites installed:
+
+```bash
+go test ./test/integration/distributed/lifecycle -count=1 -timeout=8m
+```
+
+The opt-in [real-data migration suite](../../../test/integration/distributed/lifecycle_real/README.md)
+can exercise normal migration and process-kill recovery using a locally available
+native snapshot. It is separate from the self-contained regular E2E suite.
diff --git a/docs/design/README.md b/docs/design/README.md
index 6d90070ea..2ba2bb74d 100644
--- a/docs/design/README.md
+++ b/docs/design/README.md
@@ -7,6 +7,7 @@ must be added there when it is introduced. When a milestone ships, its directory
## Current milestone — 0.12.0
+- [Trace sampling before lifecycle migration](0.12.0/lifecycle-trace-sampling.md)
- [Measure QL Catalog B: core SQL implementation plan](0.12.0/measure-ql-core-sql/README.md)
- [Tag aggregation and time bucketing in the measure query engine](0.12.0/tag-aggregation/README.md)
- [Data export and import](0.12.0/data-export-import/README.md)
diff --git a/docs/interacting/data-lifecycle.md b/docs/interacting/data-lifecycle.md
index 51e3f853b..1204275e8 100644
--- a/docs/interacting/data-lifecycle.md
+++ b/docs/interacting/data-lifecycle.md
@@ -52,6 +52,22 @@ You can also manage the Group by other clients such as the [Canopy](canopy.md) w
For more details about how they works, please refer to the [data rotation](../concept/rotation.md).
+## Trace sampling between lifecycle stages
+
+For trace groups with multiple storage stages, the lifecycle agent can sample
+traces before migrating retained spans to the next stage. Configure the group's
+`pipeline.stages` with a plugin chain for the **source** stage: a `hot` rule runs
+on hot-to-warm migration, and a `warm` rule runs when leaving warm. This is
+separate from data-node merge/finalization sampling through top-level
+`pipeline.plugins`.
+
+The agent commits whole-trace decisions across the source snapshot's parts and
+replays retained spans to the destination, where indexes are rebuilt. Prepared
+decisions and delivery checkpoints survive restarts; uncertain trace fragments
+are retained conservatively. See [trace retention sampling](../operation/lifecycle.md#trace-retention-sampling)
+for the group API example, required plugin-host flags, persistent state, and
+crash recovery behavior.
+
## [Property](../concept/data-model.md#properties)
`Property` data provides both [CRUD](./bydbctl/property.md) operations and TTL mechanism.
diff --git a/docs/operation/lifecycle.md b/docs/operation/lifecycle.md
index 7a1d66566..4fdfeba01 100644
--- a/docs/operation/lifecycle.md
+++ b/docs/operation/lifecycle.md
@@ -8,6 +8,7 @@ A dedicated lifecycle agent (an independent Go process) is responsible for:
- Identifying data eligible for migration based on lifecycle stages.
- Extracting data from the source node.
+- Applying configured source-stage trace sampling before sending retained spans.
- Migrating data to designated target nodes.
- Removing the original data after a successful migration.
@@ -36,6 +37,8 @@ The lifecycle agent performs the following steps:
- Identify target nodes for the next lifecycle stage.
- For each stream, measure or trace:
- Query data from the current stage.
+ - For traces with a matching source-stage sampling chain, persist whole-trace
+ decisions across snapshot parts and replay only retained spans.
- Transform data into write requests.
- Send write requests to target nodes.
- Delete the source data after a successful migration.
@@ -297,3 +300,170 @@ EOF
- Query both hot and warm nodes to ensure data availability.
This process automatically migrates data according to the defined lifecycle stages, optimizing your storage usage while maintaining data availability.
+
+## Trace retention sampling
+
+### Selecting a policy through the group API
+
+The lifecycle agent applies the source stage's `pipeline.stages` plugin
+chain before migrating trace rows. Migration sampling is independent of
+`enabled_events`; the destination stage and top-level plugin chain are not run.
+An applicable stage policy requires an explicitly enabled native plugin host.
+A missing plugin or incompatible ABI fails migration before sending data.
+
+For `hot -> warm`, configure `pipeline.stages[].stage` as `hot`; a `warm`
+rule applies when leaving warm for the next stage. For an implicit hot stage,
+the source identity comes from the agent's `type` node label. Stage names must
+match that resolved identity exactly. Sampling currently requires exactly one
+trace schema in the group. `schemaNames` and `schemaNameRegex`, when present,
+select that schema; either selector may match.
+
+The following example updates an existing `sw_trace` group through the liaison
+HTTP API. It assumes the group already has hot-to-warm lifecycle stages and a
+SkyWalking `segment` trace schema. It preserves the group's other settings but
+replaces its pipeline, so include any existing top-level plugins or additional
+stage rules you want to retain.
+
+```bash
+curl --fail --silent --show-error \
+ http://localhost:17913/api/v1/group/schema/sw_trace > group-current.json
+
+jq '{group: .group} | .group.pipeline = {
+ enabled: true,
+ schemaNames: ["segment"],
+ stages: [{
+ stage: "hot",
+ plugins: [{
+ name: "sw-trace-sampler",
+ sampler: {
+ path: "sw-trace-sampler.so",
+ abiVersion: 1,
+ config: {
+ keepErrors: true,
+ durationThresholdMs: 500,
+ healthySampleRate: 0.01
+ }
+ }
+ }]
+ }]
+}' group-current.json > group-sampling.json
+
+curl --fail --silent --show-error -X PUT \
+ -H 'Content-Type: application/json' \
+ --data-binary @group-sampling.json \
+ http://localhost:17913/api/v1/group/schema/sw_trace
+```
+
+The same `pipeline` can be submitted as part of a complete group document with
+`bydbctl group update -f group.yaml`, or through
+`GroupRegistryService.Update` with `GroupRegistryServiceUpdateRequest.group`.
+The example keeps errors and traces meeting the duration threshold, plus a 1%
+sample of healthy traces; completeness safeguards can retain additional traces.
+See the [sampler configuration reference](../../plugins/README.md) for supported
+schemas and rule semantics.
+
+A disabled or absent pipeline, an unmatched schema selector, or no nonempty
+source-stage chain preserves ordinary unsampled migration. `enabledEvents`
+controls data-node merge/finalization only. This example has no top-level
+`plugins`, so it configures lifecycle sampling without a data-node sampling chain.
+Deploy a capable lifecycle agent before enabling stage rules: older agents may
+ignore them. Existing unsampled migration attempts finish without retroactive
+sampling.
+
+### Enabling the lifecycle plugin host
+
+| Agent flag | Default | Purpose |
+| --- | --- | --- |
+| `--trace-sampling-enabled` | `false` | Enable native sampler hosting |
+| `--trace-sampling-trusted-dir` | empty | Directory containing trusted `.so` files |
+| `--trace-sampling-timeout` | `1s` | Batch decision deadline |
+| `--trace-sampling-circuit-break` | `3` | Consecutive timeouts before retaining all batches |
+
+Use the `-plugins` host image for the lifecycle agent and mount the matching
+plugin carrier binaries read-only at the trusted directory. That image builds
+`/lifecycle`, `/banyand`, and the plugin carrier from the same Go toolchain,
+CGO setting, SDK module graph, libc and build flags. The ordinary static image
+cannot load native Go plugins. See [plugin deployment](plugins.md) for carrier
+image volumes and init-container delivery.
+
+For example, a Compose lifecycle service can use the existing cluster networking
+and discovery options with these additional settings:
+
+```yaml
+image: apache/skywalking-banyandb:<TAG>-plugins
+entrypoint: ["/lifecycle"]
+command:
+ - --trace-sampling-enabled
+ - --trace-sampling-trusted-dir=/plugins
+ - --trace-sampling-timeout=1s
+ - --trace-sampling-circuit-break=3
+ - --progress-file=/state/lifecycle-progress.json
+volumes:
+ - ./plugins:/plugins:ro
+ - lifecycle-state:/state
+```
+
+Persist `<progress-file>.sampling` together with the progress file and immutable
+snapshots. For `/state/lifecycle-progress.json`, the artifact directory is
+`/state/lifecycle-progress.json.sampling`. Prepared decisions and delivery checkpoints remain
+required until source removal succeeds. Restart reuses those decisions even if
+schema or plugin configuration changes; do not downgrade an agent while sampled
+attempts remain pending. Sampler timeouts retain affected traces. Native calls
+that outlive their deadline retain their input buffers and worker slots until
+returning; plugins are closed only afterward.
+
+
+### Delivery, small parts, and recovery
+
+The agent assembles each trace across all core parts in a snapshot's source
+segment/shard before deciding whether to keep it. It retains traces whose
+completeness is uncertain, including source-segment boundary fragments, and
+traces that exceed decision budgets. This is a local snapshot decision, not a
+guarantee that a trace is complete across nodes, segments, or future arrivals.
+
+Sampling writes durable decision tables, indexes, manifests, and delivery
+checkpoints to disk; bounded scans can also use temporary metadata merge files.
+It does **not** write a reduced native data part for every input part. Retained
+spans flow through continuous bounded batches across source-part boundaries.
+The destination rebuilds indexes through its normal write path and merges small
+parts through normal compaction. A small final part is valid; migration does not
+wait for compaction or retain extra traces to fill it. An all-drop shard sends
+no data and counts as completed only after its decision state is durable.
+
+| Interruption point | Behavior on restart |
+| --- | --- |
+| Before a shard's decision manifest is committed | Preparation may resample that source segment/shard; completed shards are reused. |
+| After preparation or during delivery | Reuse the committed decisions and pinned schema/routing; resume after the last durably acknowledged source cursor without calling the sampler for that prepared shard. |
+| After acknowledgement but before checkpoint persistence | Some retained rows may be resent: delivery remains at-least-once. |
+| Required snapshot or committed artifact missing/corrupt | Stop the affected migration and preserve available source state; do not silently resample. |
+
+Restart with the same progress-file path and snapshot volumes. Do not delete
+`.sampling` state to retry a failed migration or change policy for an already
+prepared shard. The agent discovers pending sampling artifacts even if the
+legacy progress file is missing. Only successful delivery of all required work
+allows source removal; artifacts are cleaned up after source removal succeeds.
+
+A missing plugin, incompatible ABI, or disabled host with an applicable chain
+fails preflight before sending. Runtime plugin errors, panics, or invalid verdicts
+bypass that link; other successful links still apply. A batch timeout or open
+circuit retains the affected batch. Source read errors and artifact durability
+failures stop migration rather than treating unreadable data as a sampling drop.
+
+### Verifying sampling
+
+Use `--report-dir` to retain migration reports. The top-level `trace_sampling`
+map records each artifact's `manifest`, `manifest_bytes`, `replayed_bytes`, and
+`resume_count`. Manifest fields include `source_stage`, `target_stage`, `traces`,
+`kept_traces`, `spans`, `kept_spans`, `fail_open`, and `preparation_nanos`.
+Calculate dropped counts by subtracting kept counts from input counts. Logical
+sampling counts describe the prepared snapshot; replay counters can include
+retried deliveries.
+
+Query the destination both by trace ID and by indexed tags/time range to verify
+that retained spans are available and sampled-out traces are absent. After
+successful migration, confirm source removal and sampling artifact cleanup.
+The regular [lifecycle integration suite](../../test/integration/distributed/lifecycle)
+includes native sampling and a process-kill recovery case that restarts with the
+sampler binary unavailable, verifies reuse of committed decisions, and checks
+public destination queries. See the
+[design](../design/0.12.0/lifecycle-trace-sampling.md) for the durable protocol.
diff --git a/docs/operation/plugins-debugging.md b/docs/operation/plugins-debugging.md
index 1f16b2434..ef1a36486 100644
--- a/docs/operation/plugins-debugging.md
+++ b/docs/operation/plugins-debugging.md
@@ -17,8 +17,8 @@ See [`docs/operation/plugins.md`](plugins.md) for deployment and
## Symptom: "my data is not being sampled" (nothing is dropped)
-Work through these checks in order; each one either explains the symptom or
-rules out a cause.
+For data-node merge/finalization sampling, work through these checks in order.
+For migration sampling, see [Lifecycle migration](#lifecycle-migration) below.
1. **Is the feature enabled on this node?** `-trace-pipeline-native-plugin-enabled`
must be `true` and `-trace-pipeline-trusted-plugin-dir` must be set to a
@@ -52,13 +52,12 @@ rules out a cause.
expect to be dropped comes back `true`, the sampler's own logic (or its
config, e.g. a threshold set too permissively) is the cause, not the
engine.
-6. **Remember: plugins target the whole group, not a schema or a stage.**
- `TracePipelineConfig.schema_names` / `schema_name_regex` / `stages` are
- accepted by the proto but **not wired** into the engine today — the
- validator rejects any config that tries to use them
- (`api/validate/validate.go`), and the group as a whole is the only unit a
- pipeline can target. If you were expecting a schema-scoped or
- stage-scoped effect, that scoping does not exist yet.
+6. **Check which host owns the chain.** Data-node merge/finalization uses the
+ top-level `pipeline.plugins`. Lifecycle migration uses
+ `pipeline.stages[].plugins` for the source stage. The API accepts
+ `schema_names`, `schema_name_regex`, and `stages`; the lifecycle agent applies
+ the schema selectors when resolving its source-stage chain. A stage-only
+ configuration does not install a data-node merge sampler.
## Symptom: "unexpected data is being sampled" (wrong traces kept or dropped)
@@ -100,3 +99,32 @@ rules out a cause.
maturity and fragment-boundary range expansion. The fixed fallback is 2h when
the group omits a positive value. Verify that the deployment's maximum delay
between fragments of one trace fits within that grace.
+
+## Lifecycle migration
+
+For sampling while moving traces between storage stages, check the following:
+
+1. Enable `--trace-sampling-enabled` on the lifecycle agent and set
+ `--trace-sampling-trusted-dir` to the directory containing compatible `.so`
+ files. The data-node plugin flags do not enable the lifecycle host. A missing
+ or incompatible plugin fails migration preflight; inspect the agent's error
+ log rather than the data-node sampler metrics.
+2. Check `pipeline.enabled`, schema selectors, and the source-stage chain.
+ For hot-to-warm migration, configure `pipeline.stages[].stage: hot`.
+ Top-level `pipeline.plugins` and `enabled_events` do not select migration
+ samplers. With no applicable source-stage chain, migration remains unsampled.
+3. Inspect the lifecycle report's top-level `trace_sampling` entries for each
+ artifact's `manifest` counts and fail-open reasons, plus `resume_count`.
+ Subtract `kept_traces` from `traces` to calculate dropped traces. Migration assembles
+ traces across snapshot parts within each segment/shard, and conservatively
+ retains unsafe inputs such as oversized traces or possible segment-boundary
+ fragments. Data-node per-merge behavior described above is a separate path.
+4. After a crash, reuse the same progress and snapshot storage. Prepared sampling
+ decisions are durable and can be replayed without loading the plugin again;
+ changing plugin configuration does not resample those prepared decisions.
+ Missing or inconsistent recovery artifacts stop migration instead of silently
+ starting over. Unacknowledged delivery may be repeated after recovery.
+
+See [Trace retention sampling](lifecycle.md#trace-retention-sampling) for the
+configuration example, report fields, and recovery contract, and
+[Lifecycle-agent hosting](plugins.md#lifecycle-agent-hosting) for plugin packaging.
diff --git a/docs/operation/plugins.md b/docs/operation/plugins.md
index e1f5a8eaa..33fad6f9f 100644
--- a/docs/operation/plugins.md
+++ b/docs/operation/plugins.md
@@ -1,8 +1,8 @@
# Trace-Pipeline Sampler Plugins
-BanyanDB's trace pipeline can retain/drop traces in-merge (and at segment
-finalization) using operator-installed native Go plugins ("samplers"). This
-document covers packaging and deploying plugins on a running cluster. See
+BanyanDB's trace pipeline can retain/drop traces in-merge, at segment
+finalization, and during lifecycle migration using operator-installed native Go
+plugins ("samplers"). This document covers packaging and deploying plugins on a running cluster. See
[`docs/design/archive/0.11.0/post-trace-pipeline.md`](../design/archive/0.11.0/post-trace-pipeline.md) for
the plugin ABI/contract, and
[`plugins/README.md`](../../plugins/README.md) for authoring a first-party
@@ -10,13 +10,14 @@ plugin. For diagnosing "nothing is being sampled" / "unexpected data is being
sampled" symptoms offline, see
[`docs/operation/plugins-debugging.md`](plugins-debugging.md).
-## Overview
+## Data-node hosting
- Plugins are loaded **on demand at group-schema reconcile**, on the **data
- node role only** (the standalone service also satisfies this role; the
- liaison never hosts plugins).
+ node role** (the standalone service also satisfies this role; the
+ liaison never hosts plugins). Lifecycle agents load their own sampler chains
+ before migration; see [Lifecycle-agent hosting](#lifecycle-agent-hosting).
- The feature is gated by two flags, both off by default:
- - `-trace-pipeline-native-plugin-enabled` (bool) — the whole feature switch.
+ - `-trace-pipeline-native-plugin-enabled` (bool) — the data-node feature switch.
- `-trace-pipeline-trusted-plugin-dir` (string) — the **single** directory
`.so` paths are resolved within. There is no second flag and no
multi-directory support.
@@ -36,7 +37,7 @@ lockstep note below):
|---|---|---|---|
| default (`apache/skywalking-banyandb:<tag>`) | `busybox:stable-glibc` | `CGO_ENABLED=0` static `banyand-server` | Normal image; hosts **no** plugins, ever |
| slim (`...:<tag>-slim`) | `busybox:stable-glibc` | `CGO_ENABLED=0` static `banyand-server` | Slim default; hosts no plugins |
-| **plugins host** (`...:<tag>-plugins`) | `gcr.io/distroless/base-debian12` | `CGO_ENABLED=1` dynamic `banyand-server` + an **empty** `/plugins` | The plugin-capable host; `.so` are mounted in, not baked |
+| **plugins host** (`...:<tag>-plugins`) | `gcr.io/distroless/base-debian12` | `CGO_ENABLED=1` dynamic `banyand-server` and `/lifecycle` + an **empty** `/plugins` | The plugin-capable host; `.so` are mounted in, not baked |
| **plugins carrier** (`...:<tag>-plugins-carrier`) | `busybox:stable-glibc` | the built first-party `.so` at `/plugins` | Mounted into the host at `/plugins` to deliver plugins |
All four are tags on the **same** `apache/skywalking-banyandb` repository.
@@ -61,9 +62,9 @@ at `/plugins`.
> (the carrier tag equals the host tag). Always deploy the carrier at the same
> tag as the host image. A separate image is fully supported; independently
> *versioning* the carrier is not — any drift makes `plugin.Open` reject the
-> `.so` (fail-open, loud; see below).
+> `.so` (data-node fail-open or lifecycle preflight failure; see below).
-## Enabling plugins
+## Enabling data-node plugins
Use the `-plugins` **host** image for the **data-node** pod (liaison pods
never host plugins) and set the two flags:
@@ -108,11 +109,44 @@ After activation, import the optional
execution latency, sampling outcomes, fail-open retention, drop-set capacity, and
finalization through the FODC proxy.
+## Lifecycle-agent hosting
+
+The lifecycle agent can run the same sampler plugins before sending trace data
+from one storage stage to the next. Use the `-plugins` host image with its
+`/lifecycle` executable, mount the carrier at `/plugins`, and configure the agent:
+
+```text
+--trace-sampling-enabled=true
+--trace-sampling-trusted-dir=/plugins
+```
+
+These flags are independent of the data-node `trace-pipeline-*` flags. The
+lifecycle executable and its `.so` files must satisfy the same build compatibility
+requirements described above. Plugin execution defaults to a one-second timeout
+per batch (`--trace-sampling-timeout=1s`); three consecutive timeouts open the
+circuit (`--trace-sampling-circuit-break=3`) and retain subsequent batches.
+
+Configure the group's `pipeline.stages[].plugins` chain for the **source** stage:
+`stage: hot` controls sampling during hot-to-warm migration. The lifecycle agent
+does not use top-level `pipeline.plugins` or `enabledEvents` to choose its chain.
+`pipeline.enabled` and the schema selectors still apply. See
+[Trace retention sampling](lifecycle.md#trace-retention-sampling) for a complete
+schema example, API configuration, and recovery requirements.
+
+Unlike data-node reconciliation, a lifecycle preflight failure (including a
+missing or incompatible plugin, or a disabled lifecycle plugin host) stops the
+affected migration before sending data. During sampling, a plugin error bypasses that link while other successful links
+still apply; a timeout retains the affected batch. Completed sampling decisions are persisted
+before replay; recovery can reuse those decisions without loading the plugin
+again. Keep the agent's snapshot and progress storage across restarts. Delivery
+remains at least once, so a crash can replay rows acknowledged before their checkpoint was persisted.
+
## Delivering the carrier (first-party) and third-party plugins
-The loader supports exactly **one** trusted dir
-(`-trace-pipeline-trusted-plugin-dir`, a single flag; every
-`SamplerPlugin.path` resolves within it). First-party plugins mount at
+Each host uses exactly **one** trusted dir
+(`-trace-pipeline-trusted-plugin-dir` for data nodes or
+`--trace-sampling-trusted-dir` for lifecycle agents; every `SamplerPlugin.path`
+resolves within its host's directory). First-party plugins mount at
`/plugins` (`SamplerPlugin.path=<name>.so`); third-party plugins mount at the
reserved subdirectory `/plugins/thirdparty`
(`SamplerPlugin.path=thirdparty/<name>.so`). Because the host image ships
@@ -137,11 +171,12 @@ Two delivery mechanisms — both work for first- and third-party:
First-party parity is guaranteed by lockstep (deploy the carrier at the host's
tag). Third-party `.so` parity is the **operator's** responsibility — they
build against the pinned SDK + toolchain published for that release (this
-repo's `go.mod` at that tag; `pkg/pipeline/sdk`). Mismatch is always safe and
-loud: the loader rejects a skewed `.so` (`plugin.Open` "different version of
-package …", or an `ABIVersion` mismatch), fails open (ERROR log +
-`sampler_load_failed` metric), and the affected group simply keeps its
-previous sampler set. Other groups, and the node itself, are unaffected.
+repo's `go.mod` at that tag; `pkg/pipeline/sdk`). The loader rejects a skewed `.so`
+(`plugin.Open` "different version of package …", or an `ABIVersion` mismatch).
+On a data node, this fails open (ERROR log +
+`sampler_load_failed` metric), and the affected group keeps its previous sampler
+set. Other groups, and the node itself, are unaffected. On a lifecycle agent,
+this fails migration preflight.
## Building and verifying plugins before deploying
@@ -191,4 +226,6 @@ meantime:
mechanism.
- Optional third-party hooks mount an operator image onto the subdirectory
`/plugins/thirdparty`.
-- Target the **data-node** pod only (liaison never hosts).
+- These chart settings target the **data-node** pod (liaison never hosts).
+ Lifecycle-agent deployments need their own plugin image, mount, and
+ `trace-sampling-*` flags as described above.
diff --git a/pkg/compress/zstd/bounded_test.go b/pkg/compress/zstd/bounded_test.go
new file mode 100644
index 000000000..9f52d1a39
--- /dev/null
+++ b/pkg/compress/zstd/bounded_test.go
@@ -0,0 +1,54 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package zstd
+
+import (
+ "bytes"
+ "testing"
+
+ nativezstd "github.com/klauspost/compress/zstd"
+ "github.com/stretchr/testify/require"
+)
+
+func TestDecompressBoundedKnownSizeAllocatesActualFrame(t *testing.T) {
+ data := bytes.Repeat([]byte("x"), 300)
+ decoded, err := DecompressBounded(Compress(nil, data, 1), 32<<20)
+ require.NoError(t, err)
+ require.Equal(t, data, decoded)
+ require.Less(t, cap(decoded), 1024)
+ _, err = DecompressBounded(Compress(nil, data, 1), 100)
+ require.ErrorIs(t, err, ErrDecompressionLimit)
+}
+
+func TestDecompressBoundedUnknownFrameSize(t *testing.T) {
+ var output bytes.Buffer
+ writer, err := nativezstd.NewWriter(&output, nativezstd.WithEncoderConcurrency(1), nativezstd.WithWindowSize(1024))
+ require.NoError(t, err)
+ data := bytes.Repeat([]byte("x"), 2000)
+ _, err = writer.Write(data)
+ require.NoError(t, err)
+ require.NoError(t, writer.Close())
+ var header nativezstd.Header
+ require.NoError(t, header.Decode(output.Bytes()))
+ require.False(t, header.HasFCS)
+ decoded, err := DecompressBounded(output.Bytes(), 3000)
+ require.NoError(t, err)
+ require.Equal(t, data, decoded)
+ _, err = DecompressBounded(output.Bytes(), 1500)
+ require.ErrorIs(t, err, ErrDecompressionLimit)
+}
diff --git a/pkg/compress/zstd/zstd.go b/pkg/compress/zstd/zstd.go
index 9eaab14ee..fe0c8c124 100644
--- a/pkg/compress/zstd/zstd.go
+++ b/pkg/compress/zstd/zstd.go
@@ -19,6 +19,9 @@
package zstd
import (
+ "bytes"
+ "errors"
+ "io"
"sync"
"sync/atomic"
@@ -90,3 +93,42 @@ func newEncoder(compressionLevel int) *zstd.Encoder {
}
return e
}
+
+// ErrDecompressionLimit identifies data whose decoded size exceeds a caller's budget.
+var ErrDecompressionLimit = errors.New("decoded zstd block exceeds budget")
+
+// DecompressBounded decompresses src while limiting the decoded bytes to limit.
+// The returned buffer is independently owned. A nonpositive limit is invalid.
+func DecompressBounded(src []byte, limit int) ([]byte, error) {
+ if limit <= 0 {
+ return nil, ErrDecompressionLimit
+ }
+ var header zstd.Header
+ if err := header.Decode(src); err != nil {
+ return nil, err
+ }
+ if header.HasFCS && header.FrameContentSize > uint64(limit) {
+ return nil, ErrDecompressionLimit
+ }
+ bounded, err := zstd.NewReader(nil, zstd.WithDecoderConcurrency(1), zstd.WithDecodeAllCapLimit(true),
+ zstd.WithDecoderMaxMemory(uint64(max(limit, 1024))))
+ if err != nil {
+ return nil, err
+ }
+ defer bounded.Close()
+ if !header.HasFCS {
+ if err = bounded.Reset(bytes.NewReader(src)); err != nil {
+ return nil, err
+ }
+ decoded, readErr := io.ReadAll(io.LimitReader(bounded, int64(limit)+1))
+ if len(decoded) > limit {
+ return nil, ErrDecompressionLimit
+ }
+ return decoded, readErr
+ }
+ decoded, err := bounded.DecodeAll(src, make([]byte, 0, int(header.FrameContentSize)))
+ if errors.Is(err, zstd.ErrDecoderSizeExceeded) {
+ return nil, ErrDecompressionLimit
+ }
+ return decoded, err
+}
diff --git a/pkg/encoding/bytes.go b/pkg/encoding/bytes.go
index c267dcc22..681333f47 100644
--- a/pkg/encoding/bytes.go
+++ b/pkg/encoding/bytes.go
@@ -72,9 +72,13 @@ func EncodeBytesBlock(dst []byte, a [][]byte) []byte {
// BytesBlockDecoder decodes a block of strings from src.
// It reuses the underlying buffer for storing the decoded strings.
type BytesBlockDecoder struct {
- data []byte
+ data []byte
+ maxBytes int
}
+// SetMaxBytes bounds decoded payload bytes held by this decoder; zero disables the bound.
+func (bbd *BytesBlockDecoder) SetMaxBytes(limit int) { bbd.maxBytes = limit }
+
// Reset resets the decoder.
func (bbd *BytesBlockDecoder) Reset() {
bbd.data = bbd.data[:0]
@@ -96,7 +100,7 @@ func (bbd *BytesBlockDecoder) Decode(dst [][]byte, src []byte, itemsCount uint64
var tail []byte
var err error
- u64List.L, tail, err = DecodeUint64Block(u64List.L[:0], src, itemsCount)
+ u64List.L, tail, err = decodeUint64BlockLimit(u64List.L[:0], src, itemsCount, bbd.maxBytes)
if err != nil {
return dst, fmt.Errorf("cannot decode string lengths: %w", err)
}
@@ -104,7 +108,7 @@ func (bbd *BytesBlockDecoder) Decode(dst [][]byte, src []byte, itemsCount uint64
src = tail
dataLen := len(bbd.data)
- bbd.data, tail, err = decompressBlock(bbd.data, src)
+ bbd.data, tail, err = decompressBlockLimit(bbd.data, src, bbd.maxBytes)
if err != nil {
return dst, fmt.Errorf("cannot decode bytes block with strings: %w", err)
}
@@ -142,7 +146,7 @@ func (bbd *BytesBlockDecoder) DecodeWithTail(dst [][]byte, src []byte, itemsCoun
var tail []byte
var err error
- u64List.L, tail, err = DecodeUint64Block(u64List.L[:0], src, itemsCount)
+ u64List.L, tail, err = decodeUint64BlockLimit(u64List.L[:0], src, itemsCount, bbd.maxBytes)
if err != nil {
return dst, nil, fmt.Errorf("cannot decode string lengths: %w", err)
}
@@ -150,7 +154,7 @@ func (bbd *BytesBlockDecoder) DecodeWithTail(dst [][]byte, src []byte, itemsCoun
src = tail
dataLen := len(bbd.data)
- bbd.data, tail, err = decompressBlock(bbd.data, src)
+ bbd.data, tail, err = decompressBlockLimit(bbd.data, src, bbd.maxBytes)
if err != nil {
return dst, tail, fmt.Errorf("cannot decode bytes block with strings: %w", err)
}
@@ -189,11 +193,15 @@ func EncodeUint64Block(dst []byte, a []uint64) []byte {
// DecodeUint64Block decodes a block of uint64 values from src.
func DecodeUint64Block(dst []uint64, src []byte, itemsCount uint64) ([]uint64, []byte, error) {
+ return decodeUint64BlockLimit(dst, src, itemsCount, 0)
+}
+
+func decodeUint64BlockLimit(dst []uint64, src []byte, itemsCount uint64, limit int) ([]uint64, []byte, error) {
bb := bbPool.Generate()
defer bbPool.Release(bb)
var err error
- bb.Buf, src, err = decompressBlock(bb.Buf[:0], src)
+ bb.Buf, src, err = decompressBlockLimit(bb.Buf[:0], src, limit)
if err != nil {
return dst, src, fmt.Errorf("cannot decode bytes block: %w", err)
}
@@ -313,6 +321,10 @@ func compressBlock(dst, src []byte) []byte {
}
func decompressBlock(dst, src []byte) ([]byte, []byte, error) {
+ return decompressBlockLimit(dst, src, 0)
+}
+
+func decompressBlockLimit(dst, src []byte, limit int) ([]byte, []byte, error) {
if len(src) < 1 {
return dst, src, fmt.Errorf("cannot decode block type from empty src")
}
@@ -329,6 +341,9 @@ func decompressBlock(dst, src []byte) ([]byte, []byte, error) {
return dst, src, fmt.Errorf("cannot read plain block with the size %d bytes from %b bytes", blockLen, len(src))
}
+ if limit > 0 && len(dst)+blockLen > limit {
+ return dst, src, zstd.ErrDecompressionLimit
+ }
dst = append(dst, src[:blockLen]...)
src = src[blockLen:]
return dst, src, nil
@@ -344,8 +359,13 @@ func decompressBlock(dst, src []byte) ([]byte, []byte, error) {
// Decompress the block
var err error
bb := bbPool.Generate()
- bb.Buf, err = zstd.Decompress(bb.Buf[:0], compressedBlock)
+ if limit > 0 {
+ bb.Buf, err = zstd.DecompressBounded(compressedBlock, limit-len(dst))
+ } else {
+ bb.Buf, err = zstd.Decompress(bb.Buf[:0], compressedBlock)
+ }
if err != nil {
+ bbPool.Release(bb)
return dst, src, fmt.Errorf("cannot decompress block: %w", err)
}
diff --git a/pkg/encoding/bytes_limit_test.go b/pkg/encoding/bytes_limit_test.go
new file mode 100644
index 000000000..2dd491340
--- /dev/null
+++ b/pkg/encoding/bytes_limit_test.go
@@ -0,0 +1,53 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package encoding
+
+import (
+ "bytes"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/apache/skywalking-banyandb/pkg/compress/zstd"
+)
+
+func TestBytesBlockDecoderRejectsCompressedPayloadOverBudget(t *testing.T) {
+ encoded := EncodeBytesBlock(nil, [][]byte{bytes.Repeat([]byte("x"), 1<<20)})
+ decoder := &BytesBlockDecoder{}
+ decoder.SetMaxBytes(1024)
+ _, err := decoder.Decode(nil, encoded, 1)
+ require.ErrorIs(t, err, zstd.ErrDecompressionLimit)
+ require.LessOrEqual(t, cap(decoder.data), 1024)
+ decoder.SetMaxBytes(0)
+ decoded, err := decoder.Decode(nil, encoded, 1)
+ require.NoError(t, err)
+ require.Len(t, decoded[0], 1<<20)
+}
+
+func TestBytesBlockDecoderBudgetAccumulatesAndResetReleasesUsage(t *testing.T) {
+ decoder := &BytesBlockDecoder{}
+ decoder.SetMaxBytes(5)
+ encoded := EncodeBytesBlock(nil, [][]byte{[]byte("abc")})
+ _, err := decoder.Decode(nil, encoded, 1)
+ require.NoError(t, err)
+ _, err = decoder.Decode(nil, encoded, 1)
+ require.ErrorIs(t, err, zstd.ErrDecompressionLimit)
+ decoder.Reset()
+ _, err = decoder.Decode(nil, encoded, 1)
+ require.NoError(t, err)
+}
diff --git a/pkg/encoding/dictionary.go b/pkg/encoding/dictionary.go
index 244e19129..c6d239d80 100644
--- a/pkg/encoding/dictionary.go
+++ b/pkg/encoding/dictionary.go
@@ -101,21 +101,43 @@ func (d *Dictionary) Encode(dst []byte) []byte {
// Decode decodes the dictionary.
func (d *Dictionary) Decode(dst [][]byte, src []byte, itemsCount uint64) ([][]byte, error) {
+ return d.DecodeWithLimit(dst, src, itemsCount, 0)
+}
+
+// DecodeWithLimit bounds dictionary payload bytes; zero preserves unlimited decoding.
+func (d *Dictionary) DecodeWithLimit(dst [][]byte, src []byte, itemsCount uint64, limit int) ([][]byte, error) {
src, count := BytesToVarUint64(src)
if count == 0 {
return dst, nil
}
+ if count > maxUniqueValues {
+ return nil, fmt.Errorf("dictionary has too many unique values: %d", count)
+ }
- values, src, err := d.decodeBytesBlockWithTail(src, count)
+ values, src, err := d.decodeBytesBlockWithTail(src, count, limit)
if err != nil {
return nil, err
}
d.values = values
- d.tmp, err = decodeBitPacking(d.tmp, src)
+ if limit > 0 {
+ d.tmp, err = decodeBitPackingLimit(d.tmp, src, itemsCount*2)
+ } else {
+ d.tmp, err = decodeBitPacking(d.tmp, src)
+ }
if err != nil {
return nil, err
}
+ if len(d.tmp)%2 != 0 {
+ return nil, fmt.Errorf("dictionary run lengths have an incomplete pair")
+ }
+ var total uint64
+ for index := 0; index < len(d.tmp); index += 2 {
+ total += uint64(d.tmp[index+1])
+ if total > itemsCount || uint64(d.tmp[index]) >= count {
+ return nil, fmt.Errorf("dictionary run exceeds declared rows or values")
+ }
+ }
d.indices = decodeRLE(d.indices, d.tmp)
if uint64(len(d.indices)) != itemsCount {
return nil, fmt.Errorf("unexpected item counts; got %d; want %d", len(d.indices), itemsCount)
@@ -126,13 +148,13 @@ func (d *Dictionary) Decode(dst [][]byte, src []byte, itemsCount uint64) ([][]by
return dst, nil
}
-func (d *Dictionary) decodeBytesBlockWithTail(src []byte, itemsCount uint64) ([][]byte, []byte, error) {
+func (d *Dictionary) decodeBytesBlockWithTail(src []byte, itemsCount uint64, limit int) ([][]byte, []byte, error) {
u64List := generateBoundedUint64List()
defer ReleaseUint64List(u64List)
var tail []byte
var err error
- u64List.L, tail, err = DecodeUint64Block(u64List.L[:0], src, itemsCount)
+ u64List.L, tail, err = decodeUint64BlockLimit(u64List.L[:0], src, itemsCount, limit)
if err != nil {
return nil, nil, fmt.Errorf("cannot decode string lengths: %w", err)
}
@@ -140,7 +162,7 @@ func (d *Dictionary) decodeBytesBlockWithTail(src []byte, itemsCount uint64) ([]
src = tail
var decompressedData []byte
- decompressedData, tail, err = decompressBlock(nil, src)
+ decompressedData, tail, err = decompressBlockLimit(nil, src, limit)
if err != nil {
return nil, nil, fmt.Errorf("cannot decode bytes block with strings: %w", err)
}
@@ -238,7 +260,8 @@ func (bpe *bitPackingEncoder) encode(src []uint32) {
}
type bitPackingDecoder struct {
- br *Reader
+ br *Reader
+ maxItems uint64
}
func newBitPackingDecoder(br *Reader) *bitPackingDecoder {
@@ -255,6 +278,9 @@ func (bpd *bitPackingDecoder) decode(dst []uint32) ([]uint32, error) {
if length == 0 {
return []uint32{}, nil
}
+ if bpd.maxItems > 0 && length > bpd.maxItems {
+ return nil, fmt.Errorf("dictionary index count exceeds row budget")
+ }
bitsWidth, err := bpd.br.ReadBits(8)
if err != nil {
@@ -281,9 +307,14 @@ func encodeBitPacking(src []uint32) []byte {
}
func decodeBitPacking(dst []uint32, src []byte) ([]uint32, error) {
+ return decodeBitPackingLimit(dst, src, 0)
+}
+
+func decodeBitPackingLimit(dst []uint32, src []byte, maxItems uint64) ([]uint32, error) {
reader := bytes.NewReader(src)
br := NewReader(reader)
decoder := newBitPackingDecoder(br)
+ decoder.maxItems = maxItems
return decoder.decode(dst)
}
diff --git a/test/cases/lifecycle/lifecycle.go b/test/cases/lifecycle/lifecycle.go
index c77153ad7..a33e0d5d4 100644
--- a/test/cases/lifecycle/lifecycle.go
+++ b/test/cases/lifecycle/lifecycle.go
@@ -717,7 +717,7 @@ func crossSegmentTimestamps() (single, left, right time.Time) {
// is set by the K8s downward API to the lifecycle pod's actual pod
// name (e.g. demo-banyandb-data-hot-0); the integration test uses
// "localhost" because the data node's bind address is the loopback.
-func runLifecycleMigration(progressFile, reportDir string) observability.MetricsRegistry {
+func runLifecycleMigration(progressFile, reportDir string, additionalArgs ...string) observability.MetricsRegistry {
lifecycleCmd, reg := lifecycle.NewCommandWithRegistry()
priorPodName, priorHad := os.LookupEnv("POD_NAME")
if err := os.Setenv("POD_NAME", "localhost"); err != nil {
@@ -739,6 +739,7 @@ func runLifecycleMigration(progressFile, reportDir string) observability.Metrics
"--report-dir", reportDir,
}
args = append(args, SharedContext.MetadataFlags...)
+ args = append(args, additionalArgs...)
lifecycleCmd.SetArgs(args)
gomega.Expect(lifecycleCmd.Execute()).To(gomega.Succeed())
return reg
diff --git a/test/cases/lifecycle/orphan.go b/test/cases/lifecycle/orphan.go
index 98b435d8e..724716957 100644
--- a/test/cases/lifecycle/orphan.go
+++ b/test/cases/lifecycle/orphan.go
@@ -35,12 +35,14 @@ import (
"github.com/onsi/gomega"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
+ "google.golang.org/protobuf/types/known/durationpb"
"google.golang.org/protobuf/types/known/timestamppb"
commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
measurev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/measure/v1"
modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+ schemav1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/schema/v1"
streamv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/stream/v1"
"github.com/apache/skywalking-banyandb/banyand/backup/lifecycle"
"github.com/apache/skywalking-banyandb/pkg/grpchelper"
@@ -178,6 +180,20 @@ func readOrphanArchive(groupArchiveDir string) (recs []orphanRec, manifestRows i
return recs, manifestRows
}
+// awaitOrphanSchemasApplied ensures index writes cannot overtake schema loading
+// on the data nodes. A liaison write acknowledgement alone does not ensure this.
+func awaitOrphanSchemasApplied(ctx context.Context, conn *grpc.ClientConn, group, kind string, names ...string) {
+ keys := []*schemav1.SchemaKey{{Kind: "group", Name: group}}
+ for _, name := range names {
+ keys = append(keys, &schemav1.SchemaKey{Kind: kind, Group: group, Name: name})
+ }
+ response, err := schemav1.NewSchemaBarrierServiceClient(conn).AwaitSchemaApplied(ctx, &schemav1.AwaitSchemaAppliedRequest{
+ Keys: keys, MinRevisions: make([]int64, len(keys)), Timeout: durationpb.New(flags.EventuallyTimeout),
+ })
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ gomega.Expect(response.GetApplied()).To(gomega.BeTrue(), "orphan fixture schemas not applied; laggards=%v", response.GetLaggards())
+}
+
var _ = ginkgo.Describe("Lifecycle orphan-schema archive", ginkgo.Ordered, func() {
ginkgo.It("measure: deleted-schema rows are archived + source removed; sibling measure migrates", func() {
const (
@@ -222,6 +238,7 @@ var _ = ginkgo.Describe("Lifecycle orphan-schema archive", ginkgo.Ordered, func(
}
createMeasure(toDelete)
createMeasure(keep)
+ awaitOrphanSchemasApplied(ctx, conn, group, "measure", toDelete, keep)
ginkgo.By("writing rows on the straddling (row-replay) segment to both measures")
// The write goes through the liaison, which routes by schema. A freshly
@@ -372,6 +389,7 @@ var _ = ginkgo.Describe("Lifecycle orphan-schema archive", ginkgo.Ordered, func(
}
createStream(toDelete)
createStream(keep)
+ awaitOrphanSchemasApplied(ctx, conn, group, "stream", toDelete, keep)
ginkgo.By("writing elements on the straddling (row-replay) segment to both streams")
// Retry each element alone until the liaison has synced the freshly
diff --git a/test/cases/lifecycle/sampling.go b/test/cases/lifecycle/sampling.go
new file mode 100644
index 000000000..1dcf3aead
--- /dev/null
+++ b/test/cases/lifecycle/sampling.go
@@ -0,0 +1,274 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package lifecycle_test
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "sort"
+ "strings"
+ "time"
+
+ "github.com/onsi/ginkgo/v2"
+ "github.com/onsi/gomega"
+ "google.golang.org/grpc"
+ "google.golang.org/grpc/credentials/insecure"
+ "google.golang.org/protobuf/types/known/timestamppb"
+
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+ tracev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/trace/v1"
+ "github.com/apache/skywalking-banyandb/pkg/grpchelper"
+ "github.com/apache/skywalking-banyandb/pkg/pipeline/sdk"
+ "github.com/apache/skywalking-banyandb/pkg/test/flags"
+)
+
+const (
+ sampledTraceGroup = "sw_lifecycle_sampled_trace"
+ sampledTraceSchema = "sampled_trace"
+ sampledTraceIndex = "sampled_timestamp"
+)
+
+var _ = ginkgo.Describe("Trace lifecycle native sampling", ginkgo.Ordered, func() {
+ ginkgo.It("retains whole traces and rebuilds destination indexes before deleting the source", func() {
+ directory, err := os.MkdirTemp("", "lifecycle-native-sampling")
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ defer os.RemoveAll(directory)
+ pluginDirectory := filepath.Join(directory, "plugins")
+ gomega.Expect(os.Mkdir(pluginDirectory, 0o700)).To(gomega.Succeed())
+ buildLifecycleSampler(pluginDirectory)
+ connection, dialErr := grpchelper.Conn(SharedContext.LiaisonAddr, 10*time.Second, grpc.WithTransportCredentials(insecure.NewCredentials()))
+ gomega.Expect(dialErr).NotTo(gomega.HaveOccurred())
+ defer connection.Close()
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+ createSampledTraceSchema(ctx, connection)
+ defer func() {
+ cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cleanupCancel()
+ _, cleanupErr := databasev1.NewGroupRegistryServiceClient(connection).Delete(cleanupCtx,
+ &databasev1.GroupRegistryServiceDeleteRequest{Group: sampledTraceGroup, Force: true})
+ gomega.Expect(cleanupErr).NotTo(gomega.HaveOccurred())
+ }()
+ _, left, right := crossSegmentTimestamps()
+ client := tracev1.NewTraceServiceClient(connection)
+ for round := 0; round < 3; round++ {
+ timestamp := left.Add(time.Duration(round) * time.Second)
+ if round == 2 {
+ timestamp = right
+ }
+ gomega.Eventually(func() error { return writeSampledSpans(client, timestamp, round) }, flags.EventuallyTimeout).Should(gomega.Succeed())
+ // Wait for persistence between writes so one trace is present in several source parts.
+ gomega.Eventually(func() int { return countSampledCoreParts(SharedContext.SrcDir) }, flags.EventuallyTimeout).
+ Should(gomega.BeNumerically(">=", round+1))
+ }
+ queryRange := &modelv1.TimeRange{Begin: timestamppb.New(left.Add(-time.Hour)), End: timestamppb.New(right.Add(time.Hour))}
+ verifySampledTraceQuery(client, queryRange, "hot", []string{"sample-drop", "sample-keep"}, false)
+ verifySampledTraceQuery(client, queryRange, "hot", []string{"sample-drop", "sample-keep"}, true)
+ reports := filepath.Join(directory, "reports")
+ ginkgo.By("executing the real lifecycle command with native plugin hosting")
+ runLifecycleMigration(filepath.Join(directory, "progress.json"), reports,
+ "--trace-sampling-enabled", "--trace-sampling-trusted-dir", pluginDirectory, "--node-labels", "type=hot")
+ verifySampledTraceQuery(client, queryRange, "warm", []string{"sample-keep"}, false)
+ verifySampledTraceQuery(client, queryRange, "warm", []string{"sample-keep"}, true)
+ verifySampledTraceQuery(client, queryRange, "hot", nil, false)
+ gomega.Expect(countSampledCoreParts(SharedContext.SrcDir)).To(gomega.BeZero(), "successful sampled migration removes source parts")
+ verifySampledTraceReport(reports)
+ entries, readErr := os.ReadDir(filepath.Join(directory, "progress.json.sampling"))
+ gomega.Expect(readErr).NotTo(gomega.HaveOccurred())
+ for _, entry := range entries {
+ gomega.Expect(entry.Name()).NotTo(gomega.Equal(sampledTraceGroup), "artifacts remain until source deletion, then are reclaimed")
+ }
+ })
+})
+
+func buildLifecycleSampler(directory string) {
+ root, buildFlags := lifecycleNativeBuildContext()
+ args := append([]string{"build", "-buildmode=plugin", "-o", filepath.Join(directory, "lifecycle-sampler.so")}, buildFlags...)
+ args = append(args, "./test/cases/lifecycle/testdata/sampling-plugin")
+ command := exec.Command("go", args...)
+ command.Dir = root
+ command.Env = append(os.Environ(), "CGO_ENABLED=1")
+ output, err := command.CombinedOutput()
+ gomega.Expect(err).NotTo(gomega.HaveOccurred(), "native sampler must build with the lifecycle test toolchain: %s", output)
+}
+
+func createSampledTraceSchema(ctx context.Context, connection *grpc.ClientConn) {
+ createSampledTraceSchemaForGroup(ctx, connection, sampledTraceGroup)
+}
+
+func createSampledTraceSchemaForGroup(ctx context.Context, connection *grpc.ClientConn, group string) {
+ _, err := databasev1.NewGroupRegistryServiceClient(connection).Create(ctx, &databasev1.GroupRegistryServiceCreateRequest{Group: &commonv1.Group{
+ Metadata: &commonv1.Metadata{Name: group}, Catalog: commonv1.Catalog_CATALOG_TRACE, ResourceOpts: orphanStages(),
+ Pipeline: &commonv1.TracePipelineConfig{Enabled: true, SchemaNames: []string{sampledTraceSchema}, Stages: []*commonv1.StageRule{{
+ Stage: "hot",
+ Plugins: []*commonv1.Plugin{{Name: "whole-trace-e2e", Kind: &commonv1.Plugin_Sampler{Sampler: &commonv1.SamplerPlugin{
+ Path: "lifecycle-sampler.so", AbiVersion: sdk.ABIVersion,
+ }}}},
+ }}},
+ }})
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ _, err = databasev1.NewTraceRegistryServiceClient(connection).Create(ctx, &databasev1.TraceRegistryServiceCreateRequest{Trace: &databasev1.Trace{
+ Metadata: &commonv1.Metadata{Name: sampledTraceSchema, Group: group},
+ TraceIdTagName: "trace_id", SpanIdTagName: "span_id", TimestampTagName: "timestamp",
+ Tags: []*databasev1.TraceTagSpec{
+ {Name: "trace_id", Type: databasev1.TagType_TAG_TYPE_STRING},
+ {Name: "span_id", Type: databasev1.TagType_TAG_TYPE_STRING},
+ {Name: "timestamp", Type: databasev1.TagType_TAG_TYPE_TIMESTAMP},
+ },
+ }})
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ _, err = databasev1.NewIndexRuleRegistryServiceClient(connection).Create(ctx, &databasev1.IndexRuleRegistryServiceCreateRequest{IndexRule: &databasev1.IndexRule{
+ Metadata: &commonv1.Metadata{Name: sampledTraceIndex, Group: group}, Tags: []string{"timestamp"}, Type: databasev1.IndexRule_TYPE_TREE,
+ }})
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ _, err = databasev1.NewIndexRuleBindingRegistryServiceClient(connection).Create(ctx, &databasev1.IndexRuleBindingRegistryServiceCreateRequest{
+ IndexRuleBinding: &databasev1.IndexRuleBinding{
+ Metadata: &commonv1.Metadata{Name: "sampled-index-binding", Group: group},
+ Rules: []string{sampledTraceIndex}, Subject: &databasev1.Subject{Catalog: commonv1.Catalog_CATALOG_TRACE, Name: sampledTraceSchema},
+ BeginAt: timestamppb.New(time.Unix(0, 0)), ExpireAt: timestamppb.New(time.Now().AddDate(100, 0, 0)),
+ },
+ })
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+}
+
+func writeSampledSpans(client tracev1.TraceServiceClient, timestamp time.Time, round int) error {
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ stream, err := client.Write(ctx)
+ if err != nil {
+ return err
+ }
+ for _, id := range []string{"sample-drop", "sample-keep"} {
+ request := &tracev1.WriteRequest{
+ Metadata: &commonv1.Metadata{Name: sampledTraceSchema, Group: sampledTraceGroup}, Version: uint64(round + 1),
+ Tags: []*modelv1.TagValue{
+ {Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: id}}},
+ {Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: fmt.Sprintf("span-%d", round)}}},
+ {Value: &modelv1.TagValue_Timestamp{Timestamp: timestamppb.New(timestamp)}},
+ }, Span: []byte(fmt.Sprintf("payload-%s-%d", id, round)),
+ }
+ if err = stream.Send(request); err != nil {
+ return err
+ }
+ }
+ return drainWriteResult(stream.Recv, stream.CloseSend)
+}
+
+func countSampledCoreParts(root string) int {
+ return countSampledCorePartsForGroup(root, sampledTraceGroup)
+}
+
+func countSampledCorePartsForGroup(root, group string) int {
+ count := 0
+ _ = filepath.WalkDir(filepath.Join(root, "trace", "data", group), func(path string, entry os.DirEntry, err error) error {
+ if err != nil {
+ return nil
+ }
+ if entry.IsDir() && entry.Name() == "sidx" {
+ return filepath.SkipDir
+ }
+ if entry.IsDir() && len(entry.Name()) == 16 {
+ if _, err = os.Stat(filepath.Join(path, "primary.bin")); err == nil {
+ count++
+ }
+ }
+ return nil
+ })
+ return count
+}
+
+func verifySampledTraceQuery(client tracev1.TraceServiceClient, queryRange *modelv1.TimeRange, stage string, wantIDs []string, indexed bool) {
+ gomega.Eventually(func() error {
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ request := &tracev1.QueryRequest{
+ Groups: []string{sampledTraceGroup}, Name: sampledTraceSchema, TimeRange: queryRange,
+ Stages: []string{stage}, TagProjection: []string{"trace_id", "span_id"}, Limit: 100,
+ }
+ if indexed {
+ request.OrderBy = &modelv1.QueryOrder{IndexRuleName: sampledTraceIndex, Sort: modelv1.Sort_SORT_ASC}
+ } else {
+ request.Criteria = &modelv1.Criteria{Exp: &modelv1.Criteria_Condition{Condition: &modelv1.Condition{
+ Name: "trace_id", Op: modelv1.Condition_BINARY_OP_IN,
+ Value: &modelv1.TagValue{Value: &modelv1.TagValue_StrArray{StrArray: &modelv1.StrArray{Value: []string{"sample-drop", "sample-keep"}}}},
+ }}}
+ }
+ response, err := client.Query(ctx, request)
+ if err != nil {
+ return err
+ }
+ ids := make([]string, 0, len(response.Traces))
+ for _, trace := range response.Traces {
+ ids = append(ids, trace.TraceId)
+ spanIDs := make([]string, 0, len(trace.Spans))
+ for _, span := range trace.Spans {
+ spanIDs = append(spanIDs, span.SpanId)
+ if string(span.Span) != fmt.Sprintf("payload-%s-%s", trace.TraceId, strings.TrimPrefix(span.SpanId, "span-")) {
+ return fmt.Errorf("retained span body changed: %s", span.SpanId)
+ }
+ }
+ sort.Strings(spanIDs)
+ if fmt.Sprint(spanIDs) != "[span-0 span-1 span-2]" {
+ return fmt.Errorf("trace %s lost spans: %v", trace.TraceId, spanIDs)
+ }
+ }
+ sort.Strings(ids)
+ if fmt.Sprint(ids) != fmt.Sprint(wantIDs) {
+ return fmt.Errorf("stage %s indexed=%v: want %v got %v", stage, indexed, wantIDs, ids)
+ }
+ return nil
+ }, flags.EventuallyTimeout).Should(gomega.Succeed())
+}
+
+func verifySampledTraceReport(directory string) {
+ paths, err := filepath.Glob(filepath.Join(directory, "*.json"))
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ gomega.Expect(paths).NotTo(gomega.BeEmpty())
+ payload, err := os.ReadFile(paths[len(paths)-1])
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ var report struct {
+ Sampling map[string]struct {
+ Manifest struct {
+ Group string `json:"group"`
+ Input uint64 `json:"traces"`
+ Kept uint64 `json:"kept_traces"`
+ Spans uint64 `json:"spans"`
+ KeptSpans uint64 `json:"kept_spans"`
+ } `json:"manifest"`
+ } `json:"trace_sampling"`
+ }
+ gomega.Expect(json.Unmarshal(payload, &report)).To(gomega.Succeed())
+ found := false
+ for _, entry := range report.Sampling {
+ if entry.Manifest.Group != sampledTraceGroup {
+ continue
+ }
+ found = true
+ gomega.Expect(entry.Manifest.Input).To(gomega.Equal(uint64(2)))
+ gomega.Expect(entry.Manifest.Kept).To(gomega.Equal(uint64(1)))
+ gomega.Expect(entry.Manifest.Spans).To(gomega.Equal(uint64(6)))
+ gomega.Expect(entry.Manifest.KeptSpans).To(gomega.Equal(uint64(3)))
+ }
+ gomega.Expect(found).To(gomega.BeTrue(), "real lifecycle report must contain committed sampling decisions")
+}
diff --git a/test/cases/lifecycle/sampling_process.go b/test/cases/lifecycle/sampling_process.go
new file mode 100644
index 000000000..46668f8fa
--- /dev/null
+++ b/test/cases/lifecycle/sampling_process.go
@@ -0,0 +1,232 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package lifecycle_test
+
+import (
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "fmt"
+ "io"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "runtime/debug"
+ "strings"
+ "sync"
+ "syscall"
+ "time"
+
+ "github.com/onsi/ginkgo/v2"
+ "github.com/onsi/gomega"
+)
+
+type sampledLifecycleProcess struct {
+ command *exec.Cmd
+ done chan error
+ exited chan struct{}
+ waitErr error
+ progressFile string
+ logPath string
+ cleanupOnce sync.Once
+}
+
+type sampledProcessCursor struct {
+ Part string `json:"part"`
+ Block int `json:"block"`
+ Row int `json:"row"`
+}
+
+type sampledProcessCheckpoint struct {
+ ManifestSHA256 string `json:"manifest_sha256"`
+ Cursor sampledProcessCursor `json:"cursor"`
+ Sequence uint64 `json:"sequence"`
+ Delivered bool `json:"delivered"`
+}
+
+type sampledArtifactIdentity struct {
+ SHA256 string `json:"sha256"`
+ Inode uint64 `json:"inode"`
+ ModifiedNanos int64 `json:"modified_nanos"`
+ Bytes int64 `json:"bytes"`
+}
+
+func lifecycleNativeBuildContext() (string, []string) {
+ root, err := os.Getwd()
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ for {
+ if _, statErr := os.Stat(filepath.Join(root, "go.mod")); statErr == nil {
+ break
+ }
+ parent := filepath.Dir(root)
+ gomega.Expect(parent).NotTo(gomega.Equal(root), "native lifecycle tests require the repository module checkout")
+ root = parent
+ }
+ var flags []string
+ if info, valid := debug.ReadBuildInfo(); valid {
+ for _, setting := range info.Settings {
+ switch setting.Key {
+ case "-race", "-trimpath":
+ if setting.Value == "true" {
+ flags = append(flags, setting.Key)
+ }
+ case "-tags":
+ if setting.Value != "" {
+ flags = append(flags, "-tags", setting.Value)
+ }
+ }
+ }
+ }
+ return root, flags
+}
+
+func buildSampledLifecycleBinary(directory string) string {
+ root, buildFlags := lifecycleNativeBuildContext()
+ binary := filepath.Join(directory, "banyand-lifecycle")
+ args := append([]string{"build", "-o", binary}, buildFlags...)
+ args = append(args, "./banyand/cmd/lifecycle")
+ command := exec.Command("go", args...)
+ command.Dir = root
+ command.Env = append(os.Environ(), "CGO_ENABLED=1")
+ output, err := command.CombinedOutput()
+ gomega.Expect(err).NotTo(gomega.HaveOccurred(), "native lifecycle host must match sampler build flags: %s", output)
+ return binary
+}
+
+func startSampledLifecycle(directory, binary, pluginDir string) *sampledLifecycleProcess {
+ gomega.Expect(os.MkdirAll(directory, 0o700)).To(gomega.Succeed())
+ log, err := os.CreateTemp(directory, "lifecycle-process-*.log")
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ progress := filepath.Join(directory, "progress.json")
+ args := []string{
+ "--grpc-addr=" + SharedContext.DataAddr,
+ "--stream-root-path=" + SharedContext.SrcDir,
+ "--measure-root-path=" + SharedContext.SrcDir,
+ "--trace-root-path=" + SharedContext.SrcDir,
+ "--progress-file=" + progress, "--report-dir=" + filepath.Join(directory, "reports"),
+ "--node-labels=type=hot", "--trace-sampling-enabled", "--trace-sampling-trusted-dir=" + pluginDir,
+ "--row-replay-max-batch-rows=1",
+ }
+ args = append(args, SharedContext.MetadataFlags...)
+ command := exec.Command(binary, args...)
+ command.Dir = directory
+ command.Env = append(os.Environ(), "POD_NAME=localhost")
+ command.Stdout = log
+ command.Stderr = log
+ gomega.Expect(command.Start()).To(gomega.Succeed())
+ process := &sampledLifecycleProcess{command: command, done: make(chan error, 1), exited: make(chan struct{}), progressFile: progress, logPath: log.Name()}
+ go func() {
+ process.waitErr = command.Wait()
+ _ = log.Close()
+ process.done <- process.waitErr
+ close(process.exited)
+ }()
+ ginkgo.DeferCleanup(process.cleanup)
+ return process
+}
+
+func (process *sampledLifecycleProcess) cleanup() {
+ process.cleanupOnce.Do(func() {
+ select {
+ case <-process.exited:
+ return
+ default:
+ }
+ _ = process.command.Process.Kill()
+ gomega.Eventually(process.exited, 10*time.Second).Should(gomega.BeClosed(), "lifecycle child cleanup must reap process")
+ })
+}
+
+func readSampledProcessCheckpoint(path string) sampledProcessCheckpoint {
+ payload, err := os.ReadFile(path)
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ var record struct {
+ Checkpoint sampledProcessCheckpoint `json:"checkpoint"`
+ }
+ gomega.Expect(json.Unmarshal(payload, &record)).To(gomega.Succeed())
+ return record.Checkpoint
+}
+
+func stopAtSampledCheckpoint(process *sampledLifecycleProcess, minimum uint64) (string, sampledProcessCheckpoint) {
+ root := filepath.Join(process.progressFile+".sampling", recoveredTraceGroup)
+ var artifact string
+ gomega.Eventually(func() bool {
+ select {
+ case <-process.exited:
+ ginkgo.Fail(fmt.Sprintf("lifecycle child exited before partial delivery: %v; log=%s", process.waitErr, process.logPath))
+ default:
+ }
+ _ = filepath.WalkDir(root, func(path string, entry os.DirEntry, walkErr error) error {
+ if walkErr != nil {
+ return nil
+ }
+ if entry.IsDir() && strings.HasPrefix(entry.Name(), ".preparing-") {
+ return filepath.SkipDir
+ }
+ if entry.IsDir() || entry.Name() != "checkpoint.json" {
+ return nil
+ }
+ payload, readErr := os.ReadFile(path)
+ if readErr != nil {
+ return nil
+ }
+ var record struct {
+ Checkpoint sampledProcessCheckpoint `json:"checkpoint"`
+ }
+ if json.Unmarshal(payload, &record) == nil && record.Checkpoint.Sequence > minimum && !record.Checkpoint.Delivered {
+ artifact = filepath.Dir(path)
+ }
+ return nil
+ })
+ return artifact != ""
+ }, 90*time.Second, time.Millisecond).Should(gomega.BeTrue(), "a durable incomplete delivery checkpoint must appear; log=%s", process.logPath)
+ gomega.Expect(process.command.Process.Signal(syscall.SIGSTOP)).To(gomega.Succeed())
+ gomega.Eventually(func() bool {
+ status, err := os.ReadFile(fmt.Sprintf("/proc/%d/status", process.command.Process.Pid))
+ return err == nil && strings.Contains(string(status), "State:\tT")
+ }, time.Second, time.Millisecond).Should(gomega.BeTrue(), "lifecycle child must stop before capturing durable state")
+ checkpoint := readSampledProcessCheckpoint(filepath.Join(artifact, "checkpoint.json"))
+ gomega.Expect(checkpoint.Sequence).To(gomega.BeNumerically(">", minimum))
+ gomega.Expect(checkpoint.Delivered).To(gomega.BeFalse(), "fixture must leave a retained suffix unsent before SIGKILL")
+ return artifact, checkpoint
+}
+
+func preparedFileIdentity(directory string) map[string]sampledArtifactIdentity {
+ result := make(map[string]sampledArtifactIdentity)
+ for _, name := range []string{"manifest.json", "decisions", "offsets"} {
+ path := filepath.Join(directory, name)
+ file, err := os.Open(path)
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ digest := sha256.New()
+ _, err = io.Copy(digest, file)
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ gomega.Expect(file.Close()).To(gomega.Succeed())
+ info, statErr := os.Stat(path)
+ gomega.Expect(statErr).NotTo(gomega.HaveOccurred())
+ native, ok := info.Sys().(*syscall.Stat_t)
+ gomega.Expect(ok).To(gomega.BeTrue())
+ result[name] = sampledArtifactIdentity{SHA256: hex.EncodeToString(digest.Sum(nil)), Inode: native.Ino, ModifiedNanos: info.ModTime().UnixNano(), Bytes: info.Size()}
+ }
+ return result
+}
+
+func killSampledLifecycle(process *sampledLifecycleProcess) {
+ gomega.Expect(process.command.Process.Kill()).To(gomega.Succeed())
+ gomega.Eventually(process.exited, 10*time.Second).Should(gomega.BeClosed(), "SIGKILL child must be reaped")
+ gomega.Expect(process.waitErr).To(gomega.HaveOccurred(), "killed child cannot report successful migration completion")
+}
diff --git a/test/cases/lifecycle/sampling_recovery.go b/test/cases/lifecycle/sampling_recovery.go
new file mode 100644
index 000000000..4b0b258e8
--- /dev/null
+++ b/test/cases/lifecycle/sampling_recovery.go
@@ -0,0 +1,276 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package lifecycle_test
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "os"
+ "path/filepath"
+ "runtime"
+ "strings"
+ "syscall"
+ "time"
+
+ "github.com/onsi/ginkgo/v2"
+ "github.com/onsi/gomega"
+ "google.golang.org/grpc"
+ "google.golang.org/grpc/credentials/insecure"
+ "google.golang.org/protobuf/types/known/timestamppb"
+
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+ tracev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/trace/v1"
+ "github.com/apache/skywalking-banyandb/pkg/grpchelper"
+ "github.com/apache/skywalking-banyandb/pkg/test/flags"
+)
+
+const (
+ recoveredTraceGroup = "sw_lifecycle_sampled_recovery"
+ recoveryTracePairs = 512
+)
+
+var _ = ginkgo.Describe("Trace lifecycle native sampling recovery", func() {
+ ginkgo.It("resumes committed sampling after killing the lifecycle process without resampling", func() {
+ if runtime.GOOS != "linux" {
+ ginkgo.Skip("checkpoint interruption uses Linux SIGSTOP and /proc")
+ }
+ directory, err := os.MkdirTemp("", "lifecycle-sampling-recovery")
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ defer os.RemoveAll(directory)
+ plugins := filepath.Join(directory, "plugins")
+ gomega.Expect(os.Mkdir(plugins, 0o700)).To(gomega.Succeed())
+ buildLifecycleSampler(plugins)
+ binary := buildSampledLifecycleBinary(directory)
+ connection, dialErr := grpchelper.Conn(SharedContext.LiaisonAddr, 10*time.Second, grpc.WithTransportCredentials(insecure.NewCredentials()))
+ gomega.Expect(dialErr).NotTo(gomega.HaveOccurred())
+ defer connection.Close()
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+ createSampledTraceSchemaForGroup(ctx, connection, recoveredTraceGroup)
+ defer func() {
+ cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cleanupCancel()
+ _, cleanupErr := databasev1.NewGroupRegistryServiceClient(connection).Delete(cleanupCtx,
+ &databasev1.GroupRegistryServiceDeleteRequest{Group: recoveredTraceGroup, Force: true})
+ gomega.Expect(cleanupErr).NotTo(gomega.HaveOccurred())
+ }()
+ _, timestamp, _ := crossSegmentTimestamps()
+ client := tracev1.NewTraceServiceClient(connection)
+ for round := 0; round < 3; round++ {
+ gomega.Eventually(func() error {
+ return writeRecoveryTraceRound(client, timestamp.Add(time.Duration(round)*time.Second), round)
+ }, flags.EventuallyTimeout).
+ Should(gomega.Succeed())
+ gomega.Eventually(func() int { return countSampledCorePartsForGroup(SharedContext.SrcDir, recoveredTraceGroup) }, flags.EventuallyTimeout).
+ Should(gomega.BeNumerically(">=", round+1))
+ }
+ queryRange := &modelv1.TimeRange{Begin: timestamppb.New(timestamp.Add(-time.Hour)), End: timestamppb.New(timestamp.Add(time.Hour))}
+ verifyRecoveryTraceInventory(client, queryRange, "hot", false, true)
+ ginkgo.By("killing a separate lifecycle process after an acknowledged incomplete cursor")
+ initial := startSampledLifecycle(directory, binary, plugins)
+ defer initial.cleanup()
+ artifact, before := stopAtSampledCheckpoint(initial, 0)
+ identities := preparedFileIdentity(artifact)
+ verifyRecoveryManifest(artifact)
+ gomega.Expect(countSampledCorePartsForGroup(SharedContext.SrcDir, recoveredTraceGroup)).To(gomega.BeNumerically(">", 0))
+ killSampledLifecycle(initial)
+ gomega.Expect(countSampledCorePartsForGroup(SharedContext.SrcDir, recoveredTraceGroup)).To(gomega.BeNumerically(">", 0), "SIGKILL cannot delete the source")
+ gomega.Expect(preparedFileIdentity(artifact)).To(gomega.Equal(identities))
+ gomega.Expect(os.Rename(filepath.Join(plugins, "lifecycle-sampler.so"), filepath.Join(plugins, "lifecycle-sampler.so.unavailable"))).To(gomega.Succeed())
+ ginkgo.By("restarting with the same progress and an unavailable native sampler")
+ resumed := startSampledLifecycle(directory, binary, plugins)
+ defer resumed.cleanup()
+ resumedArtifact, after := stopAtSampledCheckpoint(resumed, before.Sequence)
+ gomega.Expect(resumedArtifact).To(gomega.Equal(artifact))
+ gomega.Expect(preparedFileIdentity(artifact)).To(gomega.Equal(identities), "committed decisions must never be regenerated")
+ gomega.Expect(after.ManifestSHA256).To(gomega.Equal(before.ManifestSHA256))
+ gomega.Expect(after.Sequence).To(gomega.BeNumerically(">", before.Sequence))
+ gomega.Expect(after.Cursor).NotTo(gomega.Equal(before.Cursor), "acknowledged source cursor must advance")
+ gomega.Expect(resumed.command.Process.Signal(syscall.SIGCONT)).To(gomega.Succeed())
+ gomega.Eventually(resumed.done, 2*time.Minute).Should(gomega.Receive(gomega.Succeed()))
+ duplicateCopies := verifyRecoveryTraceInventory(client, queryRange, "warm", false, true)
+ verifyRecoveryTraceInventory(client, queryRange, "warm", true, true)
+ verifyRecoveryTraceInventory(client, queryRange, "hot", false, false)
+ verifyRecoveryResumeReport(filepath.Join(directory, "reports"))
+ gomega.Expect(countSampledCorePartsForGroup(SharedContext.SrcDir, recoveredTraceGroup)).To(gomega.BeZero())
+ _, artifactErr := os.Stat(filepath.Join(initial.progressFile+".sampling", recoveredTraceGroup))
+ gomega.Expect(os.IsNotExist(artifactErr)).To(gomega.BeTrue(), "artifacts are reclaimed only after successful source deletion")
+ ginkgo.GinkgoWriter.Printf("sampling recovery: sequence %d -> %d, unique kept spans %d, exact retry copies %d\n",
+ before.Sequence, after.Sequence, recoveryTracePairs*3, duplicateCopies)
+ })
+})
+
+func writeRecoveryTraceRound(client tracev1.TraceServiceClient, timestamp time.Time, round int) error {
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+ stream, err := client.Write(ctx)
+ if err != nil {
+ return err
+ }
+ for index := 0; index < recoveryTracePairs; index++ {
+ for _, kind := range []string{"keep", "drop"} {
+ id := fmt.Sprintf("sample-%s-%04d", kind, index)
+ request := &tracev1.WriteRequest{
+ Metadata: &commonv1.Metadata{Name: sampledTraceSchema, Group: recoveredTraceGroup}, Version: uint64(round + 1),
+ Tags: []*modelv1.TagValue{
+ {Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: id}}},
+ {Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: fmt.Sprintf("span-%d", round)}}},
+ {Value: &modelv1.TagValue_Timestamp{Timestamp: timestamppb.New(timestamp)}},
+ }, Span: []byte(fmt.Sprintf("payload-%s-%d", id, round)),
+ }
+ if err = stream.Send(request); err != nil {
+ return err
+ }
+ }
+ }
+ return drainWriteResult(stream.Recv, stream.CloseSend)
+}
+
+func verifyRecoveryManifest(directory string) {
+ payload, err := os.ReadFile(filepath.Join(directory, "manifest.json"))
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ var manifest struct {
+ Snapshot string `json:"snapshot"`
+ Traces uint64 `json:"traces"`
+ KeptTraces uint64 `json:"kept_traces"`
+ Spans uint64 `json:"spans"`
+ KeptSpans uint64 `json:"kept_spans"`
+ }
+ gomega.Expect(json.Unmarshal(payload, &manifest)).To(gomega.Succeed())
+ gomega.Expect(manifest.Traces).To(gomega.Equal(uint64(recoveryTracePairs * 2)))
+ gomega.Expect(manifest.KeptTraces).To(gomega.Equal(uint64(recoveryTracePairs)))
+ gomega.Expect(manifest.Spans).To(gomega.Equal(uint64(recoveryTracePairs * 6)))
+ gomega.Expect(manifest.KeptSpans).To(gomega.Equal(uint64(recoveryTracePairs * 3)))
+ info, statErr := os.Stat(manifest.Snapshot)
+ gomega.Expect(statErr).NotTo(gomega.HaveOccurred())
+ gomega.Expect(info.IsDir()).To(gomega.BeTrue())
+}
+
+func verifyRecoveryTraceInventory(client tracev1.TraceServiceClient, queryRange *modelv1.TimeRange, stage string, indexed, wantRows bool) uint64 {
+ var duplicates uint64
+ for begin := 0; begin < recoveryTracePairs; begin += 64 {
+ end := begin + 64
+ ids := make([]string, 0, 128)
+ for index := begin; index < end; index++ {
+ ids = append(ids, fmt.Sprintf("sample-keep-%04d", index), fmt.Sprintf("sample-drop-%04d", index))
+ }
+ if indexed {
+ ids = nil
+ end = recoveryTracePairs
+ }
+ var batchDuplicates uint64
+ gomega.Eventually(func() error {
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ request := &tracev1.QueryRequest{
+ Groups: []string{recoveredTraceGroup}, Name: sampledTraceSchema, TimeRange: queryRange,
+ Stages: []string{stage}, TagProjection: []string{"trace_id", "span_id"}, Limit: uint32(len(ids)),
+ }
+ if indexed {
+ request.Limit = recoveryTracePairs
+ request.OrderBy = &modelv1.QueryOrder{IndexRuleName: sampledTraceIndex, Sort: modelv1.Sort_SORT_ASC}
+ } else {
+ request.Criteria = &modelv1.Criteria{Exp: &modelv1.Criteria_Condition{Condition: &modelv1.Condition{
+ Name: "trace_id", Op: modelv1.Condition_BINARY_OP_IN,
+ Value: &modelv1.TagValue{Value: &modelv1.TagValue_StrArray{StrArray: &modelv1.StrArray{Value: ids}}},
+ }}}
+ }
+ response, queryErr := client.Query(ctx, request)
+ if queryErr != nil {
+ return queryErr
+ }
+ expected := map[string]map[string]int{}
+ for index := begin; index < end; index++ {
+ for _, kind := range []string{"keep", "drop"} {
+ if !wantRows || stage == "warm" && kind == "drop" {
+ continue
+ }
+ expected[fmt.Sprintf("sample-%s-%04d", kind, index)] = map[string]int{"span-0": 0, "span-1": 0, "span-2": 0}
+ }
+ }
+ for _, result := range response.Traces {
+ spans, exists := expected[result.TraceId]
+ if !exists {
+ return fmt.Errorf("unexpected or dropped trace %s", result.TraceId)
+ }
+ for _, span := range result.Spans {
+ if _, exists = spans[span.SpanId]; !exists {
+ return fmt.Errorf("unexpected span identity %s", span.SpanId)
+ }
+ if string(span.Span) != fmt.Sprintf("payload-%s-%s", result.TraceId, strings.TrimPrefix(span.SpanId, "span-")) {
+ return fmt.Errorf("span body changed")
+ }
+ spans[span.SpanId]++
+ }
+ }
+ var copies uint64
+ for _, spans := range expected {
+ for _, count := range spans {
+ if count == 0 {
+ return fmt.Errorf("retained span missing")
+ }
+ copies += uint64(count - 1)
+ }
+ }
+ batchDuplicates = copies
+ return nil
+ }, flags.EventuallyTimeout).Should(gomega.Succeed())
+ duplicates += batchDuplicates
+ if indexed {
+ break
+ }
+ }
+ return duplicates
+}
+
+func verifyRecoveryResumeReport(directory string) {
+ paths, globErr := filepath.Glob(filepath.Join(directory, "*.json"))
+ gomega.Expect(globErr).NotTo(gomega.HaveOccurred())
+ gomega.Expect(paths).NotTo(gomega.BeEmpty())
+ payload, readErr := os.ReadFile(paths[len(paths)-1])
+ gomega.Expect(readErr).NotTo(gomega.HaveOccurred())
+ var report struct {
+ Summary struct {
+ TraceMigration struct {
+ SyncBreakdown map[string]struct {
+ Rows uint64 `json:"row_replay_rows"`
+ } `json:"sync_breakdown"`
+ } `json:"trace_migration"`
+ } `json:"summary"`
+ Sampling map[string]struct {
+ Manifest struct {
+ Group string `json:"group"`
+ } `json:"manifest"`
+ ResumeCount uint64 `json:"resume_count"`
+ } `json:"trace_sampling"`
+ }
+ gomega.Expect(json.Unmarshal(payload, &report)).To(gomega.Succeed())
+ replayedRows := report.Summary.TraceMigration.SyncBreakdown[recoveredTraceGroup].Rows
+ gomega.Expect(replayedRows).To(gomega.BeNumerically(">", 0))
+ gomega.Expect(replayedRows).To(gomega.BeNumerically("<", recoveryTracePairs*3), "resumed delivery skips acknowledged source rows")
+ var resumes uint64
+ for _, attempt := range report.Sampling {
+ if attempt.Manifest.Group == recoveredTraceGroup {
+ resumes += attempt.ResumeCount
+ }
+ }
+ gomega.Expect(resumes).To(gomega.BeNumerically(">", 0))
+}
diff --git a/test/cases/lifecycle/testdata/sampling-plugin/main.go b/test/cases/lifecycle/testdata/sampling-plugin/main.go
new file mode 100644
index 000000000..f59572680
--- /dev/null
+++ b/test/cases/lifecycle/testdata/sampling-plugin/main.go
@@ -0,0 +1,43 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+ "github.com/apache/skywalking-banyandb/pkg/pipeline/sdk"
+ "strings"
+)
+
+// ABIVersion identifies the pinned host ABI.
+var ABIVersion = sdk.ABIVersion
+
+type sampler struct{}
+
+func (*sampler) Kind() sdk.Kind { return sdk.KindSampler }
+func (*sampler) Project() sdk.Projection { return sdk.Projection{Spans: true} }
+func (*sampler) Close() error { return nil }
+func (*sampler) Decide(batch *sdk.TraceBatch) (sdk.Verdict, error) {
+ keep := make([]bool, len(batch.Traces))
+ for index, trace := range batch.Traces {
+ // An incomplete per-part scan retains this trace, making assembly failures visible.
+ keep[index] = (trace.TraceID != "sample-drop" && !strings.HasPrefix(trace.TraceID, "sample-drop-")) || len(trace.Spans) != 3
+ }
+ return sdk.Verdict{Keep: keep}, nil
+}
+
+// NewSampler constructs a deterministic whole-trace lifecycle test sampler.
+func NewSampler([]byte) (sdk.Sampler, error) { return &sampler{}, nil }
diff --git a/test/cases/measure/ql_generated_reuse_schema.go b/test/cases/measure/ql_generated_reuse_schema.go
index 4f01152ef..1a450eb54 100644
--- a/test/cases/measure/ql_generated_reuse_schema.go
+++ b/test/cases/measure/ql_generated_reuse_schema.go
@@ -38,8 +38,9 @@ func prepareGeneratedReuseFixture(base time.Time) {
ctx, cancel := context.WithTimeout(context.Background(), flags.EventuallyTimeout)
defer cancel()
registry := databasev1.NewMeasureRegistryServiceClient(SharedContext.Connection)
- _, createErr := registry.Create(ctx, &databasev1.MeasureRegistryServiceCreateRequest{Measure: qlgen.T05FixtureSchema()})
+ created, createErr := registry.Create(ctx, &databasev1.MeasureRegistryServiceCreateRequest{Measure: qlgen.T05FixtureSchema()})
gm.Expect(createErr).NotTo(gm.HaveOccurred())
+ awaitGeneratedReuseSchema(qlgen.FixtureGroup, qlgen.T05FixtureName, created.GetModRevision())
writeGeneratedReuseFixture(base)
}
@@ -67,13 +68,21 @@ func writeGeneratedReuseFixture(base time.Time) {
}, flags.EventuallyTimeout, 100*time.Millisecond).Should(gm.Succeed())
}
-func awaitGeneratedReuseRevision(revision int64) {
+func awaitGeneratedReuseSchema(group, name string, revision int64) {
ctx, cancel := context.WithTimeout(context.Background(), flags.EventuallyTimeout)
defer cancel()
- response, awaitErr := schemav1.NewSchemaBarrierServiceClient(SharedContext.Connection).AwaitRevisionApplied(ctx,
- &schemav1.AwaitRevisionAppliedRequest{MinRevision: revision, Timeout: durationpb.New(10 * time.Second)})
+ // A notification watermark can advance before the executor loads this group.
+ // Wait for the exact schema revision in the caches used by query execution.
+ response, awaitErr := schemav1.NewSchemaBarrierServiceClient(SharedContext.Connection).AwaitSchemaApplied(ctx,
+ &schemav1.AwaitSchemaAppliedRequest{
+ Keys: []*schemav1.SchemaKey{
+ {Kind: "group", Name: group},
+ {Kind: "measure", Group: group, Name: name},
+ },
+ MinRevisions: []int64{0, revision}, Timeout: durationpb.New(flags.EventuallyTimeout),
+ })
gm.Expect(awaitErr).NotTo(gm.HaveOccurred())
- gm.Expect(response.GetApplied()).To(gm.BeTrue())
+ gm.Expect(response.GetApplied()).To(gm.BeTrue(), "fixture schema not applied; laggards=%v", response.GetLaggards())
}
func replaceGeneratedReuseSchema(group string) {
@@ -89,7 +98,7 @@ func replaceGeneratedReuseSchema(group string) {
&databasev1.TagSpec{Name: "cache_dependency_probe", Type: databasev1.TagType_TAG_TYPE_STRING})
updated, updateErr := registry.Update(ctx, &databasev1.MeasureRegistryServiceUpdateRequest{Measure: schema})
gm.Expect(updateErr).NotTo(gm.HaveOccurred())
- awaitGeneratedReuseRevision(updated.GetModRevision())
+ awaitGeneratedReuseSchema(group, qlgen.T05FixtureName, updated.GetModRevision())
}
func verifyReuseDependencies(base time.Time, scenario qlgen.ReuseScenario) {
@@ -133,7 +142,7 @@ func verifyReuseSchemaLifecycle(base time.Time, scenario qlgen.ReuseScenario) {
}, flags.EventuallyTimeout, 100*time.Millisecond).Should(gm.Succeed())
created, recreateErr := registry.Create(ctx, &databasev1.MeasureRegistryServiceCreateRequest{Measure: qlgen.T05FixtureSchema()})
gm.Expect(recreateErr).NotTo(gm.HaveOccurred())
- awaitGeneratedReuseRevision(created.GetModRevision())
+ awaitGeneratedReuseSchema(qlgen.FixtureGroup, qlgen.T05FixtureName, created.GetModRevision())
writeGeneratedReuseFixture(base)
gm.Expect(executeGeneratedReuseCall(context.Background(), base, scenario.Cases[3])).To(gm.Succeed())
gm.Expect(liaisongrpc.BydbQLPlanCacheStats().Compiles).To(gm.BeNumerically(">", afterReplace.Compiles))
@@ -158,7 +167,7 @@ func verifyReuseMultiGroup(base time.Time, scenario qlgen.ReuseScenario) {
created, schemaErr := databasev1.NewMeasureRegistryServiceClient(SharedContext.Connection).
Create(ctx, &databasev1.MeasureRegistryServiceCreateRequest{Measure: schema})
gm.Expect(schemaErr).NotTo(gm.HaveOccurred())
- awaitGeneratedReuseRevision(created.GetModRevision())
+ awaitGeneratedReuseSchema(other, qlgen.T05FixtureName, created.GetModRevision())
cases := scenario.Cases
gm.Expect(executeGeneratedReuseCall(context.Background(), base, cases[0])).To(gm.Succeed())
before := liaisongrpc.BydbQLPlanCacheStats()
@@ -168,7 +177,7 @@ func verifyReuseMultiGroup(base time.Time, scenario qlgen.ReuseScenario) {
unrelatedCreated, unrelatedErr := databasev1.NewMeasureRegistryServiceClient(SharedContext.Connection).
Create(ctx, &databasev1.MeasureRegistryServiceCreateRequest{Measure: unrelated})
gm.Expect(unrelatedErr).NotTo(gm.HaveOccurred())
- awaitGeneratedReuseRevision(unrelatedCreated.GetModRevision())
+ awaitGeneratedReuseSchema(qlgen.FixtureGroup, unrelated.GetMetadata().GetName(), unrelatedCreated.GetModRevision())
gm.Expect(executeGeneratedReuseCall(context.Background(), base, cases[1])).To(gm.Succeed())
gm.Expect(liaisongrpc.BydbQLPlanCacheStats().Compiles).To(gm.Equal(before.Compiles))
replaceGeneratedReuseSchema(other)
@@ -205,7 +214,7 @@ func restoreGeneratedReuseFixture(base time.Time) {
Create(ctx, &databasev1.MeasureRegistryServiceCreateRequest{Measure: qlgen.T05FixtureSchema()})
if status.Code(createErr) != codes.AlreadyExists {
gm.Expect(createErr).NotTo(gm.HaveOccurred())
- awaitGeneratedReuseRevision(created.GetModRevision())
+ awaitGeneratedReuseSchema(qlgen.FixtureGroup, qlgen.T05FixtureName, created.GetModRevision())
}
writeGeneratedReuseFixture(base)
}
diff --git a/test/e2e-v2/cases/lifecycle/lifecycle-cases.yaml b/test/e2e-v2/cases/lifecycle/lifecycle-cases.yaml
index ec57505cd..3f580c807 100644
--- a/test/e2e-v2/cases/lifecycle/lifecycle-cases.yaml
+++ b/test/e2e-v2/cases/lifecycle/lifecycle-cases.yaml
@@ -23,6 +23,10 @@
# ${oap_host}/${oap_12800} instead, so file-discovery/e2e.yaml aliases them to
# these names in a setup step.
+# Native source-stage sampling is exercised by the distributed lifecycle suite
+# (test/cases/lifecycle/sampling.go), which runs real hot/warm nodes, the Group API,
+# and the lifecycle command. These OAP cases query pre-generated cold fixtures.
+
cases:
- query: swctl --display yaml --base-url=http://${service_skywalking_oap_host}:${service_skywalking_oap_12800}/graphql metrics exec --expression="top_n(service_resp_time,3,des,attr0='GENERAL',attr1!='Not_exist')/100"
expected: ../storage/expected/topN-OP-service.yml
diff --git a/test/integration/distributed/lifecycle_real/README.md b/test/integration/distributed/lifecycle_real/README.md
new file mode 100644
index 000000000..5048d1ee9
--- /dev/null
+++ b/test/integration/distributed/lifecycle_real/README.md
@@ -0,0 +1,30 @@
+This opt-in test imports an independently downloaded native trace segment into a
+real hot data node, starts two warm data nodes and a liaison, and runs the actual
+lifecycle executable as a separate process. It runs an uninterrupted migration
+and a second migration interrupted after an acknowledged durable replay cursor.
+The second process resumes with the sampler plugin unavailable, demonstrating
+that it reuses prepared decisions. Public trace and timestamp-index queries check
+retained payloads and dropped traces. Source cleanup is checked after delivery.
+
+The fixture directory must contain `source/sw_trace/seg-20261012`, metadata API
+wrappers in `metadata/{group,trace,index-rules,index-bindings}.json`, a matching
+native first-party sampler in `plugins/sw-trace-sampler.so`, and the lifecycle
+executable in `bin/lifecycle`. `REAL_LIFECYCLE_BIN` can override the executable.
+Build the executable and plugin with the same Go toolchain and build flags.
+
+```sh
+REAL_TRACE_DIR=/tmp/lifecycle-showcase-20261013 \
+ go test ./test/integration/distributed/lifecycle_real \
+ -run '^TestRealLifecycleAgent$' -v -count=1 -timeout=10m
+```
+
+Each run preserves its node directories, child logs, pinned source snapshot,
+prepared sampling artifacts, interruption and resumed checkpoints, and aggregate
+verification report under a new `agent-e2e-*` directory inside the fixture root.
+The original downloaded data is preserved. The local source TTL is one hour and
+the warm TTL is 3,650 hours so the historical source is eligible without changing
+its timestamps or payloads. Both stages retain the showcase's two-shard layout;
+a fixture containing only shard 1 sends data to its assigned warm node.
+
+Without `REAL_TRACE_DIR`, the test skips and requires no downloaded data or native
+plugin. No real trace payloads or fixture files belong in the checkout.
diff --git a/test/integration/distributed/lifecycle_real/real_test.go b/test/integration/distributed/lifecycle_real/real_test.go
new file mode 100644
index 000000000..22a7e3864
--- /dev/null
+++ b/test/integration/distributed/lifecycle_real/real_test.go
@@ -0,0 +1,434 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package lifecyclereal
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "fmt"
+ "io"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "syscall"
+ "testing"
+ "time"
+
+ "github.com/onsi/gomega"
+ "github.com/stretchr/testify/require"
+ "google.golang.org/grpc"
+ "google.golang.org/grpc/credentials/insecure"
+ "google.golang.org/protobuf/encoding/protojson"
+ "google.golang.org/protobuf/proto"
+ "google.golang.org/protobuf/types/known/structpb"
+
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ tracev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/trace/v1"
+ "github.com/apache/skywalking-banyandb/banyand/metadata/schema"
+ "github.com/apache/skywalking-banyandb/pkg/grpchelper"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+ "github.com/apache/skywalking-banyandb/pkg/test/setup"
+)
+
+// TestRealLifecycleAgent runs actual networking nodes and a separate lifecycle process.
+// Downloaded fixtures and persistent evidence remain outside the repository.
+func TestRealLifecycleAgent(t *testing.T) {
+ root := os.Getenv("REAL_TRACE_DIR")
+ if root == "" {
+ t.Skip("set REAL_TRACE_DIR to the downloaded showcase snapshot")
+ }
+ executable := os.Getenv("REAL_LIFECYCLE_BIN")
+ if executable == "" {
+ executable = filepath.Join(root, "bin", "lifecycle")
+ }
+ require.FileExists(t, executable)
+ gomega.RegisterTestingT(t)
+ require.NoError(t, logger.Init(logger.Logging{Env: "dev", Level: "warn"}))
+ output, err := os.MkdirTemp(root, "agent-e2e-")
+ require.NoError(t, err)
+ t.Logf("persistent real lifecycle evidence: %s", output)
+ var baseline *verificationReport
+ for _, interrupted := range []bool{false, true} {
+ name := "normal"
+ if interrupted {
+ name = "recovery"
+ }
+ t.Run(name, func(t *testing.T) {
+ report := runRealScenario(t, root, filepath.Join(output, name), executable, interrupted)
+ if interrupted && baseline != nil {
+ baselineUnique, recoveredUnique := *baseline, report
+ baselineUnique.DuplicateSpans = 0
+ recoveredUnique.DuplicateSpans = 0
+ require.Equal(t, baselineUnique, recoveredUnique, "recovery must match uninterrupted unique retained payloads")
+ }
+ if !interrupted {
+ baseline = &report
+ }
+ })
+ }
+}
+
+func runRealScenario(t *testing.T, root, output, executable string, interrupted bool) verificationReport {
+ scenarioStarted := time.Now()
+ require.NoError(t, os.MkdirAll(output, 0o750))
+ group := &commonv1.Group{}
+ traceSchema := &databasev1.Trace{}
+ readMetadata(t, filepath.Join(root, "metadata", "group.json"), "group", group)
+ readMetadata(t, filepath.Join(root, "metadata", "trace.json"), "trace", traceSchema)
+ group.Metadata.ModRevision = 0
+ traceSchema.Metadata.ModRevision = 0
+ group.ResourceOpts.Ttl = &commonv1.IntervalRule{Unit: commonv1.IntervalRule_UNIT_HOUR, Num: 1}
+ warm := proto.Clone(group.ResourceOpts.Stages[0]).(*commonv1.LifecycleStage)
+ warm.ShardNum = group.ResourceOpts.ShardNum
+ warm.Ttl = &commonv1.IntervalRule{Unit: commonv1.IntervalRule_UNIT_HOUR, Num: 3650}
+ group.ResourceOpts.Stages = []*commonv1.LifecycleStage{warm}
+ config, err := structpb.NewStruct(map[string]any{"durationThresholdMs": 500, "keepErrors": true, "healthySampleRate": 0.01})
+ require.NoError(t, err)
+ group.Pipeline.Plugins = nil
+ group.Pipeline.Stages = []*commonv1.StageRule{{Stage: "hot", Plugins: []*commonv1.Plugin{{Name: "real-sampler", Kind: &commonv1.Plugin_Sampler{
+ Sampler: &commonv1.SamplerPlugin{Path: "sw-trace-sampler.so", AbiVersion: 1, Config: config},
+ }}}}}
+ plugins := filepath.Join(output, "plugins")
+ copyTree(t, filepath.Join(root, "plugins"), plugins)
+ hotRoot := filepath.Join(output, "hot")
+ source := filepath.Join(hotRoot, "trace", "data", group.Metadata.Name, "seg-20261012")
+ copyTree(t, filepath.Join(root, "source", group.Metadata.Name, "seg-20261012"), source)
+ writer := setup.NewDiscoveryFileWriter(output)
+ cluster := setup.PropertyClusterConfig(writer)
+ hot, _, _, stopHot := setup.DataNodeFromDataDir(cluster, hotRoot, "--node-labels=type=hot", "--trace-flush-timeout=200ms")
+ defer stopHot()
+ warmAddresses := []string{}
+ for i := 0; i < 2; i++ {
+ address, _, _, stopWarm := setup.DataNodeFromDataDir(cluster, filepath.Join(output, fmt.Sprintf("warm-%d", i)),
+ "--node-labels=type=warm", "--has-meta-role=false", "--trace-flush-timeout=200ms")
+ defer stopWarm()
+ warmAddresses = append(warmAddresses, address)
+ }
+ setup.PreloadSchemaViaProperty(cluster, func(ctx context.Context, registry schema.Registry) error {
+ if _, createErr := registry.CreateGroup(ctx, group); createErr != nil {
+ return createErr
+ }
+ if _, createErr := registry.CreateTrace(ctx, traceSchema); createErr != nil {
+ return createErr
+ }
+ for _, rule := range readRules(t, root) {
+ rule.Metadata.ModRevision = 0
+ if _, createErr := registry.CreateIndexRule(ctx, rule); createErr != nil {
+ return createErr
+ }
+ }
+ for _, binding := range readBindings(t, root) {
+ binding.Metadata.ModRevision = 0
+ if _, createErr := registry.CreateIndexRuleBinding(ctx, binding); createErr != nil {
+ return createErr
+ }
+ }
+ return nil
+ })
+ cluster.AddLoadedKinds(schema.KindTrace)
+ liaison, stopLiaison := setup.LiaisonNode(cluster, "--data-node-selector=type=hot")
+ defer stopLiaison()
+ connection, connectionErr := grpchelper.Conn(liaison, 10*time.Second, grpc.WithTransportCredentials(insecure.NewCredentials()),
+ grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(64<<20)))
+ require.NoError(t, connectionErr)
+ defer connection.Close()
+ client := tracev1.NewTraceServiceClient(connection)
+ verifyRealHotSource(t, client, traceSchema, filepath.Join(root, "source", group.Metadata.Name, "seg-20261012", "shard-1"))
+ progress := filepath.Join(output, "progress.json")
+ args := []string{
+ "--node-labels=type=hot",
+ "--grpc-addr=" + hot, "--node-discovery-mode=file", "--node-discovery-file-path=" + writer.Path(),
+ "--stream-root-path=" + hotRoot, "--measure-root-path=" + hotRoot, "--trace-root-path=" + hotRoot,
+ "--progress-file=" + progress, "--report-dir=" + filepath.Join(output, "report"), "--trace-sampling-enabled",
+ "--trace-sampling-trusted-dir=" + plugins, "--row-replay-max-batch-rows=64",
+ }
+ evidence := map[string]any{"hot": hot, "warm": warmAddresses, "liaison": liaison, "command": append([]string{executable}, args...), "interrupted": interrupted}
+ process, done := startAgent(t, output, "initial", executable, args)
+ evidence["initial_pid"] = process.Pid
+ artifact := waitCheckpoint(t, progress+".sampling", process, done, 0)
+ stopAgent(t, process)
+ copyTree(t, artifact, filepath.Join(output, "verified-artifacts"))
+ manifestPayload, manifestErr := os.ReadFile(filepath.Join(artifact, "manifest.json"))
+ require.NoError(t, manifestErr)
+ var pinned struct {
+ Parts []string `json:"parts"`
+ }
+ require.NoError(t, json.Unmarshal(manifestPayload, &pinned))
+ require.NotEmpty(t, pinned.Parts)
+ verificationSource := filepath.Join(output, "verification-source")
+ copyTree(t, filepath.Dir(pinned.Parts[0]), verificationSource)
+ if interrupted {
+ before := readCheckpoint(t, artifact)
+ prepared := artifactIdentity(t, artifact)
+ require.DirExists(t, source, "source must remain before final acknowledgment")
+ require.Positive(t, nativeSpanCount(t, source), "source core parts remain before final acknowledgment")
+ copyTree(t, artifact, filepath.Join(output, "interrupted-artifacts"))
+ require.NoError(t, process.Kill())
+ require.Error(t, <-done)
+ require.Positive(t, nativeSpanCount(t, source), "SIGKILL must preserve source data")
+ evidence["interrupted_at"] = time.Now().UTC()
+ evidence["interrupted_checkpoint"] = before
+ evidence["prepared_identity"] = prepared
+ require.NoError(t, os.Rename(filepath.Join(plugins, "sw-trace-sampler.so"), filepath.Join(plugins, "sw-trace-sampler.so-unavailable")))
+ process, done = startAgent(t, output, "resumed", executable, args)
+ evidence["resumed_pid"] = process.Pid
+ waitCheckpoint(t, progress+".sampling", process, done, before.Sequence)
+ stopAgent(t, process)
+ after := readCheckpoint(t, artifact)
+ require.Equal(t, prepared, artifactIdentity(t, artifact), "restart must reuse pinned decisions without loading the now unavailable plugin")
+ require.Greater(t, after.Sequence, before.Sequence)
+ evidence["resumed_cursor_observed_at"] = time.Now().UTC()
+ evidence["resumed_checkpoint"] = after
+ require.NoError(t, process.Signal(syscall.SIGCONT))
+ } else {
+ require.NoError(t, process.Signal(syscall.SIGCONT))
+ }
+ select {
+ case waitErr := <-done:
+ require.NoError(t, waitErr)
+ case <-time.After(3 * time.Minute):
+ _ = process.Kill()
+ t.Fatal("lifecycle child exceeded deadline")
+ }
+ require.NoDirExists(t, source, "source cleanup follows successful delivery")
+ evidence["delivery_completed"] = true
+ evidence["delivery_elapsed_seconds"] = time.Since(scenarioStarted).Seconds()
+ writeEvidence(t, output, evidence)
+ verification := verifyRealTraceMigration(t, client, verificationSource, filepath.Join(output, "verified-artifacts"), traceSchema)
+ evidence["verification"] = verification
+ t.Logf("verified unique traces=%d spans=%d dropped_traces=%d duplicate_delivery_spans=%d output=%s",
+ verification.KeptTraces, verification.KeptSpans, verification.DroppedTraces, verification.DuplicateSpans, output)
+ evidence["completed"] = true
+ evidence["elapsed_seconds"] = time.Since(scenarioStarted).Seconds()
+ warmCounts := make([]uint64, 2)
+ for i := range warmCounts {
+ warmCounts[i] = nativeSpanCount(t, filepath.Join(output, fmt.Sprintf("warm-%d", i), "trace", "data", group.Metadata.Name))
+ }
+ require.Positive(t, warmCounts[0]+warmCounts[1], "real warm nodes persisted received spans")
+ evidence["warm_node_physical_spans"] = warmCounts
+ writeEvidence(t, output, evidence)
+ return verification
+}
+
+func readMetadata(t *testing.T, path, key string, message proto.Message) {
+ t.Helper()
+ payload, err := os.ReadFile(path)
+ require.NoError(t, err)
+ var wrapper map[string]json.RawMessage
+ require.NoError(t, json.Unmarshal(payload, &wrapper))
+ require.NoError(t, protojson.Unmarshal(wrapper[key], message))
+}
+
+func readRules(t *testing.T, root string) []*databasev1.IndexRule {
+ payload, err := os.ReadFile(filepath.Join(root, "metadata", "index-rules.json"))
+ require.NoError(t, err)
+ var wrapper map[string][]json.RawMessage
+ require.NoError(t, json.Unmarshal(payload, &wrapper))
+ var result []*databasev1.IndexRule
+ for _, raw := range wrapper["indexRule"] {
+ rule := &databasev1.IndexRule{}
+ require.NoError(t, protojson.Unmarshal(raw, rule))
+ result = append(result, rule)
+ }
+ return result
+}
+
+func readBindings(t *testing.T, root string) []*databasev1.IndexRuleBinding {
+ payload, err := os.ReadFile(filepath.Join(root, "metadata", "index-bindings.json"))
+ require.NoError(t, err)
+ var wrapper map[string][]json.RawMessage
+ require.NoError(t, json.Unmarshal(payload, &wrapper))
+ var result []*databasev1.IndexRuleBinding
+ for _, raw := range wrapper["indexRuleBinding"] {
+ binding := &databasev1.IndexRuleBinding{}
+ require.NoError(t, protojson.Unmarshal(raw, binding))
+ result = append(result, binding)
+ }
+ return result
+}
+
+func copyTree(t *testing.T, source, destination string) {
+ t.Helper()
+ require.NoError(t, filepath.WalkDir(source, func(path string, entry os.DirEntry, walkErr error) error {
+ if walkErr != nil {
+ return walkErr
+ }
+ relative, err := filepath.Rel(source, path)
+ if err != nil {
+ return err
+ }
+ target := filepath.Join(destination, relative)
+ if entry.IsDir() {
+ return os.MkdirAll(target, 0o750)
+ }
+ if !entry.Type().IsRegular() {
+ return fmt.Errorf("unexpected nonregular fixture file %s", path)
+ }
+ input, err := os.Open(path)
+ if err != nil {
+ return err
+ }
+ defer input.Close()
+ output, err := os.OpenFile(target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
+ if err != nil {
+ return err
+ }
+ _, err = io.Copy(output, input)
+ closeErr := output.Close()
+ if err != nil {
+ return err
+ }
+ return closeErr
+ }))
+}
+
+func startAgent(t *testing.T, output, name, executable string, args []string) (*os.Process, <-chan error) {
+ t.Helper()
+ log, err := os.Create(filepath.Join(output, name+".log"))
+ require.NoError(t, err)
+ command := exec.Command(executable, args...)
+ command.Dir = output
+ command.Env = append(os.Environ(), "POD_NAME=localhost")
+ command.Stdout = log
+ command.Stderr = log
+ require.NoError(t, command.Start())
+ done := make(chan error, 1)
+ go func() { done <- command.Wait(); _ = log.Close() }()
+ t.Cleanup(func() { _ = command.Process.Kill() })
+ return command.Process, done
+}
+
+type checkpoint struct {
+ ManifestSHA256 string `json:"manifest_sha256"`
+ Cursor json.RawMessage `json:"cursor"`
+ Sequence uint64 `json:"sequence"`
+ Delivered bool `json:"delivered"`
+}
+
+func readCheckpoint(t *testing.T, directory string) checkpoint {
+ t.Helper()
+ payload, err := os.ReadFile(filepath.Join(directory, "checkpoint.json"))
+ require.NoError(t, err)
+ var envelope struct {
+ Checkpoint checkpoint `json:"checkpoint"`
+ }
+ require.NoError(t, json.Unmarshal(payload, &envelope))
+ return envelope.Checkpoint
+}
+
+func waitCheckpoint(t *testing.T, root string, process *os.Process, done <-chan error, minimum uint64) string {
+ t.Helper()
+ deadline := time.Now().Add(90 * time.Second)
+ for time.Now().Before(deadline) {
+ select {
+ case err := <-done:
+ t.Fatalf("agent ended before interruption: %v", err)
+ default:
+ }
+ found := ""
+ _ = filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error {
+ if err != nil || entry.IsDir() || entry.Name() != "checkpoint.json" || strings.Contains(path, ".preparing-") {
+ return nil
+ }
+ payload, readErr := os.ReadFile(path)
+ if readErr != nil {
+ return nil
+ }
+ var envelope struct {
+ Checkpoint checkpoint `json:"checkpoint"`
+ }
+ if json.Unmarshal(payload, &envelope) == nil && envelope.Checkpoint.Sequence > minimum && !envelope.Checkpoint.Delivered {
+ found = filepath.Dir(path)
+ }
+ return nil
+ })
+ if found != "" {
+ return found
+ }
+ time.Sleep(time.Millisecond)
+ }
+ _ = process.Kill()
+ t.Fatal("no incomplete acknowledged checkpoint appeared")
+ return ""
+}
+
+func artifactIdentity(t *testing.T, directory string) map[string]any {
+ t.Helper()
+ result := map[string]any{}
+ for _, name := range []string{"manifest.json", "decisions", "offsets"} {
+ path := filepath.Join(directory, name)
+ payload, err := os.ReadFile(path)
+ require.NoError(t, err)
+ info, err := os.Stat(path)
+ require.NoError(t, err)
+ sum := sha256.Sum256(payload)
+ stat := info.Sys().(*syscall.Stat_t)
+ result[name] = map[string]any{"sha256": hex.EncodeToString(sum[:]), "inode": stat.Ino, "mtime": info.ModTime().UnixNano()}
+ }
+ return result
+}
+
+func nativeSpanCount(t *testing.T, root string) uint64 {
+ t.Helper()
+ var count uint64
+ walkErr := filepath.WalkDir(root, func(path string, entry os.DirEntry, entryErr error) error {
+ if os.IsNotExist(entryErr) {
+ return nil
+ }
+ if entryErr != nil {
+ return entryErr
+ }
+ if entry.IsDir() || entry.Name() != "metadata.json" || strings.Contains(path, string(filepath.Separator)+"sidx"+string(filepath.Separator)) {
+ return nil
+ }
+ payload, readErr := os.ReadFile(path)
+ if readErr != nil {
+ return readErr
+ }
+ var metadata struct {
+ TotalCount uint64 `json:"totalCount"`
+ }
+ if decodeErr := json.Unmarshal(payload, &metadata); decodeErr != nil {
+ return decodeErr
+ }
+ count += metadata.TotalCount
+ return nil
+ })
+ require.NoError(t, walkErr)
+ return count
+}
+
+func stopAgent(t *testing.T, process *os.Process) {
+ t.Helper()
+ require.NoError(t, process.Signal(syscall.SIGSTOP))
+ require.Eventually(t, func() bool {
+ status, readErr := os.ReadFile(fmt.Sprintf("/proc/%d/status", process.Pid))
+ return readErr == nil && strings.Contains(string(status), "State:\tT")
+ }, time.Second, time.Millisecond, "child must be stopped before capturing durable state")
+}
+
+func writeEvidence(t *testing.T, output string, evidence map[string]any) {
+ t.Helper()
+ payload, err := json.MarshalIndent(evidence, "", " ")
+ require.NoError(t, err)
+ require.NoError(t, os.WriteFile(filepath.Join(output, "result.json"), payload, 0o600))
+}
diff --git a/test/integration/distributed/lifecycle_real/verify_test.go b/test/integration/distributed/lifecycle_real/verify_test.go
new file mode 100644
index 000000000..51755714e
--- /dev/null
+++ b/test/integration/distributed/lifecycle_real/verify_test.go
@@ -0,0 +1,442 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package lifecyclereal
+
+import (
+ "bufio"
+ "context"
+ "crypto/sha256"
+ "encoding/binary"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+ "sort"
+ "strconv"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+ "google.golang.org/protobuf/proto"
+ "google.golang.org/protobuf/types/known/timestamppb"
+
+ databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+ tracev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/trace/v1"
+ "github.com/apache/skywalking-banyandb/banyand/trace"
+ "github.com/apache/skywalking-banyandb/pkg/pipeline/sdk"
+)
+
+const verificationBatchSize = 128
+
+type verificationReport struct {
+ BodySignature string `json:"body_signature_sha256_xor"`
+ DuplicateSpans uint64 `json:"duplicate_spans"`
+ InputTraces uint64 `json:"input_traces"`
+ KeptTraces uint64 `json:"kept_traces"`
+ DroppedTraces uint64 `json:"dropped_traces"`
+ InputSpans uint64 `json:"input_spans"`
+ KeptSpans uint64 `json:"kept_spans"`
+ DroppedSpans uint64 `json:"dropped_spans"`
+}
+
+type verificationManifest struct {
+ Group string `json:"group"`
+ TableDigest string `json:"table_digest"`
+ Schema []byte `json:"schema"`
+ Parts []string `json:"parts"`
+ Traces uint64 `json:"traces"`
+ KeptTraces uint64 `json:"kept_traces"`
+ Spans uint64 `json:"spans"`
+ KeptSpans uint64 `json:"kept_spans"`
+ SegmentStart int64 `json:"segment_start"`
+ SegmentEnd int64 `json:"segment_end"`
+}
+
+type expectedTrace struct {
+ spans map[string]int
+ id string
+ keep bool
+}
+
+// verifyRealTraceMigration checks every captured decision against public warm
+// queries, then traverses the timestamp index without the trace-ID fast path.
+func verifyRealTraceMigration(t *testing.T, client tracev1.TraceServiceClient, shardPath, artifactDir string,
+ schema *databasev1.Trace,
+) verificationReport {
+ t.Helper()
+ payload, err := os.ReadFile(filepath.Join(artifactDir, "manifest.json"))
+ require.NoError(t, err)
+ var manifest verificationManifest
+ require.NoError(t, json.Unmarshal(payload, &manifest))
+ require.Equal(t, schema.Metadata.Group, manifest.Group)
+ pinned := &databasev1.Trace{}
+ require.NoError(t, proto.Unmarshal(manifest.Schema, pinned))
+ require.True(t, proto.Equal(verificationSchema(schema), verificationSchema(pinned)), "verification must use pinned trace schema")
+ decisions, err := os.Open(filepath.Join(artifactDir, "decisions"))
+ require.NoError(t, err)
+ defer decisions.Close()
+ digest := sha256.New()
+ _, err = io.Copy(digest, decisions)
+ require.NoError(t, err)
+ require.Equal(t, manifest.TableDigest, hex.EncodeToString(digest.Sum(nil)))
+ _, err = decisions.Seek(0, io.SeekStart)
+ require.NoError(t, err)
+ reader := bufio.NewReader(decisions)
+ parts := make([]string, len(manifest.Parts))
+ for index, part := range manifest.Parts {
+ parts[index] = filepath.Join(shardPath, filepath.Base(part))
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+ defer cancel()
+ scanner, err := trace.NewProjectedSnapshotScanner(ctx, parts, sdk.Projection{SpanIDs: true, Spans: true}, trace.SnapshotScanOptions{
+ MaxOpenReaders: 16, MaxTraceBytes: 128 << 20, TempDir: t.TempDir(),
+ })
+ require.NoError(t, err)
+ defer scanner.Close()
+ var report verificationReport
+ var signature [sha256.Size]byte
+ batch := make([]expectedTrace, 0, verificationBatchSize)
+ for {
+ assembled, scanErr := scanner.Next(ctx)
+ if errors.Is(scanErr, io.EOF) {
+ break
+ }
+ require.NoError(t, scanErr)
+ require.False(t, assembled.Oversized, "verification trace exceeded its projection budget")
+ id, keep, decisionErr := readVerificationDecision(reader)
+ require.NoError(t, decisionErr)
+ require.Equal(t, id, assembled.Trace.TraceID, "decision order must match immutable source trace order")
+ require.Equal(t, len(assembled.Trace.SpanIDs), len(assembled.Trace.Spans))
+ expected := expectedTrace{id: id, keep: keep, spans: make(map[string]int)}
+ report.InputTraces++
+ report.InputSpans += assembled.SpanCount
+ if keep {
+ report.KeptTraces++
+ report.KeptSpans += assembled.SpanCount
+ for index, body := range assembled.Trace.Spans {
+ spanID := assembled.Trace.SpanIDs[index]
+ expected.spans[verificationSpanKey(spanID, body)]++
+ addVerificationSignature(&signature, id, spanID, body)
+ }
+ }
+ batch = append(batch, expected)
+ if len(batch) == verificationBatchSize {
+ report.DuplicateSpans += verifyPublicBatch(t, client, schema, manifest, batch)
+ batch = batch[:0]
+ }
+ }
+ if len(batch) > 0 {
+ report.DuplicateSpans += verifyPublicBatch(t, client, schema, manifest, batch)
+ }
+ _, _, err = readVerificationDecision(reader)
+ require.ErrorIs(t, err, io.EOF, "decision table must cover exactly the source trace inventory")
+ require.Equal(t, manifest.Traces, report.InputTraces)
+ require.Equal(t, manifest.Spans, report.InputSpans)
+ require.Equal(t, manifest.KeptTraces, report.KeptTraces)
+ require.Equal(t, manifest.KeptSpans, report.KeptSpans)
+ report.DroppedTraces = report.InputTraces - report.KeptTraces
+ report.DroppedSpans = report.InputSpans - report.KeptSpans
+ require.Positive(t, report.DroppedTraces, "real sampler verification requires intentional drops")
+ report.BodySignature = hex.EncodeToString(signature[:])
+ verifyPublicTimestampIndex(t, client, schema, manifest, report)
+ return report
+}
+
+func verificationSchema(schema *databasev1.Trace) *databasev1.Trace {
+ normalized := proto.Clone(schema).(*databasev1.Trace)
+ normalized.Metadata.Id = 0
+ normalized.Metadata.CreateRevision = 0
+ normalized.Metadata.ModRevision = 0
+ normalized.CreatedAt = nil
+ normalized.UpdatedAt = nil
+ return normalized
+}
+
+func readVerificationDecision(reader io.Reader) (string, bool, error) {
+ var header [5]byte
+ if _, err := io.ReadFull(reader, header[:]); err != nil {
+ return "", false, err
+ }
+ length := binary.LittleEndian.Uint32(header[:4])
+ if length > 1<<20 || header[4] > 1 {
+ return "", false, fmt.Errorf("invalid durable decision record")
+ }
+ id := make([]byte, length)
+ if _, err := io.ReadFull(reader, id); err != nil {
+ return "", false, err
+ }
+ return string(id), header[4] == 1, nil
+}
+
+func verificationSpanKey(id string, body []byte) string {
+ digest := sha256.Sum256(body)
+ return id + "\x00" + hex.EncodeToString(digest[:])
+}
+
+func verificationRecordDigest(traceID, spanID string, body []byte) [sha256.Size]byte {
+ digest := sha256.New()
+ var size [8]byte
+ for _, value := range [][]byte{[]byte(traceID), []byte(spanID), body} {
+ binary.LittleEndian.PutUint64(size[:], uint64(len(value)))
+ digest.Write(size[:])
+ digest.Write(value)
+ }
+ var result [sha256.Size]byte
+ copy(result[:], digest.Sum(nil))
+ return result
+}
+
+func addVerificationSignature(signature *[sha256.Size]byte, traceID, spanID string, body []byte) {
+ for index, value := range verificationRecordDigest(traceID, spanID, body) {
+ signature[index] ^= value
+ }
+}
+
+func publicVerificationRequest(schema *databasev1.Trace, manifest verificationManifest) *tracev1.QueryRequest {
+ return &tracev1.QueryRequest{
+ Groups: []string{manifest.Group}, Name: schema.Metadata.Name, Stages: []string{"warm"}, Limit: verificationBatchSize,
+ TimeRange: &modelv1.TimeRange{Begin: timestamppb.New(time.Unix(0, manifest.SegmentStart)), End: timestamppb.New(time.Unix(0, manifest.SegmentEnd))},
+ TagProjection: []string{schema.SpanIdTagName},
+ }
+}
+
+func verifyPublicBatch(t *testing.T, client tracev1.TraceServiceClient, schema *databasev1.Trace,
+ manifest verificationManifest, batch []expectedTrace,
+) uint64 {
+ t.Helper()
+ ids := make([]string, len(batch))
+ expected := make(map[string]expectedTrace, len(batch))
+ for index, item := range batch {
+ ids[index] = item.id
+ expected[item.id] = item
+ }
+ request := publicVerificationRequest(schema, manifest)
+ request.Criteria = &modelv1.Criteria{Exp: &modelv1.Criteria_Condition{Condition: &modelv1.Condition{
+ Name: schema.TraceIdTagName,
+ Op: modelv1.Condition_BINARY_OP_IN, Value: &modelv1.TagValue{Value: &modelv1.TagValue_StrArray{StrArray: &modelv1.StrArray{Value: ids}}},
+ }}}
+ var duplicateSpans uint64
+ for _, ordered := range []bool{false, true} {
+ if ordered {
+ request.OrderBy = &modelv1.QueryOrder{IndexRuleName: "start_time", Sort: modelv1.Sort_SORT_ASC}
+ }
+ var observedDuplicates uint64
+ require.EventuallyWithT(t, func(collector *assert.CollectT) {
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ response, err := client.Query(ctx, request)
+ if !assert.NoError(collector, err) {
+ return
+ }
+ copies, compareErr := publicBatchDuplicates(expected, response)
+ if assert.NoError(collector, compareErr) {
+ observedDuplicates = copies
+ }
+ }, time.Minute, 500*time.Millisecond, "public retained/dropped inventory mismatch (ordered=%t)", ordered)
+ if !ordered {
+ duplicateSpans = observedDuplicates
+ }
+ }
+ return duplicateSpans
+}
+
+func comparePublicBatch(expected map[string]expectedTrace, response *tracev1.QueryResponse) error {
+ _, err := publicBatchDuplicates(expected, response)
+ return err
+}
+
+func publicBatchDuplicates(expected map[string]expectedTrace, response *tracev1.QueryResponse) (uint64, error) {
+ actual := make(map[string]map[string]int)
+ for _, result := range response.Traces {
+ item, exists := expected[result.TraceId]
+ if !exists || !item.keep {
+ fingerprint := sha256.Sum256([]byte(result.TraceId))
+ return 0, fmt.Errorf("destination returned unexpected or dropped trace: trace_sha256=%x unexpected=%t", fingerprint[:8], !exists)
+ }
+ if actual[result.TraceId] == nil {
+ actual[result.TraceId] = make(map[string]int)
+ }
+ for _, span := range result.Spans {
+ actual[result.TraceId][verificationSpanKey(span.SpanId, span.Span)]++
+ }
+ }
+ var duplicates uint64
+ for id, item := range expected {
+ if !item.keep {
+ continue
+ }
+ expectedCopies, actualCopies, missing, extra := 0, 0, 0, 0
+ for key, count := range item.spans {
+ expectedCopies += count
+ if actual[id][key] < count {
+ missing += count - actual[id][key]
+ }
+ }
+ for key, count := range actual[id] {
+ actualCopies += count
+ if item.spans[key] == 0 {
+ extra += count
+ }
+ }
+ if missing > 0 || extra > 0 {
+ fingerprint := sha256.Sum256([]byte(id))
+ return 0, fmt.Errorf("retained span inventory differs: trace_sha256=%x expected=%d actual=%d missing=%d conflicting_or_unexpected=%d",
+ fingerprint[:8], expectedCopies, actualCopies, missing, extra)
+ }
+ // Receiver acknowledgements may precede a durable checkpoint. Retries may
+ // replay exact copies; all unique identities and bodies must still match.
+ duplicates += uint64(actualCopies - expectedCopies)
+ }
+ return duplicates, nil
+}
+
+func verifyPublicTimestampIndex(t *testing.T, client tracev1.TraceServiceClient, schema *databasev1.Trace,
+ manifest verificationManifest, expected verificationReport,
+) {
+ t.Helper()
+ const indexPageSize = 512
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+ defer cancel()
+ started := time.Now()
+ // Scan once rather than restarting hundreds of successful windows on a
+ // global polling deadline. Only individual failed query pages are retried.
+ seenTraces := make(map[[sha256.Size]byte]struct{})
+ seenSpans := make(map[[sha256.Size]byte]struct{})
+ var signature [sha256.Size]byte
+ var windows, pages uint64
+ for begin := manifest.SegmentStart; begin < manifest.SegmentEnd; {
+ require.NoError(t, ctx.Err(), "index scan budget exhausted: windows=%d pages=%d traces=%d spans=%d elapsed=%s",
+ windows, pages, len(seenTraces), len(seenSpans), time.Since(started))
+ end := begin + int64(5*time.Minute)
+ if end > manifest.SegmentEnd {
+ end = manifest.SegmentEnd
+ }
+ request := publicVerificationRequest(schema, manifest)
+ request.Limit = indexPageSize
+ request.OrderBy = &modelv1.QueryOrder{IndexRuleName: "start_time", Sort: modelv1.Sort_SORT_ASC}
+ // API bounds require millisecond precision. Inclusive window edges may
+ // repeat records; the fingerprints below remove those copies.
+ request.TimeRange.Begin = timestamppb.New(time.Unix(0, begin))
+ request.TimeRange.End = timestamppb.New(time.Unix(0, end))
+ maxPages := expected.KeptTraces/indexPageSize + 2
+ for page := uint64(0); ; page++ {
+ require.Less(t, page, maxPages, "index pagination exceeded cap: window=[%d,%d) offset=%d traces=%d spans=%d",
+ begin, end, request.Offset, len(seenTraces), len(seenSpans))
+ var response *tracev1.QueryResponse
+ require.EventuallyWithT(t, func(collector *assert.CollectT) {
+ callCtx, callCancel := context.WithTimeout(ctx, 10*time.Second)
+ defer callCancel()
+ queried, queryErr := client.Query(callCtx, request)
+ if assert.NoError(collector, queryErr, "index query failed: window=[%d,%d) page=%d offset=%d elapsed=%s",
+ begin, end, page, request.Offset, time.Since(started)) {
+ response = queried
+ }
+ }, 30*time.Second, 500*time.Millisecond, "timestamp index page must be queryable")
+ pages++
+ for _, result := range response.Traces {
+ seenTraces[sha256.Sum256([]byte(result.TraceId))] = struct{}{}
+ require.LessOrEqual(t, uint64(len(seenTraces)), expected.KeptTraces,
+ "index returned excess unique traces: window=[%d,%d) page=%d spans=%d", begin, end, page, len(seenSpans))
+ for _, span := range result.Spans {
+ fingerprint := verificationRecordDigest(result.TraceId, span.SpanId, span.Span)
+ if _, exists := seenSpans[fingerprint]; exists {
+ continue
+ }
+ seenSpans[fingerprint] = struct{}{}
+ require.LessOrEqual(t, uint64(len(seenSpans)), expected.KeptSpans,
+ "index returned excess unique spans: window=[%d,%d) page=%d traces=%d", begin, end, page, len(seenTraces))
+ for index, value := range fingerprint {
+ signature[index] ^= value
+ }
+ }
+ }
+ if len(response.Traces) < indexPageSize {
+ break
+ }
+ request.Offset += indexPageSize
+ }
+ windows++
+ begin = end
+ }
+ t.Logf("timestamp index scan: windows=%d pages=%d traces=%d spans=%d elapsed=%s",
+ windows, pages, len(seenTraces), len(seenSpans), time.Since(started))
+ require.Equal(t, expected.KeptTraces, uint64(len(seenTraces)), "timestamp index unique trace count: windows=%d pages=%d", windows, pages)
+ require.Equal(t, expected.KeptSpans, uint64(len(seenSpans)), "timestamp index unique span count: windows=%d pages=%d", windows, pages)
+ require.Equal(t, expected.BodySignature, hex.EncodeToString(signature[:]), "timestamp index unique body signature: windows=%d pages=%d", windows, pages)
+}
+
+// verifyRealHotSource checks one complete source trace through both public
+// trace-ID lookup and the independently rebuilt timestamp index before migration.
+func verifyRealHotSource(t *testing.T, client tracev1.TraceServiceClient, schema *databasev1.Trace, shardPath string) {
+ t.Helper()
+ entries, err := os.ReadDir(shardPath)
+ require.NoError(t, err)
+ var paths []string
+ for _, entry := range entries {
+ if !entry.IsDir() || len(entry.Name()) != 16 {
+ continue
+ }
+ if _, parseErr := strconv.ParseUint(entry.Name(), 16, 64); parseErr == nil {
+ paths = append(paths, filepath.Join(shardPath, entry.Name()))
+ }
+ }
+ sort.Strings(paths)
+ require.NotEmpty(t, paths, "real source must contain native core parts")
+ scanner, err := trace.NewProjectedSnapshotScanner(context.Background(), paths, sdk.Projection{SpanIDs: true, Spans: true},
+ trace.SnapshotScanOptions{MaxOpenReaders: 16, MaxTraceBytes: 128 << 20, TempDir: t.TempDir()})
+ require.NoError(t, err)
+ defer scanner.Close()
+ first, err := scanner.Next(context.Background())
+ require.NoError(t, err)
+ require.False(t, first.Oversized)
+ expected := expectedTrace{id: first.Trace.TraceID, keep: true, spans: make(map[string]int)}
+ require.Equal(t, len(first.Trace.SpanIDs), len(first.Trace.Spans))
+ for index, body := range first.Trace.Spans {
+ expected.spans[verificationSpanKey(first.Trace.SpanIDs[index], body)]++
+ }
+ manifest := verificationManifest{
+ Group: schema.Metadata.Group,
+ SegmentStart: first.Trace.MinTS - int64(time.Millisecond),
+ SegmentEnd: first.Trace.MaxTS + int64(time.Millisecond),
+ }
+ request := publicVerificationRequest(schema, manifest)
+ request.Stages = []string{"hot"}
+ request.Criteria = &modelv1.Criteria{Exp: &modelv1.Criteria_Condition{Condition: &modelv1.Condition{
+ Name: schema.TraceIdTagName,
+ Op: modelv1.Condition_BINARY_OP_IN, Value: &modelv1.TagValue{Value: &modelv1.TagValue_StrArray{StrArray: &modelv1.StrArray{Value: []string{first.Trace.TraceID}}}},
+ }}}
+ require.Eventually(t, func() bool {
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ response, queryErr := client.Query(ctx, request)
+ return queryErr == nil && comparePublicBatch(map[string]expectedTrace{expected.id: expected}, response) == nil
+ }, time.Minute, 500*time.Millisecond, "real source trace must be visible before lifecycle starts")
+ request.Criteria = nil
+ request.OrderBy = &modelv1.QueryOrder{IndexRuleName: "start_time", Sort: modelv1.Sort_SORT_ASC}
+ require.Eventually(t, func() bool {
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ response, queryErr := client.Query(ctx, request)
+ return queryErr == nil && len(response.GetTraces()) > 0
+ }, time.Minute, 500*time.Millisecond, "real hot source timestamp index must be visible before lifecycle starts")
+}