You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by li...@apache.org on 2026/10/09 22:28:09 UTC
(skywalking-banyandb) 01/01: Add the data export dry-run protocol and `bydbctl data export --dry-run`.
This is an automated email from the ASF dual-hosted git repository.
mrproliu pushed a commit to branch feat/export-plan-session
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit 04ecc9c92bae93645cbc7e1683493bac84bff6c6
Author: mrproliu <74...@qq.com>
AuthorDate: Sat Oct 10 00:27:28 2026 +0200
Add the data export dry-run protocol and `bydbctl data export --dry-run`.
---
.github/workflows/e2e.yml | 10 +-
.golangci.yml | 2 +
CHANGES.md | 1 +
api/proto/banyandb/transfer/v1/export.proto | 273 ++++
banyand/internal/storage/segment.go | 2 +-
banyand/internal/storage/segment_test.go | 4 +-
banyand/internal/storage/snapshot.go | 31 +
banyand/internal/storage/snapshot_test.go | 34 +
banyand/internal/storage/storage.go | 3 +
banyand/internal/storage/version.go | 57 +-
banyand/internal/storage/version_test.go | 47 +
banyand/liaison/grpc/export.go | 634 ++++++++
banyand/liaison/grpc/export_test.go | 1560 ++++++++++++++++++++
banyand/liaison/grpc/methodpolicy.go | 4 +
banyand/liaison/grpc/methodpolicy_contract_test.go | 13 +-
banyand/liaison/grpc/rbac_schema_contract_test.go | 2 +-
banyand/liaison/grpc/server.go | 17 +-
.../measure/export_part_reader_test.go | 19 +-
banyand/measure/snapshot.go | 17 +-
banyand/measure/svc_data.go | 32 +-
banyand/measure/svc_standalone.go | 17 +
banyand/property/db/db.go | 60 +-
banyand/property/db/snapshot_shards_test.go | 162 ++
banyand/property/listener.go | 6 +-
banyand/property/service.go | 33 +-
banyand/queue/local.go | 22 +-
banyand/queue/local_export.go | 148 ++
banyand/queue/local_export_test.go | 191 +++
banyand/queue/pub/pub.go | 10 +
banyand/queue/queue.go | 10 +
banyand/queue/sub/server.go | 9 +
.../stream/export_part_reader_test.go | 18 +-
banyand/stream/snapshot.go | 17 +-
banyand/stream/svc_standalone.go | 17 +
.../trace/export_part_reader_test.go | 18 +-
banyand/trace/export_snapshot.go | 131 ++
banyand/trace/export_snapshot_test.go | 89 ++
banyand/trace/svc_standalone.go | 35 +-
banyand/transfer/export/backend.go | 111 ++
.../cli/main.go => banyand/transfer/export/doc.go | 22 +-
banyand/transfer/export/inventory.go | 316 ++++
banyand/transfer/export/inventory_test.go | 364 +++++
banyand/transfer/export/lease.go | 108 ++
banyand/transfer/export/lease_test.go | 92 ++
banyand/transfer/export/partmeta.go | 82 +
banyand/transfer/export/partmeta_test.go | 125 ++
banyand/transfer/export/planner.go | 411 ++++++
banyand/transfer/export/planner_test.go | 562 +++++++
banyand/transfer/export/service.go | 129 ++
banyand/transfer/export/service_test.go | 202 +++
banyand/transfer/export/session.go | 433 ++++++
banyand/transfer/export/session_test.go | 934 ++++++++++++
bydbctl/cmd/cli/main.go | 6 +
bydbctl/internal/cmd/data.go | 304 ++++
bydbctl/internal/cmd/data_export.go | 116 ++
bydbctl/internal/cmd/data_export_test.go | 323 ++++
bydbctl/internal/cmd/data_test.go | 228 +++
bydbctl/internal/cmd/root.go | 82 +-
docs/api-reference.md | 434 ++++++
docs/design/0.12.0/data-export-import/README.md | 419 +++---
docs/interacting/bydbctl/bydbctl.md | 133 +-
docs/operation/configuration.md | 4 +
docs/operation/security.md | 3 +-
docs/operation/upgrade.md | 13 +
pkg/cmdsetup/data.go | 9 +
pkg/cmdsetup/standalone.go | 9 +
pkg/schema/export_snapshot.go | 62 +
pkg/schema/export_snapshot_test.go | 102 ++
pkg/test/setup/setup.go | 5 +-
pkg/test/transfer/lease.go | 83 ++
pkg/test/transfer/plan.go | 150 ++
pkg/test/transfer/tree.go | 115 ++
pkg/transfer/config.go | 96 ++
pkg/transfer/config_test.go | 159 ++
.../cli/main.go => pkg/transfer/exporter/doc.go | 21 +-
pkg/transfer/exporter/exitcode.go | 71 +
.../transfer/exporter/exitcode_test.go | 17 +-
pkg/transfer/exporter/fake_liaison_test.go | 264 ++++
pkg/transfer/exporter/plan.go | 163 ++
pkg/transfer/exporter/plan_test.go | 158 ++
pkg/transfer/exporter/preflight.go | 181 +++
pkg/transfer/exporter/preflight_test.go | 237 +++
pkg/transfer/exporter/render.go | 153 ++
pkg/transfer/exporter/report.go | 201 +++
pkg/transfer/exporter/report_test.go | 378 +++++
pkg/transfer/exporter/schema.go | 96 ++
pkg/transfer/exporter/schema_test.go | 147 ++
pkg/transfer/exporter/session.go | 72 +
pkg/transfer/exporter/session_test.go | 119 ++
.../cmd/cli/main.go => pkg/transfer/parallelism.go | 33 +-
pkg/transfer/selector.go | 103 ++
pkg/transfer/selector_test.go | 121 ++
.../cases/transfer/dry-run/docker-compose.yml | 106 ++
test/e2e-v2/cases/transfer/dry-run/e2e.yaml | 95 ++
.../cases/transfer/dry-run/expected/degraded.yaml | 152 ++
.../cases/transfer/dry-run/expected/dry-run.yaml | 105 ++
.../transfer/dry-run/expected/release-session.yaml | 19 +
test/e2e-v2/cases/transfer/dry-run/nodes.yaml | 25 +
test/e2e-v2/cases/transfer/dry-run/plan.yaml | 29 +
.../cases/transfer/dry-run/scripts/common.sh | 50 +
.../cases/transfer/dry-run/scripts/degraded.sh | 119 ++
.../transfer/dry-run/scripts/report-events.sh | 34 +
.../transfer/dry-run/scripts/run-lifecycle.sh | 41 +
.../transfer/dry-run/scripts/wait-for-data.sh | 58 +
.../distributed/data_export/plan_test.go | 624 ++++++++
.../distributed/data_export/suite_test.go | 19 +-
.../standalone/data_export/plan_test.go | 415 ++++++
.../standalone/data_export/suite_test.go | 19 +-
108 files changed, 14531 insertions(+), 415 deletions(-)
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index 602869847..0ce23b86c 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -86,6 +86,11 @@ jobs:
config: test/e2e-v2/cases/lifecycle/file-discovery/e2e.yaml
- name: Lifecycle DNS Discovery
config: test/e2e-v2/cases/lifecycle/dns-discovery/e2e.yaml
+ - name: Transfer Dry Run
+ config: test/e2e-v2/cases/transfer/dry-run/e2e.yaml
+ # Its setup builds bydbctl on the host, which needs the generated protobuf code
+ # (gitignored) from the prepare job's build artifacts.
+ download-artifacts: 'true'
env:
TAG: ${{ github.sha }}
steps:
@@ -96,7 +101,7 @@ jobs:
- name: Setup build environment
uses: ./.github/actions/setup-build-env
with:
- download-artifacts: 'false'
+ download-artifacts: ${{ matrix.test.download-artifacts || 'false' }}
setup-node: 'false'
setup-docker: 'false'
- name: Download docker image artifact
@@ -143,7 +148,8 @@ jobs:
du -sh .
docker images
- uses: actions/upload-artifact@v7
- if: ${{ failure() }}
+ # cancelled() keeps the logs of a run that hit timeout-minutes.
+ if: ${{ failure() || cancelled() }}
name: Upload Logs
with:
name: test-logs-${{ matrix.test.name }}
diff --git a/.golangci.yml b/.golangci.yml
index d015a5fe3..165074ded 100644
--- a/.golangci.yml
+++ b/.golangci.yml
@@ -71,6 +71,8 @@ linters-settings:
alias: clusterv1
- pkg: github.com/apache/skywalking-banyandb/api/proto/banyandb/trace/v1
alias: tracev1
+ - pkg: github.com/apache/skywalking-banyandb/api/proto/banyandb/transfer/v1
+ alias: transferv1
- pkg: github.com/apache/skywalking-banyandb/pkg/pb/v1
alias: pbv1
lll:
diff --git a/CHANGES.md b/CHANGES.md
index 89ad16dd1..24db73ddb 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -14,6 +14,7 @@ Release Notes.
- `measure.v1.QueryRequest.Aggregation` can now target a tag (`tag_name` + `tag_family`) instead of a field: `SUM`/`MIN`/`MAX`/`MEAN` over an `INT` tag, and `COUNT` over any scalar tag, in both standalone and distributed queries. Wire-additive only — existing clients that only ever set `field_name` are unaffected. `COUNT_DISTINCT` is not yet implemented; see the [tag aggregation and time bucketing design](docs/design/0.12.0/tag-aggregation/README.md).
- `measure.v1.QueryRequest.GroupBy.time_bucket` groups a measure query by tumbling time windows, alone or combined with a tag `GroupBy`, in both standalone and distributed queries. The bucket width comes from the request or falls back to the measure's own `interval`. The part-scan path streams bucket-by-bucket so live memory never exceeds one bucket's groups; index-mode measures (no ascending-timestamp guarantee) use a bounded non-streaming fallback instead. A mixed-version cluster where [...]
- `databasev1.Node` now reports its supported file format versions and IANA time zone name.
+- Add the data export dry-run protocol (`transfer.v1.ExportService`) and `bydbctl data export --dry-run`.
### Bug Fixes
diff --git a/api/proto/banyandb/transfer/v1/export.proto b/api/proto/banyandb/transfer/v1/export.proto
new file mode 100644
index 000000000..5063838eb
--- /dev/null
+++ b/api/proto/banyandb/transfer/v1/export.proto
@@ -0,0 +1,273 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+syntax = "proto3";
+
+package banyandb.transfer.v1;
+
+import "banyandb/common/v1/common.proto";
+import "validate/validate.proto";
+
+option go_package = "github.com/apache/skywalking-banyandb/api/proto/banyandb/transfer/v1";
+option java_package = "org.apache.skywalking.banyandb.transfer.v1";
+
+// ExportService is served by the liaison gRPC endpoint (17912) in both cluster and
+// standalone deployments. Plan fans out to every data node and relays frames one by one;
+// Sessions fans out one lifecycle action and reports per node. Every method must be
+// registered in GlobalMethodPolicies(), otherwise the liaison refuses to start.
+service ExportService {
+ // Plan enumerates the per-segment units (per-group for property) a selector hits. Each
+ // segment unit lists the shards the node holds for that segment. Without a session it reads
+ // the live directories. CreateSession snapshots first and announces the new session id
+ // in the first frame; ReadSession reads that snapshot and renews its lease.
+ rpc Plan(PlanRequest) returns (stream PlanResponse);
+ // Sessions runs one lifecycle action (list, heartbeat, release) on every data node and
+ // streams one frame per node, stamped with node_id by the liaison.
+ rpc Sessions(SessionsRequest) returns (stream SessionsResponse);
+}
+
+// Selector names a catalog and optionally its groups. Empty groups means every group of
+// that catalog; an empty selectors list means every group of every catalog.
+message Selector {
+ common.v1.Catalog catalog = 1 [(validate.rules).enum = {defined_only: true, not_in: [0]}];
+ repeated string groups = 2 [(validate.rules).repeated = {max_items: 1024, items: {string: {min_len: 1}}}];
+}
+
+// SegmentUnit keys one physical unit of a stream, measure or trace group on one node: every
+// shard the node holds for a segment. Stage is not part of the key: it is a property of the
+// node, reported read-only on the frame that carries the unit.
+message SegmentUnit {
+ common.v1.Catalog catalog = 1 [(validate.rules).enum = {defined_only: true, not_in: [0, 3]}];
+ string group = 2 [(validate.rules).string.min_len = 1];
+ // Segment directory suffix without the "seg-" prefix: 8 digits for day segments, 10 for
+ // hour segments.
+ string segment_suffix = 3 [(validate.rules).string.pattern = "^([0-9]{8}|[0-9]{10})$"];
+ // Shards that hold at least one part. Empty means the unit only carries segment-level
+ // artifacts (index-mode measure, or a segment whose shards have not flushed yet).
+ repeated uint32 shard_ids = 4;
+}
+
+// UnitInventory is one unit on one node together with its statistics. Property groups have
+// no segments, so they are inventoried per group instead of per segment.
+message UnitInventory {
+ oneof kind {
+ SegmentInventory segment = 1;
+ PropertyInventory property = 2;
+ }
+}
+
+// SegmentInventory is a SegmentUnit of a stream, measure or trace group with its statistics.
+message SegmentInventory {
+ SegmentUnit unit = 1;
+ // Version string from the segment metadata file; input of the import version gate.
+ string segment_version = 2;
+ repeated ShardStat shards = 3;
+ // Segment-level series index (the sidx/ directory).
+ SidxStat segment_level = 4;
+}
+
+// PropertyInventory is one property group on one node: its shards, each holding a document
+// store without parts or timestamps.
+message PropertyInventory {
+ string group = 1 [(validate.rules).string.min_len = 1];
+ repeated PropertyShardStat shards = 2;
+}
+
+// PropertyShardStat describes one shard of a property group.
+message PropertyShardStat {
+ uint32 shard_id = 1;
+ // On-disk size of the shard's document store; property has no compressed form, so this
+ // is both the compressed and the uncompressed estimate.
+ uint64 estimated_bytes = 2;
+ // Documents in the store, including superseded versions and tombstones.
+ uint64 doc_count = 3;
+}
+
+// ShardStat aggregates the parts of one shard. Timestamps are Unix nanoseconds.
+//
+// For trace, estimated_uncompressed_bytes counts the span payload only (tags are excluded),
+// so it can be below estimated_compressed_bytes; estimated_compressed_bytes is the bytes the
+// part writer recorded and is a lower bound of the on-disk size.
+message ShardStat {
+ uint32 shard_id = 1;
+ int64 min_timestamp = 2;
+ int64 max_timestamp = 3;
+ // Sizes of the shard's parts plus its per-shard index directories (stream idx/, trace sidx/).
+ uint64 estimated_compressed_bytes = 4;
+ uint64 estimated_uncompressed_bytes = 5;
+ uint32 parts_count = 6;
+ uint64 total_count = 7;
+ repeated PartStat parts = 8;
+}
+
+// PartStat is the per-part view of the ShardStat aggregates. Timestamps are Unix nanoseconds.
+message PartStat {
+ uint64 id = 1;
+ int64 min_timestamp = 2;
+ int64 max_timestamp = 3;
+ uint64 total_count = 4;
+}
+
+// SidxStat describes the segment-level series index.
+message SidxStat {
+ uint64 estimated_bytes = 1;
+ // Documents of the series index, computed only when the current schema declares an
+ // index-mode measure in the group (0 otherwise). Those documents are the index-mode rows
+ // plus the series entries of the group's regular measures, so the value is an upper bound
+ // of the rows.
+ uint64 doc_count = 2;
+}
+
+// PlanRequest selects what to enumerate and which view to read. Without a session it reads
+// the live directories and creates nothing on any node.
+message PlanRequest {
+ repeated Selector selectors = 1 [(validate.rules).repeated = {max_items: 1024}];
+ // Which export session to drive. Unset: inventory the live directories.
+ oneof session {
+ CreateSession create = 2;
+ ReadSession read = 3;
+ }
+}
+
+// CreateSession is the first Plan of an export that will transfer data. Every data node
+// snapshots the selected catalogs into a new session and inventories that
+// snapshot. One session id covers the whole cluster: the client leaves id empty, the liaison
+// generates it, announces it in the first frame and forwards it, so on a data node id is
+// always set.
+message CreateSession {
+ string id = 1 [(validate.rules).string = {max_len: 64, pattern: "^[0-9a-f]*$"}];
+ // Remove every session the node holds, whatever its lease says, before creating this
+ // one; the removed ids come back in preempted_session_ids. Without it any other session
+ // on the node answers ALREADY_EXISTS naming the occupants.
+ bool preempt = 2;
+}
+
+// ReadSession is every later Plan of the same export: resuming after an interruption, or
+// planning again before transferring. It inventories the session's snapshot and renews its lease,
+// so the result is the same frozen view CreateSession returned.
+message ReadSession {
+ string id = 1 [(validate.rules).string = {min_len: 1, max_len: 64, pattern: "^[0-9a-f]+$"}];
+}
+
+// PlanResponse is one frame of the Plan stream. The liaison stream is: with CreateSession a
+// `created` frame first; then `units` frames, each carrying the units of exactly one
+// (node, group); then exactly one `summary` frame. A catalog that fails to snapshot fails the
+// whole call and the liaison rolls the session back.
+//
+// A data node's own stream (the liaison's upstream) follows the same order without the
+// `created` frame: it never sets node_id or answered_nodes, and its closing `summary`, which
+// carries only preempted_session_ids, is sent only with CreateSession when anything was
+// removed.
+message PlanResponse {
+ oneof frame {
+ SessionCreated created = 1;
+ UnitFrame units = 2;
+ PlanSummary summary = 3;
+ }
+}
+
+// SessionCreated announces the id the liaison generated for a CreateSession Plan.
+message SessionCreated {
+ string session_id = 1;
+}
+
+// UnitFrame carries the units of one group on one node.
+message UnitFrame {
+ // Stamped by the liaison from the upstream connection; a data node leaves it empty.
+ string node_id = 1;
+ // The node's resolved lifecycle stage for this group. Empty for the default tier: the
+ // group declares no stages, the node has no labels, or the labels match no stage. Only
+ // groups the schema registry knows are listed, in every mode.
+ string stage = 2;
+ repeated UnitInventory units = 3;
+}
+
+// PlanSummary closes the stream.
+message PlanSummary {
+ // Sessions removed while creating because preempt was set (deduplicated, sorted).
+ repeated string preempted_session_ids = 1;
+ // Data nodes the liaison could not plan (live directories and ReadSession only;
+ // CreateSession treats any of these as a failure instead): a transport failure, no next
+ // frame within the liaison's maximum wait for a node's Plan, NotFound (the node no longer holds the requested
+ // session), or a data node without ExportService (UNIMPLEMENTED, an older version during a
+ // rolling upgrade). Unit frames already stamped with such a node_id are partial and must
+ // be dropped.
+ repeated string unreachable_nodes = 2;
+ // Data nodes whose Plan stream completed normally, whether or not they held any unit. Every
+ // node the liaison planned is in exactly one of answered_nodes and unreachable_nodes, so
+ // unreachable_nodes is the coverage gap a client must report.
+ repeated string answered_nodes = 3;
+}
+
+// SessionsRequest runs one session lifecycle action on every data node; the liaison
+// streams back one SessionsResponse per node.
+message SessionsRequest {
+ enum Action {
+ ACTION_UNSPECIFIED = 0;
+ // Read-only: every node reports the session it holds (the newest) or none. An unreachable node
+ // fails the whole call, so "clean" and "silent" cannot look alike.
+ ACTION_LIST = 1;
+ // The heartbeat: renew the lease of session_id on every node.
+ ACTION_HEARTBEAT = 2;
+ // Delete the snapshot of session_id on every node; a node that does not hold it still
+ // succeeds (answering `none`), so the action is idempotent.
+ ACTION_RELEASE = 3;
+ }
+ Action action = 1 [(validate.rules).enum = {defined_only: true, not_in: [0]}];
+ // Required by ACTION_HEARTBEAT and ACTION_RELEASE, ignored by ACTION_LIST.
+ string session_id = 2 [(validate.rules).string = {max_len: 64, pattern: "^[0-9a-f]*$"}];
+}
+
+// SessionsResponse is one data node's answer.
+message SessionsResponse {
+ string node_id = 1;
+ oneof outcome {
+ // ACTION_LIST: the session this node holds. One session id covers the whole cluster, so
+ // a node reports at most one: the newest when leftovers of older sessions exist.
+ SessionLease session = 2;
+ // ACTION_LIST: the node holds no session. ACTION_RELEASE: the node did not hold
+ // session_id, so there was nothing to delete; this is still a success.
+ Ack none = 3;
+ // ACTION_HEARTBEAT: the node renewed the lease. ACTION_RELEASE: the node held the session
+ // and deleted it.
+ Ack done = 4;
+ // ACTION_HEARTBEAT and ACTION_RELEASE: the node did not complete the action; the frame is
+ // still sent so the client can tell which nodes failed.
+ // ACTION_HEARTBEAT: the node was unreachable, does not hold the session, or is a data
+ // node without ExportService. An expired or unreadable lease, or any other node error,
+ // fails the whole call instead (FAILED_PRECONDITION for the lease).
+ // ACTION_RELEASE: the node was unreachable, removing the session failed, or it is a data
+ // node without ExportService. A node that does not hold the session answers `none`.
+ string error = 5;
+ }
+}
+
+// Ack is an outcome that carries no data.
+message Ack {}
+
+// SessionLease mirrors the .lease file of an export session plus the catalogs it covers,
+// which the data node derives from the <export-snapshot-path>/<id> directories it finds.
+// expires_at drives reclamation by the node's sweeper; last_heartbeat_at tells a refused
+// operator when the holder was last seen. Times are Unix nanoseconds of the data node's
+// clock; a session whose .lease is not readable reports zero times.
+message SessionLease {
+ string session_id = 1;
+ int64 started_at = 2;
+ int64 expires_at = 3;
+ int64 last_heartbeat_at = 4;
+ repeated common.v1.Catalog catalogs = 5;
+}
diff --git a/banyand/internal/storage/segment.go b/banyand/internal/storage/segment.go
index 00dd5b67d..7e09d2d4b 100644
--- a/banyand/internal/storage/segment.go
+++ b/banyand/internal/storage/segment.go
@@ -902,7 +902,7 @@ func (sc *segmentController[T, O]) create(ctx context.Context, start time.Time)
}
segPath := path.Join(sc.location, fmt.Sprintf(segTemplate, sc.format(start)))
sc.lfs.MkdirPanicIfExist(segPath, DirPerm)
- meta := segmentMeta{
+ meta := SegmentMetadata{
Version: currentVersion,
EndTime: end.Format(time.RFC3339Nano),
}
diff --git a/banyand/internal/storage/segment_test.go b/banyand/internal/storage/segment_test.go
index 41edf7042..4d3437eff 100644
--- a/banyand/internal/storage/segment_test.go
+++ b/banyand/internal/storage/segment_test.go
@@ -774,7 +774,7 @@ func TestOpenReadsPersistedEndTime(t *testing.T) {
customEnd := day1.Add(12 * time.Hour)
segPath1 := filepath.Join(tempDir, "seg-"+day1.Format(dayFormat))
require.NoError(t, os.MkdirAll(segPath1, DirPerm))
- meta1 := segmentMeta{Version: currentVersion, EndTime: customEnd.Format(time.RFC3339Nano)}
+ meta1 := SegmentMetadata{Version: currentVersion, EndTime: customEnd.Format(time.RFC3339Nano)}
meta1Data, marshalErr := json.Marshal(meta1)
require.NoError(t, marshalErr)
require.NoError(t, os.WriteFile(filepath.Join(segPath1, metadataFilename), meta1Data, FilePerm))
@@ -782,7 +782,7 @@ func TestOpenReadsPersistedEndTime(t *testing.T) {
// day2 with standard endTime.
segPath2 := filepath.Join(tempDir, "seg-"+day2.Format(dayFormat))
require.NoError(t, os.MkdirAll(segPath2, DirPerm))
- meta2 := segmentMeta{Version: currentVersion, EndTime: day2.Add(24 * time.Hour).Format(time.RFC3339Nano)}
+ meta2 := SegmentMetadata{Version: currentVersion, EndTime: day2.Add(24 * time.Hour).Format(time.RFC3339Nano)}
meta2Data, marshalErr := json.Marshal(meta2)
require.NoError(t, marshalErr)
require.NoError(t, os.WriteFile(filepath.Join(segPath2, metadataFilename), meta2Data, FilePerm))
diff --git a/banyand/internal/storage/snapshot.go b/banyand/internal/storage/snapshot.go
index d3eff63be..dae01a456 100644
--- a/banyand/internal/storage/snapshot.go
+++ b/banyand/internal/storage/snapshot.go
@@ -23,10 +23,12 @@ import (
"os"
"path/filepath"
"sort"
+ "strings"
"time"
"github.com/apache/skywalking-banyandb/pkg/fs"
"github.com/apache/skywalking-banyandb/pkg/logger"
+ banyandbpath "github.com/apache/skywalking-banyandb/pkg/path"
)
// ReadSnapshotPartNames reads the part directory names recorded in a ".snp"
@@ -60,6 +62,35 @@ func ParseSnapshotTimestamp(name string) (time.Time, error) {
return parsedTime, nil
}
+// ResolveExportSnapshotDir returns the export snapshot directory of a catalog: flagValue
+// resolved to an absolute path when set, otherwise <catalogRoot>/export-snapshots.
+func ResolveExportSnapshotDir(flagValue, catalogRoot string) (string, error) {
+ if flagValue == "" {
+ return filepath.Join(catalogRoot, ExportSnapshotsDir), nil
+ }
+ return banyandbpath.Get(flagValue)
+}
+
+// ExportSnapshotPathUsage is the help text of the --<catalog>-export-snapshot-path flag.
+func ExportSnapshotPathUsage(catalog string) string {
+ usage := fmt.Sprintf("the directory holding export session snapshots of %[1]s. "+
+ "If not set, <%[1]s-root-path>/%[1]s/%[2]s will be used. ", catalog, ExportSnapshotsDir)
+ if catalog == "property" {
+ return usage + "Property snapshots are full copies of the property index, so it needs as much free space as the property data"
+ }
+ return usage + "It must be on the same filesystem as the data path because snapshots are hard links"
+}
+
+// PathWithin reports whether child is parent or lies inside it. A relative and an
+// absolute path never nest.
+func PathWithin(child, parent string) bool {
+ rel, err := filepath.Rel(filepath.Clean(parent), filepath.Clean(child))
+ if err != nil {
+ return false
+ }
+ return rel == "." || (rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)))
+}
+
// DeleteStaleSnapshots deletes the stale snapshots in the root directory.
func DeleteStaleSnapshots(root string, maxNum int, minAge time.Duration, lfs fs.FileSystem) {
if maxNum <= 0 {
diff --git a/banyand/internal/storage/snapshot_test.go b/banyand/internal/storage/snapshot_test.go
index 7f219e07b..c690becb0 100644
--- a/banyand/internal/storage/snapshot_test.go
+++ b/banyand/internal/storage/snapshot_test.go
@@ -250,3 +250,37 @@ func TestReadSnapshotPartNames(t *testing.T) {
require.Contains(t, err.Error(), "cannot parse")
})
}
+
+func TestPathWithin(t *testing.T) {
+ snapshots := filepath.Join("/data", "stream", SnapshotsDir)
+ for _, tc := range []struct {
+ child string
+ want bool
+ }{
+ {filepath.Join("/data", "stream", ExportSnapshotsDir), false},
+ {"/data/stream", false},
+ {"/data/stream/snapshots-export", false},
+ {"/elsewhere/export", false},
+ {"relative/export", false},
+ {snapshots, true},
+ {snapshots + "/", true},
+ {filepath.Join(snapshots, "export"), true},
+ {filepath.Join(snapshots, "a", "..", "b"), true},
+ } {
+ assert.Equal(t, tc.want, PathWithin(tc.child, snapshots), tc.child)
+ }
+}
+
+func TestResolveExportSnapshotDir(t *testing.T) {
+ dir, err := ResolveExportSnapshotDir("", "/data/stream")
+ require.NoError(t, err)
+ assert.Equal(t, filepath.Join("/data/stream", ExportSnapshotsDir), dir)
+
+ dir, err = ResolveExportSnapshotDir("/custom/export/", "/data/stream")
+ require.NoError(t, err)
+ assert.Equal(t, "/custom/export", dir)
+
+ dir, err = ResolveExportSnapshotDir("rel", "/data/stream")
+ require.NoError(t, err)
+ assert.True(t, filepath.IsAbs(dir), dir)
+}
diff --git a/banyand/internal/storage/storage.go b/banyand/internal/storage/storage.go
index 74e72f5c0..4214e89d6 100644
--- a/banyand/internal/storage/storage.go
+++ b/banyand/internal/storage/storage.go
@@ -53,6 +53,9 @@ const (
DirPerm = 0o700
// SnapshotsDir is the directory for snapshots.
SnapshotsDir = "snapshots"
+ // ExportSnapshotsDir is the directory for export session snapshots, kept apart from
+ // SnapshotsDir so the snapshot reclaimers never touch a session.
+ ExportSnapshotsDir = "export-snapshots"
// RepairDir is the directory for repairs.
RepairDir = "repairs"
// DataDir is the directory for data.
diff --git a/banyand/internal/storage/version.go b/banyand/internal/storage/version.go
index 00071710c..6f2f1d18a 100644
--- a/banyand/internal/storage/version.go
+++ b/banyand/internal/storage/version.go
@@ -38,14 +38,17 @@ const (
// readers expect.
const SegmentMetadataFilename = metadataFilename
+// PartMetadataFilename is the per-part metadata document stream, measure and trace write
+// under <shard>/<part>/. Exported for the same reason as SegmentMetadataFilename.
+const PartMetadataFilename = partDiskMetadataFilename
+
// CurrentSegmentVersion is the version string a freshly-written segment
// must carry. Exported for the same reason as SegmentMetadataFilename.
const CurrentSegmentVersion = currentVersion
-// SegmentMetadata is the on-disk shape of <segment>/metadata. Kept in
-// sync with the (unexported) segmentMeta runtime reads. Exported so
-// out-of-package writers can produce identical bytes via the same JSON
-// tags (lowercase `version` / `endTime,omitempty`).
+// SegmentMetadata is the on-disk shape of <segment>/metadata, shared by the runtime
+// reader and out-of-package writers so both use the same JSON tags (lowercase
+// `version` / `endTime,omitempty`).
type SegmentMetadata struct {
Version string `json:"version"`
EndTime string `json:"endTime,omitempty"`
@@ -65,27 +68,45 @@ func checkVersion(version string) error {
"incompatible version %s, supported versions: %s", version, strings.Join(compatibleVersions, ", ")))
}
-type segmentMeta struct {
- Version string `json:"version"`
- EndTime string `json:"endTime,omitempty"`
-}
+// ErrEmptySegmentMetadata is returned (wrapped) by DecodeSegmentMetadata for a metadata
+// file without content, e.g. one created but not yet written by a rollover.
+var ErrEmptySegmentMetadata = errors.New("segment metadata is empty")
-func readSegmentMeta(data []byte) (segmentMeta, error) {
- var meta segmentMeta
+// DecodeSegmentMetadata parses the raw content of a <segment>/metadata file into
+// SegmentMetadata. It accepts both the JSON document written by current releases and
+// the legacy bare version string, and performs no compatibility check: callers that
+// need to reject unsupported versions go through readSegmentMeta.
+func DecodeSegmentMetadata(data []byte) (SegmentMetadata, error) {
+ var meta SegmentMetadata
trimmed := strings.TrimSpace(string(data))
- if len(trimmed) > 0 && trimmed[0] == '{' {
- if unmarshalErr := json.Unmarshal(data, &meta); unmarshalErr != nil {
- return segmentMeta{}, unmarshalErr
- }
- } else {
- meta.Version = trimmed
+ if trimmed == "" {
+ return SegmentMetadata{}, errors.WithStack(ErrEmptySegmentMetadata)
}
- if checkErr := checkVersion(meta.Version); checkErr != nil {
- return segmentMeta{}, checkErr
+ if trimmed[0] == '{' {
+ if err := json.Unmarshal(data, &meta); err != nil {
+ return SegmentMetadata{}, err
+ }
+ return meta, nil
}
+ meta.Version = trimmed
return meta, nil
}
+func readSegmentMeta(data []byte) (SegmentMetadata, error) {
+ decoded, err := DecodeSegmentMetadata(data)
+ if err != nil {
+ if errors.Is(err, ErrEmptySegmentMetadata) {
+ // Empty metadata carries no version: as incompatible as an unknown one, so retrying cannot help.
+ return SegmentMetadata{}, initerror.AsPermanent(err)
+ }
+ return SegmentMetadata{}, err
+ }
+ if checkErr := checkVersion(decoded.Version); checkErr != nil {
+ return SegmentMetadata{}, checkErr
+ }
+ return decoded, nil
+}
+
// GetCurrentVersion returns the current storage version.
func GetCurrentVersion() string {
return currentVersion
diff --git a/banyand/internal/storage/version_test.go b/banyand/internal/storage/version_test.go
index 31350d56d..93801a1e1 100644
--- a/banyand/internal/storage/version_test.go
+++ b/banyand/internal/storage/version_test.go
@@ -90,3 +90,50 @@ func TestReadSegmentMeta_NewFormatNoEndTime(t *testing.T) {
assert.Equal(t, "1.4.0", meta.Version)
assert.Equal(t, "", meta.EndTime)
}
+
+func TestDecodeSegmentMetadata(t *testing.T) {
+ tests := []struct {
+ name string
+ raw string
+ want SegmentMetadata
+ }{
+ {
+ name: "json",
+ raw: `{"version":"1.5.0","endTime":"2026-09-28T00:00:00Z"}`,
+ want: SegmentMetadata{Version: "1.5.0", EndTime: "2026-09-28T00:00:00Z"},
+ },
+ {name: "legacy raw string", raw: "1.3.0\n", want: SegmentMetadata{Version: "1.3.0"}},
+ {
+ name: "unknown version is reported not rejected",
+ raw: `{"version":"99.0.0"}`,
+ want: SegmentMetadata{Version: "99.0.0"},
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := DecodeSegmentMetadata([]byte(tt.raw))
+ require.NoError(t, err)
+ assert.Equal(t, tt.want, got)
+ })
+ }
+}
+
+func TestDecodeSegmentMetadata_MalformedJSON(t *testing.T) {
+ _, err := DecodeSegmentMetadata([]byte(`{"version":`))
+ require.Error(t, err)
+}
+
+func TestDecodeSegmentMetadata_EmptyIsAnError(t *testing.T) {
+ for _, raw := range []string{"", " \n"} {
+ _, err := DecodeSegmentMetadata([]byte(raw))
+ require.ErrorIs(t, err, ErrEmptySegmentMetadata, "raw %q", raw)
+ }
+}
+
+func TestReadSegmentMeta_WhitespaceOnlyIsPermanent(t *testing.T) {
+ for _, raw := range []string{"", " \n\t"} {
+ _, err := readSegmentMeta([]byte(raw))
+ require.Error(t, err)
+ assert.True(t, initerror.IsPermanent(err), "empty metadata %q must surface as permanent", raw)
+ }
+}
diff --git a/banyand/liaison/grpc/export.go b/banyand/liaison/grpc/export.go
new file mode 100644
index 000000000..3d37eebfb
--- /dev/null
+++ b/banyand/liaison/grpc/export.go
@@ -0,0 +1,634 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package grpc
+
+import (
+ "context"
+ "crypto/rand"
+ "encoding/hex"
+ "errors"
+ "fmt"
+ "io"
+ "slices"
+ "sort"
+ "strings"
+ "sync"
+ "time"
+
+ "golang.org/x/sync/errgroup"
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/status"
+ "google.golang.org/protobuf/proto"
+
+ transferv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/transfer/v1"
+ "github.com/apache/skywalking-banyandb/banyand/liaison/grpc/route"
+ "github.com/apache/skywalking-banyandb/banyand/queue"
+ "github.com/apache/skywalking-banyandb/pkg/grpchelper"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+)
+
+// The export timeouts are variables so tests can shorten them.
+var (
+ // exportSessionTimeout bounds one data node's answer to any Sessions action (list,
+ // heartbeat, release), including the release that undoes a half-created session. Every
+ // action waits on the node's session lock, which a create holds while it snapshots and a
+ // release holds while it removes a session, so a slow snapshot or a half-open connection
+ // must not pin the call.
+ exportSessionTimeout = 2 * time.Minute
+ // exportPlanNodeMaxWait is the longest the liaison waits for a data node's next Plan frame
+ // before giving the node up. It restarts with every frame and is paused while a frame waits
+ // on the downstream client, so a healthy node streaming a large inventory is never cut off.
+ // A dry run or ReadSession reports a node that keeps silent as unreachable; a CreateSession
+ // fails with DeadlineExceeded. The wait for the first frame also covers the data node's
+ // snapshot in a CreateSession, so a slower snapshot fails the create.
+ exportPlanNodeMaxWait = 5 * time.Minute
+)
+
+// unsupportedExport describes a data node without ExportService (an older version during a
+// rolling upgrade), which answers every export RPC with codes.Unimplemented.
+const unsupportedExport = "does not support export (older version)"
+
+// errPlanNodeMaxWait is the cancellation cause of an upstream Plan the liaison gave up on
+// after exportPlanNodeMaxWait without a frame.
+var errPlanNodeMaxWait = errors.New("export plan max wait exceeded")
+
+// exportService is the liaison-side ExportService: a server-streaming proxy. Plan fans
+// out to every registered data node, relays `units` frames 1:1 stamping node_id from the
+// upstream connection, and closes with one `summary` frame; Sessions fans one lifecycle
+// action out and answers one frame per node.
+type exportService struct {
+ transferv1.UnimplementedExportServiceServer
+ tire2 queue.Client
+ routeTable route.TableProvider
+ selfNode func() string
+ newSessionID func() string
+ l *logger.Logger
+}
+
+// newExportService fans out over the data nodes registered in routeTable. A standalone
+// process has no tire2 route table: it is its own only data node, named by selfNode, and its
+// local pipeline serves the ExportService in-process.
+func newExportService(tire2 queue.Client, routeTable route.TableProvider, selfNode func() string, l *logger.Logger) *exportService {
+ return &exportService{tire2: tire2, routeTable: routeTable, selfNode: selfNode, newSessionID: randomSessionID, l: l}
+}
+
+// randomSessionID returns 16 random bytes as lowercase hex, which satisfies the session id
+// rule enforced by the proto and by every data node.
+func randomSessionID() string {
+ var b [16]byte
+ if _, err := rand.Read(b[:]); err != nil {
+ logger.Panicf("cannot generate export session id: %v", err)
+ }
+ return hex.EncodeToString(b[:])
+}
+
+func (s *exportService) registeredDataNodes() []string {
+ rt := s.routeTable.GetRouteTable()
+ names := make([]string, 0, len(rt.GetRegistered()))
+ for _, n := range rt.GetRegistered() {
+ names = append(names, n.GetMetadata().GetName())
+ }
+ sort.Strings(names)
+ return names
+}
+
+// planNode is one target of the export fan-out, answering with the shapes a data node's
+// ExportService handlers produce.
+type planNode interface {
+ name() string
+ // plan runs one Plan and hands every upstream frame to onFrame, returning its error as is.
+ plan(ctx context.Context, req *transferv1.PlanRequest, onFrame func(*transferv1.PlanResponse) error) error
+ // sessions runs one Sessions action and returns the node's answer (node_id unset).
+ sessions(ctx context.Context, req *transferv1.SessionsRequest) (*transferv1.SessionsResponse, error)
+}
+
+// notOpenedError wraps a Plan failure that happened before the upstream stream existed and
+// that the node cannot have seen, so a rollback must not target it.
+type notOpenedError struct{ error }
+
+func (e notOpenedError) Unwrap() error { return e.error }
+
+// GRPCStatus keeps the wrapped status as is, so its message is not prefixed with the
+// wrapper's own text.
+func (e notOpenedError) GRPCStatus() *status.Status { return status.Convert(e.error) }
+
+// remoteNode reaches a data node through the tire2 client: the connection pool in a
+// cluster, the in-process local pipeline in a standalone process.
+type remoteNode struct {
+ tire2 queue.Client
+ id string
+}
+
+func (n remoteNode) name() string { return n.id }
+
+// exportClient hands out the node's ExportService client. A pipeline without one (a local
+// queue with no registered ExportService) is Unimplemented like a node predating it; any other
+// failure means the node cannot be reached.
+func (n remoteNode) exportClient() (transferv1.ExportServiceClient, error) {
+ client, err := n.tire2.NewExportClient(n.id)
+ if errors.Is(err, queue.ErrNotImplemented) {
+ return nil, status.Error(codes.Unimplemented, err.Error())
+ }
+ if err != nil {
+ return nil, status.Error(codes.Unavailable, err.Error())
+ }
+ return client, nil
+}
+
+func (n remoteNode) plan(ctx context.Context, req *transferv1.PlanRequest, onFrame func(*transferv1.PlanResponse) error) error {
+ client, err := n.exportClient()
+ if err != nil {
+ return notOpenedError{err}
+ }
+ up, err := client.Plan(ctx, req)
+ if err != nil {
+ // An open cut short by ctx may already have delivered the request; any other open
+ // failure (the connection refused it) never reached the node. A node still connecting
+ // when ctx ends is counted too, so its failed rollback may name it as leaked.
+ if code := status.Code(err); ctx.Err() != nil && (code == codes.Canceled || code == codes.DeadlineExceeded) {
+ return err
+ }
+ return notOpenedError{err}
+ }
+ for {
+ frame, recvErr := up.Recv()
+ if errors.Is(recvErr, io.EOF) {
+ return nil
+ }
+ if recvErr != nil {
+ return recvErr
+ }
+ if err = onFrame(frame); err != nil {
+ return err
+ }
+ }
+}
+
+// sessions returns the one frame a data node answers a Sessions action with. A stream
+// without an outcome, or with more than one frame, is Internal.
+func (n remoteNode) sessions(ctx context.Context, req *transferv1.SessionsRequest) (*transferv1.SessionsResponse, error) {
+ client, err := n.exportClient()
+ if err != nil {
+ return nil, err
+ }
+ up, err := client.Sessions(ctx, req)
+ if err != nil {
+ return nil, err
+ }
+ frame, err := up.Recv()
+ if errors.Is(err, io.EOF) || (err == nil && frame.GetOutcome() == nil) {
+ return nil, status.Error(codes.Internal, "no sessions outcome in the stream")
+ }
+ if err != nil {
+ return nil, err
+ }
+ if _, err = up.Recv(); !errors.Is(err, io.EOF) {
+ if err != nil {
+ return nil, err
+ }
+ return nil, status.Error(codes.Internal, "more than one sessions frame in the stream")
+ }
+ return frame, nil
+}
+
+// errorOutcome is a Sessions frame whose node did not complete the action.
+func errorOutcome(msg string) *transferv1.SessionsResponse {
+ return &transferv1.SessionsResponse{Outcome: &transferv1.SessionsResponse_Error{Error: msg}}
+}
+
+// failedOutcome reports whether a Sessions frame carries the `error` outcome.
+func failedOutcome(f *transferv1.SessionsResponse) bool {
+ _, failed := f.GetOutcome().(*transferv1.SessionsResponse_Error)
+ return failed
+}
+
+// planNodes is the fan-out target set: every registered data node or, in a standalone
+// process, the process itself. Empty when a cluster has no data node registered.
+func (s *exportService) planNodes() []planNode {
+ if s.routeTable == nil {
+ return []planNode{remoteNode{tire2: s.tire2, id: s.selfNode()}}
+ }
+ names := s.registeredDataNodes()
+ nodes := make([]planNode, 0, len(names))
+ for _, name := range names {
+ nodes = append(nodes, remoteNode{tire2: s.tire2, id: name})
+ }
+ return nodes
+}
+
+// toStatus keeps a status error as it is and maps a raw context error to Canceled or
+// DeadlineExceeded, so a downstream disconnect never surfaces as codes.Unknown.
+func toStatus(err error) *status.Status {
+ if st, ok := status.FromError(err); ok {
+ return st
+ }
+ return status.FromContextError(err)
+}
+
+// nodeErrKind is how the export fan-outs classify a data node's failure; each action applies
+// its own policy to the kinds.
+type nodeErrKind int
+
+const (
+ nodeErrOther nodeErrKind = iota
+ nodeErrTransport // the node was not reached or did not answer (failover codes)
+ nodeErrUnsupported // the node predates ExportService
+ nodeErrNotFound // the node does not hold the session
+)
+
+// classifyNodeErr returns the kind of a data node's failure and its status.
+func classifyNodeErr(err error) (nodeErrKind, *status.Status) {
+ st := toStatus(err)
+ switch {
+ case grpchelper.IsFailoverError(err):
+ return nodeErrTransport, st
+ case st.Code() == codes.Unimplemented:
+ return nodeErrUnsupported, st
+ case st.Code() == codes.NotFound:
+ return nodeErrNotFound, st
+ }
+ return nodeErrOther, st
+}
+
+// unsupportedNodeError fails a call on a data node without ExportService.
+func unsupportedNodeError(node string) error {
+ return status.Errorf(codes.Unimplemented, "data node %s %s", node, unsupportedExport)
+}
+
+// nodeError reports a data node's failure under the given code, prefixed with the node.
+func nodeError(code codes.Code, node, msg string) error {
+ return status.Errorf(code, "data node %s: %s", node, msg)
+}
+
+// createAborted is the error of a CreateSession Plan that did not complete: the cause,
+// whether the half-created session was released (or no node got the request, so there was
+// nothing to release), which nodes still hold it when releasing failed, and the sessions the
+// nodes that finished reported as preempted, which are gone either way. The preempted list
+// covers only those nodes: with preempt set, a node that did not finish may have removed
+// sessions too, so unfinishedPreempt adds a hint to check with ACTION_LIST.
+func createAborted(id string, cause error, nothingEstablished bool, leaked, preempted []string, unfinishedPreempt bool) error {
+ st := status.Convert(cause)
+ outcome := "snapshots released"
+ if nothingEstablished {
+ outcome = "nothing to release"
+ }
+ msg := fmt.Sprintf("create export session %s aborted, %s: %s", id, outcome, st.Message())
+ if len(leaked) > 0 {
+ msg = fmt.Sprintf("create export session %s aborted: %s; releasing it failed on %v, release session %s by hand",
+ id, st.Message(), leaked, id)
+ }
+ if len(preempted) > 0 {
+ msg += fmt.Sprintf("; preempted sessions %v were already removed", preempted)
+ }
+ if unfinishedPreempt {
+ msg += "; nodes that did not finish may also have removed preempted sessions, check with ACTION_LIST"
+ }
+ return status.Error(st.Code(), msg)
+}
+
+// Plan implements transferv1.ExportServiceServer.
+func (s *exportService) Plan(req *transferv1.PlanRequest, stream transferv1.ExportService_PlanServer) error {
+ if err := req.ValidateAll(); err != nil {
+ return status.Error(codes.InvalidArgument, err.Error())
+ }
+ if err := validateSession(req); err != nil {
+ return err
+ }
+ nodes := s.planNodes()
+ if len(nodes) == 0 {
+ return status.Error(codes.FailedPrecondition, "this liaison has not discovered any data node")
+ }
+ if req.GetCreate() != nil {
+ return s.planCreateSession(req, stream, nodes)
+ }
+ summary, _, _, err := s.fanOutPlan(stream.Context(), req, nodes, stream.Send)
+ if err != nil {
+ return err
+ }
+ return stream.Send(summary)
+}
+
+// validateSession applies the rule ValidateAll cannot express: a CreateSession must come
+// without an id, because the liaison generates it. No session is a dry run and ReadSession's
+// id is already validated by ValidateAll.
+func validateSession(req *transferv1.PlanRequest) error {
+ if req.GetCreate().GetId() != "" {
+ return status.Error(codes.InvalidArgument, "create session must not carry an id; the liaison generates it")
+ }
+ return nil
+}
+
+// sessionID is the id of the session req drives, empty for a dry run.
+func sessionID(req *transferv1.PlanRequest) string {
+ if c := req.GetCreate(); c != nil {
+ return c.GetId()
+ }
+ return req.GetRead().GetId()
+}
+
+// withSessionID clones req and stamps the generated id into its CreateSession.
+func withSessionID(req *transferv1.PlanRequest, id string) *transferv1.PlanRequest {
+ upstream := proto.Clone(req).(*transferv1.PlanRequest)
+ upstream.GetCreate().Id = id
+ return upstream
+}
+
+// planCreateSession generates the cluster-wide session id, announces it in the first
+// frame, then fans out strictly: any node that fails or does not answer aborts the whole
+// attempt and the session is released on every node that opened a Plan stream before the
+// error is returned.
+func (s *exportService) planCreateSession(req *transferv1.PlanRequest, stream transferv1.ExportService_PlanServer, nodes []planNode) error {
+ id := s.newSessionID()
+ created := &transferv1.PlanResponse{Frame: &transferv1.PlanResponse_Created{Created: &transferv1.SessionCreated{SessionId: id}}}
+ if err := stream.Send(created); err != nil {
+ return err
+ }
+ summary, established, unfinished, err := s.fanOutPlan(stream.Context(), withSessionID(req, id), nodes, stream.Send)
+ if err == nil {
+ err = stream.Send(summary)
+ }
+ if err != nil {
+ // Without the summary the client cannot use the session, so it is undone like any
+ // other failure.
+ leaked := s.releaseOn(context.WithoutCancel(stream.Context()), id, established)
+ return createAborted(id, toStatus(err).Err(), len(established) == 0, leaked,
+ summary.GetSummary().GetPreemptedSessionIds(), req.GetCreate().GetPreempt() && unfinished)
+ }
+ return nil
+}
+
+// fanOutPlan runs one upstream Plan per node and relays `units` frames through send
+// (serialized). Node `summary` frames are folded into the returned final summary,
+// and the nodes that may have received the request are returned so a rollback targets only
+// them and not the nodes that were unreachable before any Plan reached them. unfinished
+// reports whether some node closed without its summary. The error, if any, is a status error;
+// the summary then carries only the preempted session ids of the nodes that finished, so an
+// aborted create can still name them. A node's summary must be its last frame: anything after
+// it fails the node as Internal.
+// CreateSession is strict: every failure is fatal, transport failures as Unavailable and a
+// node silent for exportPlanNodeMaxWait as DeadlineExceeded. Otherwise transport failures, the
+// max wait, NotFound (the node does not hold the session) and Unimplemented (the node
+// predates ExportService) degrade the node to unreachable_nodes and any other status is
+// returned as-is.
+func (s *exportService) fanOutPlan(
+ ctx context.Context, req *transferv1.PlanRequest, nodes []planNode, send func(*transferv1.PlanResponse) error,
+) (*transferv1.PlanResponse, []planNode, bool, error) {
+ strict := req.GetCreate() != nil
+ ctx, cancel := context.WithCancel(ctx)
+ defer cancel()
+ var sendMu sync.Mutex
+ sendLocked := func(frame *transferv1.PlanResponse) error {
+ sendMu.Lock()
+ defer sendMu.Unlock()
+ return send(frame)
+ }
+ var mu sync.Mutex
+ var established []planNode
+ var finished int
+ var unreachable, answered, preempted []string
+ markUnreachable := func(node string) {
+ mu.Lock()
+ unreachable = append(unreachable, node)
+ mu.Unlock()
+ }
+ nodeFailed := func(node string, err error) error {
+ kind, st := classifyNodeErr(err)
+ if strict {
+ switch kind {
+ case nodeErrTransport:
+ return nodeError(codes.Unavailable, node, st.Message())
+ case nodeErrUnsupported:
+ return unsupportedNodeError(node)
+ default:
+ return nodeError(st.Code(), node, st.Message())
+ }
+ }
+ switch kind {
+ case nodeErrTransport:
+ s.l.Warn().Err(err).Str("node", node).Msg("data node unreachable during export plan; continuing with the remaining nodes")
+ case nodeErrNotFound:
+ s.l.Warn().Err(err).Str("node", node).Str("session", sessionID(req)).
+ Msg("data node does not hold the export session; reporting it as unreachable")
+ case nodeErrUnsupported:
+ s.l.Warn().Err(err).Str("node", node).Msg("data node " + unsupportedExport + "; reporting it as unreachable")
+ default:
+ return nodeError(st.Code(), node, st.Message())
+ }
+ markUnreachable(node)
+ return nil
+ }
+
+ g, gctx := errgroup.WithContext(ctx)
+ for _, n := range nodes {
+ g.Go(func() error {
+ upCtx, cancelUp := context.WithCancelCause(gctx)
+ defer cancelUp(nil)
+ wait := time.AfterFunc(exportPlanNodeMaxWait, func() { cancelUp(errPlanNodeMaxWait) })
+ defer wait.Stop()
+ var sendErr error
+ var summarized bool
+ err := n.plan(upCtx, req, func(frame *transferv1.PlanResponse) error {
+ // The window restarts after every frame and is paused while the frame waits
+ // on the downstream client, so a slow client is never blamed on the node.
+ wait.Stop()
+ defer wait.Reset(exportPlanNodeMaxWait)
+ if summarized {
+ return status.Error(codes.Internal, "frame after summary")
+ }
+ switch f := frame.GetFrame().(type) {
+ case *transferv1.PlanResponse_Summary:
+ summarized = true
+ mu.Lock()
+ finished++
+ preempted = append(preempted, f.Summary.GetPreemptedSessionIds()...)
+ mu.Unlock()
+ return nil
+ case *transferv1.PlanResponse_Units:
+ f.Units.NodeId = n.name()
+ sendErr = sendLocked(frame)
+ return sendErr
+ default:
+ return status.Errorf(codes.Internal, "unexpected plan frame %T", frame.GetFrame())
+ }
+ })
+ wait.Stop()
+ timedOut := errors.Is(context.Cause(upCtx), errPlanNodeMaxWait)
+ var notOpened notOpenedError
+ if !errors.As(err, ¬Opened) && status.Code(err) != codes.Unimplemented {
+ // The node may have received the request, so a rollback must cover it. A node
+ // without ExportService cannot hold a session.
+ mu.Lock()
+ established = append(established, n)
+ mu.Unlock()
+ }
+ switch {
+ case err == nil:
+ mu.Lock()
+ answered = append(answered, n.name())
+ mu.Unlock()
+ return nil
+ case sendErr != nil:
+ return sendErr
+ case gctx.Err() != nil:
+ return gctx.Err() // another node or the client already failed the call
+ case timedOut && strict:
+ return nodeError(codes.DeadlineExceeded, n.name(), "no frame for "+exportPlanNodeMaxWait.String())
+ case timedOut:
+ s.l.Warn().Str("node", n.name()).Msg("no frame for " + exportPlanNodeMaxWait.String() +
+ " during export plan; reporting the data node as unreachable")
+ markUnreachable(n.name())
+ return nil
+ }
+ return nodeFailed(n.name(), err)
+ })
+ }
+ err := g.Wait()
+ summary := &transferv1.PlanSummary{PreemptedSessionIds: slices.Compact(slices.Sorted(slices.Values(preempted)))}
+ frame := &transferv1.PlanResponse{Frame: &transferv1.PlanResponse_Summary{Summary: summary}}
+ unfinished := finished < len(nodes)
+ if err != nil {
+ return frame, established, unfinished, toStatus(err).Err()
+ }
+ sort.Strings(unreachable)
+ sort.Strings(answered)
+ summary.UnreachableNodes, summary.AnsweredNodes = unreachable, answered
+ return frame, established, unfinished, nil
+}
+
+// releaseOn drops the session on the given nodes and returns the ones that failed, sorted.
+func (s *exportService) releaseOn(ctx context.Context, id string, nodes []planNode) []string {
+ req := &transferv1.SessionsRequest{Action: transferv1.SessionsRequest_ACTION_RELEASE, SessionId: id}
+ frames, _ := s.fanOutSessions(ctx, req, nodes) // ACTION_RELEASE reports every failure in a frame
+ var failed []string
+ for _, f := range frames {
+ if failedOutcome(f) {
+ failed = append(failed, f.GetNodeId())
+ }
+ }
+ return failed
+}
+
+// Sessions implements transferv1.ExportServiceServer: fan the action out and answer one
+// frame per data node, stamped with node_id, in node order.
+func (s *exportService) Sessions(req *transferv1.SessionsRequest, stream transferv1.ExportService_SessionsServer) error {
+ if err := req.ValidateAll(); err != nil {
+ return status.Error(codes.InvalidArgument, err.Error())
+ }
+ if req.GetAction() != transferv1.SessionsRequest_ACTION_LIST && req.GetSessionId() == "" {
+ return status.Errorf(codes.InvalidArgument, "action %s requires the session id", req.GetAction())
+ }
+ nodes := s.planNodes()
+ if len(nodes) == 0 {
+ return status.Error(codes.FailedPrecondition, "this liaison has not discovered any data node")
+ }
+ frames, err := s.fanOutSessions(stream.Context(), req, nodes)
+ if err != nil {
+ return err
+ }
+ for _, f := range frames {
+ if err = stream.Send(f); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// fanOutSessions runs one Sessions action on every node and returns their frames sorted by
+// node. How a node's failure is handled depends on the action. ACTION_LIST is strict: a node
+// that does not answer fails the call as Unavailable, because a client cannot tell a clean
+// node from a silent one, and a node that answers with an error (including Unimplemented from
+// a node without ExportService) fails it with that node's code, or Internal for an `error`
+// frame. ACTION_HEARTBEAT reports an
+// unreachable node or one without ExportService as an `error` frame (the next heartbeat
+// retries) and fails the call with any other error, so an expired session stops the
+// client's heartbeat. ACTION_RELEASE never fails the call for a node: every failure is
+// reported as an `error` frame so a partially released session stays visible.
+func (s *exportService) fanOutSessions(ctx context.Context, req *transferv1.SessionsRequest, nodes []planNode) ([]*transferv1.SessionsResponse, error) {
+ release := req.GetAction() == transferv1.SessionsRequest_ACTION_RELEASE
+ var mu sync.Mutex
+ list := req.GetAction() == transferv1.SessionsRequest_ACTION_LIST
+ var frames []*transferv1.SessionsResponse
+ var unreached, slow []string
+ g, gctx := errgroup.WithContext(ctx)
+ for _, n := range nodes {
+ g.Go(func() error {
+ nodeCtx, cancel := context.WithTimeout(gctx, exportSessionTimeout)
+ defer cancel()
+ frame, err := n.sessions(nodeCtx, req)
+ if err != nil {
+ if !release && gctx.Err() != nil {
+ return nil // another node or the client already failed the call
+ }
+ kind, st := classifyNodeErr(err)
+ switch {
+ case kind == nodeErrUnsupported && list:
+ return unsupportedNodeError(n.name())
+ case kind == nodeErrUnsupported:
+ frame = errorOutcome("data node " + unsupportedExport)
+ case release || (kind == nodeErrTransport && req.GetAction() == transferv1.SessionsRequest_ACTION_HEARTBEAT):
+ frame = errorOutcome(st.Message())
+ case kind == nodeErrTransport:
+ s.l.Warn().Err(err).Str("node", n.name()).Msg("data node did not answer the export session list")
+ mu.Lock()
+ if errors.Is(nodeCtx.Err(), context.DeadlineExceeded) {
+ slow = append(slow, n.name())
+ } else {
+ unreached = append(unreached, n.name())
+ }
+ mu.Unlock()
+ return nil
+ default:
+ return nodeError(st.Code(), n.name(), st.Message())
+ }
+ }
+ if list && failedOutcome(frame) {
+ return nodeError(codes.Internal, n.name(), frame.GetError())
+ }
+ frame.NodeId = n.name()
+ if failedOutcome(frame) {
+ s.l.Warn().Str("node", n.name()).Str("session", req.GetSessionId()).Stringer("action", req.GetAction()).
+ Str("error", frame.GetError()).Msg("export session action failed on a data node")
+ }
+ mu.Lock()
+ frames = append(frames, frame)
+ mu.Unlock()
+ return nil
+ })
+ }
+ if err := g.Wait(); err != nil {
+ return nil, err
+ }
+ if err := ctx.Err(); err != nil && !release {
+ return nil, toStatus(err).Err()
+ }
+ var gaps []string
+ if len(unreached) > 0 {
+ sort.Strings(unreached)
+ gaps = append(gaps, "export session list did not reach data nodes "+strings.Join(unreached, ", "))
+ }
+ if len(slow) > 0 {
+ sort.Strings(slow)
+ gaps = append(gaps, fmt.Sprintf("data nodes %s did not answer within the %s sessions timeout and may be creating or releasing a session",
+ strings.Join(slow, ", "), exportSessionTimeout))
+ }
+ if len(gaps) > 0 {
+ return nil, status.Error(codes.Unavailable, strings.Join(gaps, "; "))
+ }
+ sort.Slice(frames, func(i, j int) bool { return frames[i].GetNodeId() < frames[j].GetNodeId() })
+ return frames, nil
+}
diff --git a/banyand/liaison/grpc/export_test.go b/banyand/liaison/grpc/export_test.go
new file mode 100644
index 000000000..42430a30f
--- /dev/null
+++ b/banyand/liaison/grpc/export_test.go
@@ -0,0 +1,1560 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package grpc
+
+import (
+ "context"
+ "errors"
+ "io"
+ "net"
+ "sort"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "go.uber.org/mock/gomock"
+ grpclib "google.golang.org/grpc"
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/credentials/insecure"
+ "google.golang.org/grpc/status"
+ "google.golang.org/grpc/test/bufconn"
+
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ transferv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/transfer/v1"
+ "github.com/apache/skywalking-banyandb/banyand/queue"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+ "github.com/apache/skywalking-banyandb/pkg/test/transfer"
+)
+
+const (
+ testSessionFeedID = "feed"
+ testSessionOldID = "01d1"
+ testLocalNodeAddr = "127.0.0.1:17912"
+ testNodeA = "data-a"
+ testNodeB = "data-b"
+)
+
+// fakeDataNode is an in-memory ExportService data node: it serves one Plan frame per
+// configured group, holds at most one session, and records what the liaison asked of it.
+type fakeDataNode struct {
+ transferv1.UnimplementedExportServiceServer
+ expired map[string]bool
+ lastCreate *transferv1.PlanRequest
+ createErr error // every CreateSession Plan fails with this status
+ sessionsErr error // every Sessions call fails with this status
+ extraFrame *transferv1.PlanResponse // sent upstream before the unit frames
+ started chan struct{}
+ failAfterCh <-chan struct{} // with failAfter, Plan fails only once this is closed
+ occupant string
+ actionErr string // every Sessions action answers this error frame instead
+ groups []string
+ released []string
+ frameDelay time.Duration
+ startOnce sync.Once
+ mu sync.Mutex
+ closing bool // every Plan answers Canceled, as a connection closed by the pool does
+ stall bool // Plan sends nothing and blocks until the liaison gives up
+ stallSessions bool // Sessions sends nothing and blocks until the liaison gives up
+ failAfter bool // Plan relays every unit frame and then fails as Unavailable
+}
+
+func (f *fakeDataNode) Plan(req *transferv1.PlanRequest, stream transferv1.ExportService_PlanServer) error {
+ if f.started != nil {
+ f.startOnce.Do(func() { close(f.started) })
+ }
+ if f.stall {
+ <-stream.Context().Done()
+ return stream.Context().Err()
+ }
+ f.mu.Lock()
+ if f.closing {
+ f.mu.Unlock()
+ return status.Error(codes.Canceled, "grpc: the client connection is closing")
+ }
+ if req.GetCreate() != nil {
+ f.lastCreate = req
+ if f.createErr != nil {
+ f.mu.Unlock()
+ return f.createErr
+ }
+ if f.occupant != "" && !req.GetCreate().GetPreempt() {
+ occupant := f.occupant
+ f.mu.Unlock()
+ return status.Errorf(codes.AlreadyExists, "export session %s already exists on this node", occupant)
+ }
+ }
+ if id := req.GetRead().GetId(); req.GetRead() != nil {
+ if f.expired[id] {
+ f.mu.Unlock()
+ return status.Errorf(codes.FailedPrecondition, "export session %s expired", id)
+ }
+ if f.occupant != id {
+ f.mu.Unlock()
+ return status.Errorf(codes.NotFound, "export session %s not found on this node", id)
+ }
+ }
+ var preempted []string
+ if req.GetCreate() != nil {
+ if f.occupant != "" {
+ preempted = []string{f.occupant}
+ }
+ f.occupant = req.GetCreate().GetId()
+ }
+ f.mu.Unlock()
+ if f.extraFrame != nil {
+ if err := stream.Send(f.extraFrame); err != nil {
+ return err
+ }
+ }
+ for i, g := range f.groups {
+ if i > 0 {
+ time.Sleep(f.frameDelay)
+ }
+ if err := stream.Send(unitsFrame(g)); err != nil {
+ return err
+ }
+ }
+ if f.failAfter {
+ if f.failAfterCh != nil {
+ select {
+ case <-f.failAfterCh:
+ case <-stream.Context().Done():
+ }
+ }
+ return status.Error(codes.Unavailable, "transport is closing")
+ }
+ if req.GetCreate() != nil {
+ // A real node closes its stream with this only when it removed something; sending it
+ // always proves the liaison folds it instead of relaying it.
+ return stream.Send(summaryFrame(&transferv1.PlanSummary{PreemptedSessionIds: preempted}))
+ }
+ return nil
+}
+
+// Sessions mirrors the data node's handler: ACTION_LIST answers the occupant, ACTION_HEARTBEAT
+// answers an error frame for a session this node does not hold and fails for an expired one,
+// ACTION_RELEASE clears the occupant and answers done, or none when it holds another session
+// or nothing. With actionErr every action answers that
+// error frame instead.
+func (f *fakeDataNode) Sessions(req *transferv1.SessionsRequest, stream transferv1.ExportService_SessionsServer) error {
+ if f.stallSessions {
+ <-stream.Context().Done()
+ return stream.Context().Err()
+ }
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ if f.sessionsErr != nil {
+ return f.sessionsErr
+ }
+ id := req.GetSessionId()
+ if f.actionErr != "" {
+ return stream.Send(errorOutcome(f.actionErr))
+ }
+ switch req.GetAction() {
+ case transferv1.SessionsRequest_ACTION_LIST:
+ resp := noneOutcome()
+ if f.occupant != "" {
+ resp = sessionOutcome(f.occupant)
+ }
+ return stream.Send(resp)
+ case transferv1.SessionsRequest_ACTION_HEARTBEAT:
+ if f.expired[id] {
+ return status.Errorf(codes.FailedPrecondition, "export session %s expired", id)
+ }
+ if f.occupant != id {
+ return stream.Send(errorOutcome("export session " + id + " not found on this node"))
+ }
+ return stream.Send(doneOutcome())
+ case transferv1.SessionsRequest_ACTION_RELEASE:
+ f.released = append(f.released, id)
+ if f.occupant != id {
+ return stream.Send(&transferv1.SessionsResponse{Outcome: &transferv1.SessionsResponse_None{None: &transferv1.Ack{}}})
+ }
+ f.occupant = ""
+ return stream.Send(doneOutcome())
+ default:
+ return status.Errorf(codes.InvalidArgument, "unknown action %s", req.GetAction())
+ }
+}
+
+func (f *fakeDataNode) releasedIDs() []string {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ return append([]string(nil), f.released...)
+}
+
+func (f *fakeDataNode) createRequest() *transferv1.PlanRequest {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ return f.lastCreate
+}
+
+// serveFake serves node over an in-memory connection; a nil node serves no ExportService at
+// all, as a data node of an older version does.
+func serveFake(t *testing.T, node transferv1.ExportServiceServer) *grpclib.ClientConn {
+ t.Helper()
+ lis := bufconn.Listen(1 << 20)
+ srv := grpclib.NewServer()
+ if node != nil {
+ transferv1.RegisterExportServiceServer(srv, node)
+ }
+ //panicdiag:allow-rawgo test server
+ go func() { _ = srv.Serve(lis) }()
+ t.Cleanup(srv.Stop)
+ conn, err := grpclib.NewClient("passthrough:///bufconn",
+ grpclib.WithContextDialer(func(context.Context, string) (net.Conn, error) { return lis.Dial() }),
+ grpclib.WithTransportCredentials(insecure.NewCredentials()))
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = conn.Close() })
+ return conn
+}
+
+// deadConn is a connection whose every RPC fails with Unavailable.
+func deadConn(t *testing.T) *grpclib.ClientConn {
+ t.Helper()
+ conn, err := grpclib.NewClient("passthrough:///dead",
+ grpclib.WithContextDialer(func(context.Context, string) (net.Conn, error) { return nil, errors.New("connection refused") }),
+ grpclib.WithTransportCredentials(insecure.NewCredentials()))
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = conn.Close() })
+ return conn
+}
+
+type fakeRouteTable struct{ nodes []string }
+
+func (f fakeRouteTable) GetRouteTable() *databasev1.RouteTable {
+ rt := &databasev1.RouteTable{}
+ for _, n := range f.nodes {
+ rt.Registered = append(rt.Registered, &databasev1.Node{Metadata: &commonv1.Metadata{Name: n}})
+ }
+ return rt
+}
+
+// newFanOut builds an exportService over the given fake nodes plus optional dead nodes,
+// whose connections refuse every RPC.
+func newFanOut(t *testing.T, nodes map[string]*fakeDataNode, dead ...string) *exportService {
+ t.Helper()
+ return newFanOutWith(t, nodes, fanOutExtras{dead: dead})
+}
+
+// fanOutExtras are the registered data nodes that are not fake ExportService servers.
+type fanOutExtras struct {
+ wrap func(node string, c transferv1.ExportServiceClient) transferv1.ExportServiceClient // wraps a fake node's client
+ dead []string // every RPC fails with Unavailable
+ noClient []string // the pool cannot even hand out a client
+ old []string // the node serves no ExportService: every RPC answers Unimplemented
+}
+
+func newFanOutWith(t *testing.T, nodes map[string]*fakeDataNode, extras fanOutExtras) *exportService {
+ t.Helper()
+ ctrl := gomock.NewController(t)
+ client := queue.NewMockClient(ctrl)
+ var names []string
+ for name, node := range nodes {
+ c := transferv1.NewExportServiceClient(serveFake(t, node))
+ if extras.wrap != nil {
+ c = extras.wrap(name, c)
+ }
+ client.EXPECT().NewExportClient(name).Return(c, nil).AnyTimes()
+ names = append(names, name)
+ }
+ for _, name := range extras.dead {
+ conn := deadConn(t)
+ client.EXPECT().NewExportClient(name).Return(transferv1.NewExportServiceClient(conn), nil).AnyTimes()
+ names = append(names, name)
+ }
+ for _, name := range extras.noClient {
+ client.EXPECT().NewExportClient(name).Return(nil, errors.New("node "+name+" is not in the pool")).AnyTimes()
+ names = append(names, name)
+ }
+ for _, name := range extras.old {
+ conn := serveFake(t, nil)
+ client.EXPECT().NewExportClient(name).Return(transferv1.NewExportServiceClient(conn), nil).AnyTimes()
+ names = append(names, name)
+ }
+ sort.Strings(names)
+ svc := newExportService(client, fakeRouteTable{nodes: names}, nil, logger.GetLogger("export-test"))
+ svc.newSessionID = func() string { return testSessionFeedID }
+ return svc
+}
+
+// recordingStream captures what the liaison sends downstream.
+type recordingStream struct {
+ grpclib.ServerStream
+ ctx context.Context
+ frames []*transferv1.PlanResponse
+}
+
+func (r *recordingStream) Context() context.Context { return r.ctx }
+func (r *recordingStream) Send(f *transferv1.PlanResponse) error {
+ r.frames = append(r.frames, f)
+ return nil
+}
+
+type recordingSessionsStream struct {
+ grpclib.ServerStream
+ ctx context.Context
+ frames []*transferv1.SessionsResponse
+}
+
+func (r *recordingSessionsStream) Context() context.Context { return r.ctx }
+func (r *recordingSessionsStream) Send(f *transferv1.SessionsResponse) error {
+ r.frames = append(r.frames, f)
+ return nil
+}
+
+func listReq() *transferv1.SessionsRequest {
+ return &transferv1.SessionsRequest{Action: transferv1.SessionsRequest_ACTION_LIST}
+}
+
+func heartbeatReq(id string) *transferv1.SessionsRequest {
+ return &transferv1.SessionsRequest{Action: transferv1.SessionsRequest_ACTION_HEARTBEAT, SessionId: id}
+}
+
+func releaseReq(id string) *transferv1.SessionsRequest {
+ return &transferv1.SessionsRequest{Action: transferv1.SessionsRequest_ACTION_RELEASE, SessionId: id}
+}
+
+// sessions runs one Sessions call against svc and returns its frames.
+func sessions(t *testing.T, svc *exportService, req *transferv1.SessionsRequest) []*transferv1.SessionsResponse {
+ t.Helper()
+ rs := &recordingSessionsStream{ctx: context.Background()}
+ if err := svc.Sessions(req, rs); err != nil {
+ t.Fatalf("Sessions(%+v) = %v", req, err)
+ }
+ return rs.frames
+}
+
+// errorsByNode maps every frame's node to its error text, "" for a done frame. Any other
+// outcome fails the test: heartbeat and release answer nothing else.
+func errorsByNode(t *testing.T, frames []*transferv1.SessionsResponse) map[string]string {
+ t.Helper()
+ out := map[string]string{}
+ for _, f := range frames {
+ switch o := f.GetOutcome().(type) {
+ case *transferv1.SessionsResponse_Done, *transferv1.SessionsResponse_None:
+ // none: a release on a node that did not hold the session, still a success
+ out[f.GetNodeId()] = ""
+ case *transferv1.SessionsResponse_Error:
+ out[f.GetNodeId()] = o.Error
+ default:
+ t.Fatalf("heartbeat and release frames are done, none or error, got %+v", f)
+ }
+ }
+ return out
+}
+
+// lastSummary returns the final frame of a Plan stream, failing unless it is the summary.
+func lastSummary(t *testing.T, frames []*transferv1.PlanResponse) *transferv1.PlanSummary {
+ t.Helper()
+ last := frames[len(frames)-1]
+ if last.GetSummary() == nil {
+ t.Fatalf("the last frame must be the summary, got %+v", last)
+ }
+ return last.GetSummary()
+}
+
+func summaryFrame(s *transferv1.PlanSummary) *transferv1.PlanResponse {
+ return &transferv1.PlanResponse{Frame: &transferv1.PlanResponse_Summary{Summary: s}}
+}
+
+// unitsFrame is the units frame a data node emits for one stream group (node_id empty).
+func unitsFrame(group string) *transferv1.PlanResponse {
+ return &transferv1.PlanResponse{Frame: &transferv1.PlanResponse_Units{Units: &transferv1.UnitFrame{Units: []*transferv1.UnitInventory{{
+ Kind: &transferv1.UnitInventory_Segment{Segment: &transferv1.SegmentInventory{
+ Unit: &transferv1.SegmentUnit{Catalog: commonv1.Catalog_CATALOG_STREAM, Group: group, SegmentSuffix: "20260928", ShardIds: []uint32{0}},
+ }},
+ }}}}}
+}
+
+func sessionOutcome(id string) *transferv1.SessionsResponse {
+ return &transferv1.SessionsResponse{Outcome: &transferv1.SessionsResponse_Session{Session: &transferv1.SessionLease{SessionId: id}}}
+}
+
+func noneOutcome() *transferv1.SessionsResponse {
+ return &transferv1.SessionsResponse{Outcome: &transferv1.SessionsResponse_None{None: &transferv1.Ack{}}}
+}
+
+func doneOutcome() *transferv1.SessionsResponse {
+ return &transferv1.SessionsResponse{Outcome: &transferv1.SessionsResponse_Done{Done: &transferv1.Ack{}}}
+}
+
+func TestExportPlan_FanOutStampsNodeAndReportsUnreachable(t *testing.T) {
+ svc := newFanOut(t, map[string]*fakeDataNode{
+ testNodeA: {groups: []string{"g1", "g2"}},
+ testNodeB: {groups: []string{"g1"}},
+ }, "data-dead")
+ stream := &recordingStream{ctx: context.Background()}
+ if err := svc.Plan(&transferv1.PlanRequest{}, stream); err != nil {
+ t.Fatal(err)
+ }
+ last := lastSummary(t, stream.frames)
+ if got := last.UnreachableNodes; len(got) != 1 || got[0] != "data-dead" {
+ t.Fatalf("unreachable = %v", got)
+ }
+ if got := last.AnsweredNodes; len(got) != 2 || got[0] != testNodeA || got[1] != testNodeB {
+ t.Fatalf("answered = %v", got)
+ }
+ byNode := map[string]int{}
+ for _, f := range stream.frames[:len(stream.frames)-1] {
+ if f.GetUnits() == nil {
+ t.Fatalf("every frame before the summary is a units frame: %+v", f)
+ }
+ if f.GetUnits().GetNodeId() == "" {
+ t.Fatalf("every units frame must be stamped with node_id: %+v", f)
+ }
+ byNode[f.GetUnits().GetNodeId()]++
+ }
+ if byNode[testNodeA] != 2 || byNode[testNodeB] != 1 {
+ t.Fatalf("frames per node = %v, want a:2 b:1", byNode)
+ }
+}
+
+func TestExportPlan_NodeErrorIsFatalNotUnreachable(t *testing.T) {
+ svc := newFanOut(t, map[string]*fakeDataNode{
+ testNodeA: {groups: []string{"g"}, occupant: "abcd"},
+ testNodeB: {expired: map[string]bool{"abcd": true}},
+ })
+ err := svc.Plan(transfer.ReadRequest("abcd"), &recordingStream{ctx: context.Background()})
+ if status.Code(err) != codes.FailedPrecondition || !strings.Contains(err.Error(), testNodeB) {
+ t.Fatalf("an expired session must fail the whole call naming the node, got %v", err)
+ }
+}
+
+func TestExportPlan_StandaloneUsesLocalPlanner(t *testing.T) {
+ local := &fakeLocalPlanner{frames: 2, preempted: []string{"old"}}
+ svc := newLocalExportService(local, testLocalNodeAddr)
+ svc.newSessionID = func() string { return testSessionFeedID }
+
+ stream := &recordingStream{ctx: context.Background()}
+ if err := svc.Plan(&transferv1.PlanRequest{}, stream); err != nil {
+ t.Fatal(err)
+ }
+ if len(stream.frames) != 3 || stream.frames[0].GetUnits().GetNodeId() != testLocalNodeAddr || stream.frames[1].GetUnits().GetNodeId() != testLocalNodeAddr {
+ t.Fatalf("local frames must be stamped with the liaison's own node id: %+v", stream.frames)
+ }
+ if got := lastSummary(t, stream.frames).AnsweredNodes; len(got) != 1 || got[0] != testLocalNodeAddr {
+ t.Fatalf("answered = %v", got)
+ }
+
+ created := &recordingStream{ctx: context.Background()}
+ if err := svc.Plan(transfer.CreateRequest(nil, false), created); err != nil {
+ t.Fatal(err)
+ }
+ assertAnnouncesSession(t, created.frames[0])
+ if local.lastReq.GetCreate().GetId() != testSessionFeedID {
+ t.Fatalf("local create must pass the announced id to the planner, got %q", local.lastReq.GetCreate().GetId())
+ }
+ if got := lastSummary(t, created.frames).PreemptedSessionIds; len(got) != 1 || got[0] != "old" {
+ t.Fatalf("preempted ids must move to the final frame: %+v", lastSummary(t, created.frames))
+ }
+ for _, f := range created.frames[1 : len(created.frames)-1] {
+ if f.GetUnits() == nil {
+ t.Fatalf("the planner's summary frame must not be forwarded: %+v", f)
+ }
+ }
+}
+
+// fakeLocalPlanner is a standalone process's own ExportService, which the local pipeline
+// serves in-process.
+type fakeLocalPlanner struct {
+ transferv1.UnimplementedExportServiceServer
+ listErr error // ACTION_LIST fails with this status, as the data node's handler would
+ listFrameErr string // ACTION_LIST answers an error frame with this text
+ releaseErr error // ACTION_RELEASE reports this in the frame's error
+ planErr error
+ lastReq *transferv1.PlanRequest
+ released string
+ preempted []string
+ frames int
+ stall bool // emit nothing and block until the liaison gives up
+}
+
+func (f *fakeLocalPlanner) Plan(req *transferv1.PlanRequest, stream transferv1.ExportService_PlanServer) error {
+ f.lastReq = req
+ if f.stall {
+ <-stream.Context().Done()
+ return stream.Context().Err()
+ }
+ if f.planErr != nil {
+ return f.planErr
+ }
+ for i := 0; i < f.frames; i++ {
+ if err := stream.Send(unitsFrame("g")); err != nil {
+ return err
+ }
+ }
+ if req.GetCreate() != nil {
+ return stream.Send(summaryFrame(&transferv1.PlanSummary{PreemptedSessionIds: f.preempted}))
+ }
+ return nil
+}
+
+func (f *fakeLocalPlanner) Sessions(req *transferv1.SessionsRequest, stream transferv1.ExportService_SessionsServer) error {
+ frame := doneOutcome()
+ switch req.GetAction() {
+ case transferv1.SessionsRequest_ACTION_LIST:
+ if f.listErr != nil {
+ return f.listErr
+ }
+ frame = sessionOutcome("local")
+ if f.listFrameErr != "" {
+ frame = errorOutcome(f.listFrameErr)
+ }
+ case transferv1.SessionsRequest_ACTION_RELEASE:
+ f.released = req.GetSessionId()
+ if f.releaseErr != nil {
+ frame = errorOutcome(f.releaseErr.Error())
+ }
+ default:
+ }
+ return stream.Send(frame)
+}
+
+// newLocalExportService is a standalone liaison: no tire2 route table, and a local pipeline
+// that serves planner in-process as the node named self.
+func newLocalExportService(planner transferv1.ExportServiceServer, self string) *exportService {
+ q := queue.Local()
+ q.SetExportServer(planner)
+ return newExportService(q, nil, func() string { return self }, logger.GetLogger("export-test"))
+}
+
+func TestExportPlan_NoDataNodesIsFailedPrecondition(t *testing.T) {
+ ctrl := gomock.NewController(t)
+ svc := newExportService(queue.NewMockClient(ctrl), fakeRouteTable{}, nil, logger.GetLogger("export-test"))
+ err := svc.Plan(&transferv1.PlanRequest{}, &recordingStream{ctx: context.Background()})
+ if status.Code(err) != codes.FailedPrecondition {
+ t.Fatalf("want FailedPrecondition, got %v", err)
+ }
+}
+
+func TestExportPlan_InvalidCombinations(t *testing.T) {
+ svc := newFanOut(t, map[string]*fakeDataNode{testNodeA: {}})
+ for _, req := range []*transferv1.PlanRequest{
+ {Session: &transferv1.PlanRequest_Create{Create: &transferv1.CreateSession{Id: "abcd"}}}, // the liaison generates the id
+ {Session: &transferv1.PlanRequest_Read{Read: &transferv1.ReadSession{}}}, // needs the session id
+ {Session: &transferv1.PlanRequest_Read{Read: &transferv1.ReadSession{Id: "NOT-HEX"}}},
+ } {
+ if err := svc.Plan(req, &recordingStream{ctx: context.Background()}); status.Code(err) != codes.InvalidArgument {
+ t.Fatalf("%+v must be InvalidArgument, got %v", req, err)
+ }
+ }
+}
+
+func TestExportPlan_CreateSessionFirstFrameAndFanOut(t *testing.T) {
+ a, b := &fakeDataNode{groups: []string{"g"}}, &fakeDataNode{groups: []string{"g"}}
+ svc := newFanOut(t, map[string]*fakeDataNode{testNodeA: a, testNodeB: b})
+ stream := &recordingStream{ctx: context.Background()}
+ if err := svc.Plan(transfer.CreateRequest(nil, false), stream); err != nil {
+ t.Fatal(err)
+ }
+ assertAnnouncesSession(t, stream.frames[0])
+ if a.createRequest().GetCreate().GetId() != testSessionFeedID || b.createRequest().GetCreate().GetId() != testSessionFeedID {
+ t.Fatal("every node must receive the same generated session id with CreateSession")
+ }
+ last := lastSummary(t, stream.frames)
+ if len(last.UnreachableNodes) != 0 || len(last.AnsweredNodes) != 2 {
+ t.Fatalf("create mode never degrades: %+v", last)
+ }
+ for _, f := range stream.frames[1 : len(stream.frames)-1] {
+ if f.GetUnits() == nil {
+ t.Fatalf("node summary frames must not be forwarded: %+v", f)
+ }
+ }
+}
+
+func TestExportPlan_CreateSessionRollsBackOnOccupant(t *testing.T) {
+ a, b := &fakeDataNode{groups: []string{"g"}}, &fakeDataNode{groups: []string{"g"}, occupant: testSessionOldID}
+ svc := newFanOut(t, map[string]*fakeDataNode{testNodeA: a, testNodeB: b})
+ err := svc.Plan(transfer.CreateRequest(nil, false), &recordingStream{ctx: context.Background()})
+ if status.Code(err) != codes.AlreadyExists || !strings.Contains(err.Error(), testSessionOldID) {
+ t.Fatalf("want AlreadyExists naming the occupant, got %v", err)
+ }
+ if got := a.releasedIDs(); len(got) != 1 || got[0] != testSessionFeedID {
+ t.Fatalf("liaison must release the new session on the node that accepted it, got %v", got)
+ }
+ if b.occupant != testSessionOldID {
+ t.Fatal("the occupant must survive a failed attempt")
+ }
+}
+
+// A node whose connection the pool closed mid-call (the liaison evicting a node that just
+// went down) answers Canceled; a live Plan degrades it to unreachable like any other transport
+// failure, while a CreateSession Plan stays strict and rolls back.
+func TestExportPlan_ClosingConnectionIsUnreachable(t *testing.T) {
+ a, b := &fakeDataNode{groups: []string{"g"}}, &fakeDataNode{closing: true}
+ svc := newFanOut(t, map[string]*fakeDataNode{testNodeA: a, testNodeB: b})
+ rs := &recordingStream{ctx: context.Background()}
+ if err := svc.Plan(&transferv1.PlanRequest{}, rs); err != nil {
+ t.Fatalf("a closing upstream connection must not fail a live plan: %v", err)
+ }
+ last := lastSummary(t, rs.frames)
+ if got := last.GetUnreachableNodes(); len(got) != 1 || got[0] != testNodeB {
+ t.Fatalf("unreachable = %v, want [data-b]", got)
+ }
+ if got := last.GetAnsweredNodes(); len(got) != 1 || got[0] != testNodeA {
+ t.Fatalf("answered = %v, want [data-a]", got)
+ }
+
+ rs = &recordingStream{ctx: context.Background()}
+ err := svc.Plan(transfer.CreateRequest(nil, false), rs)
+ if status.Code(err) != codes.Unavailable || !strings.Contains(err.Error(), testNodeB) {
+ t.Fatalf("CreateSession must stay strict and report transport failures as Unavailable, got %v", err)
+ }
+ if got := a.releasedIDs(); len(got) != 1 || got[0] != testSessionFeedID {
+ t.Fatalf("the accepted node must be rolled back, got %v", got)
+ }
+}
+
+func TestExportPlan_CreateSessionUnreachableIsFatal(t *testing.T) {
+ a := &fakeDataNode{groups: []string{"g"}}
+ svc := newFanOut(t, map[string]*fakeDataNode{testNodeA: a}, "data-dead")
+ err := svc.Plan(transfer.CreateRequest(nil, false), &recordingStream{ctx: context.Background()})
+ if status.Code(err) != codes.Unavailable {
+ t.Fatalf("want Unavailable, got %v", err)
+ }
+ if got := a.releasedIDs(); len(got) != 1 {
+ t.Fatalf("the accepted node must be rolled back, got %v", got)
+ }
+ if !strings.Contains(err.Error(), "snapshots released") || strings.Contains(err.Error(), "releasing it failed") {
+ t.Fatalf("a node whose Plan never opened holds nothing and must not be reported as leaked: %v", err)
+ }
+}
+
+func TestExportPlan_PreemptedIdsAggregated(t *testing.T) {
+ a, b := &fakeDataNode{groups: []string{"g"}, occupant: testSessionOldID}, &fakeDataNode{groups: []string{"g"}, occupant: testSessionOldID}
+ svc := newFanOut(t, map[string]*fakeDataNode{testNodeA: a, testNodeB: b})
+ stream := &recordingStream{ctx: context.Background()}
+ if err := svc.Plan(transfer.CreateRequest(nil, true), stream); err != nil {
+ t.Fatal(err)
+ }
+ if got := lastSummary(t, stream.frames).PreemptedSessionIds; len(got) != 1 || got[0] != testSessionOldID {
+ t.Fatalf("preempted ids must be deduplicated across nodes: %v", got)
+ }
+ assertAnnouncesSession(t, stream.frames[0])
+}
+
+// assertAnnouncesSession checks that a CreateSession first frame is the created frame
+// carrying the generated id.
+func assertAnnouncesSession(t *testing.T, first *transferv1.PlanResponse) {
+ t.Helper()
+ if first.GetCreated() == nil || first.GetCreated().GetSessionId() != testSessionFeedID {
+ t.Fatalf("first frame must be the created frame announcing the session: %+v", first)
+ }
+}
+
+func TestExportSessions_ListAndReleaseFanOut(t *testing.T) {
+ a, b := &fakeDataNode{occupant: testSessionOldID}, &fakeDataNode{}
+ svc := newFanOut(t, map[string]*fakeDataNode{testNodeA: a, testNodeB: b})
+ listed := sessions(t, svc, listReq())
+ if len(listed) != 2 || listed[0].NodeId != testNodeA || listed[1].NodeId != testNodeB {
+ t.Fatalf("one list frame per node in node order, got %+v", listed)
+ }
+ if listed[0].GetSession().GetSessionId() != testSessionOldID || listed[1].GetNone() == nil {
+ t.Fatalf("list frames are session or none: %+v", listed)
+ }
+ released := sessions(t, svc, releaseReq(testSessionOldID))
+ if got := errorsByNode(t, released); len(got) != 2 || got[testNodeA] != "" || got[testNodeB] != "" {
+ t.Fatalf("release frames = %+v", released)
+ }
+ if a.releasedIDs()[0] != testSessionOldID || b.releasedIDs()[0] != testSessionOldID {
+ t.Fatal("release must reach every node")
+ }
+}
+
+func TestExportSessions_InvalidRequests(t *testing.T) {
+ svc := newFanOut(t, map[string]*fakeDataNode{testNodeA: {}})
+ for _, req := range []*transferv1.SessionsRequest{
+ {}, // the action is required
+ heartbeatReq(""), // heartbeat needs the session id
+ releaseReq(""), // release needs the session id
+ releaseReq("NOT-HEX"), // the id rule
+ {Action: 99, SessionId: "abcd"}, // an unknown action
+ } {
+ if err := svc.Sessions(req, &recordingSessionsStream{ctx: context.Background()}); status.Code(err) != codes.InvalidArgument {
+ t.Fatalf("%+v must be InvalidArgument, got %v", req, err)
+ }
+ }
+ // ACTION_LIST ignores the id.
+ if frames := sessions(t, svc, &transferv1.SessionsRequest{Action: transferv1.SessionsRequest_ACTION_LIST, SessionId: "abcd"}); len(frames) != 1 {
+ t.Fatalf("list frames = %+v", frames)
+ }
+}
+
+func TestExportSessions_ListUnreachableFails(t *testing.T) {
+ svc := newFanOut(t, map[string]*fakeDataNode{testNodeA: {}}, "data-dead")
+ err := svc.Sessions(listReq(), &recordingSessionsStream{ctx: context.Background()})
+ if status.Code(err) != codes.Unavailable || !strings.Contains(err.Error(), "data-dead") {
+ t.Fatalf("want Unavailable naming data-dead, got %v", err)
+ }
+}
+
+// A release relays each node's own outcome: done where the node held the session, none where
+// it did not, so the client can tell an unknown session from a released one.
+func TestExportSessions_ReleaseRelaysDoneAndNone(t *testing.T) {
+ a := &fakeDataNode{occupant: testSessionOldID}
+ b := &fakeDataNode{}
+ svc := newFanOut(t, map[string]*fakeDataNode{testNodeA: a, testNodeB: b})
+ frames := sessions(t, svc, releaseReq(testSessionOldID))
+ got := map[string]string{}
+ for _, f := range frames {
+ switch f.GetOutcome().(type) {
+ case *transferv1.SessionsResponse_Done:
+ got[f.GetNodeId()] = "done"
+ case *transferv1.SessionsResponse_None:
+ got[f.GetNodeId()] = "none"
+ default:
+ t.Fatalf("unexpected release frame %+v", f)
+ }
+ }
+ if len(got) != 2 || got[testNodeA] != "done" || got[testNodeB] != "none" {
+ t.Fatalf("want done on the holder and none elsewhere, got %v", got)
+ }
+}
+
+func TestExportSessions_ReleaseReportsFailedNodeInFrame(t *testing.T) {
+ svc := newFanOut(t, map[string]*fakeDataNode{testNodeA: {}}, "data-dead")
+ got := errorsByNode(t, sessions(t, svc, releaseReq(testSessionOldID)))
+ if len(got) != 2 || got[testNodeA] != "" || got["data-dead"] == "" {
+ t.Fatalf("an unreachable node must be reported in its own frame, not fail the release: %v", got)
+ }
+}
+
+// ACTION_HEARTBEAT: a node that does not hold the session or cannot be reached answers an
+// error frame and the call succeeds, so the client reports the gap and the next tick
+// retries; an expired session fails the call, which stops the heartbeat.
+func TestExportSessions_HeartbeatShapes(t *testing.T) {
+ const id = "abcd01ef"
+ a, b := &fakeDataNode{occupant: id}, &fakeDataNode{occupant: id}
+ svc := newFanOut(t, map[string]*fakeDataNode{testNodeA: a, testNodeB: b}, "data-dead")
+ got := errorsByNode(t, sessions(t, svc, heartbeatReq(id)))
+ if len(got) != 3 || got[testNodeA] != "" || got[testNodeB] != "" || got["data-dead"] == "" {
+ t.Fatalf("heartbeat frames = %v, want an error only for the unreachable node", got)
+ }
+
+ b.mu.Lock()
+ b.occupant = ""
+ b.mu.Unlock()
+ got = errorsByNode(t, sessions(t, svc, heartbeatReq(id)))
+ if got[testNodeA] != "" || !strings.Contains(got[testNodeB], "not found on this node") {
+ t.Fatalf("a node that lost the session answers an error frame, got %v", got)
+ }
+
+ b.expired = map[string]bool{id: true}
+ err := svc.Sessions(heartbeatReq(id), &recordingSessionsStream{ctx: context.Background()})
+ if status.Code(err) != codes.FailedPrecondition || !strings.Contains(err.Error(), "data node data-b: export session "+id+" expired") {
+ t.Fatalf("an expired session must fail the heartbeat naming the node, got %v", err)
+ }
+}
+
+func TestExportLocal_ListAndRelease(t *testing.T) {
+ local := &fakeLocalPlanner{}
+ svc := newLocalExportService(local, roleLabelSelf)
+ listed := sessions(t, svc, listReq())
+ if len(listed) != 1 || listed[0].NodeId != roleLabelSelf || listed[0].GetSession().GetSessionId() != "local" {
+ t.Fatalf("local list = %+v", listed)
+ }
+ released := sessions(t, svc, releaseReq("abcd"))
+ if len(released) != 1 || released[0].NodeId != roleLabelSelf || released[0].GetDone() == nil || local.released != "abcd" {
+ t.Fatalf("local release = %+v, released %q", released, local.released)
+ }
+}
+
+// erroringStream lets the first Send succeed and fails every subsequent one, simulating a
+// client that disconnects after receiving the first frame.
+type erroringStream struct {
+ grpclib.ServerStream
+ ctx context.Context
+ sendErr error
+ frames []*transferv1.PlanResponse
+ count int
+ mu sync.Mutex
+}
+
+func (r *erroringStream) Context() context.Context { return r.ctx }
+func (r *erroringStream) Send(f *transferv1.PlanResponse) error {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ if r.count > 0 {
+ return r.sendErr
+ }
+ r.count++
+ r.frames = append(r.frames, f)
+ return nil
+}
+
+func TestExportPlan_DownstreamSendFailureAbortsAndDoesNotHang(t *testing.T) {
+ svc := newFanOut(t, map[string]*fakeDataNode{
+ testNodeA: {groups: []string{"g1", "g2"}},
+ testNodeB: {groups: []string{"g1"}},
+ })
+ stream := &erroringStream{ctx: context.Background(), sendErr: status.Error(codes.Unavailable, "client disconnected")}
+ done := make(chan error, 1)
+ //panicdiag:allow-rawgo deadline guard for goroutine-leak detection
+ go func() { done <- svc.Plan(&transferv1.PlanRequest{}, stream) }()
+ select {
+ case err := <-done:
+ if status.Code(err) != codes.Unavailable || !strings.Contains(err.Error(), "client disconnected") {
+ t.Fatalf("Plan must return the downstream Send error, got %v", err)
+ }
+ case <-time.After(5 * time.Second):
+ t.Fatal("Plan did not return within deadline; possible goroutine leak")
+ }
+ stream.mu.Lock()
+ defer stream.mu.Unlock()
+ if len(stream.frames) != 1 || stream.frames[0].GetUnits() == nil {
+ t.Fatalf("only the first units frame may reach the client and no summary, got %+v", stream.frames)
+ }
+}
+
+func TestExportPlan_CreateSessionPartialNodeFailureRollsBack(t *testing.T) {
+ a := &fakeDataNode{groups: []string{"g"}}
+ b := &fakeDataNode{createErr: status.Error(codes.Internal, "snapshot failed")}
+ svc := newFanOut(t, map[string]*fakeDataNode{testNodeA: a, testNodeB: b})
+ err := svc.Plan(transfer.CreateRequest(nil, false), &recordingStream{ctx: context.Background()})
+ if status.Code(err) != codes.Internal {
+ t.Fatalf("want codes.Internal when a data node fails CreateSession, got %v", err)
+ }
+ if got := a.releasedIDs(); len(got) != 1 || got[0] != testSessionFeedID {
+ t.Fatalf("session must be rolled back on the node that accepted it, got %v", got)
+ }
+ a.mu.Lock()
+ occ := a.occupant
+ a.mu.Unlock()
+ if occ != "" {
+ t.Fatalf("node A occupant must be cleared after rollback, got %q", occ)
+ }
+}
+
+func TestExportLocal_ErrorPropagation(t *testing.T) {
+ // A local release failure is reported like the fan-out does: an error frame, no RPC error.
+ svc1 := newLocalExportService(&fakeLocalPlanner{releaseErr: errors.New("disk full")}, roleLabelSelf)
+ released := sessions(t, svc1, releaseReq("abcd"))
+ if len(released) != 1 || released[0].NodeId != roleLabelSelf || released[0].GetError() != "disk full" {
+ t.Fatalf("local release failure must surface in the liaison's own frame, got %+v", released)
+ }
+
+ // A local list error keeps the data node's code and gains the node prefix.
+ svc2 := newLocalExportService(&fakeLocalPlanner{listErr: status.Error(codes.Internal, "list export sessions: io error")}, roleLabelSelf)
+ err := svc2.Sessions(listReq(), &recordingSessionsStream{ctx: context.Background()})
+ if status.Code(err) != codes.Internal || !strings.Contains(err.Error(), "data node "+roleLabelSelf+": list export sessions: io error") {
+ t.Fatalf("a local list error must keep codes.Internal and name the node, got %v", err)
+ }
+
+ // A non-create Plan error keeps its code and gains the node prefix, as the fan-out does.
+ svc3 := newLocalExportService(&fakeLocalPlanner{planErr: status.Error(codes.FailedPrecondition, "export session abcd expired")}, roleLabelSelf)
+ err = svc3.Plan(transfer.ReadRequest("abcd"), &recordingStream{ctx: context.Background()})
+ if status.Code(err) != codes.FailedPrecondition || !strings.Contains(err.Error(), "data node "+roleLabelSelf+": export session abcd expired") {
+ t.Fatalf("local plan error must keep its code and name the node, got %v", err)
+ }
+}
+
+// The standalone path is the fan-out itself: a NotFound from the process's own ExportService degrades
+// the liaison itself to unreachable_nodes for ReadSession, and stays fatal (with the session
+// rolled back) for CreateSession.
+func TestExportLocal_NotFoundDegradesUnlessStrict(t *testing.T) {
+ local := &fakeLocalPlanner{planErr: status.Error(codes.NotFound, "export session abcd not found on this node")}
+ svc := newLocalExportService(local, roleLabelSelf)
+ svc.newSessionID = func() string { return testSessionFeedID }
+ {
+ req := transfer.ReadRequest("abcd")
+ stream := &recordingStream{ctx: context.Background()}
+ if err := svc.Plan(req, stream); err != nil {
+ t.Fatalf("%+v: NotFound must not fail a non-strict local plan: %v", req, err)
+ }
+ if len(stream.frames) != 1 {
+ t.Fatalf("%+v: want only the summary frame, got %+v", req, stream.frames)
+ }
+ last := lastSummary(t, stream.frames)
+ if got := last.GetUnreachableNodes(); len(got) != 1 || got[0] != roleLabelSelf {
+ t.Fatalf("%+v: unreachable = %v, want [%s]", req, got, roleLabelSelf)
+ }
+ if len(last.GetAnsweredNodes()) != 0 {
+ t.Fatalf("%+v: the degraded summary must carry no answered nodes: %+v", req, last)
+ }
+ }
+ if local.released != "" {
+ t.Fatalf("a degraded read must not release anything, released %q", local.released)
+ }
+
+ err := svc.Plan(transfer.CreateRequest(nil, false), &recordingStream{ctx: context.Background()})
+ if status.Code(err) != codes.NotFound || !strings.Contains(err.Error(), "create export session "+testSessionFeedID+" aborted") {
+ t.Fatalf("create mode must keep NotFound fatal and roll back, got %v", err)
+ }
+ if local.released != testSessionFeedID {
+ t.Fatalf("the failed create must be rolled back, released %q", local.released)
+ }
+}
+
+func TestExportLocal_PlanCreateSessionRollsBackOnPlanError(t *testing.T) {
+ local := &fakeLocalPlanner{planErr: status.Error(codes.Internal, "snapshot failed")}
+ svc := newLocalExportService(local, roleLabelSelf)
+ svc.newSessionID = func() string { return testSessionFeedID }
+
+ stream := &recordingStream{ctx: context.Background()}
+ err := svc.Plan(transfer.CreateRequest(nil, false), stream)
+ if status.Code(err) != codes.Internal {
+ t.Fatalf("the abort must keep the planner's code, got %v", err)
+ }
+ if len(stream.frames) < 1 || stream.frames[0].GetCreated().GetSessionId() != testSessionFeedID {
+ t.Fatalf("first frame must announce the session id before any error, got %+v", stream.frames)
+ }
+ if local.released != testSessionFeedID {
+ t.Fatalf("a failed local create must release the announced session id; got %q, want %q", local.released, testSessionFeedID)
+ }
+ want := "create export session " + testSessionFeedID + " aborted, snapshots released: data node " + roleLabelSelf + ": snapshot failed"
+ if err.Error() != "rpc error: code = Internal desc = "+want {
+ t.Fatalf("local abort text must match the fan-out:\n got %v\nwant %s", err, want)
+ }
+
+ // When the local release fails too, the liaison itself is the leaked node.
+ local.releaseErr = errors.New("disk full")
+ err = svc.Plan(transfer.CreateRequest(nil, false), &recordingStream{ctx: context.Background()})
+ wantLeak := "releasing it failed on [" + roleLabelSelf + "], release session " + testSessionFeedID + " by hand"
+ if status.Code(err) != codes.Internal || !strings.Contains(err.Error(), wantLeak) {
+ t.Fatalf("a failed local release must name the liaison as leaked, got %v", err)
+ }
+}
+
+// A node that fails after relaying unit frames did not finish, so it belongs to
+// unreachable_nodes and not to answered_nodes; the client drops its partial rows.
+func TestExportPlan_NodeFailingAfterFramesIsUnreachable(t *testing.T) {
+ svc := newFanOut(t, map[string]*fakeDataNode{
+ testNodeA: {groups: []string{"g"}},
+ testNodeB: {groups: []string{"g"}, failAfter: true},
+ })
+ stream := &recordingStream{ctx: context.Background()}
+ if err := svc.Plan(&transferv1.PlanRequest{}, stream); err != nil {
+ t.Fatalf("a transport failure after frames must degrade, not fail: %v", err)
+ }
+ last := lastSummary(t, stream.frames)
+ if got := last.GetUnreachableNodes(); len(got) != 1 || got[0] != testNodeB {
+ t.Fatalf("unreachable = %v, want [data-b]", got)
+ }
+ if got := last.GetAnsweredNodes(); len(got) != 1 || got[0] != testNodeA {
+ t.Fatalf("answered = %v, want [data-a]", got)
+ }
+ relayed := map[string]int{}
+ for _, f := range stream.frames[:len(stream.frames)-1] {
+ relayed[f.GetUnits().GetNodeId()]++
+ }
+ if relayed[testNodeA] != 1 || relayed[testNodeB] != 1 {
+ t.Fatalf("the failed node's frames must still be relayed with its node_id, got %v", relayed)
+ }
+}
+
+func shortenPlanNodeMaxWait(t *testing.T, window time.Duration) {
+ t.Helper()
+ saved := exportPlanNodeMaxWait
+ exportPlanNodeMaxWait = window
+ t.Cleanup(func() { exportPlanNodeMaxWait = saved })
+}
+
+// A node that stays silent past the Plan max wait is cut off by the liaison itself: a dry run
+// or ReadSession reports it as unreachable, while a CreateSession fails with DeadlineExceeded
+// and rolls back.
+func TestExportPlan_MaxWaitDegradesUnlessStrict(t *testing.T) {
+ shortenPlanNodeMaxWait(t, 500*time.Millisecond)
+ a, b := &fakeDataNode{groups: []string{"g"}}, &fakeDataNode{stall: true}
+ svc := newFanOut(t, map[string]*fakeDataNode{testNodeA: a, testNodeB: b})
+ stream := &recordingStream{ctx: context.Background()}
+ if err := svc.Plan(&transferv1.PlanRequest{}, stream); err != nil {
+ t.Fatalf("a silent node must not fail a dry run: %v", err)
+ }
+ last := lastSummary(t, stream.frames)
+ if got := last.GetUnreachableNodes(); len(got) != 1 || got[0] != testNodeB {
+ t.Fatalf("unreachable = %v, want [data-b]", got)
+ }
+ if got := last.GetAnsweredNodes(); len(got) != 1 || got[0] != testNodeA {
+ t.Fatalf("answered = %v, want [data-a]", got)
+ }
+
+ err := svc.Plan(transfer.CreateRequest(nil, false), &recordingStream{ctx: context.Background()})
+ if status.Code(err) != codes.DeadlineExceeded || !strings.Contains(err.Error(), "data node data-b: no frame for 500ms") {
+ t.Fatalf("want DeadlineExceeded naming the silent node, got %v", err)
+ }
+ if got := a.releasedIDs(); len(got) != 1 || got[0] != testSessionFeedID {
+ t.Fatalf("the create attempt must be rolled back on the node that answered, got %v", got)
+ }
+
+ local := newLocalExportService(&fakeLocalPlanner{stall: true}, roleLabelSelf)
+ stream = &recordingStream{ctx: context.Background()}
+ if err = local.Plan(&transferv1.PlanRequest{}, stream); err != nil {
+ t.Fatalf("a silent local export service must not fail a dry run: %v", err)
+ }
+ if got := lastSummary(t, stream.frames).GetUnreachableNodes(); len(got) != 1 || got[0] != roleLabelSelf {
+ t.Fatalf("the local export service is bound by the same Plan max wait, unreachable = %v", got)
+ }
+}
+
+// The Plan max wait restarts with every frame: a node whose frames each arrive within it
+// answers even when the whole plan takes longer than the window.
+func TestExportPlan_MaxWaitResetsPerFrame(t *testing.T) {
+ shortenPlanNodeMaxWait(t, 250*time.Millisecond)
+ a := &fakeDataNode{groups: []string{"g1", "g2", "g3", "g4", "g5", "g6"}, frameDelay: 100 * time.Millisecond}
+ svc := newFanOut(t, map[string]*fakeDataNode{testNodeA: a})
+ stream := &recordingStream{ctx: context.Background()}
+ start := time.Now()
+ if err := svc.Plan(&transferv1.PlanRequest{}, stream); err != nil {
+ t.Fatal(err)
+ }
+ if elapsed := time.Since(start); elapsed < 2*exportPlanNodeMaxWait {
+ t.Fatalf("the plan must outlast two max waits to prove the reset, took %v", elapsed)
+ }
+ last := lastSummary(t, stream.frames)
+ if got := last.GetAnsweredNodes(); len(got) != 1 || got[0] != testNodeA || len(last.GetUnreachableNodes()) != 0 {
+ t.Fatalf("a node streaming within the window must answer: %+v", last)
+ }
+ if len(stream.frames) != 7 {
+ t.Fatalf("want 6 units frames and the summary, got %d frames", len(stream.frames))
+ }
+}
+
+// slowSendStream blocks every Send for delay, as a slow downstream client does.
+type slowSendStream struct {
+ recordingStream
+ delay time.Duration
+}
+
+func (r *slowSendStream) Send(f *transferv1.PlanResponse) error {
+ time.Sleep(r.delay)
+ return r.recordingStream.Send(f)
+}
+
+// The Plan max wait is paused while a frame waits on the downstream client, so a client slower
+// than the window is never blamed on the node.
+func TestExportPlan_SlowDownstreamIsNotUnreachable(t *testing.T) {
+ shortenPlanNodeMaxWait(t, 200*time.Millisecond)
+ svc := newFanOut(t, map[string]*fakeDataNode{testNodeA: {groups: []string{"g1", "g2"}}})
+ stream := &slowSendStream{recordingStream: recordingStream{ctx: context.Background()}, delay: 500 * time.Millisecond}
+ if err := svc.Plan(&transferv1.PlanRequest{}, stream); err != nil {
+ t.Fatal(err)
+ }
+ last := lastSummary(t, stream.frames)
+ if got := last.GetAnsweredNodes(); len(got) != 1 || got[0] != testNodeA || len(last.GetUnreachableNodes()) != 0 {
+ t.Fatalf("a slow client must not mark the node unreachable: %+v", last)
+ }
+ if len(stream.frames) != 3 {
+ t.Fatalf("want 2 units frames and the summary, got %d frames", len(stream.frames))
+ }
+}
+
+// eofSignalingClient closes eof once a Plan stream it opened ends, that is after the liaison
+// handled every frame of it, summary included.
+type eofSignalingClient struct {
+ transferv1.ExportServiceClient
+ eof chan struct{}
+}
+
+func (c eofSignalingClient) Plan(ctx context.Context, in *transferv1.PlanRequest,
+ opts ...grpclib.CallOption,
+) (grpclib.ServerStreamingClient[transferv1.PlanResponse], error) {
+ up, err := c.ExportServiceClient.Plan(ctx, in, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return eofSignalingStream{ServerStreamingClient: up, eof: c.eof}, nil
+}
+
+type eofSignalingStream struct {
+ grpclib.ServerStreamingClient[transferv1.PlanResponse]
+ eof chan struct{}
+}
+
+func (s eofSignalingStream) Recv() (*transferv1.PlanResponse, error) {
+ frame, err := s.ServerStreamingClient.Recv()
+ if errors.Is(err, io.EOF) {
+ close(s.eof)
+ }
+ return frame, err
+}
+
+// A create that fails after one node finished, preempting a session there, names the preempted
+// session, which is gone even though the new one was rolled back, and warns that the node that
+// did not finish may have removed sessions too. Node b fails only once the liaison handled node
+// a's whole stream.
+func TestExportPlan_CreateSessionAbortNamesPreempted(t *testing.T) {
+ aDone := make(chan struct{})
+ a := &fakeDataNode{groups: []string{"g"}, occupant: testSessionOldID}
+ b := &fakeDataNode{groups: []string{"g1"}, failAfter: true, failAfterCh: aDone}
+ svc := newFanOutWith(t, map[string]*fakeDataNode{testNodeA: a, testNodeB: b}, fanOutExtras{
+ wrap: func(node string, c transferv1.ExportServiceClient) transferv1.ExportServiceClient {
+ if node != testNodeA {
+ return c
+ }
+ return eofSignalingClient{ExportServiceClient: c, eof: aDone}
+ },
+ })
+ err := svc.Plan(transfer.CreateRequest(nil, true), &recordingStream{ctx: context.Background()})
+ if status.Code(err) != codes.Unavailable || !strings.Contains(err.Error(), "preempted sessions ["+testSessionOldID+"] were already removed") {
+ t.Fatalf("want Unavailable naming the preempted session, got %v", err)
+ }
+ if !strings.Contains(err.Error(), "nodes that did not finish may also have removed preempted sessions, check with ACTION_LIST") {
+ t.Fatalf("an unfinished node may have preempted sessions too, got %v", err)
+ }
+ if got := a.releasedIDs(); len(got) != 1 || got[0] != testSessionFeedID {
+ t.Fatalf("the failed create must be rolled back, got %v", got)
+ }
+}
+
+// A node that never held the session answers NotFound: a coverage gap for a ReadSession Plan
+// (the client reports it as unreachable), still fatal when creating.
+func TestExportPlan_NotFoundDegradesUnlessStrict(t *testing.T) {
+ a, b := &fakeDataNode{groups: []string{"g"}, occupant: "abcd"}, &fakeDataNode{}
+ svc := newFanOut(t, map[string]*fakeDataNode{testNodeA: a, testNodeB: b})
+ {
+ req := transfer.ReadRequest("abcd")
+ stream := &recordingStream{ctx: context.Background()}
+ if err := svc.Plan(req, stream); err != nil {
+ t.Fatalf("%+v: NotFound must not fail a non-strict plan: %v", req, err)
+ }
+ last := lastSummary(t, stream.frames)
+ if got := last.GetUnreachableNodes(); len(got) != 1 || got[0] != testNodeB {
+ t.Fatalf("%+v: unreachable = %v, want [data-b]", req, got)
+ }
+ if got := last.GetAnsweredNodes(); len(got) != 1 || got[0] != testNodeA {
+ t.Fatalf("%+v: answered = %v, want [data-a]", req, got)
+ }
+ }
+
+ a.mu.Lock()
+ a.occupant = ""
+ a.mu.Unlock()
+ b.createErr = status.Error(codes.NotFound, "no such thing")
+ err := svc.Plan(transfer.CreateRequest(nil, false), &recordingStream{ctx: context.Background()})
+ if status.Code(err) != codes.NotFound || !strings.Contains(err.Error(), "data node data-b") {
+ t.Fatalf("create mode must keep NotFound fatal, got %v", err)
+ }
+ if got := a.releasedIDs(); len(got) != 1 || got[0] != testSessionFeedID {
+ t.Fatalf("the accepted node must be rolled back, got %v", got)
+ }
+}
+
+// Canceling the downstream stream mid-create releases the session on every node whose Plan
+// stream was opened, and the error carries the context's own code rather than Unknown.
+func TestExportPlan_CreateSessionCanceledDownstreamReleasesEveryNode(t *testing.T) {
+ a, b := &fakeDataNode{groups: []string{"g"}}, &fakeDataNode{stall: true, started: make(chan struct{})}
+ svc := newFanOut(t, map[string]*fakeDataNode{testNodeA: a, testNodeB: b})
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ //panicdiag:allow-rawgo test driver cancels once the stalled node has opened its stream
+ go func() {
+ <-b.started
+ cancel()
+ }()
+ err := svc.Plan(transfer.CreateRequest(nil, false), &recordingStream{ctx: ctx})
+ if status.Code(err) != codes.Canceled || !strings.Contains(err.Error(), "create export session "+testSessionFeedID+" aborted, snapshots released") {
+ t.Fatalf("want Canceled with the abort text, got %v", err)
+ }
+ for name, node := range map[string]*fakeDataNode{testNodeA: a, testNodeB: b} {
+ if got := node.releasedIDs(); len(got) != 1 || got[0] != testSessionFeedID {
+ t.Fatalf("%s must be released after a canceled create, got %v", name, got)
+ }
+ }
+}
+
+func TestExportSessions_ListNodeErrorKeepsCode(t *testing.T) {
+ svc := newFanOut(t, map[string]*fakeDataNode{
+ testNodeA: {},
+ testNodeB: {sessionsErr: status.Error(codes.PermissionDenied, "not allowed")},
+ })
+ err := svc.Sessions(listReq(), &recordingSessionsStream{ctx: context.Background()})
+ if status.Code(err) != codes.PermissionDenied || !strings.Contains(err.Error(), "data node data-b: not allowed") {
+ t.Fatalf("a node that answers with an error must fail the list with its own code, got %v", err)
+ }
+}
+
+// summaryFailingStream records every frame and fails the summary, as a client that drops
+// right before the end of a CreateSession does.
+type summaryFailingStream struct {
+ recordingStream
+ mu sync.Mutex
+}
+
+func (r *summaryFailingStream) Send(f *transferv1.PlanResponse) error {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ if f.GetSummary() != nil {
+ return status.Error(codes.Unavailable, "client went away")
+ }
+ return r.recordingStream.Send(f)
+}
+
+// Without the summary the client cannot use the session, so a failed final Send rolls the
+// session back on every node like any other failure.
+func TestExportPlan_CreateSessionSummarySendFailureRollsBack(t *testing.T) {
+ a, b := &fakeDataNode{groups: []string{"g"}}, &fakeDataNode{groups: []string{"g"}}
+ svc := newFanOut(t, map[string]*fakeDataNode{testNodeA: a, testNodeB: b})
+ err := svc.Plan(transfer.CreateRequest(nil, false), &summaryFailingStream{recordingStream: recordingStream{ctx: context.Background()}})
+ if status.Code(err) != codes.Unavailable || !strings.Contains(err.Error(), "create export session "+testSessionFeedID+" aborted, snapshots released: client went away") {
+ t.Fatalf("want Unavailable with the abort text, got %v", err)
+ }
+ for name, node := range map[string]*fakeDataNode{testNodeA: a, testNodeB: b} {
+ if got := node.releasedIDs(); len(got) != 1 || got[0] != testSessionFeedID {
+ t.Fatalf("%s must be released when the summary cannot be sent, got %v", name, got)
+ }
+ }
+}
+
+// A node the pool cannot hand a client for never saw the request: a dry run reports it as
+// unreachable, and a failed create neither releases nor reports it as leaked even when other
+// nodes already failed the call.
+func TestExportPlan_NodeWithoutClient(t *testing.T) {
+ a := &fakeDataNode{groups: []string{"g"}}
+ svc := newFanOutWith(t, map[string]*fakeDataNode{testNodeA: a}, fanOutExtras{noClient: []string{"data-gone1", "data-gone2"}})
+ stream := &recordingStream{ctx: context.Background()}
+ if err := svc.Plan(&transferv1.PlanRequest{}, stream); err != nil {
+ t.Fatal(err)
+ }
+ last := lastSummary(t, stream.frames)
+ if got := last.GetUnreachableNodes(); strings.Join(got, ",") != "data-gone1,data-gone2" {
+ t.Fatalf("unreachable = %v, want both nodes without a client", got)
+ }
+ if got := last.GetAnsweredNodes(); len(got) != 1 || got[0] != testNodeA {
+ t.Fatalf("answered = %v, want [data-a]", got)
+ }
+
+ err := svc.Plan(transfer.CreateRequest(nil, false), &recordingStream{ctx: context.Background()})
+ if status.Code(err) != codes.Unavailable || !strings.Contains(err.Error(), "aborted, snapshots released") || strings.Contains(err.Error(), "releasing it failed") {
+ t.Fatalf("nodes that never got a client must not be reported as leaked, got %v", err)
+ }
+ if !strings.Contains(err.Error(), "is not in the pool") || strings.Count(err.Error(), "rpc error") != 1 {
+ t.Fatalf("the pool error must appear once, without a nested status prefix, got %v", err)
+ }
+ if got := a.releasedIDs(); len(got) != 1 || got[0] != testSessionFeedID {
+ t.Fatalf("the accepted node must be rolled back, got %v", got)
+ }
+}
+
+// A node that releases with an error is named in the abort as leaked.
+func TestExportPlan_CreateSessionReleaseFailureIsLeaked(t *testing.T) {
+ a := &fakeDataNode{groups: []string{"g"}, actionErr: "remove lease: busy"}
+ b := &fakeDataNode{createErr: status.Error(codes.Internal, "snapshot failed")}
+ svc := newFanOut(t, map[string]*fakeDataNode{testNodeA: a, testNodeB: b})
+ err := svc.Plan(transfer.CreateRequest(nil, false), &recordingStream{ctx: context.Background()})
+ wantLeak := "releasing it failed on [" + testNodeA + "], release session " + testSessionFeedID + " by hand"
+ if status.Code(err) != codes.Internal || !strings.Contains(err.Error(), wantLeak) {
+ t.Fatalf("want Internal naming data-a as leaked, got %v", err)
+ }
+
+ got := errorsByNode(t, sessions(t, svc, releaseReq(testSessionFeedID)))
+ if got[testNodeA] != "remove lease: busy" || got[testNodeB] != "" {
+ t.Fatalf("release must answer the node's error frame, got %v", got)
+ }
+ got = errorsByNode(t, sessions(t, svc, heartbeatReq(testSessionFeedID)))
+ if got[testNodeA] != "remove lease: busy" {
+ t.Fatalf("heartbeat must answer the node's error frame, got %v", got)
+ }
+}
+
+// A data node of an older version serves no ExportService and answers Unimplemented: a dry
+// run reports it as unreachable, heartbeat and release answer an error frame for it, while
+// list and create fail.
+func TestExport_DataNodeWithoutExportService(t *testing.T) {
+ a := &fakeDataNode{groups: []string{"g"}}
+ svc := newFanOutWith(t, map[string]*fakeDataNode{testNodeA: a}, fanOutExtras{old: []string{"data-old"}})
+ stream := &recordingStream{ctx: context.Background()}
+ if err := svc.Plan(&transferv1.PlanRequest{}, stream); err != nil {
+ t.Fatalf("an old data node must not fail a dry run: %v", err)
+ }
+ last := lastSummary(t, stream.frames)
+ if got := last.GetUnreachableNodes(); len(got) != 1 || got[0] != "data-old" {
+ t.Fatalf("unreachable = %v, want [data-old]", got)
+ }
+
+ err := svc.Plan(transfer.CreateRequest(nil, false), &recordingStream{ctx: context.Background()})
+ if status.Code(err) != codes.Unimplemented || !strings.Contains(err.Error(), "aborted, snapshots released: data node data-old does not support export (older version)") {
+ t.Fatalf("create must fail with Unimplemented naming the old node and not treat it as leaked, got %v", err)
+ }
+
+ err = svc.Sessions(listReq(), &recordingSessionsStream{ctx: context.Background()})
+ if status.Code(err) != codes.Unimplemented || !strings.Contains(err.Error(), "data node data-old does not support export (older version)") {
+ t.Fatalf("list must fail naming the old node, got %v", err)
+ }
+ for _, req := range []*transferv1.SessionsRequest{heartbeatReq(testSessionFeedID), releaseReq(testSessionFeedID)} {
+ got := errorsByNode(t, sessions(t, svc, req))
+ if !strings.Contains(got["data-old"], "does not support export") {
+ t.Fatalf("%s must answer an error frame for the old node, got %v", req.GetAction(), got)
+ }
+ }
+}
+
+// An error frame in a node's ACTION_LIST answer must fail the list, not be hidden by `none`.
+func TestExportSessions_ListUpstreamErrorFrameFails(t *testing.T) {
+ svc := newFanOut(t, map[string]*fakeDataNode{
+ testNodeA: {},
+ testNodeB: {actionErr: "read lease: io error"},
+ })
+ err := svc.Sessions(listReq(), &recordingSessionsStream{ctx: context.Background()})
+ if status.Code(err) != codes.Internal || !strings.Contains(err.Error(), "data node data-b: read lease: io error") {
+ t.Fatalf("want Internal naming data-b, got %v", err)
+ }
+}
+
+// A node that never answers a Sessions call is cut off by the per-node timeouts: list fails
+// as Unavailable, heartbeat and release answer an error frame for it.
+func TestExportSessions_StalledNode(t *testing.T) {
+ saved := exportSessionTimeout
+ exportSessionTimeout = 200 * time.Millisecond
+ t.Cleanup(func() { exportSessionTimeout = saved })
+ svc := newFanOut(t, map[string]*fakeDataNode{
+ testNodeA: {occupant: testSessionOldID},
+ testNodeB: {stallSessions: true},
+ })
+ err := svc.Sessions(listReq(), &recordingSessionsStream{ctx: context.Background()})
+ want := "data nodes data-b did not answer within the 200ms sessions timeout and may be creating or releasing a session"
+ if status.Code(err) != codes.Unavailable || !strings.Contains(err.Error(), want) {
+ t.Fatalf("want Unavailable naming data-b as slow, got %v", err)
+ }
+ for _, req := range []*transferv1.SessionsRequest{heartbeatReq(testSessionOldID), releaseReq(testSessionOldID)} {
+ got := errorsByNode(t, sessions(t, svc, req))
+ if len(got) != 2 || got[testNodeA] != "" || got[testNodeB] == "" {
+ t.Fatalf("%s must answer an error frame only for the stalled node, got %v", req.GetAction(), got)
+ }
+ }
+}
+
+// A data node only streams units and summary frames; anything else is a broken node.
+func TestExportPlan_UnexpectedUpstreamFrameIsInternal(t *testing.T) {
+ created := &transferv1.PlanResponse{Frame: &transferv1.PlanResponse_Created{Created: &transferv1.SessionCreated{SessionId: "abcd"}}}
+ svc := newFanOut(t, map[string]*fakeDataNode{testNodeA: {groups: []string{"g"}, extraFrame: created}})
+ err := svc.Plan(&transferv1.PlanRequest{}, &recordingStream{ctx: context.Background()})
+ if status.Code(err) != codes.Internal || !strings.Contains(err.Error(), "data node data-a: unexpected plan frame") {
+ t.Fatalf("want Internal naming the node, got %v", err)
+ }
+}
+
+// Any heartbeat failure other than an unreachable node fails the call with the node's code.
+func TestExportSessions_HeartbeatInternalFails(t *testing.T) {
+ svc := newFanOut(t, map[string]*fakeDataNode{
+ testNodeA: {occupant: testSessionOldID},
+ testNodeB: {sessionsErr: status.Error(codes.Internal, "read lease: io error")},
+ })
+ err := svc.Sessions(heartbeatReq(testSessionOldID), &recordingSessionsStream{ctx: context.Background()})
+ if status.Code(err) != codes.Internal || !strings.Contains(err.Error(), "data node data-b: read lease: io error") {
+ t.Fatalf("want Internal naming data-b, got %v", err)
+ }
+}
+
+// A node the pool cannot hand a client for: list fails naming it, heartbeat and release answer
+// an error frame for it.
+func TestExportSessions_NodeWithoutClient(t *testing.T) {
+ svc := newFanOutWith(t, map[string]*fakeDataNode{testNodeA: {occupant: testSessionOldID}}, fanOutExtras{noClient: []string{"data-gone"}})
+ err := svc.Sessions(listReq(), &recordingSessionsStream{ctx: context.Background()})
+ if status.Code(err) != codes.Unavailable || !strings.Contains(err.Error(), "did not reach data nodes data-gone") {
+ t.Fatalf("want Unavailable naming data-gone, got %v", err)
+ }
+ for _, req := range []*transferv1.SessionsRequest{heartbeatReq(testSessionOldID), releaseReq(testSessionOldID)} {
+ got := errorsByNode(t, sessions(t, svc, req))
+ if len(got) != 2 || got[testNodeA] != "" || !strings.Contains(got["data-gone"], "is not in the pool") {
+ t.Fatalf("%s must answer an error frame for data-gone, got %v", req.GetAction(), got)
+ }
+ }
+}
+
+func TestExportSessions_ListCanceled(t *testing.T) {
+ svc := newFanOut(t, map[string]*fakeDataNode{testNodeA: {}, testNodeB: {}})
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+ if err := svc.Sessions(listReq(), &recordingSessionsStream{ctx: ctx}); status.Code(err) != codes.Canceled {
+ t.Fatalf("a canceled list must answer Canceled, got %v", err)
+ }
+}
+
+func TestExportSessions_NoDataNodesIsFailedPrecondition(t *testing.T) {
+ ctrl := gomock.NewController(t)
+ svc := newExportService(queue.NewMockClient(ctrl), fakeRouteTable{}, nil, logger.GetLogger("export-test"))
+ if err := svc.Sessions(listReq(), &recordingSessionsStream{ctx: context.Background()}); status.Code(err) != codes.FailedPrecondition {
+ t.Fatalf("want FailedPrecondition, got %v", err)
+ }
+}
+
+// The local node fails a list on an error frame like a remote one.
+func TestExportLocal_ListErrorFrameFails(t *testing.T) {
+ svc := newLocalExportService(&fakeLocalPlanner{listFrameErr: "read lease: io error"}, roleLabelSelf)
+ err := svc.Sessions(listReq(), &recordingSessionsStream{ctx: context.Background()})
+ if status.Code(err) != codes.Internal || !strings.Contains(err.Error(), "data node "+roleLabelSelf+": read lease: io error") {
+ t.Fatalf("want Internal naming the liaison, got %v", err)
+ }
+}
+
+// emptySessionsNode answers every Sessions call with an end of stream and no frame.
+type emptySessionsNode struct {
+ transferv1.UnimplementedExportServiceServer
+}
+
+func (emptySessionsNode) Sessions(*transferv1.SessionsRequest, transferv1.ExportService_SessionsServer) error {
+ return nil
+}
+
+// twoFramesSessionsNode answers every Sessions call with two frames.
+type twoFramesSessionsNode struct {
+ transferv1.UnimplementedExportServiceServer
+}
+
+func (twoFramesSessionsNode) Sessions(_ *transferv1.SessionsRequest, stream transferv1.ExportService_SessionsServer) error {
+ if err := stream.Send(noneOutcome()); err != nil {
+ return err
+ }
+ return stream.Send(sessionOutcome(testSessionOldID))
+}
+
+// A data node answers exactly one frame; a second one is a protocol violation.
+func TestExportSessions_SecondFrameIsInternal(t *testing.T) {
+ ctrl := gomock.NewController(t)
+ client := queue.NewMockClient(ctrl)
+ client.EXPECT().NewExportClient(testNodeA).Return(transferv1.NewExportServiceClient(serveFake(t, twoFramesSessionsNode{})), nil).AnyTimes()
+ svc := newExportService(client, fakeRouteTable{nodes: []string{testNodeA}}, nil, logger.GetLogger("export-test"))
+ err := svc.Sessions(listReq(), &recordingSessionsStream{ctx: context.Background()})
+ if status.Code(err) != codes.Internal || !strings.Contains(err.Error(), "data node data-a: more than one sessions frame in the stream") {
+ t.Fatalf("want Internal naming data-a, got %v", err)
+ }
+}
+
+func TestExportSessions_NoOutcomeIsInternal(t *testing.T) {
+ ctrl := gomock.NewController(t)
+ client := queue.NewMockClient(ctrl)
+ client.EXPECT().NewExportClient(testNodeA).Return(transferv1.NewExportServiceClient(serveFake(t, emptySessionsNode{})), nil).AnyTimes()
+ svc := newExportService(client, fakeRouteTable{nodes: []string{testNodeA}}, nil, logger.GetLogger("export-test"))
+ err := svc.Sessions(listReq(), &recordingSessionsStream{ctx: context.Background()})
+ if status.Code(err) != codes.Internal || !strings.Contains(err.Error(), "data node data-a: no sessions outcome in the stream") {
+ t.Fatalf("want Internal naming data-a, got %v", err)
+ }
+ if got := errorsByNode(t, sessions(t, svc, releaseReq(testSessionOldID))); !strings.Contains(got[testNodeA], "no sessions outcome") {
+ t.Fatalf("release must answer an error frame for a node without an outcome, got %v", got)
+ }
+}
+
+func TestExportLocal_Heartbeat(t *testing.T) {
+ svc := newLocalExportService(&fakeLocalPlanner{}, roleLabelSelf)
+ frames := sessions(t, svc, heartbeatReq("abcd"))
+ if len(frames) != 1 || frames[0].NodeId != roleLabelSelf || frames[0].GetDone() == nil {
+ t.Fatalf("local heartbeat = %+v", frames)
+ }
+}
+
+func TestMethodPolicies_CoverExportService(t *testing.T) {
+ var got []string
+ for _, p := range GlobalMethodPolicies() {
+ if strings.HasPrefix(p.FullMethod, "/banyandb.transfer.v1.ExportService/") {
+ got = append(got, p.FullMethod)
+ }
+ }
+ sort.Strings(got)
+ want := []string{
+ "/banyandb.transfer.v1.ExportService/Plan",
+ "/banyandb.transfer.v1.ExportService/Sessions",
+ }
+ if strings.Join(got, ",") != strings.Join(want, ",") {
+ t.Fatalf("export policies = %v, want %v", got, want)
+ }
+}
+
+// When every node finished, the preempted list is complete and the abort carries no hint.
+func TestExportPlan_CreateSessionAbortAfterEveryNodeFinishedHasNoHint(t *testing.T) {
+ a, b := &fakeDataNode{groups: []string{"g"}, occupant: testSessionOldID}, &fakeDataNode{groups: []string{"g"}}
+ svc := newFanOut(t, map[string]*fakeDataNode{testNodeA: a, testNodeB: b})
+ err := svc.Plan(transfer.CreateRequest(nil, true), &summaryFailingStream{recordingStream: recordingStream{ctx: context.Background()}})
+ if status.Code(err) != codes.Unavailable || !strings.Contains(err.Error(), "preempted sessions ["+testSessionOldID+"] were already removed") {
+ t.Fatalf("want Unavailable naming the preempted session, got %v", err)
+ }
+ if strings.Contains(err.Error(), "did not finish") {
+ t.Fatalf("every node finished, so the preempted list is complete: %v", err)
+ }
+}
+
+// A create that reached no node established nothing and must not claim snapshots were released.
+func TestExportPlan_CreateSessionReachingNoNodeHasNothingToRelease(t *testing.T) {
+ svc := newFanOutWith(t, nil, fanOutExtras{noClient: []string{"data-gone"}})
+ err := svc.Plan(transfer.CreateRequest(nil, false), &recordingStream{ctx: context.Background()})
+ want := "create export session " + testSessionFeedID + " aborted, nothing to release: data node data-gone: "
+ if status.Code(err) != codes.Unavailable || !strings.Contains(err.Error(), want) {
+ t.Fatalf("want Unavailable saying nothing to release, got %v", err)
+ }
+}
+
+// planFramesNode answers every Plan with the given frames.
+type planFramesNode struct {
+ transferv1.UnimplementedExportServiceServer
+ frames []*transferv1.PlanResponse
+}
+
+func (n planFramesNode) Plan(_ *transferv1.PlanRequest, stream transferv1.ExportService_PlanServer) error {
+ for _, f := range n.frames {
+ if err := stream.Send(f); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// A data node's summary is its last frame: any frame after it, a second summary included,
+// fails the node as Internal.
+func TestExportPlan_FrameAfterSummaryIsInternal(t *testing.T) {
+ for name, after := range map[string]*transferv1.PlanResponse{
+ "units": unitsFrame("g"),
+ "summary": summaryFrame(&transferv1.PlanSummary{}),
+ } {
+ t.Run(name, func(t *testing.T) {
+ ctrl := gomock.NewController(t)
+ client := queue.NewMockClient(ctrl)
+ node := planFramesNode{frames: []*transferv1.PlanResponse{summaryFrame(&transferv1.PlanSummary{}), after}}
+ client.EXPECT().NewExportClient(testNodeA).Return(transferv1.NewExportServiceClient(serveFake(t, node)), nil).AnyTimes()
+ svc := newExportService(client, fakeRouteTable{nodes: []string{testNodeA}}, nil, logger.GetLogger("export-test"))
+ err := svc.Plan(&transferv1.PlanRequest{}, &recordingStream{ctx: context.Background()})
+ if status.Code(err) != codes.Internal || !strings.Contains(err.Error(), "data node data-a: frame after summary") {
+ t.Fatalf("want Internal naming data-a, got %v", err)
+ }
+ })
+ }
+}
+
+// A standalone pipeline without a registered ExportService is unsupported, like a data node
+// predating it, not unreachable.
+func TestExportLocal_WithoutExportServerIsUnsupported(t *testing.T) {
+ svc := newExportService(queue.Local(), nil, func() string { return roleLabelSelf }, logger.GetLogger("export-test"))
+ svc.newSessionID = func() string { return testSessionFeedID }
+ stream := &recordingStream{ctx: context.Background()}
+ if err := svc.Plan(&transferv1.PlanRequest{}, stream); err != nil {
+ t.Fatalf("an unsupported node must not fail a dry run: %v", err)
+ }
+ if got := lastSummary(t, stream.frames).GetUnreachableNodes(); len(got) != 1 || got[0] != roleLabelSelf {
+ t.Fatalf("unreachable = %v, want [%s]", got, roleLabelSelf)
+ }
+
+ err := svc.Plan(transfer.CreateRequest(nil, false), &recordingStream{ctx: context.Background()})
+ want := "aborted, nothing to release: data node " + roleLabelSelf + " " + unsupportedExport
+ if status.Code(err) != codes.Unimplemented || !strings.Contains(err.Error(), want) {
+ t.Fatalf("create must fail as Unimplemented, got %v", err)
+ }
+
+ err = svc.Sessions(listReq(), &recordingSessionsStream{ctx: context.Background()})
+ if status.Code(err) != codes.Unimplemented || !strings.Contains(err.Error(), unsupportedExport) {
+ t.Fatalf("list must fail as Unimplemented, got %v", err)
+ }
+ got := errorsByNode(t, sessions(t, svc, heartbeatReq(testSessionFeedID)))
+ if got[roleLabelSelf] != "data node "+unsupportedExport {
+ t.Fatalf("heartbeat must answer the unsupported error frame, got %v", got)
+ }
+}
diff --git a/banyand/liaison/grpc/methodpolicy.go b/banyand/liaison/grpc/methodpolicy.go
index a41e4ee6d..058fb261a 100644
--- a/banyand/liaison/grpc/methodpolicy.go
+++ b/banyand/liaison/grpc/methodpolicy.go
@@ -39,6 +39,7 @@ import (
schemav1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/schema/v1"
streamv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/stream/v1"
tracev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/trace/v1"
+ transferv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/transfer/v1"
"github.com/apache/skywalking-banyandb/banyand/liaison/pkg/auth"
)
@@ -217,6 +218,7 @@ func registeredServiceDescriptors(activation ServiceActivation) []grpclib.Servic
databasev1.TraceRegistryService_ServiceDesc,
databasev1.ClusterStateService_ServiceDesc,
databasev1.NodeQueryService_ServiceDesc,
+ transferv1.ExportService_ServiceDesc,
grpc_health_v1.Health_ServiceDesc,
}
if activation.SchemaBarrier {
@@ -246,6 +248,8 @@ func GlobalMethodPolicies() MethodPolicyTable {
permissionPolicy("/banyandb.database.v1.GroupRegistryService/Inspect", auth.PermissionClusterRead, ScopeGlobal),
permissionPolicy("/banyandb.database.v1.GroupRegistryService/Query", auth.PermissionClusterRead, ScopeGlobal),
permissionPolicy("/banyandb.database.v1.SnapshotService/Snapshot", auth.PermissionClusterAdmin, ScopeGlobal),
+ permissionPolicy("/banyandb.transfer.v1.ExportService/Plan", auth.PermissionClusterAdmin, ScopeGlobal),
+ permissionPolicy("/banyandb.transfer.v1.ExportService/Sessions", auth.PermissionClusterAdmin, ScopeGlobal),
permissionPolicy("/banyandb.stream.v1.StreamService/DeleteExpiredSegments", auth.PermissionClusterAdmin, ScopeGlobal),
permissionPolicy("/banyandb.measure.v1.MeasureService/DeleteExpiredSegments", auth.PermissionClusterAdmin, ScopeGlobal),
permissionPolicy("/banyandb.trace.v1.TraceService/DeleteExpiredSegments", auth.PermissionClusterAdmin, ScopeGlobal),
diff --git a/banyand/liaison/grpc/methodpolicy_contract_test.go b/banyand/liaison/grpc/methodpolicy_contract_test.go
index e547686d0..acb8c4eff 100644
--- a/banyand/liaison/grpc/methodpolicy_contract_test.go
+++ b/banyand/liaison/grpc/methodpolicy_contract_test.go
@@ -84,6 +84,8 @@ const (
mGroupInspect = "/banyandb.database.v1.GroupRegistryService/Inspect"
mGroupTaskQuery = "/banyandb.database.v1.GroupRegistryService/Query"
mSnapshot = "/banyandb.database.v1.SnapshotService/Snapshot"
+ mExportPlan = "/banyandb.transfer.v1.ExportService/Plan"
+ mExportSessions = "/banyandb.transfer.v1.ExportService/Sessions"
mGetMaxRevision = "/banyandb.cluster.v1.NodeSchemaStatusService/GetMaxRevision"
mGetKeyRevisions = "/banyandb.cluster.v1.NodeSchemaStatusService/GetKeyRevisions"
mGetAbsentKeys = "/banyandb.cluster.v1.NodeSchemaStatusService/GetAbsentKeys"
@@ -109,6 +111,8 @@ var globalMethods = map[string]auth.Permission{
mGroupInspect: auth.PermissionClusterRead,
mGroupTaskQuery: auth.PermissionClusterRead,
mSnapshot: auth.PermissionClusterAdmin,
+ mExportPlan: auth.PermissionClusterAdmin,
+ mExportSessions: auth.PermissionClusterAdmin,
mGetMaxRevision: auth.PermissionClusterAdmin,
mGetKeyRevisions: auth.PermissionClusterAdmin,
mGetAbsentKeys: auth.PermissionClusterAdmin,
@@ -165,8 +169,8 @@ func policyTable(t *testing.T) liaisongrpc.MethodPolicyTable {
// from the registered set alone.
func TestR1_MethodPolicyCoversExactlyTheRegisteredMethods(t *testing.T) {
table := policyTable(t)
- if len(table) != 78 {
- t.Fatalf("GlobalMethodPolicies() returned %d rows, want the fixed 78-method oracle", len(table))
+ if len(table) != 80 {
+ t.Fatalf("GlobalMethodPolicies() returned %d rows, want the fixed 80-method oracle", len(table))
}
registered := liaisongrpc.RegisteredMethods(fullActivation)
if len(registered) == 0 {
@@ -199,6 +203,7 @@ func TestR1_RegisteredMethodsTracksTheServiceDescriptors(t *testing.T) {
mGetMaxRevision, mGetKeyRevisions, mGetAbsentKeys, mAwaitRevision,
mStreamDeleteSeg, mMeasureDeleteSeg, mTraceDeleteSeg, mMeasureInternalQ,
mMeasureQuery, mStreamWrite, mGroupCreate, mGroupList, mPropertyApply, mBydbQLQuery,
+ mExportPlan, mExportSessions,
} {
if !present[m] {
t.Errorf("RegisteredMethods(full activation) is missing %q, which the liaison serves", m)
@@ -333,7 +338,7 @@ func TestR6_GlobalMethodsAreActivatedAndTheRestFailClosed(t *testing.T) {
}
}
for permission, want := range map[auth.Permission]int{
- auth.PermissionClusterRead: 4, auth.PermissionClusterAdmin: 8,
+ auth.PermissionClusterRead: 4, auth.PermissionClusterAdmin: 10,
auth.PermissionSchemaRead: 27, auth.PermissionSchemaWrite: 24,
auth.PermissionDataRead: 6, auth.PermissionDataWrite: 5,
} {
@@ -379,6 +384,8 @@ func TestR3_GlobalDecisionMatrix(t *testing.T) {
{mHealthCheck, allow, allow, allow, allow, allow},
// cluster:admin — admin only.
{mSnapshot, allow, deny, deny, deny, deny},
+ {mExportPlan, allow, deny, deny, deny, deny},
+ {mExportSessions, allow, deny, deny, deny, deny},
{mGetMaxRevision, allow, deny, deny, deny, deny},
{mGetKeyRevisions, allow, deny, deny, deny, deny},
{mGetAbsentKeys, allow, deny, deny, deny, deny},
diff --git a/banyand/liaison/grpc/rbac_schema_contract_test.go b/banyand/liaison/grpc/rbac_schema_contract_test.go
index 33914757a..c5db3046e 100644
--- a/banyand/liaison/grpc/rbac_schema_contract_test.go
+++ b/banyand/liaison/grpc/rbac_schema_contract_test.go
@@ -523,7 +523,7 @@ func TestSchemaR2_ScopeFamilyClassification(t *testing.T) {
// with no scope family at all, so a permission-bearing method can no longer be
// unclassified and the loop below has nothing left to forgive.
for family, want := range map[liaisongrpc.ScopeFamily]int{
- liaisongrpc.ScopeGlobal: 13,
+ liaisongrpc.ScopeGlobal: 15,
liaisongrpc.ScopeDirectGroup: 11,
liaisongrpc.ScopeGroupBodyName: 2,
liaisongrpc.ScopeMetadataGroup: 21,
diff --git a/banyand/liaison/grpc/server.go b/banyand/liaison/grpc/server.go
index 19148b48e..80407f918 100644
--- a/banyand/liaison/grpc/server.go
+++ b/banyand/liaison/grpc/server.go
@@ -46,6 +46,7 @@ import (
schemav1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/schema/v1"
streamv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/stream/v1"
tracev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/trace/v1"
+ transferv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/transfer/v1"
"github.com/apache/skywalking-banyandb/banyand/internal/storage"
"github.com/apache/skywalking-banyandb/banyand/liaison/grpc/route"
"github.com/apache/skywalking-banyandb/banyand/liaison/pkg/auth"
@@ -111,6 +112,7 @@ type server struct {
streamSVC *streamService
barrierSVC *barrierService
nodeStatusSVC *property.NodeSchemaStatusServer
+ exportSVC *exportService
*streamRegistryServer
measureSVC *measureService
bydbQLSVC *bydbQLService
@@ -276,19 +278,21 @@ func NewServer(_ context.Context, tir1Client, tir2Client, broadcaster queue.Clie
// initCurrentNode leaves curNode nil; an empty selfName makes the
// barrier exclude self from the watched set, which is the correct
// behavior for headless test fixtures.
+ selfName := func() string {
+ if s.curNode == nil {
+ return ""
+ }
+ return s.curNode.GetMetadata().GetName()
+ }
if cacheProvider != nil {
s.barrierSVC = newBarrierServiceCluster(
cacheProvider,
func() queue.Client { return tir1Client },
func() queue.Client { return tir2Client },
- func() string {
- if s.curNode == nil {
- return ""
- }
- return s.curNode.GetMetadata().GetName()
- },
+ selfName,
)
}
+ s.exportSVC = newExportService(tir2Client, routeProviders["tire2"], selfName, logger.GetLogger("liaison-export"))
s.accessLogRecorders = []accessLogRecorder{streamSVC, measureSVC, traceSVC, s.propertyServer}
s.queryAccessLogRecorders = []queryAccessLogRecorder{streamSVC, measureSVC, traceSVC, s.propertyServer, bydbQLSVC}
@@ -661,6 +665,7 @@ func (s *server) registerServices(grpcServer *grpclib.Server) {
databasev1.RegisterTraceRegistryServiceServer(grpcServer, s.traceRegistryServer)
databasev1.RegisterClusterStateServiceServer(grpcServer, s)
databasev1.RegisterNodeQueryServiceServer(grpcServer, s)
+ transferv1.RegisterExportServiceServer(grpcServer, s.exportSVC)
if s.barrierSVC != nil {
schemav1.RegisterSchemaBarrierServiceServer(grpcServer, s.barrierSVC)
}
diff --git a/bydbctl/cmd/cli/main.go b/banyand/measure/export_part_reader_test.go
similarity index 74%
copy from bydbctl/cmd/cli/main.go
copy to banyand/measure/export_part_reader_test.go
index 557ef1464..f787e7409 100644
--- a/bydbctl/cmd/cli/main.go
+++ b/banyand/measure/export_part_reader_test.go
@@ -15,19 +15,12 @@
// specific language governing permissions and limitations
// under the License.
-// Package main provides main entry for the command-line toolkit, i.e. bydbctl.
-package main
+package measure
-import (
- "fmt"
- "os"
+import "github.com/apache/skywalking-banyandb/banyand/transfer/export"
- "github.com/apache/skywalking-banyandb/bydbctl/internal/cmd"
+// The export unit reads part metadata through these services.
+var (
+ _ export.PartReader = (*standalone)(nil)
+ _ export.PartReader = (*dataSVC)(nil)
)
-
-func main() {
- if err := cmd.Execute(); err != nil {
- _, _ = fmt.Fprintln(os.Stderr, err)
- os.Exit(1)
- }
-}
diff --git a/banyand/measure/snapshot.go b/banyand/measure/snapshot.go
index fa8323a94..971bed1c7 100644
--- a/banyand/measure/snapshot.go
+++ b/banyand/measure/snapshot.go
@@ -25,7 +25,6 @@ import (
"path/filepath"
"sort"
"strings"
- "sync"
"sync/atomic"
"time"
@@ -229,6 +228,17 @@ func (s *standalone) takeGroupSnapshot(dstDir string, groupName string) (bool, e
return created, nil
}
+// TakeExportSnapshot implements export.Backend: one hard-link snapshot of every group under
+// <exportSnapshotDir>/<name>. It shares the TopicSnapshot listener's mutex so an export session
+// and a periodic backup never snapshot the same groups concurrently, but unlike the listener
+// it neither reclaims old snapshots nor invents a name: the export session owns the
+// directory's lifetime.
+func (s *standalone) TakeExportSnapshot(ctx context.Context, name string) error {
+ s.snapshotMux.Lock()
+ defer s.snapshotMux.Unlock()
+ return schema.SnapshotGroups(ctx, s.schemaRepo, s.l, filepath.Join(s.exportSnapshotDir, name), s.takeGroupSnapshot)
+}
+
// collectSegDirs walks a directory tree and collects all seg-* directory paths.
// It only collects directories matching the "seg-*" pattern and ignores all files.
// Returns a map of relative paths to seg-* directories.
@@ -344,7 +354,6 @@ type snapshotListener struct {
*bus.UnImplementedHealthyListener
s *standalone
snapshotSeq uint64
- snapshotMux sync.Mutex
}
// Rev takes a snapshot of the database.
@@ -368,8 +377,8 @@ func (s *snapshotListener) Rev(ctx context.Context, message bus.Message) bus.Mes
if len(gg) == 0 {
return bus.NewMessage(bus.MessageID(time.Now().UnixNano()), nil)
}
- s.snapshotMux.Lock()
- defer s.snapshotMux.Unlock()
+ s.s.snapshotMux.Lock()
+ defer s.s.snapshotMux.Unlock()
storage.DeleteStaleSnapshots(s.s.snapshotDir, s.s.maxFileSnapshotNum, s.s.minFileSnapshotAge, s.s.lfs)
sn := s.snapshotName()
var err error
diff --git a/banyand/measure/svc_data.go b/banyand/measure/svc_data.go
index dfebb524e..d09564e4a 100644
--- a/banyand/measure/svc_data.go
+++ b/banyand/measure/svc_data.go
@@ -72,11 +72,13 @@ type dataSVC struct {
root string
dataPath string
snapshotDir string
+ exportSnapshotDir string
option option
retentionConfig storage.RetentionConfig
cc storage.CacheConfig
maxFileSnapshotNum int
minFileSnapshotAge time.Duration
+ snapshotMux sync.Mutex
}
func (s *dataSVC) Measure(metadata *commonv1.Metadata) (Measure, error) {
@@ -101,6 +103,11 @@ func (s *dataSVC) GetRemovalSegmentsTimeRange(group string) *timestamp.TimeRange
// RetentionService interface implementation.
+// ReadPartMetadata implements export.PartReader with this catalog's own metadata.json reader.
+func (s *dataSVC) ReadPartMetadata(partDir string) (queue.StreamingPartData, error) {
+ return ParsePartMetadata(s.lfs, partDir)
+}
+
func (s *dataSVC) GetDataPath() string {
return s.dataPath
}
@@ -109,6 +116,11 @@ func (s *dataSVC) GetSnapshotDir() string {
return s.snapshotDir
}
+// GetExportSnapshotDir returns the directory that holds the export session snapshots of measure.
+func (s *dataSVC) GetExportSnapshotDir() string {
+ return s.exportSnapshotDir
+}
+
func (s *dataSVC) LoadAllGroups() []resourceSchema.Group {
return s.schemaRepo.LoadAllGroups()
}
@@ -169,6 +181,7 @@ func (s *dataSVC) FlagSet() *run.FlagSet {
flagS := run.NewFlagSet("storage")
flagS.StringVar(&s.root, "measure-root-path", "/tmp", "the root path of measure")
flagS.StringVar(&s.dataPath, "measure-data-path", "", "the data directory path of measure. If not set, <measure-root-path>/measure/data will be used")
+ flagS.StringVar(&s.exportSnapshotDir, "measure-export-snapshot-path", "", storage.ExportSnapshotPathUsage("measure"))
flagS.DurationVar(&s.option.flushTimeout, "measure-flush-timeout", defaultFlushTimeout, "the memory data timeout of measure")
flagS.DurationVar(&s.option.memWaitTimeout, "measure-lifecycle-receive-mem-wait-timeout", 5*time.Minute,
"max time the migration receiver waits for memory to recover before introducing an external segment")
@@ -262,6 +275,9 @@ func (s *dataSVC) PreRun(ctx context.Context) error {
}
path := path.Join(s.root, s.Name())
s.snapshotDir = filepath.Join(path, storage.SnapshotsDir)
+ if s.exportSnapshotDir, err = storage.ResolveExportSnapshotDir(s.exportSnapshotDir, path); err != nil {
+ return err
+ }
obsservice.UpdatePath(path)
if s.dataPath == "" {
s.dataPath = filepath.Join(path, storage.DataDir)
@@ -492,11 +508,21 @@ func newDataSupplier(path string, svc *dataSVC, sr *schemaRepo, nodeLabels map[s
}
}
+// TakeExportSnapshot implements export.Backend: one hard-link snapshot of every group under
+// <exportSnapshotDir>/<name>. It shares the TopicSnapshot listener's mutex so an export session
+// and a periodic backup never snapshot the same groups concurrently, but unlike the listener
+// it neither reclaims old snapshots nor invents a name: the export session owns the
+// directory's lifetime.
+func (s *dataSVC) TakeExportSnapshot(ctx context.Context, name string) error {
+ s.snapshotMux.Lock()
+ defer s.snapshotMux.Unlock()
+ return resourceSchema.SnapshotGroups(ctx, s.schemaRepo, s.l, filepath.Join(s.exportSnapshotDir, name), s.takeGroupSnapshot)
+}
+
type dataSnapshotListener struct {
*bus.UnImplementedHealthyListener
s *dataSVC
snapshotSeq uint64
- snapshotMux sync.Mutex
}
func (d *dataSnapshotListener) Rev(ctx context.Context, message bus.Message) bus.Message {
@@ -529,8 +555,8 @@ func (d *dataSnapshotListener) Rev(ctx context.Context, message bus.Message) bus
if len(gg) == 0 {
return bus.NewMessage(bus.MessageID(time.Now().UnixNano()), nil)
}
- d.snapshotMux.Lock()
- defer d.snapshotMux.Unlock()
+ d.s.snapshotMux.Lock()
+ defer d.s.snapshotMux.Unlock()
storage.DeleteStaleSnapshots(d.s.snapshotDir, d.s.maxFileSnapshotNum, d.s.minFileSnapshotAge, d.s.lfs)
sn := d.snapshotName()
var err error
diff --git a/banyand/measure/svc_standalone.go b/banyand/measure/svc_standalone.go
index 4ccc5f7cd..71d2e5b34 100644
--- a/banyand/measure/svc_standalone.go
+++ b/banyand/measure/svc_standalone.go
@@ -23,6 +23,7 @@ import (
"path"
"path/filepath"
"strings"
+ "sync"
"time"
"github.com/pkg/errors"
@@ -78,12 +79,14 @@ type standalone struct {
diskMonitor *storage.DiskMonitor
root string
snapshotDir string
+ exportSnapshotDir string
dataPath string
option option
retentionConfig storage.RetentionConfig
cc storage.CacheConfig
maxFileSnapshotNum int
minFileSnapshotAge time.Duration
+ snapshotMux sync.Mutex
}
func (s *standalone) Measure(metadata *commonv1.Metadata) (Measure, error) {
@@ -108,6 +111,11 @@ func (s *standalone) GetRemovalSegmentsTimeRange(group string) *timestamp.TimeRa
// RetentionService interface implementation.
+// ReadPartMetadata implements export.PartReader with this catalog's own metadata.json reader.
+func (s *standalone) ReadPartMetadata(partDir string) (queue.StreamingPartData, error) {
+ return ParsePartMetadata(s.lfs, partDir)
+}
+
func (s *standalone) GetDataPath() string {
return s.dataPath
}
@@ -116,6 +124,11 @@ func (s *standalone) GetSnapshotDir() string {
return s.snapshotDir
}
+// GetExportSnapshotDir returns the directory that holds the export session snapshots of measure.
+func (s *standalone) GetExportSnapshotDir() string {
+ return s.exportSnapshotDir
+}
+
func (s *standalone) LoadAllGroups() []resourceSchema.Group {
return s.schemaRepo.LoadAllGroups()
}
@@ -176,6 +189,7 @@ func (s *standalone) FlagSet() *run.FlagSet {
flagS := run.NewFlagSet("storage")
flagS.StringVar(&s.root, "measure-root-path", "/tmp", "the root path of measure")
flagS.StringVar(&s.dataPath, "measure-data-path", "", "the data directory path of measure. If not set, <measure-root-path>/measure/data will be used")
+ flagS.StringVar(&s.exportSnapshotDir, "measure-export-snapshot-path", "", storage.ExportSnapshotPathUsage("measure"))
flagS.DurationVar(&s.option.flushTimeout, "measure-flush-timeout", defaultFlushTimeout, "the memory data timeout of measure")
flagS.DurationVar(&s.option.memWaitTimeout, "measure-lifecycle-receive-mem-wait-timeout", 5*time.Minute,
"max time the migration receiver waits for memory to recover before introducing an external segment")
@@ -272,6 +286,9 @@ func (s *standalone) PreRun(ctx context.Context) error {
}
path := path.Join(s.root, s.Name())
s.snapshotDir = filepath.Join(path, storage.SnapshotsDir)
+ if s.exportSnapshotDir, err = storage.ResolveExportSnapshotDir(s.exportSnapshotDir, path); err != nil {
+ return err
+ }
obsservice.UpdatePath(path)
if s.dataPath == "" {
s.dataPath = filepath.Join(path, storage.DataDir)
diff --git a/banyand/property/db/db.go b/banyand/property/db/db.go
index 78c678b02..72818fe0b 100644
--- a/banyand/property/db/db.go
+++ b/banyand/property/db/db.go
@@ -22,6 +22,7 @@ import (
"context"
"errors"
"fmt"
+ "os"
"path"
"path/filepath"
"strconv"
@@ -75,8 +76,10 @@ type Database interface {
Query(ctx context.Context, request *propertyv1.QueryRequest) ([]QueriedProperty, error)
// Repair repairs a property in the database.
Repair(ctx context.Context, id []byte, shardID uint64, property *propertyv1.Property, deleteTime int64) error
- // TakeSnapShot takes a snapshot of the database.
+ // TakeSnapShot takes a snapshot of the database under the configured snapshot directory.
TakeSnapShot(ctx context.Context, sn string) *databasev1.Snapshot
+ // SnapshotShards copies every shard into <dstDir>/<group>/shard-N.
+ SnapshotShards(ctx context.Context, dstDir string) error
// Drop closes and removes all shards for the given group and deletes the group directory.
Drop(groupName string) error
// RegisterGossip registers the repair scheduler's gossip services with the given messenger.
@@ -471,7 +474,20 @@ func (db *database) Repair(ctx context.Context, id []byte, shardID uint64, prope
}
func (db *database) TakeSnapShot(ctx context.Context, sn string) *databasev1.Snapshot {
- var snapshotResult *databasev1.Snapshot
+ snp := &databasev1.Snapshot{Name: sn, Catalog: commonv1.Catalog_CATALOG_PROPERTY}
+ if err := db.SnapshotShards(ctx, path.Join(db.snapshotDir, sn, storage.DataDir)); err != nil {
+ snp.Error = err.Error()
+ }
+ return snp
+}
+
+// SnapshotShards writes every shard into <dstDir>/<group>/shard-N through the index's
+// backup, which copies the segment files (they are not hard links), so a snapshot needs as
+// much free space as the shards it copies. A destination shard directory that already
+// exists is an error. It stops at the first failure or when ctx is done, returning
+// ctx.Err() in the latter case.
+func (db *database) SnapshotShards(ctx context.Context, dstDir string) error {
+ var err error
db.groups.Range(func(_, value any) bool {
gs := value.(*groupShards)
sLst := gs.shards.Load()
@@ -479,42 +495,26 @@ func (db *database) TakeSnapShot(ctx context.Context, sn string) *databasev1.Sna
return true
}
for _, shardRef := range *sLst {
- select {
- case <-ctx.Done():
- // Context canceled: record an error snapshot and stop iteration.
- if ctxErr := ctx.Err(); ctxErr != nil {
- snapshotResult = &databasev1.Snapshot{
- Name: sn,
- Catalog: commonv1.Catalog_CATALOG_PROPERTY,
- Error: ctxErr.Error(),
- }
- }
+ if err = ctx.Err(); err != nil {
+ return false
+ }
+ snpDir := path.Join(dstDir, shardRef.group, filepath.Base(shardRef.location))
+ if err = os.MkdirAll(filepath.Dir(snpDir), storage.DirPerm); err != nil {
return false
- default:
}
- snpDir := path.Join(db.snapshotDir, sn, storage.DataDir, shardRef.group, filepath.Base(shardRef.location))
- db.lfs.MkdirPanicIfExist(snpDir, storage.DirPerm)
- snapshotErr := shardRef.store.TakeFileSnapshot(snpDir)
- if snapshotErr != nil {
- db.logger.Error().Err(snapshotErr).Str("group", shardRef.group).
+ if err = os.Mkdir(snpDir, storage.DirPerm); err != nil {
+ err = fmt.Errorf("create shard snapshot directory: %w", err)
+ return false
+ }
+ if err = shardRef.store.TakeFileSnapshot(snpDir); err != nil {
+ db.logger.Error().Err(err).Str("group", shardRef.group).
Str("shard", filepath.Base(shardRef.location)).Msg("fail to take shard snapshot")
- snapshotResult = &databasev1.Snapshot{
- Name: sn,
- Catalog: commonv1.Catalog_CATALOG_PROPERTY,
- Error: snapshotErr.Error(),
- }
return false
}
}
return true
})
- if snapshotResult != nil {
- return snapshotResult
- }
- return &databasev1.Snapshot{
- Name: sn,
- Catalog: commonv1.Catalog_CATALOG_PROPERTY,
- }
+ return err
}
type walkFn func(suffix string) error
diff --git a/banyand/property/db/snapshot_shards_test.go b/banyand/property/db/snapshot_shards_test.go
new file mode 100644
index 000000000..e8ef155a9
--- /dev/null
+++ b/banyand/property/db/snapshot_shards_test.go
@@ -0,0 +1,162 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package db
+
+import (
+ "context"
+ "errors"
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+
+ "github.com/blugelabs/bluge"
+
+ "github.com/apache/skywalking-banyandb/banyand/observability"
+ "github.com/apache/skywalking-banyandb/pkg/fs"
+ "github.com/apache/skywalking-banyandb/pkg/test"
+)
+
+func openSnapshotTestDB(t *testing.T) *database {
+ t.Helper()
+ dataDir, dataDefer, err := test.NewSpace()
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(dataDefer)
+ repairDir, repairDefer, err := test.NewSpace()
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(repairDefer)
+ dbInstance, err := OpenDB(context.Background(), Config{
+ Location: dataDir,
+ MetricsScopeName: "property_snapshot_test",
+ FlushInterval: time.Hour,
+ ExpireToDeleteDuration: time.Hour,
+ Repair: RepairConfig{Location: repairDir, BuildTreeCron: "@every 1h", TreeSlotCount: 32},
+ Snapshot: SnapshotConfig{Func: func(context.Context) (string, error) { return "", nil }},
+ }, observability.BypassRegistry, fs.NewLocalFileSystem())
+ if err != nil {
+ t.Fatal(err)
+ }
+ d := dbInstance.(*database)
+ t.Cleanup(func() { _ = d.Close() })
+ return d
+}
+
+func TestSnapshotShards_CopiesEveryShardWithItsDocuments(t *testing.T) {
+ d := openSnapshotTestDB(t)
+ sh, err := d.loadShard(context.Background(), defaultGroupName, 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, id := range []string{"1", "2", "3"} {
+ p := buildProperties(propertyBuilder{id: id, version: 1})
+ if err = sh.update(GetPropertyID(p), p); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ // Wait for a persisted segment so the snapshot shares file names with the live shard and
+ // the hard-link check below has something to compare.
+ waitForPersistedSegment(t, sh.location)
+ dst := filepath.Join(t.TempDir(), "session")
+ if err = d.SnapshotShards(context.Background(), dst); err != nil {
+ t.Fatal(err)
+ }
+ shardDir := filepath.Join(dst, defaultGroupName, "shard-0")
+ reader, err := bluge.OpenReader(bluge.DefaultConfig(shardDir))
+ if err != nil {
+ t.Fatalf("the shard snapshot must be a readable index: %v", err)
+ }
+ count, err := reader.Count()
+ _ = reader.Close()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if count != 3 {
+ t.Fatalf("the snapshot must hold every written document, including unflushed ones: got %d, want 3", count)
+ }
+
+ entries, err := os.ReadDir(shardDir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(entries) == 0 {
+ t.Fatal("the shard snapshot holds no files")
+ }
+ matched := 0
+ for _, e := range entries {
+ snapInfo, statErr := os.Stat(filepath.Join(shardDir, e.Name()))
+ if statErr != nil {
+ t.Fatal(statErr)
+ }
+ liveInfo, statErr := os.Stat(filepath.Join(sh.location, e.Name()))
+ if statErr != nil {
+ continue
+ }
+ matched++
+ if os.SameFile(liveInfo, snapInfo) {
+ t.Fatalf("%s is a hard link of the live shard; the index backup is expected to copy it", e.Name())
+ }
+ }
+ if matched == 0 {
+ t.Fatalf("no snapshot file also exists in the live shard %s, so the hard-link check proved nothing", sh.location)
+ }
+}
+
+func waitForPersistedSegment(t *testing.T, dir string) {
+ t.Helper()
+ deadline := time.Now().Add(10 * time.Second)
+ for {
+ if segs, _ := filepath.Glob(filepath.Join(dir, "*.seg")); len(segs) > 0 {
+ return
+ }
+ if time.Now().After(deadline) {
+ t.Fatalf("no persisted segment in %s", dir)
+ }
+ time.Sleep(10 * time.Millisecond)
+ }
+}
+
+func TestSnapshotShards_ExistingDestinationIsAnError(t *testing.T) {
+ d := openSnapshotTestDB(t)
+ if _, err := d.loadShard(context.Background(), defaultGroupName, 0); err != nil {
+ t.Fatal(err)
+ }
+ dst := filepath.Join(t.TempDir(), "session")
+ if err := os.MkdirAll(filepath.Join(dst, defaultGroupName, "shard-0"), 0o700); err != nil {
+ t.Fatal(err)
+ }
+ if err := d.SnapshotShards(context.Background(), dst); err == nil {
+ t.Fatal("an existing shard snapshot directory must be reported as an error")
+ }
+}
+
+func TestSnapshotShards_CanceledContext(t *testing.T) {
+ d := openSnapshotTestDB(t)
+ if _, err := d.loadShard(context.Background(), defaultGroupName, 0); err != nil {
+ t.Fatal(err)
+ }
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+ if err := d.SnapshotShards(ctx, filepath.Join(t.TempDir(), "session")); !errors.Is(err, context.Canceled) {
+ t.Fatalf("got %v, want context.Canceled", err)
+ }
+}
diff --git a/banyand/property/listener.go b/banyand/property/listener.go
index 7814df63a..b88e698e8 100644
--- a/banyand/property/listener.go
+++ b/banyand/property/listener.go
@@ -21,7 +21,6 @@ import (
"context"
"fmt"
"runtime/debug"
- "sync"
"time"
"google.golang.org/protobuf/proto"
@@ -229,7 +228,6 @@ type snapshotListener struct {
*bus.UnImplementedHealthyListener
s *service
snapshotSeq uint64
- snapshotMux sync.Mutex
}
// Rev takes a snapshot of the database.
@@ -249,8 +247,8 @@ func (s *snapshotListener) Rev(ctx context.Context, message bus.Message) bus.Mes
if !toTake {
return bus.NewMessage(bus.MessageID(time.Now().UnixNano()), nil)
}
- s.snapshotMux.Lock()
- defer s.snapshotMux.Unlock()
+ s.s.snapshotMux.Lock()
+ defer s.s.snapshotMux.Unlock()
storage.DeleteStaleSnapshots(s.s.snapshotDir, s.s.maxFileSnapshotNum, s.s.minFileSnapshotAge, s.s.lfs)
sn := s.snapshotName()
return bus.NewMessage(bus.MessageID(time.Now().UnixNano()), s.s.db.TakeSnapShot(ctx, sn))
diff --git a/banyand/property/service.go b/banyand/property/service.go
index 4b6943ef9..7974c1b3f 100644
--- a/banyand/property/service.go
+++ b/banyand/property/service.go
@@ -23,6 +23,7 @@ import (
"fmt"
"path"
"path/filepath"
+ "sync"
"time"
"github.com/robfig/cron/v3"
@@ -70,6 +71,7 @@ type service struct {
l *logger.Logger
repairScheduler *repairScheduler
snapshotDir string
+ exportSnapshotDir string
root string
repairDir string
repairBuildTreeCron string
@@ -82,12 +84,14 @@ type service struct {
maxDiskUsagePercent int
maxFileSnapshotNum int
minFileSnapshotAge time.Duration
+ snapshotMux sync.Mutex
repairEnabled bool
}
func (s *service) FlagSet() *run.FlagSet {
flagS := run.NewFlagSet("storage")
flagS.StringVar(&s.root, "property-root-path", "/tmp", "the root path of database")
+ flagS.StringVar(&s.exportSnapshotDir, "property-export-snapshot-path", "", storage.ExportSnapshotPathUsage("property"))
flagS.DurationVar(&s.flushTimeout, "property-flush-timeout", defaultFlushTimeout, "the memory data timeout of measure")
flagS.IntVar(&s.maxDiskUsagePercent, "property-max-disk-usage-percent", 95, "the maximum disk usage percentage allowed")
flagS.IntVar(&s.maxFileSnapshotNum, "property-max-file-snapshot-num", 10, "the maximum number of file snapshots allowed")
@@ -129,6 +133,30 @@ func (s *service) Name() string {
return "property"
}
+// GetDataPath returns the directory that holds the property groups on disk.
+func (s *service) GetDataPath() string {
+ return filepath.Join(path.Join(s.root, s.Name()), storage.DataDir)
+}
+
+// GetSnapshotDir returns the directory that holds the property backup snapshots.
+func (s *service) GetSnapshotDir() string {
+ return s.snapshotDir
+}
+
+// GetExportSnapshotDir returns the directory that holds the export session snapshots of property.
+func (s *service) GetExportSnapshotDir() string {
+ return s.exportSnapshotDir
+}
+
+// TakeExportSnapshot implements export.Backend: every shard is copied (not hard-linked)
+// into <exportSnapshotDir>/<name>/<group>/shard-N. It shares the TopicSnapshot listener's mutex
+// so an export session and a periodic backup never snapshot concurrently.
+func (s *service) TakeExportSnapshot(ctx context.Context, name string) error {
+ s.snapshotMux.Lock()
+ defer s.snapshotMux.Unlock()
+ return s.db.SnapshotShards(ctx, filepath.Join(s.exportSnapshotDir, name))
+}
+
func (s *service) Role() databasev1.Role {
return databasev1.Role_ROLE_DATA
}
@@ -143,6 +171,9 @@ func (s *service) PreRun(ctx context.Context) error {
}
path := path.Join(s.root, s.Name())
s.snapshotDir = filepath.Join(path, storage.SnapshotsDir)
+ if s.exportSnapshotDir, err = storage.ResolveExportSnapshotDir(s.exportSnapshotDir, path); err != nil {
+ return err
+ }
s.repairDir = filepath.Join(path, storage.RepairDir)
obsservice.UpdatePath(path)
val := ctx.Value(common.ContextNodeKey)
@@ -154,7 +185,7 @@ func (s *service) PreRun(ctx context.Context) error {
snapshotLis := &snapshotListener{s: s}
s.db, err = db.OpenDB(ctx, db.Config{
- Location: filepath.Join(path, storage.DataDir),
+ Location: s.GetDataPath(),
MetricsScopeName: "property",
FlushInterval: s.flushTimeout,
ExpireToDeleteDuration: s.expireTimeout,
diff --git a/banyand/queue/local.go b/banyand/queue/local.go
index c9502be9e..433c16488 100644
--- a/banyand/queue/local.go
+++ b/banyand/queue/local.go
@@ -26,6 +26,7 @@ import (
"github.com/apache/skywalking-banyandb/api/common"
clusterv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/cluster/v1"
databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ transferv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/transfer/v1"
"github.com/apache/skywalking-banyandb/banyand/liaison/grpc/route"
"github.com/apache/skywalking-banyandb/banyand/metadata"
"github.com/apache/skywalking-banyandb/banyand/metadata/schema"
@@ -39,8 +40,9 @@ var (
)
type local struct {
- local *bus.Bus
- stopCh chan struct{}
+ local *bus.Bus
+ exportServer transferv1.ExportServiceServer
+ stopCh chan struct{}
}
// Local return a new local Queue.
@@ -195,6 +197,22 @@ func (*local) NewNodeSchemaStatusClient(_ string) (clusterv1.NodeSchemaStatusSer
return nil, ErrNotImplemented
}
+// NewExportClient returns an in-process client of the ExportService registered with
+// SetExportServer, whatever the node name: a standalone process is its own only data
+// node. It returns ErrNotImplemented when no server has been registered.
+func (l *local) NewExportClient(_ string) (transferv1.ExportServiceClient, error) {
+ if l.exportServer == nil {
+ return nil, ErrNotImplemented
+ }
+ return localExportClient{srv: l.exportServer}, nil
+}
+
+// SetExportServer implements Server: it keeps the ExportService that NewExportClient
+// serves in-process.
+func (l *local) SetExportServer(srv transferv1.ExportServiceServer) {
+ l.exportServer = srv
+}
+
type localChunkedSyncClient struct {
local *bus.Bus
node string
diff --git a/banyand/queue/local_export.go b/banyand/queue/local_export.go
new file mode 100644
index 000000000..6de0d1482
--- /dev/null
+++ b/banyand/queue/local_export.go
@@ -0,0 +1,148 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package queue
+
+import (
+ "context"
+ "io"
+
+ "google.golang.org/grpc"
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/metadata"
+ "google.golang.org/grpc/status"
+
+ transferv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/transfer/v1"
+)
+
+// localExportClient is the standalone pipeline's transferv1.ExportServiceClient: like
+// localChunkedSyncClient it stays in-process, running the handler SetExportServer
+// registered on a goroutine and handing its frames to the caller over a channel.
+type localExportClient struct {
+ srv transferv1.ExportServiceServer
+}
+
+func (c localExportClient) Plan(ctx context.Context, in *transferv1.PlanRequest,
+ _ ...grpc.CallOption,
+) (grpc.ServerStreamingClient[transferv1.PlanResponse], error) {
+ return startLocalStream(ctx, func(s grpc.ServerStreamingServer[transferv1.PlanResponse]) error {
+ return c.srv.Plan(in, s)
+ }), nil
+}
+
+func (c localExportClient) Sessions(ctx context.Context, in *transferv1.SessionsRequest,
+ _ ...grpc.CallOption,
+) (grpc.ServerStreamingClient[transferv1.SessionsResponse], error) {
+ return startLocalStream(ctx, func(s grpc.ServerStreamingServer[transferv1.SessionsResponse]) error {
+ return c.srv.Sessions(in, s)
+ }), nil
+}
+
+// localStream is both ends of one in-process server-streaming call. Send blocks until the
+// caller receives the frame or the call's context ends, which gives the handler the same
+// back-pressure a gRPC stream does.
+type localStream[T any] struct {
+ ctx context.Context
+ err error // the handler's result, set before done is closed
+ cancel context.CancelFunc
+ frames chan *T
+ done chan struct{}
+}
+
+func startLocalStream[T any](ctx context.Context, handler func(grpc.ServerStreamingServer[T]) error) *localStream[T] {
+ callCtx, cancel := context.WithCancel(ctx)
+ s := &localStream[T]{ctx: callCtx, cancel: cancel, frames: make(chan *T), done: make(chan struct{})}
+ //panicdiag:allow-rawgo the goroutine recovers a handler panic into the call's status
+ go func() {
+ defer close(s.done)
+ // Recover before done closes, as the cluster's recovery interceptor does, so a panic
+ // fails the call instead of the process.
+ defer func() {
+ if r := recover(); r != nil {
+ s.err = status.Errorf(codes.Internal, "export handler panic: %v", r)
+ }
+ }()
+ s.err = handler(s)
+ }()
+ return s
+}
+
+// Send implements grpc.ServerStreamingServer.
+func (s *localStream[T]) Send(m *T) error {
+ select {
+ case s.frames <- m:
+ return nil
+ case <-s.ctx.Done():
+ return status.FromContextError(s.ctx.Err()).Err()
+ }
+}
+
+// Recv implements grpc.ServerStreamingClient: the next frame, io.EOF once the handler
+// returned nil, or its error as a status the way a gRPC client sees it.
+func (s *localStream[T]) Recv() (*T, error) {
+ select {
+ case m := <-s.frames:
+ return m, nil
+ case <-s.done:
+ s.cancel()
+ if s.err == nil {
+ return nil, io.EOF
+ }
+ if _, ok := status.FromError(s.err); ok {
+ return nil, s.err
+ }
+ if ctxErr := status.FromContextError(s.err); ctxErr.Code() != codes.Unknown {
+ return nil, ctxErr.Err()
+ }
+ return nil, status.Error(codes.Unknown, s.err.Error())
+ case <-s.ctx.Done():
+ return nil, status.FromContextError(s.ctx.Err()).Err()
+ }
+}
+
+// Context implements grpc.ServerStream and grpc.ClientStream.
+func (s *localStream[T]) Context() context.Context { return s.ctx }
+
+// Header implements grpc.ClientStream.
+func (*localStream[T]) Header() (metadata.MD, error) { return nil, nil }
+
+// Trailer implements grpc.ClientStream.
+func (*localStream[T]) Trailer() metadata.MD { return nil }
+
+// CloseSend implements grpc.ClientStream.
+func (*localStream[T]) CloseSend() error { return nil }
+
+// SetHeader implements grpc.ServerStream.
+func (*localStream[T]) SetHeader(metadata.MD) error { return nil }
+
+// SendHeader implements grpc.ServerStream.
+func (*localStream[T]) SendHeader(metadata.MD) error { return nil }
+
+// SetTrailer implements grpc.ServerStream.
+func (*localStream[T]) SetTrailer(metadata.MD) {}
+
+// SendMsg implements grpc.ServerStream and grpc.ClientStream; the typed Send is the only
+// way frames travel.
+func (*localStream[T]) SendMsg(any) error {
+ return status.Error(codes.Unimplemented, "local export stream carries typed frames only")
+}
+
+// RecvMsg implements grpc.ServerStream and grpc.ClientStream; the typed Recv is the only
+// way frames travel.
+func (*localStream[T]) RecvMsg(any) error {
+ return status.Error(codes.Unimplemented, "local export stream carries typed frames only")
+}
diff --git a/banyand/queue/local_export_test.go b/banyand/queue/local_export_test.go
new file mode 100644
index 000000000..802a53c07
--- /dev/null
+++ b/banyand/queue/local_export_test.go
@@ -0,0 +1,191 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package queue
+
+import (
+ "context"
+ "errors"
+ "io"
+ "testing"
+ "time"
+
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/status"
+
+ transferv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/transfer/v1"
+)
+
+type fakeExportServer struct {
+ transferv1.UnimplementedExportServiceServer
+ planErr error
+ exited chan struct{}
+ lastPlan *transferv1.PlanRequest
+ frames int
+ block bool // send frames forever, until Send fails
+}
+
+func (f *fakeExportServer) Plan(req *transferv1.PlanRequest, stream transferv1.ExportService_PlanServer) error {
+ f.lastPlan = req
+ if f.exited != nil {
+ defer close(f.exited)
+ }
+ for i := 0; f.block || i < f.frames; i++ {
+ frame := &transferv1.PlanResponse{Frame: &transferv1.PlanResponse_Units{Units: &transferv1.UnitFrame{Stage: "hot"}}}
+ if err := stream.Send(frame); err != nil {
+ return err
+ }
+ }
+ return f.planErr
+}
+
+func (f *fakeExportServer) Sessions(_ *transferv1.SessionsRequest, stream transferv1.ExportService_SessionsServer) error {
+ return stream.Send(&transferv1.SessionsResponse{Outcome: &transferv1.SessionsResponse_Done{Done: &transferv1.Ack{}}})
+}
+
+func exportClient(t *testing.T, srv transferv1.ExportServiceServer) transferv1.ExportServiceClient {
+ t.Helper()
+ q := Local()
+ q.SetExportServer(srv)
+ client, err := q.NewExportClient("any-node")
+ if err != nil {
+ t.Fatal(err)
+ }
+ return client
+}
+
+func TestLocalExportClient_WithoutServerIsNotImplemented(t *testing.T) {
+ if _, err := Local().NewExportClient("self"); !errors.Is(err, ErrNotImplemented) {
+ t.Fatalf("got %v, want ErrNotImplemented", err)
+ }
+}
+
+func TestLocalExportClient_PlanStreamsEveryFrameThenEOF(t *testing.T) {
+ srv := &fakeExportServer{frames: 3}
+ req := &transferv1.PlanRequest{Selectors: []*transferv1.Selector{{Groups: []string{"g"}}}}
+ stream, err := exportClient(t, srv).Plan(context.Background(), req)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var got int
+ for {
+ frame, recvErr := stream.Recv()
+ if errors.Is(recvErr, io.EOF) {
+ break
+ }
+ if recvErr != nil {
+ t.Fatal(recvErr)
+ }
+ if frame.GetUnits().GetStage() != "hot" {
+ t.Fatalf("unexpected frame %+v", frame)
+ }
+ got++
+ }
+ if got != 3 {
+ t.Fatalf("got %d frames, want 3", got)
+ }
+ if srv.lastPlan != req {
+ t.Fatal("the handler must receive the caller's request")
+ }
+}
+
+func TestLocalExportClient_PlanErrorsArriveAsStatus(t *testing.T) {
+ tests := []struct {
+ err error
+ name string
+ want codes.Code
+ }{
+ {name: "status", err: status.Error(codes.FailedPrecondition, "expired"), want: codes.FailedPrecondition},
+ {name: "context", err: context.Canceled, want: codes.Canceled},
+ {name: "plain", err: errors.New("boom"), want: codes.Unknown},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ stream, err := exportClient(t, &fakeExportServer{frames: 1, planErr: tt.err}).Plan(context.Background(), &transferv1.PlanRequest{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err = stream.Recv(); err != nil {
+ t.Fatalf("the frame before the error must arrive: %v", err)
+ }
+ _, err = stream.Recv()
+ if status.Code(err) != tt.want {
+ t.Fatalf("got %v, want %s", err, tt.want)
+ }
+ })
+ }
+}
+
+func TestLocalExportClient_CancelUnblocksTheHandler(t *testing.T) {
+ srv := &fakeExportServer{block: true, exited: make(chan struct{})}
+ ctx, cancel := context.WithCancel(context.Background())
+ stream, err := exportClient(t, srv).Plan(ctx, &transferv1.PlanRequest{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err = stream.Recv(); err != nil {
+ t.Fatal(err)
+ }
+ cancel()
+ select {
+ case <-srv.exited:
+ case <-time.After(5 * time.Second):
+ t.Fatal("the handler must stop once the caller cancels")
+ }
+ for {
+ if _, err = stream.Recv(); err != nil {
+ break
+ }
+ }
+ if status.Code(err) != codes.Canceled {
+ t.Fatalf("got %v, want Canceled", err)
+ }
+}
+
+func TestLocalExportClient_SessionsAnswersOneFrame(t *testing.T) {
+ stream, err := exportClient(t, &fakeExportServer{}).Sessions(context.Background(), &transferv1.SessionsRequest{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ frame, err := stream.Recv()
+ if err != nil || frame.GetDone() == nil {
+ t.Fatalf("got %+v, %v", frame, err)
+ }
+ if _, err = stream.Recv(); !errors.Is(err, io.EOF) {
+ t.Fatalf("got %v, want io.EOF", err)
+ }
+}
+
+type panickingExportServer struct {
+ transferv1.UnimplementedExportServiceServer
+}
+
+func (panickingExportServer) Plan(*transferv1.PlanRequest, transferv1.ExportService_PlanServer) error {
+ panic("boom")
+}
+
+// A panicking handler fails the call as Internal instead of crashing the process.
+func TestLocalExportClient_HandlerPanicIsInternal(t *testing.T) {
+ stream, err := exportClient(t, panickingExportServer{}).Plan(context.Background(), &transferv1.PlanRequest{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ _, err = stream.Recv()
+ if status.Code(err) != codes.Internal || status.Convert(err).Message() != "export handler panic: boom" {
+ t.Fatalf("got %v, want Internal naming the panic", err)
+ }
+}
diff --git a/banyand/queue/pub/pub.go b/banyand/queue/pub/pub.go
index f062128c7..e6890924a 100644
--- a/banyand/queue/pub/pub.go
+++ b/banyand/queue/pub/pub.go
@@ -39,6 +39,7 @@ import (
clusterv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/cluster/v1"
databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+ transferv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/transfer/v1"
"github.com/apache/skywalking-banyandb/banyand/internal/storage"
"github.com/apache/skywalking-banyandb/banyand/metadata"
"github.com/apache/skywalking-banyandb/banyand/metadata/schema"
@@ -780,3 +781,12 @@ func (p *pub) NewNodeSchemaStatusClient(node string) (clusterv1.NodeSchemaStatus
}
return clusterv1.NewNodeSchemaStatusServiceClient(c.conn), nil
}
+
+// NewExportClient implements queue.Client on top of the shared connection pool.
+func (p *pub) NewExportClient(node string) (transferv1.ExportServiceClient, error) {
+ c, ok := p.connMgr.GetClient(node)
+ if !ok {
+ return nil, fmt.Errorf("no active client for node %s", node)
+ }
+ return transferv1.NewExportServiceClient(c.conn), nil
+}
diff --git a/banyand/queue/queue.go b/banyand/queue/queue.go
index be84b883e..d2c820275 100644
--- a/banyand/queue/queue.go
+++ b/banyand/queue/queue.go
@@ -25,6 +25,7 @@ import (
"github.com/apache/skywalking-banyandb/api/common"
clusterv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/cluster/v1"
+ transferv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/transfer/v1"
"github.com/apache/skywalking-banyandb/banyand/liaison/grpc/route"
"github.com/apache/skywalking-banyandb/banyand/metadata"
"github.com/apache/skywalking-banyandb/banyand/metadata/schema"
@@ -77,6 +78,11 @@ type Client interface {
// The standalone local pipeline returns ErrNotImplemented so callers can
// treat it as "no peer to probe" without a type-switch.
NewNodeSchemaStatusClient(node string) (clusterv1.NodeSchemaStatusServiceClient, error)
+ // NewExportClient borrows the pooled connection to node and wraps it as an
+ // transferv1.ExportServiceClient so the liaison can fan Plan/Sessions out
+ // without dialing. The standalone local pipeline answers with an in-process
+ // client of the ExportService registered through SetExportServer.
+ NewExportClient(node string) (transferv1.ExportServiceClient, error)
Register(bus.Topic, schema.EventHandler)
OnAddOrUpdate(md schema.Metadata)
GracefulStop()
@@ -97,6 +103,10 @@ type Server interface {
// per-node by design (reports the local schema cache); liaison and
// data-node processes both call it.
SetNodeSchemaStatusRepo(svc metadata.Service)
+ // SetExportServer enables transfer.v1.ExportService on the data-node gRPC server.
+ // The planner is served as-is; the liaison stamps node identity on its frames.
+ // The standalone local pipeline keeps it for NewExportClient.
+ SetExportServer(srv transferv1.ExportServiceServer)
}
// BatchPublisher is the interface for publishing data in batch.
diff --git a/banyand/queue/sub/server.go b/banyand/queue/sub/server.go
index 97ecab701..eb6a59225 100644
--- a/banyand/queue/sub/server.go
+++ b/banyand/queue/sub/server.go
@@ -43,6 +43,7 @@ import (
measurev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/measure/v1"
streamv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/stream/v1"
tracev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/trace/v1"
+ transferv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/transfer/v1"
"github.com/apache/skywalking-banyandb/banyand/liaison/grpc/route"
"github.com/apache/skywalking-banyandb/banyand/metadata"
"github.com/apache/skywalking-banyandb/banyand/metadata/schema/property"
@@ -84,6 +85,7 @@ type server struct {
creds credentials.TransportCredentials
metadataRepo metadata.Repo
nodeSchemaStatusRepo metadata.Service
+ exportServer transferv1.ExportServiceServer
clientCloser context.CancelFunc
ser *grpclib.Server
listeners map[bus.Topic][]bus.MessageListener
@@ -126,6 +128,7 @@ func NewServerWithPorts(omr observability.MetricsRegistry, flagNamePrefix string
chunkedSyncHandlers: make(map[bus.Topic]queue.ChunkedSyncHandler),
omr: omr,
maxRecvMsgSize: defaultRecvSize,
+ exportServer: transferv1.UnimplementedExportServiceServer{},
flagNamePrefix: flagNamePrefix,
port: port,
httpPort: httpPort,
@@ -285,6 +288,7 @@ func (s *server) Serve() run.StopNotify {
databasev1.RegisterSnapshotServiceServer(s.ser, s)
databasev1.RegisterNodeQueryServiceServer(s.ser, s)
databasev1.RegisterClusterStateServiceServer(s.ser, s)
+ transferv1.RegisterExportServiceServer(s.ser, s.exportServer)
streamv1.RegisterStreamServiceServer(s.ser, &streamService{ser: s})
measurev1.RegisterMeasureServiceServer(s.ser, &measureService{ser: s})
tracev1.RegisterTraceServiceServer(s.ser, &traceService{ser: s})
@@ -428,6 +432,11 @@ func (s *server) SetMetadataRepo(repo metadata.Repo) {
s.metadataRepo = repo
}
+// SetExportServer implements queue.Server.
+func (s *server) SetExportServer(srv transferv1.ExportServiceServer) {
+ s.exportServer = srv
+}
+
// SetNodeSchemaStatusRepo wires the metadata.Service whose SchemaRegistry +
// NodeRepoRegistry back the per-node NodeSchemaStatusService.
func (s *server) SetNodeSchemaStatusRepo(svc metadata.Service) {
diff --git a/bydbctl/cmd/cli/main.go b/banyand/stream/export_part_reader_test.go
similarity index 74%
copy from bydbctl/cmd/cli/main.go
copy to banyand/stream/export_part_reader_test.go
index 557ef1464..52b832bb5 100644
--- a/bydbctl/cmd/cli/main.go
+++ b/banyand/stream/export_part_reader_test.go
@@ -15,19 +15,11 @@
// specific language governing permissions and limitations
// under the License.
-// Package main provides main entry for the command-line toolkit, i.e. bydbctl.
-package main
+package stream
-import (
- "fmt"
- "os"
+import "github.com/apache/skywalking-banyandb/banyand/transfer/export"
- "github.com/apache/skywalking-banyandb/bydbctl/internal/cmd"
+// The export unit reads part metadata through these services.
+var (
+ _ export.PartReader = (*standalone)(nil)
)
-
-func main() {
- if err := cmd.Execute(); err != nil {
- _, _ = fmt.Fprintln(os.Stderr, err)
- os.Exit(1)
- }
-}
diff --git a/banyand/stream/snapshot.go b/banyand/stream/snapshot.go
index e3fd0d9ff..22d27bf3d 100644
--- a/banyand/stream/snapshot.go
+++ b/banyand/stream/snapshot.go
@@ -25,7 +25,6 @@ import (
"path/filepath"
"sort" // added for sorting parts
"strings"
- "sync"
"sync/atomic"
"time"
@@ -279,6 +278,17 @@ func (s *standalone) takeGroupSnapshot(dstDir string, groupName string) (bool, e
return created, nil
}
+// TakeExportSnapshot implements export.Backend: one hard-link snapshot of every group under
+// <exportSnapshotDir>/<name>. It shares the TopicSnapshot listener's mutex so an export session
+// and a periodic backup never snapshot the same groups concurrently, but unlike the listener
+// it neither reclaims old snapshots nor invents a name: the export session owns the
+// directory's lifetime.
+func (s *standalone) TakeExportSnapshot(ctx context.Context, name string) error {
+ s.snapshotMux.Lock()
+ defer s.snapshotMux.Unlock()
+ return schema.SnapshotGroups(ctx, &s.schemaRepo, s.l, filepath.Join(s.exportSnapshotDir, name), s.takeGroupSnapshot)
+}
+
// collectSegDirs walks a directory tree and collects all seg-* directory paths.
// It only collects directories matching the "seg-*" pattern and ignores all files.
// Returns a map of relative paths to seg-* directories.
@@ -394,7 +404,6 @@ type snapshotListener struct {
*bus.UnImplementedHealthyListener
s *standalone
snapshotSeq uint64
- snapshotMux sync.Mutex
}
// Rev takes a snapshot of the database.
@@ -418,8 +427,8 @@ func (s *snapshotListener) Rev(ctx context.Context, message bus.Message) bus.Mes
if len(gg) == 0 {
return bus.NewMessage(bus.MessageID(time.Now().UnixNano()), nil)
}
- s.snapshotMux.Lock()
- defer s.snapshotMux.Unlock()
+ s.s.snapshotMux.Lock()
+ defer s.s.snapshotMux.Unlock()
storage.DeleteStaleSnapshots(s.s.snapshotDir, s.s.maxFileSnapshotNum, s.s.minFileSnapshotAge, s.s.lfs)
sn := s.snapshotName()
var err error
diff --git a/banyand/stream/svc_standalone.go b/banyand/stream/svc_standalone.go
index a901a931b..651192756 100644
--- a/banyand/stream/svc_standalone.go
+++ b/banyand/stream/svc_standalone.go
@@ -23,6 +23,7 @@ import (
"path"
"path/filepath"
"strings"
+ "sync"
"time"
"github.com/pkg/errors"
@@ -81,10 +82,12 @@ type standalone struct {
root string
dataPath string
snapshotDir string
+ exportSnapshotDir string
option option
retentionConfig storage.RetentionConfig
maxFileSnapshotNum int
minFileSnapshotAge time.Duration
+ snapshotMux sync.Mutex
}
func (s *standalone) Stream(metadata *commonv1.Metadata) (Stream, error) {
@@ -109,6 +112,11 @@ func (s *standalone) GetRemovalSegmentsTimeRange(group string) *timestamp.TimeRa
// RetentionService interface implementation.
+// ReadPartMetadata implements export.PartReader with this catalog's own metadata.json reader.
+func (s *standalone) ReadPartMetadata(partDir string) (queue.StreamingPartData, error) {
+ return ParsePartMetadata(s.lfs, partDir)
+}
+
func (s *standalone) GetDataPath() string {
return s.dataPath
}
@@ -117,6 +125,11 @@ func (s *standalone) GetSnapshotDir() string {
return s.snapshotDir
}
+// GetExportSnapshotDir returns the directory that holds the export session snapshots of stream.
+func (s *standalone) GetExportSnapshotDir() string {
+ return s.exportSnapshotDir
+}
+
func (s *standalone) LoadAllGroups() []resourceSchema.Group {
return s.schemaRepo.LoadAllGroups()
}
@@ -182,6 +195,7 @@ func (s *standalone) FlagSet() *run.FlagSet {
flagS := run.NewFlagSet("storage")
flagS.StringVar(&s.root, "stream-root-path", "/tmp", "the root path of stream")
flagS.StringVar(&s.dataPath, "stream-data-path", "", "the data directory path of stream. If not set, <stream-root-path>/stream/data will be used")
+ flagS.StringVar(&s.exportSnapshotDir, "stream-export-snapshot-path", "", storage.ExportSnapshotPathUsage("stream"))
flagS.DurationVar(&s.option.flushTimeout, "stream-flush-timeout", defaultFlushTimeout, "the memory data timeout of stream")
flagS.DurationVar(&s.option.elementIndexFlushTimeout, "element-index-flush-timeout", defaultFlushTimeout, "the elementIndex timeout of stream")
flagS.DurationVar(&s.option.memWaitTimeout, "stream-lifecycle-receive-mem-wait-timeout", 5*time.Minute,
@@ -259,6 +273,9 @@ func (s *standalone) PreRun(ctx context.Context) error {
}
path := path.Join(s.root, s.Name())
s.snapshotDir = filepath.Join(path, storage.SnapshotsDir)
+ if s.exportSnapshotDir, err = storage.ResolveExportSnapshotDir(s.exportSnapshotDir, path); err != nil {
+ return err
+ }
obsservice.UpdatePath(path)
val := ctx.Value(common.ContextNodeKey)
if val == nil {
diff --git a/bydbctl/cmd/cli/main.go b/banyand/trace/export_part_reader_test.go
similarity index 74%
copy from bydbctl/cmd/cli/main.go
copy to banyand/trace/export_part_reader_test.go
index 557ef1464..5996b90b5 100644
--- a/bydbctl/cmd/cli/main.go
+++ b/banyand/trace/export_part_reader_test.go
@@ -15,19 +15,11 @@
// specific language governing permissions and limitations
// under the License.
-// Package main provides main entry for the command-line toolkit, i.e. bydbctl.
-package main
+package trace
-import (
- "fmt"
- "os"
+import "github.com/apache/skywalking-banyandb/banyand/transfer/export"
- "github.com/apache/skywalking-banyandb/bydbctl/internal/cmd"
+// The export unit reads part metadata through these services.
+var (
+ _ export.PartReader = (*standalone)(nil)
)
-
-func main() {
- if err := cmd.Execute(); err != nil {
- _, _ = fmt.Fprintln(os.Stderr, err)
- os.Exit(1)
- }
-}
diff --git a/banyand/trace/export_snapshot.go b/banyand/trace/export_snapshot.go
new file mode 100644
index 000000000..333f672d4
--- /dev/null
+++ b/banyand/trace/export_snapshot.go
@@ -0,0 +1,131 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package trace
+
+import (
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "github.com/apache/skywalking-banyandb/banyand/internal/dump"
+ "github.com/apache/skywalking-banyandb/banyand/internal/storage"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+)
+
+// exportSnapshotAttempts bounds how often an export snapshot is retaken when a flush or
+// merge between the core and the secondary-index snapshots leaves them inconsistent
+// (design §3.5: the trace snapshot has no publication fence).
+const exportSnapshotAttempts = 3
+
+// takeConsistentSnapshot runs take into dst until verifyExportSnapshot accepts the result,
+// at most exportSnapshotAttempts times. dst is emptied before every retake.
+func takeConsistentSnapshot(dst string, l *logger.Logger, take func() error) error {
+ for attempt := 1; ; attempt++ {
+ if err := take(); err != nil {
+ return err
+ }
+ verifyErr := verifyExportSnapshot(dst)
+ if verifyErr == nil {
+ return nil
+ }
+ if attempt >= exportSnapshotAttempts {
+ return verifyErr
+ }
+ l.Warn().Err(verifyErr).Int("attempt", attempt).Msg("trace export snapshot inconsistent; retaking")
+ if err := os.RemoveAll(dst); err != nil {
+ return err
+ }
+ if err := os.MkdirAll(dst, storage.DirPerm); err != nil {
+ return err
+ }
+ }
+}
+
+// verifyExportSnapshot checks that every secondary-index part under
+// <root>/<group>/seg-*/shard-*/sidx/<rule> has a core part with the same id in its shard.
+func verifyExportSnapshot(root string) error {
+ groups, err := readSubDirs(root, "")
+ if err != nil {
+ return err
+ }
+ for _, group := range groups {
+ segs, segErr := readSubDirs(group, "seg-")
+ if segErr != nil {
+ return segErr
+ }
+ for _, seg := range segs {
+ shards, shardErr := readSubDirs(seg, "shard-")
+ if shardErr != nil {
+ return shardErr
+ }
+ for _, shard := range shards {
+ if err = verifyExportShard(shard); err != nil {
+ return err
+ }
+ }
+ }
+ }
+ return nil
+}
+
+func verifyExportShard(shard string) error {
+ coreIDs, err := dump.DiscoverPartIDs(shard)
+ if err != nil {
+ return err
+ }
+ core := make(map[uint64]struct{}, len(coreIDs))
+ for _, id := range coreIDs {
+ core[id] = struct{}{}
+ }
+ rules, err := readSubDirs(filepath.Join(shard, sidxDirName), "")
+ if err != nil {
+ return err
+ }
+ for _, rule := range rules {
+ sidxIDs, idErr := dump.DiscoverPartIDs(rule)
+ if idErr != nil {
+ return idErr
+ }
+ for _, id := range sidxIDs {
+ if _, ok := core[id]; !ok {
+ return fmt.Errorf("trace snapshot %s: sidx %s part %016x has no core part", shard, filepath.Base(rule), id)
+ }
+ }
+ }
+ return nil
+}
+
+// readSubDirs returns the paths of the directories under dir whose names start with
+// prefix; a missing dir has none.
+func readSubDirs(dir, prefix string) ([]string, error) {
+ entries, err := os.ReadDir(dir)
+ if err != nil {
+ if os.IsNotExist(err) {
+ return nil, nil
+ }
+ return nil, err
+ }
+ var out []string
+ for _, e := range entries {
+ if e.IsDir() && strings.HasPrefix(e.Name(), prefix) {
+ out = append(out, filepath.Join(dir, e.Name()))
+ }
+ }
+ return out, nil
+}
diff --git a/banyand/trace/export_snapshot_test.go b/banyand/trace/export_snapshot_test.go
new file mode 100644
index 000000000..8f8242410
--- /dev/null
+++ b/banyand/trace/export_snapshot_test.go
@@ -0,0 +1,89 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package trace
+
+import (
+ "errors"
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+)
+
+func mkdirs(t *testing.T, dirs ...string) {
+ t.Helper()
+ for _, d := range dirs {
+ require.NoError(t, os.MkdirAll(d, 0o755))
+ }
+}
+
+func TestVerifyExportSnapshot(t *testing.T) {
+ root := t.TempDir()
+ shard := filepath.Join(root, "g", "seg-20260928", "shard-0")
+ mkdirs(t, filepath.Join(shard, "0000000000000001"), filepath.Join(shard, sidxDirName, "rule_a", "0000000000000001"))
+ require.NoError(t, verifyExportSnapshot(root))
+ require.NoError(t, verifyExportSnapshot(filepath.Join(root, "missing")), "an empty snapshot is consistent")
+
+ mkdirs(t, filepath.Join(shard, sidxDirName, "rule_a", "0000000000000002"))
+ require.ErrorContains(t, verifyExportSnapshot(root), "has no core part")
+}
+
+func TestTakeConsistentSnapshot_RetakesWhenSidxIsAhead(t *testing.T) {
+ dst := t.TempDir()
+ shard := filepath.Join(dst, "g", "seg-20260928", "shard-0")
+ attempts := 0
+ err := takeConsistentSnapshot(dst, logger.GetLogger("test"), func() error {
+ attempts++
+ mkdirs(t, filepath.Join(shard, "0000000000000001"))
+ if attempts == 1 {
+ // The first snapshot catches a sidx part whose core part is not published yet.
+ mkdirs(t, filepath.Join(shard, sidxDirName, "rule", "0000000000000002"))
+ }
+ return nil
+ })
+ require.NoError(t, err)
+ require.Equal(t, 2, attempts, "an inconsistent snapshot must be retaken once")
+ _, err = os.Stat(filepath.Join(shard, sidxDirName))
+ require.True(t, os.IsNotExist(err), "the retaken snapshot must not carry the stale sidx part")
+}
+
+func TestTakeConsistentSnapshot_GivesUpAfterAttempts(t *testing.T) {
+ dst := t.TempDir()
+ attempts := 0
+ err := takeConsistentSnapshot(dst, logger.GetLogger("test"), func() error {
+ attempts++
+ mkdirs(t, filepath.Join(dst, "g", "seg-20260928", "shard-0", sidxDirName, "rule", "0000000000000002"))
+ return nil
+ })
+ require.ErrorContains(t, err, "has no core part")
+ require.Equal(t, exportSnapshotAttempts, attempts)
+}
+
+func TestTakeConsistentSnapshot_ReturnsTakeError(t *testing.T) {
+ injected := errors.New("injected")
+ attempts := 0
+ err := takeConsistentSnapshot(t.TempDir(), logger.GetLogger("test"), func() error {
+ attempts++
+ return injected
+ })
+ require.ErrorIs(t, err, injected)
+ require.Equal(t, 1, attempts)
+}
diff --git a/banyand/trace/svc_standalone.go b/banyand/trace/svc_standalone.go
index 3c5945d8d..5b0f67e13 100644
--- a/banyand/trace/svc_standalone.go
+++ b/banyand/trace/svc_standalone.go
@@ -76,18 +76,21 @@ type standalone struct {
finalizeCloser *run.Closer
schemaRepo schemaRepo
snapshotDir string
+ exportSnapshotDir string
root string
dataPath string
option option
retentionConfig storage.RetentionConfig
maxFileSnapshotNum int
minFileSnapshotAge time.Duration
+ snapshotMux sync.Mutex
}
func (s *standalone) FlagSet() *run.FlagSet {
fs := run.NewFlagSet("trace")
fs.StringVar(&s.root, "trace-root-path", "/tmp", "the root path for trace data")
fs.StringVar(&s.dataPath, "trace-data-path", "", "the path for trace data (optional)")
+ fs.StringVar(&s.exportSnapshotDir, "trace-export-snapshot-path", "", storage.ExportSnapshotPathUsage("trace"))
fs.DurationVar(&s.option.flushTimeout, "trace-flush-timeout", defaultFlushTimeout, "the timeout for trace data flush")
fs.DurationVar(&s.option.memWaitTimeout, "trace-lifecycle-receive-mem-wait-timeout", 5*time.Minute,
"max time the migration receiver waits for memory to recover before introducing an external segment")
@@ -166,6 +169,9 @@ func (s *standalone) PreRun(ctx context.Context) error {
}
path := path.Join(s.root, s.Name())
s.snapshotDir = filepath.Join(path, storage.SnapshotsDir)
+ if s.exportSnapshotDir, err = storage.ResolveExportSnapshotDir(s.exportSnapshotDir, path); err != nil {
+ return err
+ }
obsservice.UpdatePath(path)
val := ctx.Value(common.ContextNodeKey)
if val == nil {
@@ -294,6 +300,11 @@ func (s *standalone) GetRemovalSegmentsTimeRange(group string) *timestamp.TimeRa
// RetentionService interface implementation.
+// ReadPartMetadata implements export.PartReader with this catalog's own metadata.json reader.
+func (s *standalone) ReadPartMetadata(partDir string) (queue.StreamingPartData, error) {
+ return ParsePartMetadata(s.lfs, partDir)
+}
+
func (s *standalone) GetDataPath() string {
return s.dataPath
}
@@ -302,6 +313,11 @@ func (s *standalone) GetSnapshotDir() string {
return s.snapshotDir
}
+// GetExportSnapshotDir returns the directory that holds the export session snapshots of trace.
+func (s *standalone) GetExportSnapshotDir() string {
+ return s.exportSnapshotDir
+}
+
func (s *standalone) LoadAllGroups() []resourceSchema.Group {
return s.schemaRepo.LoadAllGroups()
}
@@ -486,11 +502,24 @@ func (d *standaloneSnapshotListener) compareSnapshotWithData(snapshotDir, dataDi
groupName, dataDir, snapshotDir)
}
+// TakeExportSnapshot implements export.Backend: one hard-link snapshot of every group under
+// <exportSnapshotDir>/<name>, retaken until its secondary indexes match its core parts, or
+// an error. It shares the TopicSnapshot listener's mutex so an export session and a periodic
+// backup never snapshot the same groups concurrently, but unlike the listener it neither
+// reclaims old snapshots nor invents a name: the export session owns the directory's lifetime.
+func (s *standalone) TakeExportSnapshot(ctx context.Context, name string) error {
+ s.snapshotMux.Lock()
+ defer s.snapshotMux.Unlock()
+ dst := filepath.Join(s.exportSnapshotDir, name)
+ return takeConsistentSnapshot(dst, s.l, func() error {
+ return resourceSchema.SnapshotGroups(ctx, &s.schemaRepo, s.l, dst, s.takeGroupSnapshot)
+ })
+}
+
type standaloneSnapshotListener struct {
*bus.UnImplementedHealthyListener
s *standalone
snapshotSeq uint64
- snapshotMux sync.Mutex
}
func (d *standaloneSnapshotListener) Rev(ctx context.Context, message bus.Message) bus.Message {
@@ -513,8 +542,8 @@ func (d *standaloneSnapshotListener) Rev(ctx context.Context, message bus.Messag
if len(gg) == 0 {
return bus.NewMessage(bus.MessageID(time.Now().UnixNano()), nil)
}
- d.snapshotMux.Lock()
- defer d.snapshotMux.Unlock()
+ d.s.snapshotMux.Lock()
+ defer d.s.snapshotMux.Unlock()
storage.DeleteStaleSnapshots(d.s.snapshotDir, d.s.maxFileSnapshotNum, d.s.minFileSnapshotAge, d.s.lfs)
sn := d.snapshotName()
var err error
diff --git a/banyand/transfer/export/backend.go b/banyand/transfer/export/backend.go
new file mode 100644
index 000000000..646298c24
--- /dev/null
+++ b/banyand/transfer/export/backend.go
@@ -0,0 +1,111 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package export
+
+import (
+ "context"
+ "fmt"
+
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ "github.com/apache/skywalking-banyandb/banyand/internal/storage"
+ "github.com/apache/skywalking-banyandb/banyand/queue"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+)
+
+// Backend is what the export unit needs from one catalog's storage service: where the
+// live groups are, where export snapshots go, and how to take a snapshot under a chosen
+// name. stream/measure/trace/property implement it; the wiring in pkg/cmdsetup asserts it.
+type Backend interface {
+ GetDataPath() string
+ // GetSnapshotDir is the backup snapshot directory of this catalog; no export snapshot
+ // directory may be it or lie inside it.
+ GetSnapshotDir() string
+ // GetExportSnapshotDir is the directory that holds this catalog's export session
+ // snapshots (--<catalog>-export-snapshot-path). It is separate from the backup snapshot
+ // directory, so the snapshot reclaimers never see a session.
+ GetExportSnapshotDir() string
+ // TakeExportSnapshot snapshots every group of the catalog as <GetExportSnapshotDir()>/<name>/<group>.
+ // It never runs the generic reclaimers and never renames. A catalog or group without
+ // data returns nil; it may still create empty directories (stream and trace do), so
+ // an inventory of the snapshot must tolerate groups without any segment.
+ TakeExportSnapshot(ctx context.Context, name string) error
+}
+
+// PartReader is implemented by the backends whose groups hold parts (stream, measure,
+// trace): each reads a part's metadata.json with its own catalog's reader. A missing file
+// is an error matching fs.ErrNotExist.
+type PartReader interface {
+ ReadPartMetadata(partDir string) (queue.StreamingPartData, error)
+}
+
+// Backends maps every catalog to its Backend. All four catalogs must be present.
+type Backends map[commonv1.Catalog]Backend
+
+// validatePartReaders rejects a stream, measure or trace backend that cannot read its parts.
+func (bs Backends) validatePartReaders() error {
+ for catalog, b := range bs {
+ if catalog == commonv1.Catalog_CATALOG_PROPERTY {
+ continue
+ }
+ if _, ok := b.(PartReader); !ok {
+ return fmt.Errorf("%s: the storage service cannot read part metadata", catalog)
+ }
+ }
+ return nil
+}
+
+// validateDirs rejects export snapshot directories that would collide: two catalogs
+// sharing one (or one nested in another) would see each other's sessions, one inside
+// any catalog's backup snapshot directory would be reclaimed as a stale backup, and one
+// equal to, inside or containing any catalog's data directory would mix sessions with
+// live groups (the orphan sweeper could remove a live group).
+func (bs Backends) validateDirs() error {
+ for catalog, b := range bs {
+ exportDir := b.GetExportSnapshotDir()
+ for other, ob := range bs {
+ if nested(exportDir, ob.GetDataPath()) {
+ return fmt.Errorf("%s: export snapshot path %q and the data path %q of %s must not be equal or nested",
+ catalog, exportDir, ob.GetDataPath(), other)
+ }
+ if storage.PathWithin(exportDir, ob.GetSnapshotDir()) {
+ return fmt.Errorf("%s: export snapshot path %q must not be the snapshot directory %q or inside it",
+ catalog, exportDir, ob.GetSnapshotDir())
+ }
+ if other != catalog && nested(exportDir, ob.GetExportSnapshotDir()) {
+ return fmt.Errorf("export snapshot paths of %s (%q) and %s (%q) must not be equal or nested",
+ catalog, exportDir, other, ob.GetExportSnapshotDir())
+ }
+ }
+ }
+ return nil
+}
+
+// nested reports whether a and b are the same directory or one lies inside the other.
+func nested(a, b string) bool {
+ return storage.PathWithin(a, b) || storage.PathWithin(b, a)
+}
+
+// MustBackend asserts that a storage service implements Backend, panicking at wiring time
+// otherwise. Only the data-bearing service flavors (data node, standalone) do.
+func MustBackend(svc any, name string) Backend {
+ b, ok := svc.(Backend)
+ if !ok {
+ logger.Panicf("%s service %T does not implement export.Backend", name, svc)
+ }
+ return b
+}
diff --git a/bydbctl/cmd/cli/main.go b/banyand/transfer/export/doc.go
similarity index 64%
copy from bydbctl/cmd/cli/main.go
copy to banyand/transfer/export/doc.go
index 557ef1464..2772e93dc 100644
--- a/bydbctl/cmd/cli/main.go
+++ b/banyand/transfer/export/doc.go
@@ -15,19 +15,9 @@
// specific language governing permissions and limitations
// under the License.
-// Package main provides main entry for the command-line toolkit, i.e. bydbctl.
-package main
-
-import (
- "fmt"
- "os"
-
- "github.com/apache/skywalking-banyandb/bydbctl/internal/cmd"
-)
-
-func main() {
- if err := cmd.Execute(); err != nil {
- _, _ = fmt.Fprintln(os.Stderr, err)
- os.Exit(1)
- }
-}
+// Package export implements the data-node side of the export service: a read-only
+// inventory of the on-disk units a selector hits, and the export-session snapshots that
+// freeze one point in time for a whole export. The inventory never opens parts, never
+// takes directory locks and never writes; every directory listing goes through os.ReadDir
+// so a concurrent merge or retention run surfaces as an error instead of a panic.
+package export
diff --git a/banyand/transfer/export/inventory.go b/banyand/transfer/export/inventory.go
new file mode 100644
index 000000000..3e453b05e
--- /dev/null
+++ b/banyand/transfer/export/inventory.go
@@ -0,0 +1,316 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package export
+
+import (
+ "errors"
+ "fmt"
+ "io/fs"
+ "os"
+ "path/filepath"
+ "regexp"
+ "slices"
+ "sort"
+ "strconv"
+ "strings"
+
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ transferv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/transfer/v1"
+ "github.com/apache/skywalking-banyandb/banyand/internal/dump"
+ "github.com/apache/skywalking-banyandb/banyand/internal/sidx"
+ "github.com/apache/skywalking-banyandb/banyand/queue"
+ "github.com/apache/skywalking-banyandb/pkg/index/inverted"
+)
+
+const (
+ segDirPrefix = "seg-"
+ sidxDirName = "sidx"
+ shardDirPrefix = "shard-"
+ // streamIdxDirName is the per-shard element index of a stream (stream/part.go).
+ streamIdxDirName = "idx"
+)
+
+// segSuffixPattern is the day (YYYYMMDD) or hour (YYYYMMDDHH) suffix of a segment
+// directory; anything else under a group (seg-bak, scratch copies) is not a unit.
+var segSuffixPattern = regexp.MustCompile(`^[0-9]{8}$|^[0-9]{10}$`)
+
+// partReadFunc reads one part's metadata.json; see PartReader.
+type partReadFunc func(partDir string) (queue.StreamingPartData, error)
+
+// statTSDBGroup lists <groupDir>/seg-*/shard-*/<partID>/metadata.json for stream,
+// measure and trace groups. indexMode turns on the bluge document count of the
+// segment-level series index, which is the only place index-mode measures store rows.
+// The numbers describe the directory as it is: parts that are on disk but not yet
+// published, or left behind by an interrupted merge, are counted too (design §2.3 calls
+// the live-directory inventory an estimate). A segment that vanishes mid-walk, or whose
+// metadata is still being written by a rollover, is skipped as a whole.
+func statTSDBGroup(groupDir string, catalog commonv1.Catalog, group string, indexMode bool, read partReadFunc) ([]*transferv1.UnitInventory, error) {
+ segEntries, err := readDirOrEmpty(groupDir)
+ if err != nil {
+ return nil, err
+ }
+ var units []*transferv1.UnitInventory
+ for _, se := range segEntries {
+ suffix, ok := segmentSuffixOf(se)
+ if !ok {
+ continue
+ }
+ unit, statErr := statSegment(filepath.Join(groupDir, se.Name()), suffix, catalog, group, indexMode, read)
+ if statErr != nil {
+ if errors.Is(statErr, fs.ErrNotExist) {
+ continue // segment deleted by retention, or not yet fully created, between the two listings
+ }
+ return nil, statErr
+ }
+ units = append(units, unit)
+ }
+ sort.Slice(units, func(i, j int) bool {
+ return units[i].GetSegment().GetUnit().GetSegmentSuffix() < units[j].GetSegment().GetUnit().GetSegmentSuffix()
+ })
+ return units, nil
+}
+
+// hasSegmentDir reports whether groupDir holds at least one segment directory.
+func hasSegmentDir(groupDir string) (bool, error) {
+ entries, err := readDirOrEmpty(groupDir)
+ if err != nil {
+ return false, err
+ }
+ return slices.ContainsFunc(entries, func(e os.DirEntry) bool {
+ _, ok := segmentSuffixOf(e)
+ return ok
+ }), nil
+}
+
+// segmentSuffixOf returns the time suffix of a seg-<suffix> directory entry.
+func segmentSuffixOf(e os.DirEntry) (string, bool) {
+ if !e.IsDir() || !strings.HasPrefix(e.Name(), segDirPrefix) {
+ return "", false
+ }
+ suffix := strings.TrimPrefix(e.Name(), segDirPrefix)
+ return suffix, segSuffixPattern.MatchString(suffix)
+}
+
+// statSegment inventories one segment directory as a SegmentInventory unit.
+func statSegment(segDir, suffix string, catalog commonv1.Catalog, group string, indexMode bool, read partReadFunc) (*transferv1.UnitInventory, error) {
+ version, err := readSegmentVersion(segDir)
+ if err != nil {
+ return nil, err
+ }
+ sidxDir := filepath.Join(segDir, sidxDirName)
+ sidxBytes, err := dirBytes(sidxDir)
+ if err != nil {
+ return nil, err
+ }
+ segLevel := &transferv1.SidxStat{EstimatedBytes: sidxBytes}
+ if indexMode {
+ if segLevel.DocCount, err = docCount(sidxDir); err != nil {
+ return nil, err
+ }
+ }
+ // ENOENT here is the segment itself going away; it must propagate so the caller drops
+ // the segment instead of reporting a phantom unit without shards.
+ entries, err := os.ReadDir(segDir)
+ if err != nil {
+ return nil, err
+ }
+ seg := &transferv1.SegmentInventory{
+ Unit: &transferv1.SegmentUnit{Catalog: catalog, Group: group, SegmentSuffix: suffix},
+ SegmentVersion: version,
+ SegmentLevel: segLevel,
+ }
+ for _, e := range entries {
+ id, ok := shardIDOf(e)
+ if !ok {
+ continue
+ }
+ stat, statErr := statShard(filepath.Join(segDir, e.Name()), id, catalog, read)
+ if statErr != nil {
+ if errors.Is(statErr, fs.ErrNotExist) {
+ continue
+ }
+ return nil, statErr
+ }
+ if stat.PartsCount == 0 {
+ continue // a shard directory without flushed parts holds nothing to export
+ }
+ seg.Unit.ShardIds = append(seg.Unit.ShardIds, id)
+ seg.Shards = append(seg.Shards, stat)
+ }
+ sort.Slice(seg.Shards, func(i, j int) bool { return seg.Shards[i].ShardId < seg.Shards[j].ShardId })
+ slices.Sort(seg.Unit.ShardIds)
+ return &transferv1.UnitInventory{Kind: &transferv1.UnitInventory_Segment{Segment: seg}}, nil
+}
+
+// statShard aggregates the parts of one shard. The per-shard index that travels with the
+// parts is added to both byte estimates: a stream's idx/ is copied as-is, so its size counts
+// on both sides; a trace's sidx/<rule>/<part> carries its own manifest.json sizes.
+func statShard(shardDir string, shardID uint32, catalog commonv1.Catalog, read partReadFunc) (*transferv1.ShardStat, error) {
+ partIDs, err := dump.DiscoverPartIDs(shardDir)
+ if err != nil {
+ return nil, err
+ }
+ stat := &transferv1.ShardStat{ShardId: shardID}
+ for _, id := range partIDs {
+ pm, readErr := read(filepath.Join(shardDir, fmt.Sprintf("%016x", id)))
+ if readErr != nil {
+ if errors.Is(readErr, fs.ErrNotExist) {
+ continue // part merged away between ReadDir and ReadFile
+ }
+ return nil, readErr
+ }
+ if stat.PartsCount == 0 || pm.MinTimestamp < stat.MinTimestamp {
+ stat.MinTimestamp = pm.MinTimestamp
+ }
+ if pm.MaxTimestamp > stat.MaxTimestamp {
+ stat.MaxTimestamp = pm.MaxTimestamp
+ }
+ stat.PartsCount++
+ stat.TotalCount += pm.TotalCount
+ stat.EstimatedCompressedBytes += pm.CompressedSizeBytes
+ stat.EstimatedUncompressedBytes += pm.UncompressedSizeBytes
+ stat.Parts = append(stat.Parts, &transferv1.PartStat{
+ Id: id, MinTimestamp: pm.MinTimestamp, MaxTimestamp: pm.MaxTimestamp, TotalCount: pm.TotalCount,
+ })
+ }
+ switch catalog {
+ case commonv1.Catalog_CATALOG_STREAM:
+ idxBytes, idxErr := dirBytes(filepath.Join(shardDir, streamIdxDirName))
+ if idxErr != nil {
+ return nil, idxErr
+ }
+ stat.EstimatedCompressedBytes += idxBytes
+ stat.EstimatedUncompressedBytes += idxBytes
+ case commonv1.Catalog_CATALOG_TRACE:
+ compressed, uncompressed, sidxErr := sidxBytes(filepath.Join(shardDir, sidxDirName))
+ if sidxErr != nil {
+ return nil, sidxErr
+ }
+ stat.EstimatedCompressedBytes += compressed
+ stat.EstimatedUncompressedBytes += uncompressed
+ default: // measure parts travel without a per-shard index
+ }
+ return stat, nil
+}
+
+// sidxBytes sums the manifest.json sizes of every sidx part under <sidxDir>/<rule>/. A missing
+// directory or a part merged away mid-walk counts as nothing.
+func sidxBytes(sidxDir string) (uint64, uint64, error) {
+ rules, err := readDirOrEmpty(sidxDir)
+ if err != nil {
+ return 0, 0, err
+ }
+ var compressed, uncompressed uint64
+ for _, r := range rules {
+ if !r.IsDir() {
+ continue
+ }
+ ruleDir := filepath.Join(sidxDir, r.Name())
+ partIDs, idErr := dump.DiscoverPartIDs(ruleDir)
+ if idErr != nil {
+ if errors.Is(idErr, fs.ErrNotExist) {
+ continue
+ }
+ return 0, 0, idErr
+ }
+ for _, id := range partIDs {
+ pm, readErr := sidx.ParsePartMetadata(localFS, filepath.Join(ruleDir, fmt.Sprintf("%016x", id)))
+ if readErr != nil {
+ if errors.Is(readErr, fs.ErrNotExist) {
+ continue
+ }
+ return 0, 0, readErr
+ }
+ compressed += pm.CompressedSizeBytes
+ uncompressed += pm.UncompressedSizeBytes
+ }
+ }
+ return compressed, uncompressed, nil
+}
+
+// statPropertyGroup lists <groupDir>/shard-*/ bluge directories. Property has no segment
+// layer, so it yields exactly one PropertyInventory unit per group, or nil when the group
+// has no shard directory on this node. doc_count is the cheap bluge document count: it
+// includes every version of an entity and its tombstones (design §3.1).
+func statPropertyGroup(groupDir, group string) (*transferv1.UnitInventory, error) {
+ entries, err := readDirOrEmpty(groupDir)
+ if err != nil {
+ return nil, err
+ }
+ prop := &transferv1.PropertyInventory{Group: group}
+ for _, e := range entries {
+ id, ok := shardIDOf(e)
+ if !ok {
+ continue
+ }
+ shardDir := filepath.Join(groupDir, e.Name())
+ size, sizeErr := dirBytes(shardDir)
+ if sizeErr != nil {
+ return nil, sizeErr
+ }
+ count, countErr := docCount(shardDir)
+ if countErr != nil {
+ return nil, countErr
+ }
+ stat := &transferv1.PropertyShardStat{ShardId: id, EstimatedBytes: size, DocCount: count}
+ prop.Shards = append(prop.Shards, stat)
+ }
+ if len(prop.Shards) == 0 {
+ return nil, nil
+ }
+ sort.Slice(prop.Shards, func(i, j int) bool { return prop.Shards[i].ShardId < prop.Shards[j].ShardId })
+ return &transferv1.UnitInventory{Kind: &transferv1.UnitInventory_Property{Property: prop}}, nil
+}
+
+// docCount is the committed document count of the bluge index in dir. An index that was
+// never flushed (or is absent) holds no committed generation and counts 0; any other
+// failure, a corrupt index among them, fails the plan rather than reporting 0 rows.
+func docCount(dir string) (uint64, error) {
+ count, err := inverted.ReadOnlyDocCount(dir)
+ if err != nil {
+ if errors.Is(err, inverted.ErrNoCommittedIndex) || errors.Is(err, fs.ErrNotExist) {
+ return 0, nil
+ }
+ return 0, fmt.Errorf("count documents of index %s: %w", dir, err)
+ }
+ return uint64(count), nil
+}
+
+func shardIDOf(e os.DirEntry) (uint32, bool) {
+ if !e.IsDir() || !strings.HasPrefix(e.Name(), shardDirPrefix) {
+ return 0, false
+ }
+ id, err := strconv.ParseUint(strings.TrimPrefix(e.Name(), shardDirPrefix), 10, 32)
+ if err != nil {
+ return 0, false
+ }
+ return uint32(id), true
+}
+
+// readDirOrEmpty lists dir, treating a missing directory as empty. It deliberately uses
+// os.ReadDir: pkg/fs panics on ENOENT, which a concurrent merge or retention can cause.
+func readDirOrEmpty(dir string) ([]os.DirEntry, error) {
+ entries, err := os.ReadDir(dir)
+ if err != nil {
+ if errors.Is(err, fs.ErrNotExist) {
+ return nil, nil
+ }
+ return nil, err
+ }
+ return entries, nil
+}
diff --git a/banyand/transfer/export/inventory_test.go b/banyand/transfer/export/inventory_test.go
new file mode 100644
index 000000000..b01a97e1a
--- /dev/null
+++ b/banyand/transfer/export/inventory_test.go
@@ -0,0 +1,364 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package export
+
+import (
+ "errors"
+ "io/fs"
+ "os"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "testing"
+
+ "github.com/apache/skywalking-banyandb/api/common"
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ "github.com/apache/skywalking-banyandb/pkg/index"
+ "github.com/apache/skywalking-banyandb/pkg/index/inverted"
+)
+
+func writeFile(t *testing.T, p string, body []byte) {
+ t.Helper()
+ if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(p, body, 0o600); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func partJSON(comp, uncomp, count uint64, minTS, maxTS int64) []byte {
+ // Trace's reader takes the uncompressed size from uncompressedSpanSizeBytes, the others
+ // from uncompressedSizeBytes; each ignores the other key.
+ return []byte(`{"compressedSizeBytes":` + strconv.FormatUint(comp, 10) +
+ `,"uncompressedSizeBytes":` + strconv.FormatUint(uncomp, 10) +
+ `,"uncompressedSpanSizeBytes":` + strconv.FormatUint(uncomp, 10) +
+ `,"totalCount":` + strconv.FormatUint(count, 10) +
+ `,"blocksCount":1,"minTimestamp":` + strconv.FormatInt(minTS, 10) +
+ `,"maxTimestamp":` + strconv.FormatInt(maxTS, 10) + `}`)
+}
+
+func TestStatTSDBGroup_AggregatesShardsAndSegments(t *testing.T) {
+ group := filepath.Join(t.TempDir(), "sw_record")
+ seg := filepath.Join(group, "seg-20260928")
+ writeFile(t, filepath.Join(seg, "metadata"), []byte(`{"version":"1.5.0"}`))
+ writeFile(t, filepath.Join(seg, "sidx", "000000.seg"), make([]byte, 100))
+ writeFile(t, filepath.Join(seg, "shard-0", "0000000000000001", "metadata.json"), partJSON(10, 40, 7, 100, 200))
+ writeFile(t, filepath.Join(seg, "shard-0", "0000000000000002", "metadata.json"), partJSON(20, 80, 3, 50, 300))
+ writeFile(t, filepath.Join(seg, "shard-0", "idx", "x"), make([]byte, 16)) // stream element index: bytes, not a part
+ writeFile(t, filepath.Join(seg, "shard-1", "0000000000000003", "metadata.json"), partJSON(1, 2, 1, 400, 500))
+ // A segment without shards (only metadata) must still be reported.
+ seg2 := filepath.Join(group, "seg-20260927")
+ writeFile(t, filepath.Join(seg2, "metadata"), []byte("1.4.0"))
+ writeFile(t, filepath.Join(group, "not-a-segment"), []byte("ignored"))
+ // Only day and hour suffixes are segments; a backup copy or a rollover scratch is not.
+ for _, name := range []string{"seg-bak", "seg-2026092", "seg-202609281", "seg-20260928.tmp"} {
+ writeFile(t, filepath.Join(group, name, "metadata"), []byte(`{"version":"1.5.0"}`))
+ }
+
+ units, err := statTSDBGroup(group, commonv1.Catalog_CATALOG_STREAM, "sw_record", false, testPartReader(commonv1.Catalog_CATALOG_STREAM))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(units) != 2 {
+ t.Fatalf("want 2 segments, got %d", len(units))
+ }
+ if units[0].GetSegment().Unit.SegmentSuffix != "20260927" || units[1].GetSegment().Unit.SegmentSuffix != "20260928" {
+ t.Fatalf("segments not sorted: %q %q", units[0].GetSegment().Unit.SegmentSuffix, units[1].GetSegment().Unit.SegmentSuffix)
+ }
+ u := units[1].GetSegment()
+ if u.SegmentVersion != "1.5.0" || u.SegmentLevel.GetEstimatedBytes() != 100 || u.SegmentLevel.GetDocCount() != 0 {
+ t.Fatalf("segment-level stats wrong: %+v", u.SegmentLevel)
+ }
+ if len(u.Unit.ShardIds) != 2 || u.Unit.ShardIds[0] != 0 || u.Unit.ShardIds[1] != 1 {
+ t.Fatalf("shard ids = %v", u.Unit.ShardIds)
+ }
+ s0 := u.Shards[0]
+ if s0.PartsCount != 2 || s0.TotalCount != 10 || s0.EstimatedCompressedBytes != 30+16 || s0.EstimatedUncompressedBytes != 120+16 ||
+ s0.MinTimestamp != 50 || s0.MaxTimestamp != 300 || len(s0.Parts) != 2 || s0.Parts[0].Id != 1 || s0.Parts[1].Id != 2 {
+ t.Fatalf("shard-0 stats must include the element index bytes: %+v", s0)
+ }
+ if units[0].GetSegment().SegmentVersion != "1.4.0" || len(units[0].GetSegment().Unit.ShardIds) != 0 {
+ t.Fatalf("legacy segment wrong: %+v", units[0])
+ }
+}
+
+func TestStatTSDBGroup_PartVanishedMidWalkIsSkipped(t *testing.T) {
+ group := filepath.Join(t.TempDir(), "g")
+ seg := filepath.Join(group, "seg-20260928")
+ writeFile(t, filepath.Join(seg, "metadata"), []byte(`{"version":"1.5.0"}`))
+ // A part directory without metadata.json simulates a part merged away between ReadDir and ReadFile.
+ if err := os.MkdirAll(filepath.Join(seg, "shard-0", "0000000000000009"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ writeFile(t, filepath.Join(seg, "shard-0", "000000000000000a", "metadata.json"), partJSON(1, 1, 1, 1, 2))
+ units, err := statTSDBGroup(group, commonv1.Catalog_CATALOG_MEASURE, "g", false, testPartReader(commonv1.Catalog_CATALOG_MEASURE))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if units[0].GetSegment().Shards[0].PartsCount != 1 || units[0].GetSegment().Shards[0].Parts[0].Id != 10 {
+ t.Fatalf("vanished part must be skipped, got %+v", units[0].GetSegment().Shards[0])
+ }
+}
+
+func TestStatTSDBGroup_PartlessShardIsExcluded(t *testing.T) {
+ group := filepath.Join(t.TempDir(), "g")
+ seg := filepath.Join(group, "seg-20260928")
+ writeFile(t, filepath.Join(seg, "metadata"), []byte(`{"version":"1.5.0"}`))
+ if err := os.MkdirAll(filepath.Join(seg, "shard-0"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ writeFile(t, filepath.Join(seg, "shard-1", "0000000000000001", "metadata.json"), partJSON(1, 1, 1, 1, 2))
+ units, err := statTSDBGroup(group, commonv1.Catalog_CATALOG_STREAM, "g", false, testPartReader(commonv1.Catalog_CATALOG_STREAM))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(units) != 1 || len(units[0].GetSegment().Shards) != 1 || units[0].GetSegment().Shards[0].ShardId != 1 ||
+ len(units[0].GetSegment().Unit.ShardIds) != 1 || units[0].GetSegment().Unit.ShardIds[0] != 1 {
+ t.Fatalf("a shard directory without flushed parts must not appear in shards or shard_ids: %+v", units)
+ }
+}
+
+func TestStatTSDBGroup_TraceShardCountsSidxBytes(t *testing.T) {
+ group := filepath.Join(t.TempDir(), "g")
+ seg := filepath.Join(group, "seg-2026092810")
+ writeFile(t, filepath.Join(seg, "metadata"), []byte(`{"version":"1.5.0"}`))
+ writeFile(t, filepath.Join(seg, "shard-0", "0000000000000001", "metadata.json"), partJSON(10, 40, 7, 100, 200))
+ // sidx parts describe themselves in manifest.json (banyand/internal/sidx/part.go).
+ writeFile(t, filepath.Join(seg, "shard-0", "sidx", "rule_a", "0000000000000001", "manifest.json"), partJSON(25, 60, 7, 100, 200))
+ writeFile(t, filepath.Join(seg, "shard-0", "sidx", "rule_b", "0000000000000001", "manifest.json"), partJSON(5, 10, 7, 100, 200))
+ writeFile(t, filepath.Join(seg, "shard-0", "sidx", "rule_b", "0000000000000001", "data"), make([]byte, 999)) // sizes come from the manifest, not files
+ writeFile(t, filepath.Join(seg, "shard-0", "idx", "x"), make([]byte, 7)) // not a trace artifact; must not count
+ units, err := statTSDBGroup(group, commonv1.Catalog_CATALOG_TRACE, "g", false, testPartReader(commonv1.Catalog_CATALOG_TRACE))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(units) != 1 || units[0].GetSegment().Unit.SegmentSuffix != "2026092810" {
+ t.Fatalf("hourly segment must be reported: %+v", units)
+ }
+ s0 := units[0].GetSegment().Shards[0]
+ if s0.EstimatedCompressedBytes != 10+25+5 || s0.EstimatedUncompressedBytes != 40+60+10 || s0.PartsCount != 1 {
+ t.Fatalf("trace shard bytes must include the sidx part metadata sizes: %+v", s0)
+ }
+ // A measure shard carries neither index directory.
+ units, err = statTSDBGroup(group, commonv1.Catalog_CATALOG_MEASURE, "g", false, testPartReader(commonv1.Catalog_CATALOG_MEASURE))
+ if err != nil || units[0].GetSegment().Shards[0].EstimatedCompressedBytes != 10 {
+ t.Fatalf("measure shard bytes must be the parts only: %+v, %v", units[0].GetSegment().Shards[0], err)
+ }
+}
+
+func TestStatTSDBGroup_SegmentBeingCreatedIsSkipped(t *testing.T) {
+ group := filepath.Join(t.TempDir(), "g")
+ // Storage creates the metadata file before writing it: an empty one is a rollover in flight.
+ writeFile(t, filepath.Join(group, "seg-20260929", "metadata"), nil)
+ writeFile(t, filepath.Join(group, "seg-20260929", "shard-0", "0000000000000001", "metadata.json"), partJSON(1, 1, 1, 1, 2))
+ seg := filepath.Join(group, "seg-20260928")
+ writeFile(t, filepath.Join(seg, "metadata"), []byte(`{"version":"1.5.0"}`))
+ writeFile(t, filepath.Join(seg, "shard-0", "0000000000000001", "metadata.json"), partJSON(1, 1, 1, 1, 2))
+ units, err := statTSDBGroup(group, commonv1.Catalog_CATALOG_STREAM, "g", false, testPartReader(commonv1.Catalog_CATALOG_STREAM))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(units) != 1 || units[0].GetSegment().Unit.SegmentSuffix != "20260928" {
+ t.Fatalf("the segment with empty metadata must be skipped, got %+v", units)
+ }
+}
+
+func TestReadSegmentVersion_EmptyIsNotExist(t *testing.T) {
+ seg := t.TempDir()
+ writeFile(t, filepath.Join(seg, "metadata"), []byte(" \n"))
+ _, err := readSegmentVersion(seg)
+ if !errors.Is(err, fs.ErrNotExist) {
+ t.Fatalf("empty metadata must read as not-yet-existing, got %v", err)
+ }
+}
+
+func TestStatTSDBGroup_MissingGroupDirIsEmpty(t *testing.T) {
+ units, err := statTSDBGroup(filepath.Join(t.TempDir(), "absent"), commonv1.Catalog_CATALOG_STREAM, "absent", false, testPartReader(commonv1.Catalog_CATALOG_STREAM))
+ if err != nil || len(units) != 0 {
+ t.Fatalf("want no units and no error, got %d, %v", len(units), err)
+ }
+}
+
+func TestStatPropertyGroup(t *testing.T) {
+ group := filepath.Join(t.TempDir(), "sw_prop")
+ writeFile(t, filepath.Join(group, "shard-0", "index.bin"), make([]byte, 64))
+ writeFile(t, filepath.Join(group, "shard-2", "index.bin"), make([]byte, 32))
+ unit, err := statPropertyGroup(group, "sw_prop")
+ if err != nil {
+ t.Fatal(err)
+ }
+ p := unit.GetProperty()
+ if p == nil || unit.GetSegment() != nil || p.Group != "sw_prop" {
+ t.Fatalf("a property group is a PropertyInventory unit: %+v", unit)
+ }
+ if len(p.Shards) != 2 || p.Shards[0].ShardId != 0 || p.Shards[1].ShardId != 2 || p.Shards[0].EstimatedBytes != 64 || p.Shards[1].EstimatedBytes != 32 {
+ t.Fatalf("property shard stats wrong: %+v", p.Shards)
+ }
+ // No bluge snapshot in the directory: the document count falls back to 0 without failing.
+ if p.Shards[0].DocCount != 0 {
+ t.Fatalf("doc_count must be 0 for a shard without a committed index snapshot, got %d", p.Shards[0].DocCount)
+ }
+}
+
+func TestStatPropertyGroup_NoShardsIsNil(t *testing.T) {
+ u, err := statPropertyGroup(filepath.Join(t.TempDir(), "absent"), "absent")
+ if err != nil || u != nil {
+ t.Fatalf("want nil unit, got %+v, %v", u, err)
+ }
+}
+
+func TestShardIDOf(t *testing.T) {
+ dir := t.TempDir()
+ for _, name := range []string{"shard-0", "shard-abc", "shard-", "not-shard"} {
+ if err := os.MkdirAll(filepath.Join(dir, name), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ }
+ // shard-1 is a regular file, not a directory.
+ writeFile(t, filepath.Join(dir, "shard-1"), []byte("regular file"))
+ entries, err := os.ReadDir(dir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ byName := make(map[string]os.DirEntry, len(entries))
+ for _, e := range entries {
+ byName[e.Name()] = e
+ }
+ cases := []struct {
+ name string
+ wantOK bool
+ wantID uint32
+ }{
+ {"shard-0", true, 0},
+ {"shard-abc", false, 0},
+ {"shard-", false, 0},
+ {"not-shard", false, 0},
+ {"shard-1", false, 0},
+ }
+ for _, tc := range cases {
+ e, found := byName[tc.name]
+ if !found {
+ t.Fatalf("entry %q not found in test dir", tc.name)
+ }
+ id, ok := shardIDOf(e)
+ if ok != tc.wantOK {
+ t.Errorf("shardIDOf(%q) ok=%v, want %v", tc.name, ok, tc.wantOK)
+ }
+ if ok && id != tc.wantID {
+ t.Errorf("shardIDOf(%q) id=%d, want %d", tc.name, id, tc.wantID)
+ }
+ }
+}
+
+// buildIndex writes a committed bluge index with docs documents into dir, the way the
+// segment-level series index and the property shards are written.
+func buildIndex(t *testing.T, dir string, docs int) {
+ t.Helper()
+ store, err := inverted.NewStore(inverted.StoreOpts{Path: dir})
+ if err != nil {
+ t.Fatal(err)
+ }
+ key := index.FieldKey{Analyzer: index.AnalyzerKeyword, SeriesID: common.SeriesID(1), IndexRuleID: 1}
+ documents := make(index.Documents, 0, docs)
+ for i := 0; i < docs; i++ {
+ field := index.NewStringField(key, "v"+strconv.Itoa(i))
+ field.Index = true
+ field.Store = true
+ documents = append(documents, index.Document{DocID: uint64(i + 1), Fields: []index.Field{field}})
+ }
+ if err = store.Batch(index.Batch{Documents: documents}); err != nil {
+ t.Fatal(err)
+ }
+ if err = store.Close(); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// corruptIndex overwrites every committed snapshot manifest in dir with garbage.
+func corruptIndex(t *testing.T, dir string) {
+ t.Helper()
+ entries, err := os.ReadDir(dir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ corrupted := 0
+ for _, e := range entries {
+ if strings.HasSuffix(e.Name(), ".snp") {
+ writeFile(t, filepath.Join(dir, e.Name()), []byte("garbage"))
+ corrupted++
+ }
+ }
+ if corrupted == 0 {
+ t.Fatalf("no snapshot manifest to corrupt in %s", dir)
+ }
+}
+
+func TestStatTSDBGroup_IndexModeCountsCommittedDocuments(t *testing.T) {
+ group := filepath.Join(t.TempDir(), "sw_metadata")
+ seg := filepath.Join(group, "seg-20260928")
+ writeFile(t, filepath.Join(seg, "metadata"), []byte(`{"version":"1.5.0"}`))
+ buildIndex(t, filepath.Join(seg, sidxDirName), 3)
+
+ units, err := statTSDBGroup(group, commonv1.Catalog_CATALOG_MEASURE, "sw_metadata", true, testPartReader(commonv1.Catalog_CATALOG_MEASURE))
+ if err != nil {
+ t.Fatal(err)
+ }
+ segLevel := units[0].GetSegment().GetSegmentLevel()
+ if segLevel.GetDocCount() != 3 || segLevel.GetEstimatedBytes() == 0 {
+ t.Fatalf("an index-mode segment must report the committed document count and the index bytes: %+v", segLevel)
+ }
+ // A plain measure group never opens the index.
+ units, err = statTSDBGroup(group, commonv1.Catalog_CATALOG_MEASURE, "sw_metadata", false, testPartReader(commonv1.Catalog_CATALOG_MEASURE))
+ if err != nil || units[0].GetSegment().GetSegmentLevel().GetDocCount() != 0 {
+ t.Fatalf("a plain measure group must report doc_count 0, got %+v, %v", units, err)
+ }
+ // A corrupt index fails the plan instead of reporting zero rows.
+ corruptIndex(t, filepath.Join(seg, sidxDirName))
+ if _, err = statTSDBGroup(group, commonv1.Catalog_CATALOG_MEASURE, "sw_metadata", true, testPartReader(commonv1.Catalog_CATALOG_MEASURE)); err == nil ||
+ !errors.Is(err, inverted.ErrCorruptIndex) {
+ t.Fatalf("a corrupt index must fail the plan, got %v", err)
+ }
+}
+
+func TestStatTSDBGroup_IndexModeWithoutCommittedIndexCountsZero(t *testing.T) {
+ group := filepath.Join(t.TempDir(), "sw_metadata")
+ seg := filepath.Join(group, "seg-20260928")
+ writeFile(t, filepath.Join(seg, "metadata"), []byte(`{"version":"1.5.0"}`))
+ units, err := statTSDBGroup(group, commonv1.Catalog_CATALOG_MEASURE, "sw_metadata", true, testPartReader(commonv1.Catalog_CATALOG_MEASURE))
+ if err != nil || units[0].GetSegment().GetSegmentLevel().GetDocCount() != 0 {
+ t.Fatalf("a segment without a flushed index must count 0, got %+v, %v", units, err)
+ }
+}
+
+func TestStatPropertyGroup_CountsCommittedDocuments(t *testing.T) {
+ group := filepath.Join(t.TempDir(), "sw_prop")
+ buildIndex(t, filepath.Join(group, "shard-0"), 4)
+ unit, err := statPropertyGroup(group, "sw_prop")
+ if err != nil {
+ t.Fatal(err)
+ }
+ shards := unit.GetProperty().GetShards()
+ if len(shards) != 1 || shards[0].GetDocCount() != 4 || shards[0].GetEstimatedBytes() == 0 {
+ t.Fatalf("a property shard must report its committed document count: %+v", shards)
+ }
+ corruptIndex(t, filepath.Join(group, "shard-0"))
+ if _, err = statPropertyGroup(group, "sw_prop"); !errors.Is(err, inverted.ErrCorruptIndex) {
+ t.Fatalf("a corrupt property shard must fail the plan, got %v", err)
+ }
+}
diff --git a/banyand/transfer/export/lease.go b/banyand/transfer/export/lease.go
new file mode 100644
index 000000000..69a9721c8
--- /dev/null
+++ b/banyand/transfer/export/lease.go
@@ -0,0 +1,108 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package export
+
+import (
+ "encoding/json"
+ "errors"
+ "io/fs"
+ "os"
+ "path/filepath"
+ "regexp"
+ "time"
+
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/status"
+
+ pkgfs "github.com/apache/skywalking-banyandb/pkg/fs"
+)
+
+const (
+ // LeaseFileName is the lease document inside every <exportSnapshotDir>/<id> session directory.
+ LeaseFileName = ".lease"
+ // defaultLease is how long a session survives without a heartbeat before the sweeper
+ // reclaims it: every renewal resets expiresAt to now + defaultLease, so the task length
+ // is irrelevant and nothing on the wire carries it.
+ defaultLease = 5 * 24 * time.Hour
+ // orphanGrace is how long a session directory without a readable .lease may exist
+ // before the sweeper treats it as the leftover of a crashed creation and removes it.
+ orphanGrace = 10 * time.Minute
+)
+
+// SweepInterval is how often a data node scans for expired sessions. It is a variable only
+// so integration suites can raise it; production never changes it.
+var SweepInterval = time.Minute
+
+var (
+ errNoLease = errors.New("export session has no lease file")
+ sessionIDPattern = regexp.MustCompile(`^[0-9a-f]{1,64}$`)
+)
+
+// Lease is the on-disk <session dir>/.lease document (design §3.5). Times are UnixNano;
+// lastHeartbeatAt records the last heartbeat (create counts as the first) and a
+// document without expiresAt reads as expired.
+type Lease struct {
+ SessionID string `json:"sessionId"`
+ StartedAt int64 `json:"startedAt"`
+ ExpiresAt int64 `json:"expiresAt"`
+ LastHeartbeatAt int64 `json:"lastHeartbeatAt"`
+}
+
+func (l Lease) expired(now time.Time) bool { return now.UnixNano() >= l.ExpiresAt }
+
+// ReadLease parses <sessionDir>/.lease; it exists for test helpers outside this package.
+func ReadLease(sessionDir string) (Lease, error) { return readLease(sessionDir) }
+
+// readLease parses <sessionDir>/.lease. Unknown keys are ignored, so a document written by
+// an earlier build (which also recorded client and catalogs) still loads.
+func readLease(sessionDir string) (Lease, error) {
+ raw, err := os.ReadFile(filepath.Join(sessionDir, LeaseFileName))
+ if err != nil {
+ if errors.Is(err, fs.ErrNotExist) {
+ return Lease{}, errNoLease
+ }
+ return Lease{}, err
+ }
+ var l Lease
+ if err := json.Unmarshal(raw, &l); err != nil {
+ return Lease{}, err
+ }
+ return l, nil
+}
+
+// WriteLease atomically replaces <sessionDir>/.lease; it exists for test helpers outside this package.
+func WriteLease(sessionDir string, l Lease) error { return writeLease(sessionDir, l) }
+
+// writeLease atomically replaces <sessionDir>/.lease (tmp + fsync + rename).
+func writeLease(sessionDir string, l Lease) error {
+ raw, err := json.Marshal(l)
+ if err != nil {
+ return err
+ }
+ _, err = pkgfs.NewLocalFileSystem().WriteAtomic(raw, filepath.Join(sessionDir, LeaseFileName), 0o600)
+ return err
+}
+
+// validateSessionID rejects anything that could escape the export snapshot directory. The proto
+// carries the same rule; this is the defense for the data-node port, which has no auth.
+func validateSessionID(id string) error {
+ if !sessionIDPattern.MatchString(id) {
+ return status.Errorf(codes.InvalidArgument, "invalid export session id %q: want 1-64 lowercase hex characters", id)
+ }
+ return nil
+}
diff --git a/banyand/transfer/export/lease_test.go b/banyand/transfer/export/lease_test.go
new file mode 100644
index 000000000..820643d62
--- /dev/null
+++ b/banyand/transfer/export/lease_test.go
@@ -0,0 +1,92 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package export
+
+import (
+ "errors"
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+)
+
+func TestLeaseRoundTripAndState(t *testing.T) {
+ dir := t.TempDir()
+ now := time.Unix(1_700_000_000, 0)
+ l := Lease{SessionID: "9f2a", StartedAt: now.UnixNano(), ExpiresAt: now.Add(time.Hour).UnixNano(), LastHeartbeatAt: now.UnixNano()}
+ if err := writeLease(dir, l); err != nil {
+ t.Fatal(err)
+ }
+ got, err := readLease(dir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got != l {
+ t.Fatalf("readLease() = %+v, want %+v", got, l)
+ }
+ if got.expired(now.Add(30 * time.Minute)) {
+ t.Fatal("lease must not be expired before expiresAt")
+ }
+ if !got.expired(now.Add(2 * time.Hour)) {
+ t.Fatal("lease must be expired after expiresAt")
+ }
+ if _, err := readLease(t.TempDir()); !errors.Is(err, errNoLease) {
+ t.Fatalf("missing lease must be errNoLease, got %v", err)
+ }
+}
+
+func TestReadLease_IgnoresKeysOfEarlierBuilds(t *testing.T) {
+ dir := t.TempDir()
+ raw := []byte(`{"sessionId":"9f2a","client":"legacy-client","catalogs":["CATALOG_STREAM"],` +
+ `"startedAt":1,"expiresAt":3,"lastHeartbeatAt":2}`)
+ if err := os.WriteFile(filepath.Join(dir, LeaseFileName), raw, 0o600); err != nil {
+ t.Fatal(err)
+ }
+ got, err := readLease(dir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if want := (Lease{SessionID: "9f2a", StartedAt: 1, ExpiresAt: 3, LastHeartbeatAt: 2}); got != want {
+ t.Fatalf("readLease() = %+v, want %+v", got, want)
+ }
+}
+
+func TestValidateSessionID(t *testing.T) {
+ for _, ok := range []string{"a", "9f2ac1d4", "0123456789abcdef0123456789abcdef"} {
+ if err := validateSessionID(ok); err != nil {
+ t.Fatalf("%q must be accepted: %v", ok, err)
+ }
+ }
+ for _, bad := range []string{"", "/../../data", "ABC", "9f2a-1", "..", "x y"} {
+ if err := validateSessionID(bad); err == nil {
+ t.Fatalf("%q must be rejected", bad)
+ }
+ }
+}
+
+func TestReadLease_CorruptedFileIsNotErrNoLease(t *testing.T) {
+ dir := t.TempDir()
+ writeFile(t, filepath.Join(dir, LeaseFileName), []byte(`{broken`))
+ _, err := readLease(dir)
+ if err == nil {
+ t.Fatal("corrupted lease file must return an error")
+ }
+ if errors.Is(err, errNoLease) {
+ t.Fatalf("corrupted lease file must not be errNoLease, got %v", err)
+ }
+}
diff --git a/banyand/transfer/export/partmeta.go b/banyand/transfer/export/partmeta.go
new file mode 100644
index 000000000..7d9b80ed1
--- /dev/null
+++ b/banyand/transfer/export/partmeta.go
@@ -0,0 +1,82 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package export
+
+import (
+ "errors"
+ "fmt"
+ "io/fs"
+ "os"
+ "path/filepath"
+
+ "github.com/apache/skywalking-banyandb/banyand/internal/storage"
+ pkgfs "github.com/apache/skywalking-banyandb/pkg/fs"
+)
+
+// localFS is the file system the sidx part metadata reader goes through.
+var localFS = pkgfs.NewLocalFileSystem()
+
+// readSegmentVersion returns the version recorded in <segDir>/metadata without judging
+// its compatibility: the import side owns that gate. Storage creates the file before it
+// writes it, so an empty file is a rollover in progress and reported as fs.ErrNotExist,
+// which callers treat like a segment that is not there yet.
+func readSegmentVersion(segDir string) (string, error) {
+ path := filepath.Join(segDir, storage.SegmentMetadataFilename)
+ raw, err := os.ReadFile(path)
+ if err != nil {
+ return "", err
+ }
+ meta, err := storage.DecodeSegmentMetadata(raw)
+ if errors.Is(err, storage.ErrEmptySegmentMetadata) {
+ return "", fmt.Errorf("segment metadata %s is still being written: %w", path, fs.ErrNotExist)
+ }
+ if err != nil {
+ return "", err
+ }
+ return meta.Version, nil
+}
+
+// dirBytes sums the sizes of every regular file below dir. A missing dir counts as 0
+// so a segment without sidx/ or a shard that vanished mid-walk is not an error.
+func dirBytes(dir string) (uint64, error) {
+ var total uint64
+ err := filepath.WalkDir(dir, func(_ string, d fs.DirEntry, walkErr error) error {
+ if walkErr != nil {
+ if errors.Is(walkErr, fs.ErrNotExist) {
+ return nil
+ }
+ return walkErr
+ }
+ if d.IsDir() {
+ return nil
+ }
+ info, infoErr := d.Info()
+ if infoErr != nil {
+ if errors.Is(infoErr, fs.ErrNotExist) {
+ return nil
+ }
+ return infoErr
+ }
+ total += uint64(info.Size())
+ return nil
+ })
+ if errors.Is(err, fs.ErrNotExist) {
+ return 0, nil
+ }
+ return total, err
+}
diff --git a/banyand/transfer/export/partmeta_test.go b/banyand/transfer/export/partmeta_test.go
new file mode 100644
index 000000000..0a3b1dd4a
--- /dev/null
+++ b/banyand/transfer/export/partmeta_test.go
@@ -0,0 +1,125 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package export
+
+import (
+ "errors"
+ "io/fs"
+ "os"
+ "path/filepath"
+ "testing"
+
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ "github.com/apache/skywalking-banyandb/banyand/internal/storage"
+ "github.com/apache/skywalking-banyandb/banyand/measure"
+ "github.com/apache/skywalking-banyandb/banyand/queue"
+ "github.com/apache/skywalking-banyandb/banyand/stream"
+ "github.com/apache/skywalking-banyandb/banyand/trace"
+ pkgfs "github.com/apache/skywalking-banyandb/pkg/fs"
+)
+
+// testPartReader is the PartReader a catalog's storage service provides: its own
+// ParsePartMetadata over the local file system.
+func testPartReader(catalog commonv1.Catalog) partReadFunc {
+ lfs := pkgfs.NewLocalFileSystem()
+ return func(partDir string) (queue.StreamingPartData, error) {
+ switch catalog {
+ case commonv1.Catalog_CATALOG_TRACE:
+ return trace.ParsePartMetadata(lfs, partDir)
+ case commonv1.Catalog_CATALOG_MEASURE:
+ return measure.ParsePartMetadata(lfs, partDir)
+ default:
+ return stream.ParsePartMetadata(lfs, partDir)
+ }
+ }
+}
+
+func TestPartReaders_UseEachCatalogsReader(t *testing.T) {
+ dir := t.TempDir()
+ streamPart := filepath.Join(dir, "0000000000000001")
+ writeFile(t, filepath.Join(streamPart, storage.PartMetadataFilename),
+ []byte(`{"compressedSizeBytes":10,"uncompressedSizeBytes":40,"totalCount":7,"blocksCount":1,"minTimestamp":100,"maxTimestamp":200}`))
+ tracePart := filepath.Join(dir, "0000000000000002")
+ writeFile(t, filepath.Join(tracePart, storage.PartMetadataFilename),
+ []byte(`{"compressedSizeBytes":11,"uncompressedSpanSizeBytes":41,"totalCount":8,"blocksCount":1,"minTimestamp":101,"maxTimestamp":201}`))
+ measurePart := filepath.Join(dir, "0000000000000003")
+ writeFile(t, filepath.Join(measurePart, storage.PartMetadataFilename),
+ []byte(`{"compressedSizeBytes":12,"uncompressedSizeBytes":42,"totalCount":9,"blocksCount":1,"minTimestamp":102,"maxTimestamp":202}`))
+
+ got, err := testPartReader(commonv1.Catalog_CATALOG_STREAM)(streamPart)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got.CompressedSizeBytes != 10 || got.UncompressedSizeBytes != 40 || got.TotalCount != 7 || got.MinTimestamp != 100 || got.MaxTimestamp != 200 {
+ t.Fatalf("unexpected stream metadata %+v", got)
+ }
+ got, err = testPartReader(commonv1.Catalog_CATALOG_TRACE)(tracePart)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got.UncompressedSizeBytes != 41 || got.TotalCount != 8 {
+ t.Fatalf("trace span size must be mapped onto UncompressedSizeBytes, got %+v", got)
+ }
+ got, err = testPartReader(commonv1.Catalog_CATALOG_MEASURE)(measurePart)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got.CompressedSizeBytes != 12 || got.UncompressedSizeBytes != 42 || got.TotalCount != 9 || got.MinTimestamp != 102 || got.MaxTimestamp != 202 {
+ t.Fatalf("unexpected measure metadata %+v", got)
+ }
+}
+
+func TestPartReaders_MissingIsNotExist(t *testing.T) {
+ for _, catalog := range []commonv1.Catalog{commonv1.Catalog_CATALOG_STREAM, commonv1.Catalog_CATALOG_MEASURE, commonv1.Catalog_CATALOG_TRACE} {
+ if _, err := testPartReader(catalog)(filepath.Join(t.TempDir(), "gone")); !errors.Is(err, fs.ErrNotExist) {
+ t.Fatalf("%s: a missing metadata.json must match fs.ErrNotExist, got %v", catalog, err)
+ }
+ }
+}
+
+func TestReadSegmentVersion(t *testing.T) {
+ seg := t.TempDir()
+ if err := os.WriteFile(filepath.Join(seg, "metadata"), []byte(`{"version":"1.5.0"}`), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ v, err := readSegmentVersion(seg)
+ if err != nil || v != "1.5.0" {
+ t.Fatalf("readSegmentVersion() = %q, %v", v, err)
+ }
+}
+
+func TestDirBytes(t *testing.T) {
+ dir := t.TempDir()
+ if err := os.MkdirAll(filepath.Join(dir, "sub"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filepath.Join(dir, "a"), make([]byte, 3), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filepath.Join(dir, "sub", "b"), make([]byte, 5), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ got, err := dirBytes(dir)
+ if err != nil || got != 8 {
+ t.Fatalf("dirBytes() = %d, %v; want 8", got, err)
+ }
+ got, err = dirBytes(filepath.Join(dir, "missing"))
+ if err != nil || got != 0 {
+ t.Fatalf("missing dir must count as 0, got %d, %v", got, err)
+ }
+}
diff --git a/banyand/transfer/export/planner.go b/banyand/transfer/export/planner.go
new file mode 100644
index 000000000..f22cd57aa
--- /dev/null
+++ b/banyand/transfer/export/planner.go
@@ -0,0 +1,411 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package export
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "maps"
+ "path/filepath"
+ "slices"
+ "sort"
+
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/status"
+
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ transferv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/transfer/v1"
+ "github.com/apache/skywalking-banyandb/banyand/metadata"
+ "github.com/apache/skywalking-banyandb/banyand/metadata/schema"
+ "github.com/apache/skywalking-banyandb/banyand/queue/pub"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+)
+
+// Planner produces the read-only export inventory of this node, either from the live data
+// directories or from an export-session snapshot.
+type Planner struct {
+ meta metadata.Repo
+ backends Backends
+ sessions *sessionManager
+ nodeLabels map[string]string
+ l *logger.Logger
+}
+
+// NewPlanner wires a Planner. nodeLabels drive lifecycle-stage resolution.
+func NewPlanner(meta metadata.Repo, backends Backends, nodeLabels map[string]string, l *logger.Logger) *Planner {
+ return &Planner{
+ meta: meta,
+ backends: backends,
+ sessions: newSessionManager(backends, l),
+ nodeLabels: nodeLabels,
+ l: l,
+ }
+}
+
+// PlanInventory serves one PlanRequest. Without a session it walks the live data
+// directories of the groups the schema registry knows; ReadSession walks (and renews) the
+// session snapshot and takes its group list from the snapshot itself; CreateSession snapshots
+// first and, when it preempted anything, closes the stream with a summary frame carrying those
+// ids after the group frames (the liaison announces the session id itself). Every emitted unit
+// frame covers exactly one group and leaves node_id empty for the liaison to stamp.
+func (p *Planner) PlanInventory(ctx context.Context, req *transferv1.PlanRequest, emit func(*transferv1.PlanResponse) error) error {
+ roots, summary, err := p.resolveRoots(ctx, req)
+ if err != nil {
+ return err
+ }
+ if err = p.planRoots(ctx, req, roots, emit); err != nil {
+ if summary != nil {
+ // The preempted sessions are gone even though the walk failed: report them
+ // best-effort as the last frame before the error.
+ if emitErr := emit(summary); emitErr != nil {
+ p.l.Warn().Err(emitErr).Msg("cannot report the preempted sessions of a failed create")
+ }
+ }
+ return err
+ }
+ if summary != nil {
+ return emit(summary)
+ }
+ return nil
+}
+
+// planRoots emits one frame per selected group under roots. In session mode it finally
+// checks that the session survived the walk, since a snapshot removed meanwhile reads as
+// missing directories rather than as an error.
+func (p *Planner) planRoots(ctx context.Context, req *transferv1.PlanRequest, roots map[commonv1.Catalog]string,
+ emit func(*transferv1.PlanResponse) error,
+) error {
+ groups, err := p.planGroups(ctx, req, roots)
+ if err != nil {
+ return err
+ }
+ for _, g := range selectGroups(groups, req.GetSelectors()) {
+ if err = ctx.Err(); err != nil {
+ return status.FromContextError(err).Err()
+ }
+ root, ok := roots[g.GetCatalog()]
+ if !ok {
+ continue // this catalog has no root on this node
+ }
+ frame, buildErr := p.planGroup(ctx, g, root)
+ if buildErr != nil {
+ if errors.Is(buildErr, context.Canceled) || errors.Is(buildErr, context.DeadlineExceeded) {
+ return status.FromContextError(buildErr).Err()
+ }
+ return status.Errorf(codes.Internal, "plan group %s: %v", g.GetMetadata().GetName(), buildErr)
+ }
+ if frame == nil {
+ continue
+ }
+ if err = emit(frame); err != nil {
+ return err
+ }
+ }
+ if req.GetSession() != nil {
+ return p.sessions.verify(sessionID(req))
+ }
+ return nil
+}
+
+// resolveRoots picks the groups root of every catalog for this request: the live
+// directories, an existing session's snapshot (renewing it), or a freshly created one.
+func (p *Planner) resolveRoots(ctx context.Context, req *transferv1.PlanRequest) (map[commonv1.Catalog]string, *transferv1.PlanResponse, error) {
+ switch sess := req.GetSession().(type) {
+ case nil:
+ roots := make(map[commonv1.Catalog]string, len(p.backends))
+ for c, b := range p.backends {
+ roots[c] = b.GetDataPath()
+ }
+ return roots, nil, nil
+ case *transferv1.PlanRequest_Create:
+ if sess.Create.GetId() == "" {
+ return nil, nil, status.Error(codes.InvalidArgument, "create session requires the session id the liaison generated")
+ }
+ created, err := p.sessions.create(ctx, sess.Create.GetId(), p.selectedCatalogs(req.GetSelectors()), sess.Create.GetPreempt())
+ if err != nil {
+ return nil, nil, err
+ }
+ if len(created.preempted) == 0 {
+ return created.roots, nil, nil
+ }
+ return created.roots, &transferv1.PlanResponse{Frame: &transferv1.PlanResponse_Summary{
+ Summary: &transferv1.PlanSummary{PreemptedSessionIds: created.preempted},
+ }}, nil
+ case *transferv1.PlanRequest_Read:
+ if sess.Read.GetId() == "" {
+ return nil, nil, status.Error(codes.InvalidArgument, "read session requires the session id")
+ }
+ roots, err := p.sessions.locateAndHeartbeat(sess.Read.GetId())
+ return roots, nil, err
+ default:
+ return nil, nil, status.Errorf(codes.InvalidArgument, "unknown session kind %T", sess)
+ }
+}
+
+// Sessions serves one SessionsRequest with the single frame this node answers. ACTION_LIST
+// answers `session` with the lease held here, or `none`. ACTION_HEARTBEAT extends the lease of
+// session_id and answers `done`; a session this node does not hold is answered as `error` so
+// the liaison can tell which nodes lost it, while an expired or unreadable lease fails the
+// call (FailedPrecondition). ACTION_RELEASE drops the session, idempotently, and answers
+// `done` when the node held it and `none` when it did not; a failure is answered as `error`
+// too, since a half-released session must stay visible rather than fail the fan-out.
+func (p *Planner) Sessions(req *transferv1.SessionsRequest) (*transferv1.SessionsResponse, error) {
+ id := req.GetSessionId()
+ if req.GetAction() != transferv1.SessionsRequest_ACTION_LIST && id == "" {
+ return nil, status.Errorf(codes.InvalidArgument, "action %s requires the session id", req.GetAction())
+ }
+ switch req.GetAction() {
+ case transferv1.SessionsRequest_ACTION_LIST:
+ lease, err := p.sessions.probe()
+ if err != nil {
+ return nil, status.Errorf(codes.Internal, "list export sessions: %v", err)
+ }
+ if lease == nil {
+ return &transferv1.SessionsResponse{Outcome: &transferv1.SessionsResponse_None{None: &transferv1.Ack{}}}, nil
+ }
+ return &transferv1.SessionsResponse{Outcome: &transferv1.SessionsResponse_Session{Session: lease}}, nil
+ case transferv1.SessionsRequest_ACTION_HEARTBEAT:
+ _, err := p.sessions.locateAndHeartbeat(id)
+ switch {
+ case err == nil:
+ return doneFrame(), nil
+ case status.Code(err) == codes.NotFound:
+ return errorFrame(err), nil
+ default:
+ return nil, err
+ }
+ case transferv1.SessionsRequest_ACTION_RELEASE:
+ held, err := p.sessions.release(id)
+ if err != nil {
+ return errorFrame(err), nil
+ }
+ if !held {
+ return noneFrame(), nil
+ }
+ return doneFrame(), nil
+ default:
+ return nil, status.Errorf(codes.InvalidArgument, "unknown sessions action %s", req.GetAction())
+ }
+}
+
+func noneFrame() *transferv1.SessionsResponse {
+ return &transferv1.SessionsResponse{Outcome: &transferv1.SessionsResponse_None{None: &transferv1.Ack{}}}
+}
+
+func doneFrame() *transferv1.SessionsResponse {
+ return &transferv1.SessionsResponse{Outcome: &transferv1.SessionsResponse_Done{Done: &transferv1.Ack{}}}
+}
+
+func errorFrame(err error) *transferv1.SessionsResponse {
+ return &transferv1.SessionsResponse{Outcome: &transferv1.SessionsResponse_Error{Error: status.Convert(err).Message()}}
+}
+
+// selectedCatalogs returns the catalogs a selector list touches; empty means all four.
+func (p *Planner) selectedCatalogs(selectors []*transferv1.Selector) map[commonv1.Catalog]struct{} {
+ out := map[commonv1.Catalog]struct{}{}
+ if len(selectors) == 0 {
+ for c := range p.backends {
+ out[c] = struct{}{}
+ }
+ return out
+ }
+ for _, s := range selectors {
+ if _, ok := p.backends[s.GetCatalog()]; ok {
+ out[s.GetCatalog()] = struct{}{}
+ }
+ }
+ return out
+}
+
+// sessionID is the id of the session a request drives, empty for a dry run.
+func sessionID(req *transferv1.PlanRequest) string {
+ if c := req.GetCreate(); c != nil {
+ return c.GetId()
+ }
+ return req.GetRead().GetId()
+}
+
+// planGroups lists the groups a request inventories: the group directories under the root of
+// every catalog the selectors name (every catalog without selectors) that the schema registry
+// still knows. Only the root differs between the modes: the live data directory for a dry run,
+// the session snapshot otherwise. A directory whose group the registry no longer knows is
+// skipped with a warning. A selector naming a catalog without a root is refused: in a session
+// that catalog holds no snapshot, and silently skipping it would hide a half-lost session.
+func (p *Planner) planGroups(ctx context.Context, req *transferv1.PlanRequest, roots map[commonv1.Catalog]string) ([]*commonv1.Group, error) {
+ for _, s := range req.GetSelectors() {
+ if _, ok := roots[s.GetCatalog()]; !ok {
+ return nil, status.Errorf(codes.FailedPrecondition, "export session %s has no snapshot for catalog %s on this node",
+ sessionID(req), s.GetCatalog())
+ }
+ }
+ registered, err := p.meta.GroupRegistry().ListGroup(ctx)
+ if err != nil {
+ return nil, status.Errorf(codes.Internal, "list groups: %v", err)
+ }
+ known := make(map[commonv1.Catalog]map[string]*commonv1.Group, len(roots))
+ for _, g := range registered {
+ if known[g.GetCatalog()] == nil {
+ known[g.GetCatalog()] = map[string]*commonv1.Group{}
+ }
+ known[g.GetCatalog()][g.GetMetadata().GetName()] = g
+ }
+ var out []*commonv1.Group
+ for _, catalog := range slices.Sorted(maps.Keys(p.selectedCatalogs(req.GetSelectors()))) {
+ root, ok := roots[catalog]
+ if !ok {
+ continue
+ }
+ entries, readErr := readDirOrEmpty(root)
+ if readErr != nil {
+ return nil, status.Errorf(codes.Internal, "list groups of %s under %s: %v", catalog, root, readErr)
+ }
+ for _, e := range entries {
+ if !e.IsDir() {
+ continue
+ }
+ g, registeredGroup := known[catalog][e.Name()]
+ if !registeredGroup {
+ p.l.Warn().Str("group", e.Name()).Stringer("catalog", catalog).Str("dir", root).
+ Msg("skipping a group directory the schema registry does not know")
+ continue
+ }
+ out = append(out, g)
+ }
+ }
+ sortGroups(out)
+ return out, nil
+}
+
+func sortGroups(groups []*commonv1.Group) {
+ sort.Slice(groups, func(i, j int) bool {
+ if groups[i].GetCatalog() != groups[j].GetCatalog() {
+ return groups[i].GetCatalog() < groups[j].GetCatalog()
+ }
+ return groups[i].GetMetadata().GetName() < groups[j].GetMetadata().GetName()
+ })
+}
+
+// selectGroups narrows groups to what the selectors name: a selector without groups takes
+// the whole catalog. No selectors means every group.
+func selectGroups(groups []*commonv1.Group, selectors []*transferv1.Selector) []*commonv1.Group {
+ if len(selectors) == 0 {
+ return groups
+ }
+ wholeCatalog := make(map[commonv1.Catalog]bool, len(selectors))
+ named := make(map[commonv1.Catalog]map[string]struct{}, len(selectors))
+ for _, s := range selectors {
+ if len(s.GetGroups()) == 0 {
+ wholeCatalog[s.GetCatalog()] = true
+ continue
+ }
+ if named[s.GetCatalog()] == nil {
+ named[s.GetCatalog()] = map[string]struct{}{}
+ }
+ for _, g := range s.GetGroups() {
+ named[s.GetCatalog()][g] = struct{}{}
+ }
+ }
+ var out []*commonv1.Group
+ for _, g := range groups {
+ if wholeCatalog[g.GetCatalog()] {
+ out = append(out, g)
+ continue
+ }
+ if _, ok := named[g.GetCatalog()][g.GetMetadata().GetName()]; ok {
+ out = append(out, g)
+ }
+ }
+ return out
+}
+
+// planGroup builds the single frame of one group under root, or nil when the group has no
+// data there.
+func (p *Planner) planGroup(ctx context.Context, g *commonv1.Group, root string) (*transferv1.PlanResponse, error) {
+ name := g.GetMetadata().GetName()
+ stage, err := p.resolveStage(g)
+ if err != nil {
+ return nil, err
+ }
+ groupDir := filepath.Join(root, name)
+ var units []*transferv1.UnitInventory
+ var read partReadFunc
+ if pr, ok := p.backends[g.GetCatalog()].(PartReader); ok {
+ read = pr.ReadPartMetadata
+ }
+ switch g.GetCatalog() {
+ case commonv1.Catalog_CATALOG_STREAM, commonv1.Catalog_CATALOG_TRACE:
+ units, err = statTSDBGroup(groupDir, g.GetCatalog(), name, false, read)
+ case commonv1.Catalog_CATALOG_MEASURE:
+ hasSegments, segErr := hasSegmentDir(groupDir)
+ if segErr != nil {
+ return nil, segErr
+ }
+ indexMode := false
+ if hasSegments { // a group without segments has nothing to count: skip the registry
+ if indexMode, err = p.hasIndexModeMeasure(ctx, name); err != nil {
+ return nil, err
+ }
+ }
+ units, err = statTSDBGroup(groupDir, g.GetCatalog(), name, indexMode, read)
+ case commonv1.Catalog_CATALOG_PROPERTY:
+ var unit *transferv1.UnitInventory
+ unit, err = statPropertyGroup(groupDir, name)
+ if unit != nil {
+ units = []*transferv1.UnitInventory{unit}
+ }
+ default:
+ return nil, fmt.Errorf("unsupported catalog %s", g.GetCatalog())
+ }
+ if err != nil {
+ return nil, err
+ }
+ if len(units) == 0 {
+ return nil, nil
+ }
+ return &transferv1.PlanResponse{Frame: &transferv1.PlanResponse_Units{Units: &transferv1.UnitFrame{Stage: stage, Units: units}}}, nil
+}
+
+// resolveStage returns the matched lifecycle stage name, or "" for the default tier.
+func (p *Planner) resolveStage(g *commonv1.Group) (string, error) {
+ ro := g.GetResourceOpts()
+ if ro == nil || len(ro.GetStages()) == 0 || len(p.nodeLabels) == 0 {
+ return "", nil
+ }
+ _, matched, _, err := pub.ResolveStageResourceOpts(ro, p.nodeLabels)
+ if err != nil {
+ return "", err
+ }
+ return matched.GetName(), nil
+}
+
+// hasIndexModeMeasure reports whether the group holds an index-mode measure, whose rows live
+// only in the segment-level series index. The index also carries the series documents of the
+// group's regular measures, so the resulting doc_count is an upper bound of the rows.
+func (p *Planner) hasIndexModeMeasure(ctx context.Context, group string) (bool, error) {
+ measures, err := p.meta.MeasureRegistry().ListMeasure(ctx, schema.ListOpt{Group: group})
+ if err != nil {
+ return false, err
+ }
+ for _, m := range measures {
+ if m.GetIndexMode() {
+ return true, nil
+ }
+ }
+ return false, nil
+}
diff --git a/banyand/transfer/export/planner_test.go b/banyand/transfer/export/planner_test.go
new file mode 100644
index 000000000..554c47a06
--- /dev/null
+++ b/banyand/transfer/export/planner_test.go
@@ -0,0 +1,562 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package export
+
+import (
+ "context"
+ "os"
+ "path/filepath"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "go.uber.org/mock/gomock"
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/status"
+
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ transferv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/transfer/v1"
+ "github.com/apache/skywalking-banyandb/banyand/metadata"
+ "github.com/apache/skywalking-banyandb/banyand/metadata/schema"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+)
+
+const testSessionID = "aaaa"
+
+func dayRule(num uint32) *commonv1.IntervalRule {
+ return &commonv1.IntervalRule{Unit: commonv1.IntervalRule_UNIT_DAY, Num: num}
+}
+
+func stagedGroup(name string, catalog commonv1.Catalog) *commonv1.Group {
+ return &commonv1.Group{
+ Metadata: &commonv1.Metadata{Name: name},
+ Catalog: catalog,
+ ResourceOpts: &commonv1.ResourceOpts{
+ ShardNum: 2,
+ SegmentInterval: dayRule(1),
+ Ttl: dayRule(7),
+ Stages: []*commonv1.LifecycleStage{{
+ Name: "warm", ShardNum: 4, SegmentInterval: dayRule(1), Ttl: dayRule(30), NodeSelector: "tier=warm",
+ }},
+ },
+ }
+}
+
+type plannerFixture struct {
+ planner *Planner
+ backends Backends
+ clock *fakeClock
+ groupLists *atomic.Int32
+}
+
+// newTestPlanner wires a Planner over fake backends and mocked registries. The stream
+// catalog holds "sw_record" (one part) and "sw_log" (one part); property holds "sw_prop".
+// groupLists counts the registry enumerations so a test can prove a path did not list.
+func newTestPlanner(t *testing.T, labels map[string]string, groups []*commonv1.Group, measures map[string][]*databasev1.Measure) plannerFixture {
+ t.Helper()
+ ctrl := gomock.NewController(t)
+ repo := metadata.NewMockRepo(ctrl)
+ groupReg := schema.NewMockGroup(ctrl)
+ measureReg := schema.NewMockMeasure(ctrl)
+ repo.EXPECT().GroupRegistry().Return(groupReg).AnyTimes()
+ repo.EXPECT().MeasureRegistry().Return(measureReg).AnyTimes()
+ groupLists := &atomic.Int32{}
+ groupReg.EXPECT().ListGroup(gomock.Any()).DoAndReturn(func(context.Context) ([]*commonv1.Group, error) {
+ groupLists.Add(1)
+ return groups, nil
+ }).AnyTimes()
+ measureReg.EXPECT().ListMeasure(gomock.Any(), gomock.Any()).DoAndReturn(
+ func(_ context.Context, opt schema.ListOpt) ([]*databasev1.Measure, error) {
+ return measures[opt.Group], nil
+ }).AnyTimes()
+
+ backends, clock := newTestBackends(t, "sw_record", "sw_log")
+ writeFile(t, filepath.Join(backends[commonv1.Catalog_CATALOG_PROPERTY].GetDataPath(), "sw_prop", "shard-0", "x"), []byte("p"))
+
+ p := NewPlanner(repo, backends, labels, logger.GetLogger("export-test"))
+ p.sessions.now = clock.now
+ return plannerFixture{planner: p, backends: backends, clock: clock, groupLists: groupLists}
+}
+
+func defaultGroups() []*commonv1.Group {
+ return []*commonv1.Group{
+ stagedGroup("sw_record", commonv1.Catalog_CATALOG_STREAM),
+ stagedGroup("sw_log", commonv1.Catalog_CATALOG_STREAM),
+ {Metadata: &commonv1.Metadata{Name: "sw_prop"}, Catalog: commonv1.Catalog_CATALOG_PROPERTY},
+ }
+}
+
+func collect(t *testing.T, p *Planner, req *transferv1.PlanRequest) []*transferv1.PlanResponse {
+ t.Helper()
+ frames, err := collectErr(p, req)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return frames
+}
+
+func collectErr(p *Planner, req *transferv1.PlanRequest) ([]*transferv1.PlanResponse, error) {
+ var frames []*transferv1.PlanResponse
+ err := p.PlanInventory(context.Background(), req, func(r *transferv1.PlanResponse) error {
+ frames = append(frames, r)
+ return nil
+ })
+ return frames, err
+}
+
+// unitFrames keeps the `units` frames of a data node stream.
+func unitFrames(frames []*transferv1.PlanResponse) []*transferv1.UnitFrame {
+ var out []*transferv1.UnitFrame
+ for _, f := range frames {
+ if u := f.GetUnits(); u != nil {
+ out = append(out, u)
+ }
+ }
+ return out
+}
+
+// segmentGroup is the group of a segment unit.
+func segmentGroup(u *transferv1.UnitInventory) string { return u.GetSegment().GetUnit().GetGroup() }
+
+func TestPlanInventory_OneFramePerGroupAndSelectorFilter(t *testing.T) {
+ fx := newTestPlanner(t, nil, defaultGroups(), nil)
+
+ all := collect(t, fx.planner, &transferv1.PlanRequest{})
+ if len(all) != 3 {
+ t.Fatalf("want one frame per group (3), got %d", len(all))
+ }
+ for _, f := range unitFrames(all) {
+ if f.NodeId != "" {
+ t.Fatalf("node_id is stamped by the liaison, node must leave it empty: %q", f.NodeId)
+ }
+ seen := map[string]struct{}{}
+ for _, u := range f.Units {
+ if u.GetProperty() != nil {
+ seen[u.GetProperty().GetGroup()] = struct{}{}
+ continue
+ }
+ seen[segmentGroup(u)] = struct{}{}
+ }
+ if len(seen) != 1 {
+ t.Fatalf("a frame must not span groups: %v", seen)
+ }
+ }
+
+ only := collect(t, fx.planner, &transferv1.PlanRequest{Selectors: []*transferv1.Selector{
+ {Catalog: commonv1.Catalog_CATALOG_STREAM, Groups: []string{"sw_log"}},
+ }})
+ if len(only) != 1 || segmentGroup(only[0].GetUnits().Units[0]) != "sw_log" {
+ t.Fatalf("selector must narrow to sw_log, got %+v", only)
+ }
+
+ catalogOnly := collect(t, fx.planner, &transferv1.PlanRequest{Selectors: []*transferv1.Selector{{Catalog: commonv1.Catalog_CATALOG_PROPERTY}}})
+ if len(catalogOnly) != 1 || catalogOnly[0].GetUnits().Units[0].GetProperty().GetGroup() != "sw_prop" {
+ t.Fatalf("catalog-only selector must return the property group as a PropertyInventory, got %+v", catalogOnly)
+ }
+}
+
+func TestPlanInventory_StageFollowsNodeLabels(t *testing.T) {
+ groups := []*commonv1.Group{stagedGroup("sw_record", commonv1.Catalog_CATALOG_STREAM)}
+ hot := collect(t, newTestPlanner(t, nil, groups, nil).planner, &transferv1.PlanRequest{})
+ if hot[0].GetUnits().Stage != "" {
+ t.Fatalf("unlabeled node is the default tier and must report an empty stage, got %q", hot[0].GetUnits().Stage)
+ }
+ warm := collect(t, newTestPlanner(t, map[string]string{"tier": "warm"}, groups, nil).planner, &transferv1.PlanRequest{})
+ if warm[0].GetUnits().Stage != "warm" {
+ t.Fatalf("labeled node must report the matched stage, got %q", warm[0].GetUnits().Stage)
+ }
+ cold := collect(t, newTestPlanner(t, map[string]string{"tier": "cold"}, groups, nil).planner, &transferv1.PlanRequest{})
+ if cold[0].GetUnits().Stage != "" {
+ t.Fatalf("labels matching no stage must report an empty stage, got %q", cold[0].GetUnits().Stage)
+ }
+}
+
+func TestPlanInventory_IndexModeMeasureCarriesSegmentLevelOnly(t *testing.T) {
+ measureGroup := func(name string) *commonv1.Group {
+ return &commonv1.Group{
+ Metadata: &commonv1.Metadata{Name: name}, Catalog: commonv1.Catalog_CATALOG_MEASURE,
+ ResourceOpts: stagedGroup("", commonv1.Catalog_CATALOG_MEASURE).ResourceOpts,
+ }
+ }
+ groups := []*commonv1.Group{measureGroup("sw_idx"), measureGroup("sw_plain")}
+ measures := map[string][]*databasev1.Measure{
+ "sw_idx": {{Metadata: &commonv1.Metadata{Name: "m", Group: "sw_idx"}, IndexMode: true}},
+ "sw_plain": {{Metadata: &commonv1.Metadata{Name: "p", Group: "sw_plain"}}},
+ }
+ fx := newTestPlanner(t, nil, groups, measures)
+ for _, g := range []string{"sw_idx", "sw_plain"} {
+ seg := filepath.Join(fx.backends[commonv1.Catalog_CATALOG_MEASURE].GetDataPath(), g, "seg-20260928")
+ writeFile(t, filepath.Join(seg, "metadata"), []byte(`{"version":"1.5.0"}`))
+ buildIndex(t, filepath.Join(seg, sidxDirName), 2)
+ }
+
+ frames := collect(t, fx.planner, &transferv1.PlanRequest{Selectors: []*transferv1.Selector{{Catalog: commonv1.Catalog_CATALOG_MEASURE}}})
+ if len(frames) != 2 {
+ t.Fatalf("want one frame per measure group, got %d", len(frames))
+ }
+ docCounts := map[string]uint64{}
+ for _, f := range frames {
+ u := f.GetUnits().Units[0].GetSegment()
+ if len(u.Unit.ShardIds) != 0 || u.SegmentLevel == nil || u.SegmentLevel.EstimatedBytes == 0 {
+ t.Fatalf("a shardless unit must carry segment-level stats: %+v", u)
+ }
+ docCounts[u.Unit.Group] = u.SegmentLevel.DocCount
+ }
+ if docCounts["sw_idx"] != 2 || docCounts["sw_plain"] != 0 {
+ t.Fatalf("only the index-mode group counts its series documents: %v", docCounts)
+ }
+}
+
+// createReq builds the CreateSession PlanRequest a liaison forwards: id already generated.
+func createReq(id string, preempt bool, selectors ...*transferv1.Selector) *transferv1.PlanRequest {
+ return &transferv1.PlanRequest{
+ Selectors: selectors,
+ Session: &transferv1.PlanRequest_Create{Create: &transferv1.CreateSession{Id: id, Preempt: preempt}},
+ }
+}
+
+// readReq builds the ReadSession PlanRequest driving the session named by id.
+func readReq(id string, selectors ...*transferv1.Selector) *transferv1.PlanRequest {
+ return &transferv1.PlanRequest{
+ Selectors: selectors,
+ Session: &transferv1.PlanRequest_Read{Read: &transferv1.ReadSession{Id: id}},
+ }
+}
+
+func TestPlanInventory_SessionModes(t *testing.T) {
+ fx := newTestPlanner(t, nil, defaultGroups(), nil)
+ // CreateSession on an empty node: group frames read from the snapshot and nothing else
+ // (no summary frame when nothing was preempted; the liaison announces the session id).
+ frames := collect(t, fx.planner, createReq(testSessionID, false))
+ if got := unitFrames(frames); len(got) != 3 || len(frames) != 3 {
+ t.Fatalf("snapshot inventory must be exactly the three group frames, got %+v", frames)
+ }
+ for _, f := range frames {
+ if f.GetUnits() == nil {
+ t.Fatalf("a data node sends neither a created frame nor an empty summary: %+v", f)
+ }
+ }
+
+ // Mutating the live directory must not change what the session reports.
+ if err := os.RemoveAll(filepath.Join(fx.backends[commonv1.Catalog_CATALOG_STREAM].GetDataPath(), "sw_record")); err != nil {
+ t.Fatal(err)
+ }
+ fx.clock.advance(time.Hour)
+ again := collect(t, fx.planner, readReq(testSessionID))
+ if got := unitFrames(again); len(got) != 3 {
+ t.Fatalf("session plan must read the snapshot, got %d frames", len(got))
+ }
+ for catalog := range allCatalogs() {
+ lease, readErr := readLease(mustSessionDir(t, fx.planner.sessions, catalog, testSessionID))
+ if readErr != nil || lease.LastHeartbeatAt != fx.clock.now().UnixNano() ||
+ lease.ExpiresAt != fx.clock.now().Add(defaultLease).UnixNano() {
+ t.Fatalf("ReadSession must renew the %s lease: %+v %v", catalog, lease, readErr)
+ }
+ }
+
+ // ACTION_HEARTBEAT: a done frame, lease rewritten, no group enumeration.
+ fx.clock.advance(time.Minute)
+ lists := fx.groupLists.Load()
+ frame, err := fx.planner.Sessions(&transferv1.SessionsRequest{Action: transferv1.SessionsRequest_ACTION_HEARTBEAT, SessionId: testSessionID})
+ if err != nil || frame.GetDone() == nil {
+ t.Fatalf("heartbeat must answer done, got %+v, %v", frame, err)
+ }
+ if fx.groupLists.Load() != lists {
+ t.Fatal("heartbeat must not enumerate groups")
+ }
+ for catalog := range allCatalogs() {
+ lease, readErr := readLease(mustSessionDir(t, fx.planner.sessions, catalog, testSessionID))
+ if readErr != nil || lease.LastHeartbeatAt != fx.clock.now().UnixNano() {
+ t.Fatalf("heartbeat must rewrite the %s lease: %+v %v", catalog, lease, readErr)
+ }
+ }
+
+ // Live-directory mode reflects the removal.
+ if live := unitFrames(collect(t, fx.planner, &transferv1.PlanRequest{})); len(live) != 2 {
+ t.Fatalf("live plan must reflect the live directory, got %d frames", len(live))
+ }
+}
+
+func TestPlanInventory_CreateSessionOnlySnapshotsSelectedCatalogs(t *testing.T) {
+ fx := newTestPlanner(t, nil, defaultGroups(), nil)
+ frames := collect(t, fx.planner, createReq(testSessionID, false,
+ &transferv1.Selector{Catalog: commonv1.Catalog_CATALOG_STREAM}))
+ if got := unitFrames(frames); len(got) != 2 {
+ t.Fatalf("want the two stream groups, got %d", len(got))
+ }
+ lease, _ := fx.planner.sessions.probe()
+ if lease == nil || len(lease.Catalogs) != 1 || lease.Catalogs[0] != commonv1.Catalog_CATALOG_STREAM {
+ t.Fatalf("only the selected catalog must carry a session dir: %+v", lease)
+ }
+}
+
+func TestPlanInventory_PreemptReportsOldSessions(t *testing.T) {
+ fx := newTestPlanner(t, nil, defaultGroups(), nil)
+ _ = collect(t, fx.planner, createReq(testSessionID, false))
+ _, err := collectErr(fx.planner, createReq("bbbb", false))
+ if status.Code(err) != codes.AlreadyExists {
+ t.Fatalf("want AlreadyExists, got %v", err)
+ }
+ frames := collect(t, fx.planner, createReq("bbbb", true))
+ last := frames[len(frames)-1]
+ if got := last.GetSummary().GetPreemptedSessionIds(); last.GetSummary() == nil || len(got) != 1 || got[0] != testSessionID {
+ t.Fatalf("the summary frame must close the stream and list the preempted session: %+v", last)
+ }
+ if got := unitFrames(frames); len(got) != 3 || len(got) != len(frames)-1 {
+ t.Fatalf("the unit frames must precede the summary, got %d of %d frames", len(got), len(frames))
+ }
+ // How long ago the occupant last heartbeat does not change the answer: create takes no
+ // liveness decision, so even a long-silent occupant is refused without preempt.
+ fx.clock.advance(time.Hour)
+ _, err = collectErr(fx.planner, createReq("cccc", false))
+ if status.Code(err) != codes.AlreadyExists || !strings.Contains(err.Error(), "bbbb") {
+ t.Fatalf("want AlreadyExists naming bbbb, got %v", err)
+ }
+}
+
+func TestPlanInventory_PreemptReportsOldSessionsWhenTheWalkFails(t *testing.T) {
+ fx := newTestPlanner(t, nil, defaultGroups(), nil)
+ _ = collect(t, fx.planner, createReq(testSessionID, false))
+ // The new snapshot copies a corrupt part, so the walk fails after the preemption.
+ meta := filepath.Join(fx.backends[commonv1.Catalog_CATALOG_STREAM].GetDataPath(),
+ "sw_record", "seg-20260928", "shard-0", "0000000000000001", "metadata.json")
+ writeFile(t, meta, []byte(`{broken`))
+ frames, err := collectErr(fx.planner, createReq("bbbb", true))
+ if status.Code(err) != codes.Internal {
+ t.Fatalf("a corrupt part must answer Internal, got %v", err)
+ }
+ if len(frames) == 0 {
+ t.Fatal("the preempted sessions must still be reported")
+ }
+ last := frames[len(frames)-1]
+ if got := last.GetSummary().GetPreemptedSessionIds(); len(got) != 1 || got[0] != testSessionID {
+ t.Fatalf("the summary frame must close the stream and list the preempted session: %+v", last)
+ }
+}
+
+func TestPlanInventory_SessionModeRefusesCatalogOutsideSession(t *testing.T) {
+ fx := newTestPlanner(t, nil, defaultGroups(), nil)
+ _ = collect(t, fx.planner, createReq(testSessionID, false,
+ &transferv1.Selector{Catalog: commonv1.Catalog_CATALOG_STREAM}))
+ _, err := collectErr(fx.planner, readReq(testSessionID,
+ &transferv1.Selector{Catalog: commonv1.Catalog_CATALOG_PROPERTY, Groups: []string{"sw_prop"}}))
+ if status.Code(err) != codes.FailedPrecondition || !strings.Contains(err.Error(), "no snapshot for catalog CATALOG_PROPERTY") {
+ t.Fatalf("a selector naming a catalog the session does not hold must be FailedPrecondition, got %v", err)
+ }
+ // Without selectors the missing catalog is simply not part of the session.
+ if got := unitFrames(collect(t, fx.planner, readReq(testSessionID))); len(got) != 2 {
+ t.Fatalf("want the two stream groups, got %d", len(got))
+ }
+}
+
+func TestPlanInventory_SkipsGroupDirectoriesTheRegistryDoesNotKnow(t *testing.T) {
+ // The registry knows only sw_record; the data directory also holds sw_log. Both modes read
+ // the group directories of their root and keep only the registered groups.
+ fx := newTestPlanner(t, map[string]string{"tier": "warm"}, []*commonv1.Group{stagedGroup("sw_record", commonv1.Catalog_CATALOG_STREAM)}, nil)
+ streamOnly := &transferv1.Selector{Catalog: commonv1.Catalog_CATALOG_STREAM}
+ // Ordered: the read needs the session the create makes.
+ for _, tc := range []struct {
+ req *transferv1.PlanRequest
+ name string
+ }{
+ {name: "live", req: &transferv1.PlanRequest{Selectors: []*transferv1.Selector{streamOnly}}},
+ {name: "create", req: createReq(testSessionID, false, streamOnly)},
+ {name: "read", req: readReq(testSessionID, streamOnly)},
+ } {
+ frames := unitFrames(collect(t, fx.planner, tc.req))
+ if len(frames) != 1 || segmentGroup(frames[0].Units[0]) != "sw_record" || frames[0].Stage != "warm" {
+ t.Fatalf("%s: want only the registered group sw_record (warm), got %+v", tc.name, frames)
+ }
+ }
+ named := unitFrames(collect(t, fx.planner, readReq(testSessionID,
+ &transferv1.Selector{Catalog: commonv1.Catalog_CATALOG_STREAM, Groups: []string{"sw_log"}})))
+ if len(named) != 0 {
+ t.Fatalf("a selector naming an unregistered group finds nothing, got %+v", named)
+ }
+}
+
+func TestHasIndexModeMeasure(t *testing.T) {
+ idx := func(name string, indexMode bool) *databasev1.Measure {
+ return &databasev1.Measure{Metadata: &commonv1.Metadata{Name: name, Group: "g"}, IndexMode: indexMode}
+ }
+ for name, tc := range map[string]struct {
+ measures []*databasev1.Measure
+ want bool
+ }{
+ "none": {nil, false},
+ "all": {[]*databasev1.Measure{idx("a", true), idx("b", true)}, true},
+ "mixed": {[]*databasev1.Measure{idx("a", true), idx("b", false)}, true},
+ "plain": {[]*databasev1.Measure{idx("a", false)}, false},
+ } {
+ fx := newTestPlanner(t, nil, nil, map[string][]*databasev1.Measure{"g": tc.measures})
+ got, err := fx.planner.hasIndexModeMeasure(context.Background(), "g")
+ if err != nil || got != tc.want {
+ t.Fatalf("%s: hasIndexModeMeasure = %v, %v; want %v", name, got, err, tc.want)
+ }
+ }
+}
+
+func TestPlanGroup_MeasureWithoutSegmentsSkipsRegistry(t *testing.T) {
+ // No MeasureRegistry expectation: any registry lookup fails the test.
+ p := NewPlanner(metadata.NewMockRepo(gomock.NewController(t)), nil, nil, logger.GetLogger("export-test"))
+ g := &commonv1.Group{Metadata: &commonv1.Metadata{Name: "m"}, Catalog: commonv1.Catalog_CATALOG_MEASURE}
+ root := t.TempDir()
+ writeFile(t, filepath.Join(root, "m", "seg-bak", "x"), []byte("x")) // a file and a non-seg dir are not segments
+ writeFile(t, filepath.Join(root, "m", "x"), []byte("x"))
+ for _, dir := range []string{root, filepath.Join(root, "missing")} {
+ frame, err := p.planGroup(context.Background(), g, dir)
+ if err != nil || frame != nil {
+ t.Fatalf("a measure group without segments must yield no frame, got %v, %v", frame, err)
+ }
+ }
+}
+
+func TestPlanInventory_InvalidSessionRequests(t *testing.T) {
+ fx := newTestPlanner(t, nil, defaultGroups(), nil)
+ for _, req := range []*transferv1.PlanRequest{
+ createReq("", false), // liaison must supply the id
+ readReq(""), // needs the session id
+ } {
+ if _, err := collectErr(fx.planner, req); status.Code(err) != codes.InvalidArgument {
+ t.Fatalf("%+v must be InvalidArgument, got %v", req, err)
+ }
+ }
+ if _, err := collectErr(fx.planner, readReq("deadbeef")); status.Code(err) != codes.NotFound {
+ t.Fatalf("unknown session must be NotFound, got %v", err)
+ }
+}
+
+func TestSessions_Actions(t *testing.T) {
+ fx := newTestPlanner(t, nil, defaultGroups(), nil)
+ heartbeat := func(id string) *transferv1.SessionsRequest {
+ return &transferv1.SessionsRequest{Action: transferv1.SessionsRequest_ACTION_HEARTBEAT, SessionId: id}
+ }
+ release := func(id string) *transferv1.SessionsRequest {
+ return &transferv1.SessionsRequest{Action: transferv1.SessionsRequest_ACTION_RELEASE, SessionId: id}
+ }
+ list := &transferv1.SessionsRequest{Action: transferv1.SessionsRequest_ACTION_LIST}
+
+ for _, req := range []*transferv1.SessionsRequest{{}, heartbeat(""), release("")} {
+ if _, err := fx.planner.Sessions(req); status.Code(err) != codes.InvalidArgument {
+ t.Fatalf("%+v must be InvalidArgument, got %v", req, err)
+ }
+ }
+ if frame, err := fx.planner.Sessions(list); err != nil || frame.GetNone() == nil {
+ t.Fatalf("an empty node lists none, got %+v, %v", frame, err)
+ }
+ // A session this node does not hold is an error frame, not a failure: the liaison relays
+ // it so the client learns which nodes lost the session.
+ frame, err := fx.planner.Sessions(heartbeat("deadbeef"))
+ if err != nil || !strings.Contains(frame.GetError(), "deadbeef not found on this node") {
+ t.Fatalf("heartbeat of an unknown session must answer an error frame, got %+v, %v", frame, err)
+ }
+ // Releasing what is not there succeeds and says so: none, not done.
+ if frame, err = fx.planner.Sessions(release("deadbeef")); err != nil || frame.GetNone() == nil {
+ t.Fatalf("release of a session the node does not hold must answer none, got %+v, %v", frame, err)
+ }
+
+ _ = collect(t, fx.planner, createReq(testSessionID, false))
+ frame, err = fx.planner.Sessions(list)
+ if err != nil || frame.GetSession().GetSessionId() != testSessionID {
+ t.Fatalf("list must report the created session, got %+v, %v", frame, err)
+ }
+ // An expired lease fails the call: the client must stop its heartbeat.
+ fx.clock.advance(defaultLease + time.Minute)
+ if _, err = fx.planner.Sessions(heartbeat(testSessionID)); status.Code(err) != codes.FailedPrecondition {
+ t.Fatalf("heartbeat of an expired session must be FailedPrecondition, got %v", err)
+ }
+ if os.Geteuid() == 0 {
+ return // root ignores the read-only directory the rest relies on
+ }
+ // A release that cannot remove the session answers an error frame, so a half-released
+ // session stays visible instead of failing the fan-out.
+ streamDir := mustSessionDir(t, fx.planner.sessions, commonv1.Catalog_CATALOG_STREAM, testSessionID)
+ if err = os.Chmod(streamDir, 0o555); err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = os.Chmod(streamDir, 0o755) })
+ if frame, err = fx.planner.Sessions(release(testSessionID)); err != nil || !strings.Contains(frame.GetError(), "release export session "+testSessionID) {
+ t.Fatalf("a failed release must answer an error frame, got %+v, %v", frame, err)
+ }
+ if err = os.Chmod(streamDir, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if frame, err = fx.planner.Sessions(release(testSessionID)); err != nil || frame.GetDone() == nil {
+ t.Fatalf("release of a held session must answer done, got %+v, %v", frame, err)
+ }
+ if frame, err = fx.planner.Sessions(release(testSessionID)); err != nil || frame.GetNone() == nil {
+ t.Fatalf("a second release must answer none, got %+v, %v", frame, err)
+ }
+ if frame, err = fx.planner.Sessions(list); err != nil || frame.GetNone() == nil {
+ t.Fatalf("the released session must be gone and listed as none, got %+v, %v", frame, err)
+ }
+}
+
+func TestPlanInventory_CreateSessionWritesDefaultLease(t *testing.T) {
+ fx := newTestPlanner(t, nil, defaultGroups(), nil)
+ _ = collect(t, fx.planner, createReq("aabb", false))
+ streamDir := mustSessionDir(t, fx.planner.sessions, commonv1.Catalog_CATALOG_STREAM, "aabb")
+ diskLease, err := readLease(streamDir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ // Creation writes expiresAt = creation time + defaultLease; nothing in the request can change it.
+ if want := fx.clock.now().Add(defaultLease).UnixNano(); diskLease.ExpiresAt != want {
+ t.Fatalf("on-disk lease ExpiresAt=%d, want creation time + default lease %d", diskLease.ExpiresAt, want)
+ }
+}
+
+func TestPlanInventory_ReadSessionRemovedDuringWalkIsNotFound(t *testing.T) {
+ fx := newTestPlanner(t, nil, defaultGroups(), nil)
+ collect(t, fx.planner, createReq(testSessionID, false))
+ req := readReq(testSessionID)
+ roots, _, err := fx.planner.resolveRoots(context.Background(), req)
+ if err != nil {
+ t.Fatal(err)
+ }
+ // The session goes between resolving its roots and walking them.
+ if _, err = fx.planner.sessions.release(testSessionID); err != nil {
+ t.Fatal(err)
+ }
+ err = fx.planner.planRoots(context.Background(), req, roots, func(*transferv1.PlanResponse) error { return nil })
+ if status.Code(err) != codes.NotFound {
+ t.Fatalf("a session removed during the plan must answer NotFound, got %v", err)
+ }
+}
+
+func TestPlanInventory_CorruptPartMetadataIsInternal(t *testing.T) {
+ fx := newTestPlanner(t, nil, defaultGroups(), nil)
+ meta := filepath.Join(fx.backends[commonv1.Catalog_CATALOG_STREAM].GetDataPath(),
+ "sw_record", "seg-20260928", "shard-0", "0000000000000001", "metadata.json")
+ writeFile(t, meta, []byte(`{broken`))
+ if _, err := collectErr(fx.planner, &transferv1.PlanRequest{}); status.Code(err) != codes.Internal {
+ t.Fatalf("a corrupt part must answer Internal, got %v", err)
+ }
+}
+
+func TestPlanInventory_CanceledIsCanceled(t *testing.T) {
+ fx := newTestPlanner(t, nil, defaultGroups(), nil)
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+ err := fx.planner.PlanInventory(ctx, &transferv1.PlanRequest{}, func(*transferv1.PlanResponse) error { return nil })
+ if status.Code(err) != codes.Canceled {
+ t.Fatalf("a canceled plan must answer Canceled, got %v", err)
+ }
+}
diff --git a/banyand/transfer/export/service.go b/banyand/transfer/export/service.go
new file mode 100644
index 000000000..bb5f09c83
--- /dev/null
+++ b/banyand/transfer/export/service.go
@@ -0,0 +1,129 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package export
+
+import (
+ "context"
+ "time"
+
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/status"
+
+ "github.com/apache/skywalking-banyandb/api/common"
+ transferv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/transfer/v1"
+ "github.com/apache/skywalking-banyandb/banyand/metadata"
+ "github.com/apache/skywalking-banyandb/banyand/queue"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+ "github.com/apache/skywalking-banyandb/pkg/run"
+)
+
+// Service is the data-node export unit. It builds the Planner once node labels are known
+// (PreRun), registers it as the gRPC ExportService of the data-node pipeline, exposes the
+// same operations in-process for the standalone liaison, and sweeps expired sessions.
+type Service struct {
+ transferv1.UnimplementedExportServiceServer
+ meta metadata.Repo
+ pipeline queue.Server
+ planner *Planner
+ closer *run.Closer
+ l *logger.Logger
+ backends Backends
+}
+
+var (
+ _ run.PreRunner = (*Service)(nil)
+ _ run.Service = (*Service)(nil)
+ _ transferv1.ExportServiceServer = (*Service)(nil)
+)
+
+// NewService creates the export unit. pipeline is the data-node gRPC server the
+// ExportService registers on; the local queue ignores the registration.
+func NewService(meta metadata.Repo, pipeline queue.Server, backends Backends) *Service {
+ return &Service{meta: meta, pipeline: pipeline, backends: backends, closer: run.NewCloser(1)}
+}
+
+// Name implements run.Unit.
+func (s *Service) Name() string { return "export" }
+
+// PreRun implements run.PreRunner: reject colliding export snapshot directories (the
+// storage services resolve them in their own PreRun, which runs first), read the node
+// labels from the context, build the planner and register the gRPC service.
+func (s *Service) PreRun(ctx context.Context) error {
+ s.l = logger.GetLogger(s.Name())
+ if err := s.backends.validateDirs(); err != nil {
+ return err
+ }
+ if err := s.backends.validatePartReaders(); err != nil {
+ return err
+ }
+ var labels map[string]string
+ if v := ctx.Value(common.ContextNodeKey); v != nil {
+ labels = v.(common.Node).Labels
+ }
+ s.planner = NewPlanner(s.meta, s.backends, labels, s.l)
+ s.pipeline.SetExportServer(s)
+ return nil
+}
+
+// Serve implements run.Service: the sweep of expired export sessions, first right away to
+// reclaim sessions that expired while the node was down (live ones are kept, which is how
+// a restarted node recovers its sessions), then every SweepInterval.
+func (s *Service) Serve() run.StopNotify {
+ run.Go(s.closer.Ctx(), "export-session-sweep", s.l, func(ctx context.Context) {
+ defer s.closer.Done()
+ s.planner.sessions.sweep()
+ ticker := time.NewTicker(SweepInterval)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-ticker.C:
+ s.planner.sessions.sweep()
+ }
+ }
+ })
+ return s.closer.CloseNotify()
+}
+
+// GracefulStop implements run.Service.
+func (s *Service) GracefulStop() {
+ s.closer.CloseThenWait()
+}
+
+// Plan implements transferv1.ExportServiceServer. The data-node stream chain has no
+// validator interceptor, so the request is validated here.
+func (s *Service) Plan(req *transferv1.PlanRequest, stream transferv1.ExportService_PlanServer) error {
+ if err := req.ValidateAll(); err != nil {
+ return status.Error(codes.InvalidArgument, err.Error())
+ }
+ return s.planner.PlanInventory(stream.Context(), req, stream.Send)
+}
+
+// Sessions implements transferv1.ExportServiceServer on a data node: a single frame per
+// call (the liaison stamps node_id).
+func (s *Service) Sessions(req *transferv1.SessionsRequest, stream transferv1.ExportService_SessionsServer) error {
+ if err := req.ValidateAll(); err != nil {
+ return status.Error(codes.InvalidArgument, err.Error())
+ }
+ frame, err := s.planner.Sessions(req)
+ if err != nil {
+ return err
+ }
+ return stream.Send(frame)
+}
diff --git a/banyand/transfer/export/service_test.go b/banyand/transfer/export/service_test.go
new file mode 100644
index 000000000..1e82d07ed
--- /dev/null
+++ b/banyand/transfer/export/service_test.go
@@ -0,0 +1,202 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package export
+
+import (
+ "context"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+
+ "google.golang.org/grpc"
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/status"
+
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ transferv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/transfer/v1"
+ "github.com/apache/skywalking-banyandb/banyand/queue"
+)
+
+// fakePipeline records the ExportService registration; nothing else of queue.Server is used.
+type fakePipeline struct {
+ queue.Server
+ registered transferv1.ExportServiceServer
+}
+
+func (p *fakePipeline) SetExportServer(srv transferv1.ExportServiceServer) { p.registered = srv }
+
+// fakeServerStream is the server side of a streaming call that records every sent frame.
+type fakeServerStream[T any] struct {
+ grpc.ServerStream
+ frames []*T
+}
+
+func (s *fakeServerStream[T]) Context() context.Context { return context.Background() }
+
+func (s *fakeServerStream[T]) Send(frame *T) error {
+ s.frames = append(s.frames, frame)
+ return nil
+}
+
+func TestService_RejectsInvalidSessionIDs(t *testing.T) {
+ // No planner: a request that got past validation would panic.
+ s := &Service{}
+ for _, req := range []*transferv1.PlanRequest{
+ createReq("ABC", false),
+ readReq("../data"),
+ } {
+ stream := &fakeServerStream[transferv1.PlanResponse]{}
+ if err := s.Plan(req, stream); status.Code(err) != codes.InvalidArgument {
+ t.Fatalf("Plan(%+v) = %v, want InvalidArgument", req, err)
+ }
+ if len(stream.frames) != 0 {
+ t.Fatalf("a rejected Plan must send nothing, got %+v", stream.frames)
+ }
+ }
+ for _, req := range []*transferv1.SessionsRequest{
+ {Action: transferv1.SessionsRequest_ACTION_HEARTBEAT, SessionId: "ABC"},
+ {Action: transferv1.SessionsRequest_ACTION_RELEASE, SessionId: "../data"},
+ } {
+ stream := &fakeServerStream[transferv1.SessionsResponse]{}
+ if err := s.Sessions(req, stream); status.Code(err) != codes.InvalidArgument {
+ t.Fatalf("Sessions(%+v) = %v, want InvalidArgument", req, err)
+ }
+ if len(stream.frames) != 0 {
+ t.Fatalf("a rejected Sessions call must send nothing, got %+v", stream.frames)
+ }
+ }
+}
+
+func TestService_SessionsSendsExactlyOneFrame(t *testing.T) {
+ fx := newTestPlanner(t, nil, defaultGroups(), nil)
+ s := &Service{planner: fx.planner}
+ stream := &fakeServerStream[transferv1.SessionsResponse]{}
+ if err := s.Sessions(&transferv1.SessionsRequest{Action: transferv1.SessionsRequest_ACTION_LIST}, stream); err != nil {
+ t.Fatal(err)
+ }
+ if len(stream.frames) != 1 || stream.frames[0].GetNone() == nil {
+ t.Fatalf("a data node answers exactly one frame, got %+v", stream.frames)
+ }
+}
+
+func TestService_ServeSweepsExpiredAndKeepsLiveSessions(t *testing.T) {
+ backends, _ := newTestBackends(t)
+ exportDir := backends[commonv1.Catalog_CATALOG_STREAM].GetExportSnapshotDir()
+ now := time.Now()
+ plant := func(id string, expiresAt time.Time) string {
+ dir := filepath.Join(exportDir, id)
+ if err := os.MkdirAll(dir, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := writeLease(dir, Lease{SessionID: id, StartedAt: now.Add(-time.Hour).UnixNano(), ExpiresAt: expiresAt.UnixNano()}); err != nil {
+ t.Fatal(err)
+ }
+ return dir
+ }
+ expired := plant("dead", now.Add(-time.Second))
+ live := plant("beef", now.Add(time.Hour))
+
+ pipeline := &fakePipeline{}
+ s := NewService(nil, pipeline, backends)
+ if err := s.PreRun(context.Background()); err != nil {
+ t.Fatal(err)
+ }
+ if pipeline.registered != s {
+ t.Fatal("PreRun must register the service on the data-node pipeline")
+ }
+ s.Serve()
+ defer s.GracefulStop()
+ for deadline := time.Now().Add(10 * time.Second); ; time.Sleep(10 * time.Millisecond) {
+ if _, err := os.Stat(expired); os.IsNotExist(err) {
+ break
+ }
+ if time.Now().After(deadline) {
+ t.Fatal("the startup sweep must remove an expired session")
+ }
+ }
+ if _, err := readLease(live); err != nil {
+ t.Fatalf("the startup sweep must keep a live session: %v", err)
+ }
+}
+
+func TestService_PreRunRejectsCollidingExportDirs(t *testing.T) {
+ stream := commonv1.Catalog_CATALOG_STREAM
+ measure := commonv1.Catalog_CATALOG_MEASURE
+ for name, tamper := range map[string]func(bs Backends){
+ "shared export dir": func(bs Backends) {
+ bs[measure].(*fakeBackend).exportSnapshotDir = bs[stream].GetExportSnapshotDir()
+ },
+ "nested export dir": func(bs Backends) {
+ bs[measure].(*fakeBackend).exportSnapshotDir = filepath.Join(bs[stream].GetExportSnapshotDir(), "measure")
+ },
+ "export dir is its own snapshot dir": func(bs Backends) {
+ bs[measure].(*fakeBackend).exportSnapshotDir = bs[measure].GetSnapshotDir()
+ },
+ "export dir inside another catalog's snapshot dir": func(bs Backends) {
+ bs[measure].(*fakeBackend).exportSnapshotDir = filepath.Join(bs[stream].GetSnapshotDir(), "export")
+ },
+ "export dir is another catalog's data dir": func(bs Backends) {
+ bs[measure].(*fakeBackend).exportSnapshotDir = bs[stream].GetDataPath()
+ },
+ "export dir inside its own data dir": func(bs Backends) {
+ bs[measure].(*fakeBackend).exportSnapshotDir = filepath.Join(bs[measure].GetDataPath(), "export")
+ },
+ "data dir inside the export dir": func(bs Backends) {
+ bs[stream].(*fakeBackend).dataDir = filepath.Join(bs[measure].GetExportSnapshotDir(), "data")
+ },
+ } {
+ t.Run(name, func(t *testing.T) {
+ backends, _ := newTestBackends(t)
+ tamper(backends)
+ pipeline := &fakePipeline{}
+ if err := NewService(nil, pipeline, backends).PreRun(context.Background()); err == nil {
+ t.Fatal("PreRun must reject colliding export snapshot directories")
+ }
+ if pipeline.registered != nil {
+ t.Fatal("a rejected PreRun must not register the service")
+ }
+ })
+ }
+ backends, _ := newTestBackends(t)
+ if err := backends.validateDirs(); err != nil {
+ t.Fatalf("distinct per-catalog directories must pass: %v", err)
+ }
+}
+
+// onlyBackend hides every method but Backend's, so it cannot read part metadata.
+type onlyBackend struct{ Backend }
+
+func TestService_PreRunRejectsABackendThatCannotReadParts(t *testing.T) {
+ backends, _ := newTestBackends(t)
+ backends[commonv1.Catalog_CATALOG_TRACE] = onlyBackend{backends[commonv1.Catalog_CATALOG_TRACE]}
+ pipeline := &fakePipeline{}
+ err := NewService(nil, pipeline, backends).PreRun(context.Background())
+ if err == nil || !strings.Contains(err.Error(), "CATALOG_TRACE") {
+ t.Fatalf("PreRun must name the backend that cannot read parts, got %v", err)
+ }
+ if pipeline.registered != nil {
+ t.Fatal("a rejected PreRun must not register the service")
+ }
+ backends[commonv1.Catalog_CATALOG_PROPERTY] = onlyBackend{backends[commonv1.Catalog_CATALOG_PROPERTY]}
+ delete(backends, commonv1.Catalog_CATALOG_TRACE)
+ if err = backends.validatePartReaders(); err != nil {
+ t.Fatalf("property holds no parts and needs no reader: %v", err)
+ }
+}
diff --git a/banyand/transfer/export/session.go b/banyand/transfer/export/session.go
new file mode 100644
index 000000000..4a69e9b69
--- /dev/null
+++ b/banyand/transfer/export/session.go
@@ -0,0 +1,433 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package export
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io/fs"
+ "maps"
+ "os"
+ "path/filepath"
+ "slices"
+ "strings"
+ "sync"
+ "time"
+
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/status"
+
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ transferv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/transfer/v1"
+ "github.com/apache/skywalking-banyandb/banyand/internal/storage"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+)
+
+// sessionManager owns the export snapshot directories of this node, one <id> per session
+// under every catalog's export snapshot directory. All state is on disk (directory +
+// .lease); the struct only serializes create/release/sweep and validates every id before
+// it touches the file system.
+type sessionManager struct {
+ backends Backends
+ now func() time.Time
+ l *logger.Logger
+ mu sync.Mutex
+}
+
+func newSessionManager(backends Backends, l *logger.Logger) *sessionManager {
+ return &sessionManager{backends: backends, now: time.Now, l: l}
+}
+
+// sessionDir returns <exportSnapshotDir>/<id> for a catalog after checking that the id is
+// well formed.
+func (m *sessionManager) sessionDir(catalog commonv1.Catalog, id string) (string, error) {
+ if err := validateSessionID(id); err != nil {
+ return "", err
+ }
+ b, ok := m.backends[catalog]
+ if !ok {
+ return "", status.Errorf(codes.InvalidArgument, "catalog %s has no export backend on this node", catalog)
+ }
+ return sessionPath(b, id), nil
+}
+
+// sessionPath joins an id that validateSessionID accepted under the export snapshot
+// directory of b; such an id is plain hex, so the result stays directly under it.
+func sessionPath(b Backend, id string) string {
+ return filepath.Join(b.GetExportSnapshotDir(), id)
+}
+
+// existing lists the well-formed <id> directories across the four catalogs' export
+// snapshot directories as id -> catalog -> dir. Anything else in there is not a session:
+// it is left alone and logged at Debug.
+func (m *sessionManager) existing() (map[string]map[commonv1.Catalog]string, error) {
+ sessions := map[string]map[commonv1.Catalog]string{}
+ for catalog, b := range m.backends {
+ root := b.GetExportSnapshotDir()
+ entries, err := readDirOrEmpty(root)
+ if err != nil {
+ return nil, err
+ }
+ for _, e := range entries {
+ if !e.IsDir() {
+ continue
+ }
+ id := e.Name()
+ if !sessionIDPattern.MatchString(id) {
+ m.l.Debug().Str("dir", filepath.Join(root, id)).Msg("export snapshot directory is not a well-formed export session; ignored")
+ continue
+ }
+ if sessions[id] == nil {
+ sessions[id] = map[commonv1.Catalog]string{}
+ }
+ sessions[id][catalog] = filepath.Join(root, id)
+ }
+ }
+ return sessions, nil
+}
+
+// probe is the read-only ACTION_LIST body: the one session this node holds, or nil when it
+// holds none. The catalogs come from the directories that hold the id and the times from
+// the first readable .lease; a session whose .lease is unreadable everywhere reports zero
+// times. One session id covers the whole cluster, so several ids mean a half-won create or
+// an interrupted release: the one with the newest started_at is reported and the others are
+// logged, since the sweeper or a preempting create reclaims them. It never renews and never
+// removes anything. It holds m.mu, so it never observes a create or release halfway.
+func (m *sessionManager) probe() (*transferv1.SessionLease, error) {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ sessions, err := m.existing()
+ if err != nil {
+ return nil, err
+ }
+ var newest *transferv1.SessionLease
+ var others []string
+ for _, id := range slices.Sorted(maps.Keys(sessions)) {
+ catalogs := slices.Sorted(maps.Keys(sessions[id]))
+ lease := &transferv1.SessionLease{SessionId: id, Catalogs: catalogs}
+ if l, ok := firstLease(sessions[id], catalogs); ok {
+ lease.StartedAt, lease.ExpiresAt, lease.LastHeartbeatAt = l.StartedAt, l.ExpiresAt, l.LastHeartbeatAt
+ }
+ switch {
+ case newest == nil:
+ newest = lease
+ case lease.StartedAt > newest.StartedAt:
+ others = append(others, newest.SessionId)
+ newest = lease
+ default:
+ others = append(others, id)
+ }
+ }
+ if len(others) > 0 {
+ m.l.Warn().Str("reported", newest.SessionId).Strs("others", others).
+ Msg("node holds more than one export session; reporting the newest, the others await reclamation")
+ }
+ return newest, nil
+}
+
+// firstLease returns the first readable .lease of a session in catalog order; catalogs is
+// the sorted key set of dirs. Every catalog of a committed session carries the same
+// document, so one copy is enough.
+func firstLease(dirs map[commonv1.Catalog]string, catalogs []commonv1.Catalog) (Lease, bool) {
+ for _, catalog := range catalogs {
+ if l, err := readLease(dirs[catalog]); err == nil {
+ return l, true
+ }
+ }
+ return Lease{}, false
+}
+
+// stale reports whether the newest directory of a session is older than orphanGrace. A
+// session without any readable lease is the leftover of a crash or of a failed rollback
+// (the sweeper never sees an in-flight create, which holds m.mu). Directories that
+// vanished meanwhile count as stale: there is nothing left to protect.
+func stale(dirs map[commonv1.Catalog]string, now time.Time) bool {
+ var newest time.Time
+ for _, dir := range dirs {
+ if info, err := os.Stat(dir); err == nil && info.ModTime().After(newest) {
+ newest = info.ModTime()
+ }
+ }
+ return now.Sub(newest) > orphanGrace
+}
+
+// reclaimable is the sweeper's rule for a whole session and mirrors locateAndHeartbeat: when
+// any copy lacks a readable lease the session can no longer be renewed, so it goes once
+// its newest directory is older than orphanGrace; otherwise it goes when any lease expired.
+func reclaimable(dirs map[commonv1.Catalog]string, now time.Time) bool {
+ expired := false
+ for _, dir := range dirs {
+ l, err := readLease(dir)
+ if err != nil {
+ return stale(dirs, now)
+ }
+ expired = expired || l.expired(now)
+ }
+ return expired
+}
+
+// createdSession is what create hands back: the session directory (the groups root) of
+// every snapshotted catalog and the sessions it removed because preempt was set.
+type createdSession struct {
+ roots map[commonv1.Catalog]string
+ preempted []string
+}
+
+// create snapshots the selected catalogs under <id> and commits their leases. Any other
+// session directory on this node, whatever its lease says, answers ALREADY_EXISTS naming
+// the occupants unless preempt is set, in which case every one of them is removed and
+// reported back; create takes no liveness decision, the sweeper alone reclaims expired
+// and abandoned sessions. A failure in any catalog removes whatever this id created, so a
+// session is all-or-nothing per node.
+func (m *sessionManager) create(
+ ctx context.Context, id string, catalogs map[commonv1.Catalog]struct{}, preempt bool,
+) (*createdSession, error) {
+ if err := validateSessionID(id); err != nil {
+ return nil, err
+ }
+ // A sibling node may already have failed and the liaison's rollback may be on its way:
+ // a create canceled before it starts must not destroy the occupants it would otherwise
+ // have replaced. Once preemption began, a failure reports the occupants already removed.
+ if err := ctx.Err(); err != nil {
+ return nil, status.FromContextError(err).Err()
+ }
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ // The lock may have been held by a long create or release: check again before touching
+ // any occupant.
+ if err := ctx.Err(); err != nil {
+ return nil, status.FromContextError(err).Err()
+ }
+ sessions, err := m.existing()
+ if err != nil {
+ return nil, status.Errorf(codes.Internal, "scan export sessions: %v", err)
+ }
+ now := m.now()
+ occupants := slices.Sorted(maps.Keys(sessions))
+ if len(occupants) > 0 && !preempt {
+ return nil, status.Errorf(codes.AlreadyExists, "export session %s already exists on this node", strings.Join(occupants, ", "))
+ }
+ for i, occupant := range occupants {
+ if rmErr := removeDirs(sessions[occupant]); rmErr != nil {
+ return nil, withPreempted(status.Errorf(codes.Internal, "preempt export session %s: %v", occupant, rmErr), occupants[:i])
+ }
+ m.l.Warn().Str("preempted", occupant).Str("session", id).Msg("export session preempted")
+ }
+ dirs := make(map[commonv1.Catalog]string, len(catalogs))
+ // Sorted so the snapshot order, and therefore what a rollback has to undo, is deterministic.
+ for _, catalog := range slices.Sorted(maps.Keys(catalogs)) {
+ dir, snapErr := m.snapshotCatalog(ctx, catalog, id)
+ if snapErr != nil {
+ m.rollback(id)
+ if errors.Is(snapErr, context.Canceled) || errors.Is(snapErr, context.DeadlineExceeded) {
+ return nil, withPreempted(status.FromContextError(snapErr).Err(), occupants)
+ }
+ return nil, withPreempted(status.Errorf(codes.Internal, "snapshot %s for export session %s: %v", catalog, id, snapErr), occupants)
+ }
+ dirs[catalog] = dir
+ }
+ // Single commit point: the lease is written only after the last snapshot, so a crash
+ // before this leaves lease-less directories that the sweeper reclaims after orphanGrace.
+ heartbeat := m.now()
+ l := Lease{SessionID: id, StartedAt: now.UnixNano(), ExpiresAt: heartbeat.Add(defaultLease).UnixNano(), LastHeartbeatAt: heartbeat.UnixNano()}
+ for _, dir := range dirs {
+ if writeErr := writeLease(dir, l); writeErr != nil {
+ m.rollback(id)
+ return nil, withPreempted(status.Errorf(codes.Internal, "commit lease of export session %s: %v", id, writeErr), occupants)
+ }
+ }
+ return &createdSession{roots: dirs, preempted: occupants}, nil
+}
+
+// withPreempted appends the sessions a failed create already removed to its status error,
+// keeping the status code.
+func withPreempted(err error, preempted []string) error {
+ if len(preempted) == 0 {
+ return err
+ }
+ st := status.Convert(err)
+ return status.Errorf(st.Code(), "%s; preempted sessions [%s] were already removed", st.Message(), strings.Join(preempted, ", "))
+}
+
+// rollback removes whatever a failed create left behind under id, logging instead of
+// failing: the caller already has the error that matters.
+func (m *sessionManager) rollback(id string) {
+ if err := m.removeAll(id); err != nil {
+ m.l.Warn().Err(err).Str("session", id).Msg("rollback of a failed export session left directories behind")
+ }
+}
+
+// snapshotCatalog creates the session directory of one catalog and takes the export
+// snapshot into it, returning the directory. The lease is committed by the caller.
+func (m *sessionManager) snapshotCatalog(ctx context.Context, catalog commonv1.Catalog, id string) (string, error) {
+ b, ok := m.backends[catalog]
+ if !ok {
+ return "", fmt.Errorf("no backend for catalog %s", catalog)
+ }
+ dir := sessionPath(b, id)
+ if err := os.MkdirAll(dir, storage.DirPerm); err != nil {
+ return "", err
+ }
+ if err := b.TakeExportSnapshot(ctx, id); err != nil {
+ return "", err
+ }
+ return dir, nil
+}
+
+// locateAndHeartbeat resolves the per-catalog groups roots of a live session and extends its
+// lease on every copy. A session this node does not hold is NotFound; an expired or
+// unreadable lease is FailedPrecondition (design §3.5). Every copy is validated before
+// any is written, so a refused renewal leaves the copies in step.
+func (m *sessionManager) locateAndHeartbeat(id string) (map[commonv1.Catalog]string, error) {
+ if err := validateSessionID(id); err != nil {
+ return nil, err
+ }
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ sessions, err := m.existing()
+ if err != nil {
+ return nil, status.Errorf(codes.Internal, "scan export sessions: %v", err)
+ }
+ dirs, ok := sessions[id]
+ if !ok {
+ return nil, status.Errorf(codes.NotFound, "export session %s not found on this node", id)
+ }
+ now := m.now()
+ leases := make(map[commonv1.Catalog]Lease, len(dirs))
+ for catalog, dir := range dirs {
+ l, readErr := readLease(dir)
+ if readErr != nil {
+ return nil, status.Errorf(codes.FailedPrecondition, "export session %s has no readable lease: %v", id, readErr)
+ }
+ if l.expired(now) {
+ return nil, status.Errorf(codes.FailedPrecondition, "export session %s expired at %s",
+ id, time.Unix(0, l.ExpiresAt).UTC().Format(time.RFC3339))
+ }
+ leases[catalog] = l
+ }
+ for catalog, dir := range dirs {
+ l := leases[catalog]
+ l.LastHeartbeatAt = now.UnixNano()
+ l.ExpiresAt = now.Add(defaultLease).UnixNano()
+ if writeErr := writeLease(dir, l); writeErr != nil {
+ return nil, status.Errorf(codes.Internal, "renew lease of export session %s: %v", id, writeErr)
+ }
+ }
+ return dirs, nil
+}
+
+// verify reports NotFound unless session id still has a directory and a readable lease in
+// every catalog; a plan calls it after its walk to detect a session removed meanwhile.
+func (m *sessionManager) verify(id string) error {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ sessions, err := m.existing()
+ if err != nil {
+ return status.Errorf(codes.Internal, "scan export sessions: %v", err)
+ }
+ dirs, ok := sessions[id]
+ if !ok {
+ return status.Errorf(codes.NotFound, "export session %s was removed during the plan", id)
+ }
+ for _, dir := range dirs {
+ if _, err = readLease(dir); err != nil {
+ return status.Errorf(codes.NotFound, "export session %s was removed during the plan: %v", id, err)
+ }
+ }
+ return nil
+}
+
+// release deletes <id> in every catalog's export snapshot directory and reports whether
+// this node held it in any catalog. Releasing a session the node does not hold is not an
+// error, so a rollback or a retry can target nodes that never had it or already dropped it.
+func (m *sessionManager) release(id string) (bool, error) {
+ if err := validateSessionID(id); err != nil {
+ return false, err
+ }
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ sessions, err := m.existing()
+ if err != nil {
+ return false, status.Errorf(codes.Internal, "release export session %s: %v", id, err)
+ }
+ if err = m.removeAll(id); err != nil {
+ return false, status.Errorf(codes.Internal, "release export session %s: %v", id, err)
+ }
+ return len(sessions[id]) > 0, nil
+}
+
+// removeAll removes id, which the caller already validated, from every catalog.
+func (m *sessionManager) removeAll(id string) error {
+ dirs := make(map[commonv1.Catalog]string, len(m.backends))
+ for catalog, b := range m.backends {
+ dirs[catalog] = sessionPath(b, id)
+ }
+ return removeDirs(dirs)
+}
+
+// removeDirs deletes every directory of a session in two phases. First the .lease of
+// every directory goes; a failure there stops before any tree is touched, but leases
+// already removed stay removed, so a session can be left with leases in only some
+// catalogs. Then every tree goes. Either way a session missing any lease is refused by
+// renew and reclaimed by the sweeper after orphanGrace. Missing leases and directories
+// are fine; every other error is reported.
+func removeDirs(dirs map[commonv1.Catalog]string) error {
+ for _, dir := range dirs {
+ if err := os.Remove(filepath.Join(dir, LeaseFileName)); err != nil && !errors.Is(err, fs.ErrNotExist) {
+ return err
+ }
+ }
+ var err error
+ for _, dir := range dirs {
+ if rmErr := os.RemoveAll(dir); rmErr != nil {
+ err = errors.Join(err, rmErr)
+ }
+ }
+ return err
+}
+
+// sweep reclaims sessions that expired or were abandoned mid-creation. It runs at start-up,
+// which is how a restarted node recovers its sessions (live ones are kept, the rest
+// reclaimed), and every SweepInterval after that. The lock covers only the scan; the
+// deletions run outside it so a large tree never stalls create or renew. A create racing
+// those deletions may therefore still see the doomed session and answer ALREADY_EXISTS,
+// and a preempting create may remove it too and list it among its preempted sessions.
+func (m *sessionManager) sweep() {
+ for id, dirs := range m.doomed() {
+ if err := removeDirs(dirs); err != nil {
+ m.l.Warn().Err(err).Str("session", id).Msg("export session sweep: remove failed, retrying next sweep")
+ continue
+ }
+ m.l.Info().Str("session", id).Msg("export session sweep: removed")
+ }
+}
+
+// doomed is the locked half of sweep: the directories of every reclaimable session.
+func (m *sessionManager) doomed() map[string]map[commonv1.Catalog]string {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ sessions, err := m.existing()
+ if err != nil {
+ m.l.Warn().Err(err).Msg("export session sweep: cannot list snapshot directories")
+ return nil
+ }
+ now := m.now()
+ maps.DeleteFunc(sessions, func(_ string, dirs map[commonv1.Catalog]string) bool { return !reclaimable(dirs, now) })
+ return sessions
+}
diff --git a/banyand/transfer/export/session_test.go b/banyand/transfer/export/session_test.go
new file mode 100644
index 000000000..4a0c364a3
--- /dev/null
+++ b/banyand/transfer/export/session_test.go
@@ -0,0 +1,934 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package export
+
+import (
+ "context"
+ "errors"
+ "maps"
+ "os"
+ "path/filepath"
+ "slices"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/status"
+
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ "github.com/apache/skywalking-banyandb/banyand/internal/storage"
+ "github.com/apache/skywalking-banyandb/banyand/queue"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+)
+
+// fakeBackend is a Backend over a temp directory: TakeExportSnapshot copies the live
+// group directories into <exportSnapshotDir>/<name>. onSnapshot, when set, runs after the
+// copy with that session directory so a test can tamper with it.
+type fakeBackend struct {
+ onSnapshot func(sessionDir string, attempt int32)
+ dataDir string
+ snapshotDir string
+ exportSnapshotDir string
+ catalog commonv1.Catalog
+ snapshots atomic.Int32
+ fail bool
+}
+
+func newFakeBackend(t *testing.T, root string, catalog commonv1.Catalog) *fakeBackend {
+ t.Helper()
+ name := strings.ToLower(strings.TrimPrefix(catalog.String(), "CATALOG_"))
+ b := &fakeBackend{
+ dataDir: filepath.Join(root, name, storage.DataDir),
+ snapshotDir: filepath.Join(root, name, storage.SnapshotsDir),
+ exportSnapshotDir: filepath.Join(root, name, storage.ExportSnapshotsDir),
+ catalog: catalog,
+ }
+ if err := os.MkdirAll(b.dataDir, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ return b
+}
+
+func (b *fakeBackend) GetDataPath() string { return b.dataDir }
+
+// ReadPartMetadata implements PartReader with the catalog's real reader.
+func (b *fakeBackend) ReadPartMetadata(partDir string) (queue.StreamingPartData, error) {
+ return testPartReader(b.catalog)(partDir)
+}
+func (b *fakeBackend) GetSnapshotDir() string { return b.snapshotDir }
+func (b *fakeBackend) GetExportSnapshotDir() string { return b.exportSnapshotDir }
+
+func (b *fakeBackend) TakeExportSnapshot(_ context.Context, name string) error {
+ attempt := b.snapshots.Add(1)
+ if b.fail {
+ return errors.New("injected snapshot failure")
+ }
+ dst := filepath.Join(b.exportSnapshotDir, name)
+ entries, err := os.ReadDir(b.dataDir)
+ if err != nil {
+ return err
+ }
+ for _, e := range entries {
+ if !e.IsDir() {
+ continue
+ }
+ if err := os.CopyFS(filepath.Join(dst, e.Name()), os.DirFS(filepath.Join(b.dataDir, e.Name()))); err != nil {
+ return err
+ }
+ }
+ if b.onSnapshot != nil {
+ b.onSnapshot(dst, attempt)
+ }
+ return nil
+}
+
+type fakeClock struct{ t time.Time }
+
+func (c *fakeClock) now() time.Time { return c.t }
+func (c *fakeClock) advance(d time.Duration) { c.t = c.t.Add(d) }
+
+func allCatalogs() map[commonv1.Catalog]struct{} {
+ return catalogSet(commonv1.Catalog_CATALOG_STREAM, commonv1.Catalog_CATALOG_MEASURE,
+ commonv1.Catalog_CATALOG_TRACE, commonv1.Catalog_CATALOG_PROPERTY)
+}
+
+func catalogSet(cs ...commonv1.Catalog) map[commonv1.Catalog]struct{} {
+ out := make(map[commonv1.Catalog]struct{}, len(cs))
+ for _, c := range cs {
+ out[c] = struct{}{}
+ }
+ return out
+}
+
+// skipIfRoot skips a test that relies on a read-only directory: root ignores the mode bits.
+func skipIfRoot(t *testing.T) {
+ t.Helper()
+ if os.Geteuid() == 0 {
+ t.Skip("root ignores directory permissions")
+ }
+}
+
+// newTestBackends builds four fake backends over one temp dir and seeds the stream catalog
+// with the given groups, each holding one flushed part so snapshots have something to copy.
+func newTestBackends(t *testing.T, streamGroups ...string) (Backends, *fakeClock) {
+ t.Helper()
+ root := t.TempDir()
+ backends := Backends{}
+ for _, c := range []commonv1.Catalog{
+ commonv1.Catalog_CATALOG_STREAM, commonv1.Catalog_CATALOG_MEASURE,
+ commonv1.Catalog_CATALOG_TRACE, commonv1.Catalog_CATALOG_PROPERTY,
+ } {
+ backends[c] = newFakeBackend(t, root, c)
+ }
+ for _, g := range streamGroups {
+ seg := filepath.Join(backends[commonv1.Catalog_CATALOG_STREAM].GetDataPath(), g, "seg-20260928")
+ writeFile(t, filepath.Join(seg, "metadata"), []byte(`{"version":"1.5.0"}`))
+ writeFile(t, filepath.Join(seg, "shard-0", "0000000000000001", "metadata.json"), partJSON(1, 2, 3, 4, 5))
+ }
+ return backends, &fakeClock{t: time.Unix(1_700_000_000, 0)}
+}
+
+// newTestManager builds a manager over newTestBackends with the single stream group "sw_record".
+func newTestManager(t *testing.T) (*sessionManager, Backends, *fakeClock) {
+ t.Helper()
+ backends, clock := newTestBackends(t, "sw_record")
+ m := newSessionManager(backends, logger.GetLogger("export-test"))
+ m.now = clock.now
+ return m, backends, clock
+}
+
+func mustCreate(t *testing.T, m *sessionManager, id string, catalogs map[commonv1.Catalog]struct{}) *createdSession {
+ t.Helper()
+ return mustCreateWith(t, m, id, catalogs, false)
+}
+
+func mustCreateWith(t *testing.T, m *sessionManager, id string, catalogs map[commonv1.Catalog]struct{}, preempt bool) *createdSession {
+ t.Helper()
+ created, err := m.create(context.Background(), id, catalogs, preempt)
+ if err != nil {
+ t.Fatalf("create(%s, preempt=%t): %v", id, preempt, err)
+ }
+ return created
+}
+
+func mustSessionDir(t *testing.T, m *sessionManager, catalog commonv1.Catalog, id string) string {
+ t.Helper()
+ dir, err := m.sessionDir(catalog, id)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return dir
+}
+
+// sessionIDs returns the ids of every session directory set on disk, in id order.
+func sessionIDs(t *testing.T, m *sessionManager) []string {
+ t.Helper()
+ sessions, err := m.existing()
+ if err != nil {
+ t.Fatal(err)
+ }
+ return slices.Sorted(maps.Keys(sessions))
+}
+
+// age sets the mtime of p relative to the fake clock, since the real file system stamps
+// directories with wall-clock time.
+func age(t *testing.T, p string, at time.Time) {
+ t.Helper()
+ if err := os.Chtimes(p, at, at); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestSession_CreateProbeRenewRelease(t *testing.T) {
+ ctx := context.Background()
+ m, _, clock := newTestManager(t)
+ created := mustCreate(t, m, testSessionID, catalogSet(commonv1.Catalog_CATALOG_STREAM))
+ if len(created.preempted) != 0 {
+ t.Fatalf("nothing to preempt on an empty node, got %v", created.preempted)
+ }
+ streamDir := mustSessionDir(t, m, commonv1.Catalog_CATALOG_STREAM, testSessionID)
+ if got := created.roots[commonv1.Catalog_CATALOG_STREAM]; got != streamDir {
+ t.Fatalf("create must hand back the stream groups root %s, got %s", streamDir, got)
+ }
+ lease, err := readLease(streamDir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if want := clock.now().UnixNano(); lease.SessionID != testSessionID || lease.StartedAt != want || lease.LastHeartbeatAt != want ||
+ lease.ExpiresAt != clock.now().Add(defaultLease).UnixNano() {
+ t.Fatalf("create must write the lease at creation time with the default length: %+v", lease)
+ }
+ _, err = os.Stat(filepath.Join(streamDir, "sw_record", "seg-20260928", "shard-0", "0000000000000001", "metadata.json"))
+ if err != nil {
+ t.Fatalf("snapshot content missing: %v", err)
+ }
+ measureDir := mustSessionDir(t, m, commonv1.Catalog_CATALOG_MEASURE, testSessionID)
+ if _, err = os.Stat(measureDir); !os.IsNotExist(err) {
+ t.Fatal("unselected catalog must not get a session dir")
+ }
+
+ probed, err := m.probe()
+ if err != nil || probed.GetSessionId() != testSessionID ||
+ len(probed.GetCatalogs()) != 1 || probed.GetCatalogs()[0] != commonv1.Catalog_CATALOG_STREAM {
+ t.Fatalf("probe = %+v, %v", probed, err)
+ }
+
+ clock.advance(10 * time.Minute)
+ roots, err := m.locateAndHeartbeat(testSessionID)
+ if err != nil || roots[commonv1.Catalog_CATALOG_STREAM] != streamDir || len(roots) != 1 {
+ t.Fatalf("locate = %+v, %v", roots, err)
+ }
+ lease, _ = readLease(streamDir)
+ if lease.LastHeartbeatAt != clock.now().UnixNano() || lease.ExpiresAt != clock.now().Add(defaultLease).UnixNano() {
+ t.Fatalf("renew must rewrite lastHeartbeatAt and expiresAt: %+v", lease)
+ }
+
+ if _, err = m.release(testSessionID); err != nil {
+ t.Fatal(err)
+ }
+ if _, err = m.release(testSessionID); err != nil {
+ t.Fatal("release must be idempotent:", err)
+ }
+ _, err = m.locateAndHeartbeat(testSessionID)
+ if status.Code(err) != codes.NotFound || !strings.Contains(err.Error(), "not found") {
+ t.Fatalf("released session must be NotFound: %v", err)
+ }
+ if _, err = m.create(ctx, "bbbb", allCatalogs(), false); err != nil {
+ t.Fatalf("a released node must accept a new session: %v", err)
+ }
+}
+
+func TestSession_CreateCommitsLeaseAfterLastSnapshot(t *testing.T) {
+ m, backends, clock := newTestManager(t)
+ // Every snapshot takes a minute; the committed lease must be stamped after the last one.
+ for _, b := range backends {
+ b.(*fakeBackend).onSnapshot = func(string, int32) { clock.advance(time.Minute) }
+ }
+ start := clock.now()
+ mustCreate(t, m, testSessionID, allCatalogs())
+ for catalog := range allCatalogs() {
+ l, err := readLease(mustSessionDir(t, m, catalog, testSessionID))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if l.StartedAt != start.UnixNano() || l.LastHeartbeatAt != clock.now().UnixNano() ||
+ l.ExpiresAt != clock.now().Add(defaultLease).UnixNano() {
+ t.Fatalf("%s lease must be committed with the time of the last snapshot: %+v (start %d, end %d)",
+ catalog, l, start.UnixNano(), clock.now().UnixNano())
+ }
+ }
+}
+
+func TestSession_CreateRefusesOccupantUnlessPreempt(t *testing.T) {
+ ctx := context.Background()
+ m, _, clock := newTestManager(t)
+ mustCreate(t, m, testSessionID, allCatalogs())
+ clock.advance(time.Hour) // how long ago the occupant last heartbeat is not create's business
+ _, err := m.create(ctx, "bbbb", allCatalogs(), false)
+ if status.Code(err) != codes.AlreadyExists || !strings.Contains(err.Error(), testSessionID) {
+ t.Fatalf("want AlreadyExists naming aaaa, got %v", err)
+ }
+ if got := sessionIDs(t, m); !slices.Equal(got, []string{testSessionID}) {
+ t.Fatalf("the loser must leave no trace: %v", got)
+ }
+ created, err := m.create(ctx, "bbbb", allCatalogs(), true)
+ if err != nil || !slices.Equal(created.preempted, []string{testSessionID}) {
+ t.Fatalf("preempt = %v, %v", created, err)
+ }
+ if got := sessionIDs(t, m); !slices.Equal(got, []string{"bbbb"}) {
+ t.Fatalf("old session must be gone: %v", got)
+ }
+}
+
+func TestSession_CreateRefusesExpiredOccupantWithoutPreempt(t *testing.T) {
+ ctx := context.Background()
+ m, _, clock := newTestManager(t)
+ mustCreate(t, m, testSessionID, allCatalogs())
+ clock.advance(defaultLease + time.Second)
+ _, err := m.create(ctx, "bbbb", catalogSet(commonv1.Catalog_CATALOG_STREAM), false)
+ if status.Code(err) != codes.AlreadyExists || !strings.Contains(err.Error(), testSessionID) {
+ t.Fatalf("an expired occupant is the sweeper's business, create must still refuse: %v", err)
+ }
+ if got := sessionIDs(t, m); !slices.Equal(got, []string{testSessionID}) {
+ t.Fatalf("the refused create must leave the occupant alone: %v", got)
+ }
+ created, err := m.create(ctx, "bbbb", catalogSet(commonv1.Catalog_CATALOG_STREAM), true)
+ if err != nil || !slices.Equal(created.preempted, []string{testSessionID}) {
+ t.Fatalf("preempt = %v, %v", created, err)
+ }
+ if got := sessionIDs(t, m); !slices.Equal(got, []string{"bbbb"}) {
+ t.Fatalf("only the new session may remain: %v", got)
+ }
+}
+
+func TestSession_PreemptRemovesEveryOccupantAndReportsAll(t *testing.T) {
+ ctx := context.Background()
+ m, _, clock := newTestManager(t)
+ mustCreate(t, m, "aaaa", catalogSet(commonv1.Catalog_CATALOG_STREAM))
+ // A second occupant can only get here through preempt; a lease-less leftover is the
+ // other shape a node may hold. Both go, whatever their age or lease says.
+ mustCreateWith(t, m, "bbbb", catalogSet(commonv1.Catalog_CATALOG_MEASURE), true)
+ orphan := mustSessionDir(t, m, commonv1.Catalog_CATALOG_TRACE, "cccc")
+ if err := os.MkdirAll(orphan, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ age(t, orphan, clock.now())
+ _, err := m.create(ctx, "dddd", allCatalogs(), false)
+ if status.Code(err) != codes.AlreadyExists || !strings.Contains(err.Error(), "bbbb") || !strings.Contains(err.Error(), "cccc") {
+ t.Fatalf("the refusal must name every occupant, got %v", err)
+ }
+ created := mustCreateWith(t, m, "dddd", allCatalogs(), true)
+ if !slices.Equal(created.preempted, []string{"bbbb", "cccc"}) {
+ t.Fatalf("preempt must remove and report every occupant, sorted, got %v", created.preempted)
+ }
+ if got := sessionIDs(t, m); !slices.Equal(got, []string{"dddd"}) {
+ t.Fatalf("only the new session may remain: %v", got)
+ }
+}
+
+func TestSession_CreateRefusesLeaselessOccupantWithoutPreempt(t *testing.T) {
+ ctx := context.Background()
+ m, _, clock := newTestManager(t)
+ orphan := mustSessionDir(t, m, commonv1.Catalog_CATALOG_STREAM, "dead")
+ if err := os.MkdirAll(orphan, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ for _, at := range []time.Time{clock.now(), clock.now().Add(-orphanGrace - time.Minute)} {
+ age(t, orphan, at)
+ _, err := m.create(ctx, testSessionID, allCatalogs(), false)
+ if status.Code(err) != codes.AlreadyExists || !strings.Contains(err.Error(), "dead") {
+ t.Fatalf("a directory without lease aged %s is an occupant and must be refused, got %v", clock.now().Sub(at), err)
+ }
+ }
+ created := mustCreateWith(t, m, testSessionID, allCatalogs(), true)
+ if !slices.Equal(created.preempted, []string{"dead"}) {
+ t.Fatalf("preempt must remove the lease-less directory, got %v", created.preempted)
+ }
+}
+
+func TestSession_CreateCanceledBeforeStartRemovesNothing(t *testing.T) {
+ m, _, _ := newTestManager(t)
+ mustCreate(t, m, testSessionID, allCatalogs())
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+ _, err := m.create(ctx, "bbbb", allCatalogs(), true)
+ if status.Code(err) != codes.Canceled {
+ t.Fatalf("want Canceled, got %v", err)
+ }
+ if got := sessionIDs(t, m); !slices.Equal(got, []string{testSessionID}) {
+ t.Fatalf("a canceled create must not touch the occupant: %v", got)
+ }
+}
+
+func TestSession_CreateCanceledWhileWaitingForLockRemovesNothing(t *testing.T) {
+ m, _, _ := newTestManager(t)
+ mustCreate(t, m, testSessionID, allCatalogs())
+ ctx, cancel := context.WithCancel(context.Background())
+ m.mu.Lock()
+ errCh := make(chan error, 1)
+ //panicdiag:allow-rawgo test goroutine blocked on the manager lock; its result is read from errCh
+ go func() {
+ _, err := m.create(ctx, "bbbb", allCatalogs(), true)
+ errCh <- err
+ }()
+ // Give the goroutine time to pass the first ctx check and block on the lock; the result
+ // is the same if it has not got there yet.
+ time.Sleep(50 * time.Millisecond)
+ cancel()
+ m.mu.Unlock()
+ if err := <-errCh; status.Code(err) != codes.Canceled {
+ t.Fatalf("want Canceled, got %v", err)
+ }
+ if got := sessionIDs(t, m); !slices.Equal(got, []string{testSessionID}) {
+ t.Fatalf("a create canceled while waiting for the lock must not touch the occupant: %v", got)
+ }
+ if _, err := m.locateAndHeartbeat(testSessionID); err != nil {
+ t.Fatalf("the occupant must still be renewable: %v", err)
+ }
+}
+
+// TestSession_CreateSingleCatalogOccupantRemovalFailureKeepsOccupant covers an occupant held
+// in one catalog only: its lease removal fails first, so nothing of it is touched.
+func TestSession_CreateSingleCatalogOccupantRemovalFailureKeepsOccupant(t *testing.T) {
+ skipIfRoot(t)
+ ctx := context.Background()
+ m, _, _ := newTestManager(t)
+ mustCreate(t, m, testSessionID, catalogSet(commonv1.Catalog_CATALOG_STREAM))
+ dir := mustSessionDir(t, m, commonv1.Catalog_CATALOG_STREAM, testSessionID)
+ // A read-only session directory refuses the removal of its lease, which comes first.
+ if err := os.Chmod(dir, 0o555); err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = os.Chmod(dir, 0o755) })
+ _, err := m.create(ctx, "bbbb", catalogSet(commonv1.Catalog_CATALOG_STREAM), true)
+ if status.Code(err) != codes.Internal || !strings.Contains(err.Error(), "preempt export session "+testSessionID) {
+ t.Fatalf("want Internal naming the occupant, got %v", err)
+ }
+ if err = os.Chmod(dir, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if got := sessionIDs(t, m); !slices.Equal(got, []string{testSessionID}) {
+ t.Fatalf("the occupant must still be intact: %v", got)
+ }
+ if _, err = m.locateAndHeartbeat(testSessionID); err != nil {
+ t.Fatalf("the occupant must still be renewable: %v", err)
+ }
+}
+
+func TestSession_ExpiredIsDistinctFromNotFound(t *testing.T) {
+ m, _, clock := newTestManager(t)
+ _, err := m.locateAndHeartbeat("deadbeef")
+ if status.Code(err) != codes.NotFound {
+ t.Fatalf("unknown session must be NotFound, got %v", err)
+ }
+ mustCreate(t, m, testSessionID, allCatalogs())
+ clock.advance(defaultLease + time.Second)
+ _, err = m.locateAndHeartbeat(testSessionID)
+ if status.Code(err) != codes.FailedPrecondition || !strings.Contains(err.Error(), "expired") {
+ t.Fatalf("want expired FailedPrecondition, got %v", err)
+ }
+ m.sweep()
+ if got := sessionIDs(t, m); len(got) != 0 {
+ t.Fatalf("sweep must delete expired sessions: %v", got)
+ }
+}
+
+func TestSession_RenewRewritesEveryCatalogCopy(t *testing.T) {
+ m, _, clock := newTestManager(t)
+ mustCreate(t, m, testSessionID, allCatalogs())
+ clock.advance(time.Hour)
+ if _, err := m.locateAndHeartbeat(testSessionID); err != nil {
+ t.Fatal(err)
+ }
+ for catalog := range allCatalogs() {
+ l, err := readLease(mustSessionDir(t, m, catalog, testSessionID))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if l.LastHeartbeatAt != clock.now().UnixNano() || l.ExpiresAt != clock.now().Add(defaultLease).UnixNano() {
+ t.Fatalf("%s copy was not renewed: %+v", catalog, l)
+ }
+ }
+}
+
+func TestSession_RenewRefusesBeforeWritingWhenOneCopyIsUnreadable(t *testing.T) {
+ m, _, clock := newTestManager(t)
+ mustCreate(t, m, testSessionID, catalogSet(commonv1.Catalog_CATALOG_STREAM, commonv1.Catalog_CATALOG_MEASURE))
+ streamDir := mustSessionDir(t, m, commonv1.Catalog_CATALOG_STREAM, testSessionID)
+ measureDir := mustSessionDir(t, m, commonv1.Catalog_CATALOG_MEASURE, testSessionID)
+ before, err := readLease(streamDir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ writeFile(t, filepath.Join(measureDir, LeaseFileName), []byte(`{broken`))
+ clock.advance(time.Hour)
+ _, err = m.locateAndHeartbeat(testSessionID)
+ if status.Code(err) != codes.FailedPrecondition || !strings.Contains(err.Error(), "no readable lease") {
+ t.Fatalf("want FailedPrecondition about the lease, got %v", err)
+ }
+ after, err := readLease(streamDir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if after != before {
+ t.Fatalf("a refused renewal must not advance any copy: before %+v, after %+v", before, after)
+ }
+}
+
+func TestSession_SweepKeepsLiveSessions(t *testing.T) {
+ m, _, clock := newTestManager(t)
+ mustCreate(t, m, testSessionID, allCatalogs())
+ clock.advance(30 * time.Minute)
+ m.sweep() // the restart path calls this first
+ if got := sessionIDs(t, m); !slices.Equal(got, []string{testSessionID}) {
+ t.Fatalf("a live session must survive the sweep: %v", got)
+ }
+}
+
+func TestSession_SweepDeletesOrphansAfterGrace(t *testing.T) {
+ m, _, clock := newTestManager(t)
+ orphan := mustSessionDir(t, m, commonv1.Catalog_CATALOG_STREAM, "dead")
+ if err := os.MkdirAll(orphan, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ age(t, orphan, clock.now().Add(-orphanGrace+time.Minute))
+ m.sweep()
+ if _, err := os.Stat(orphan); err != nil {
+ t.Fatal("fresh orphan must survive (creation may be in flight)")
+ }
+ age(t, orphan, clock.now().Add(-orphanGrace-time.Minute))
+ m.sweep()
+ if _, err := os.Stat(orphan); !os.IsNotExist(err) {
+ t.Fatal("stale orphan must be removed")
+ }
+}
+
+func TestSession_SweepReclaimsCorruptLeaseAfterGrace(t *testing.T) {
+ m, _, clock := newTestManager(t)
+ mustCreate(t, m, testSessionID, catalogSet(commonv1.Catalog_CATALOG_STREAM))
+ dir := mustSessionDir(t, m, commonv1.Catalog_CATALOG_STREAM, testSessionID)
+ writeFile(t, filepath.Join(dir, LeaseFileName), []byte(`{broken`))
+ age(t, dir, clock.now())
+ m.sweep()
+ if _, err := os.Stat(dir); err != nil {
+ t.Fatal("a corrupt lease younger than the grace must be left alone")
+ }
+ age(t, dir, clock.now().Add(-orphanGrace-time.Minute))
+ m.sweep()
+ if _, err := os.Stat(dir); !os.IsNotExist(err) {
+ t.Fatal("a corrupt lease older than the grace must be reclaimed")
+ }
+}
+
+func TestSession_SweepDecidesPerSession(t *testing.T) {
+ m, _, clock := newTestManager(t)
+ mustCreate(t, m, testSessionID, catalogSet(commonv1.Catalog_CATALOG_STREAM, commonv1.Catalog_CATALOG_MEASURE))
+ streamDir := mustSessionDir(t, m, commonv1.Catalog_CATALOG_STREAM, testSessionID)
+ measureDir := mustSessionDir(t, m, commonv1.Catalog_CATALOG_MEASURE, testSessionID)
+ // One copy expired, the other is alive: every copy goes.
+ l, err := readLease(streamDir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ l.ExpiresAt = clock.now().Add(-time.Second).UnixNano()
+ if err = writeLease(streamDir, l); err != nil {
+ t.Fatal(err)
+ }
+ m.sweep()
+ for _, dir := range []string{streamDir, measureDir} {
+ if _, err = os.Stat(dir); !os.IsNotExist(err) {
+ t.Fatalf("%s must be removed with the expired session", dir)
+ }
+ }
+}
+
+func TestSession_SweepReclaimsPartiallyLeasedSessionAfterGrace(t *testing.T) {
+ m, _, clock := newTestManager(t)
+ mustCreate(t, m, testSessionID, catalogSet(commonv1.Catalog_CATALOG_STREAM, commonv1.Catalog_CATALOG_MEASURE))
+ streamDir := mustSessionDir(t, m, commonv1.Catalog_CATALOG_STREAM, testSessionID)
+ measureDir := mustSessionDir(t, m, commonv1.Catalog_CATALOG_MEASURE, testSessionID)
+ // A removal that stopped after dropping one lease: renew refuses the session from now on.
+ if err := os.Remove(filepath.Join(measureDir, LeaseFileName)); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := m.locateAndHeartbeat(testSessionID); status.Code(err) != codes.FailedPrecondition {
+ t.Fatalf("renew of a partially leased session: got %v, want FailedPrecondition", err)
+ }
+ // The remaining lease is still live; the session is kept while its newest directory is fresh.
+ age(t, streamDir, clock.now())
+ age(t, measureDir, clock.now().Add(-orphanGrace-time.Hour))
+ m.sweep()
+ for _, dir := range []string{streamDir, measureDir} {
+ if _, err := os.Stat(dir); err != nil {
+ t.Fatalf("%s must survive while the session is younger than the grace: %v", dir, err)
+ }
+ }
+ // Once every directory is older than the grace, the session goes despite the live lease.
+ age(t, streamDir, clock.now().Add(-orphanGrace-time.Minute))
+ m.sweep()
+ for _, dir := range []string{streamDir, measureDir} {
+ if _, err := os.Stat(dir); !os.IsNotExist(err) {
+ t.Fatalf("%s must be reclaimed once the partially leased session is older than the grace", dir)
+ }
+ }
+}
+
+func TestSession_SweepIgnoresMalformedNames(t *testing.T) {
+ m, backends, clock := newTestManager(t)
+ exportDir := backends[commonv1.Catalog_CATALOG_STREAM].GetExportSnapshotDir()
+ stray := filepath.Join(exportDir, "UPPER")
+ writeFile(t, filepath.Join(stray, "leftover"), []byte("x"))
+ age(t, stray, clock.now().Add(-orphanGrace-time.Hour))
+ m.sweep()
+ if _, err := os.Stat(stray); err != nil {
+ t.Fatal("a directory that is not a well-formed session is none of the sweeper's business")
+ }
+}
+
+func TestSession_RemovalDropsLeaseBeforeTree(t *testing.T) {
+ skipIfRoot(t)
+ m, _, _ := newTestManager(t)
+ mustCreate(t, m, testSessionID, catalogSet(commonv1.Catalog_CATALOG_STREAM))
+ dir := mustSessionDir(t, m, commonv1.Catalog_CATALOG_STREAM, testSessionID)
+ // An unwritable subdirectory makes RemoveAll fail after the lease is gone.
+ locked := filepath.Join(dir, "locked")
+ writeFile(t, filepath.Join(locked, "x"), []byte("x"))
+ if err := os.Chmod(locked, 0o555); err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = os.Chmod(locked, 0o755) })
+ if _, err := m.release(testSessionID); status.Code(err) != codes.Internal {
+ t.Fatalf("want Internal from the interrupted removal, got %v", err)
+ }
+ if _, err := os.Stat(filepath.Join(dir, LeaseFileName)); !os.IsNotExist(err) {
+ t.Fatal("the lease must be removed before the tree")
+ }
+ // A half-deleted directory is a session without lease: refused by renew, reported by
+ // probe with zero times, and reclaimed by the sweeper once older than the grace.
+ if _, err := m.locateAndHeartbeat(testSessionID); status.Code(err) != codes.FailedPrecondition || !strings.Contains(err.Error(), "no readable lease") {
+ t.Fatalf("want FailedPrecondition about the lease, got %v", err)
+ }
+ lease, err := m.probe()
+ if err != nil || lease == nil || lease.LastHeartbeatAt != 0 {
+ t.Fatalf("probe() = %+v, %v", lease, err)
+ }
+ if err = os.Chmod(locked, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if _, err = m.release(testSessionID); err != nil {
+ t.Fatalf("release must finish the job once the tree is writable: %v", err)
+ }
+}
+
+func TestSession_RemovalLeaseFailureDeletesNoTree(t *testing.T) {
+ skipIfRoot(t)
+ m, _, _ := newTestManager(t)
+ mustCreate(t, m, testSessionID, allCatalogs())
+ measureDir := mustSessionDir(t, m, commonv1.Catalog_CATALOG_MEASURE, testSessionID)
+ // A read-only session directory refuses the removal of its lease.
+ if err := os.Chmod(measureDir, 0o555); err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = os.Chmod(measureDir, 0o755) })
+ if _, err := m.release(testSessionID); status.Code(err) != codes.Internal {
+ t.Fatalf("want Internal from the refused lease removal, got %v", err)
+ }
+ for catalog := range allCatalogs() {
+ if _, err := os.Stat(mustSessionDir(t, m, catalog, testSessionID)); err != nil {
+ t.Fatalf("no tree may be deleted while a lease removal fails, %s: %v", catalog, err)
+ }
+ }
+ if _, err := readLease(measureDir); err != nil {
+ t.Fatalf("the refused lease must still be there: %v", err)
+ }
+ if err := os.Chmod(measureDir, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := m.release(testSessionID); err != nil {
+ t.Fatalf("release must finish the job once the lease is removable: %v", err)
+ }
+ if got := sessionIDs(t, m); len(got) != 0 {
+ t.Fatalf("every catalog must be gone: %v", got)
+ }
+}
+
+func TestSession_RemovalTreeFailureLeavesNoLease(t *testing.T) {
+ skipIfRoot(t)
+ m, _, _ := newTestManager(t)
+ mustCreate(t, m, testSessionID, allCatalogs())
+ streamDir := mustSessionDir(t, m, commonv1.Catalog_CATALOG_STREAM, testSessionID)
+ locked := filepath.Join(streamDir, "locked")
+ writeFile(t, filepath.Join(locked, "x"), []byte("x"))
+ if err := os.Chmod(locked, 0o555); err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = os.Chmod(locked, 0o755) })
+ if _, err := m.release(testSessionID); status.Code(err) != codes.Internal {
+ t.Fatalf("want Internal from the interrupted tree removal, got %v", err)
+ }
+ for catalog := range allCatalogs() {
+ dir := mustSessionDir(t, m, catalog, testSessionID)
+ if _, err := os.Stat(filepath.Join(dir, LeaseFileName)); !os.IsNotExist(err) {
+ t.Fatalf("every lease must be gone before any tree is removed, %s: %v", catalog, err)
+ }
+ if catalog == commonv1.Catalog_CATALOG_STREAM {
+ continue
+ }
+ if _, err := os.Stat(dir); !os.IsNotExist(err) {
+ t.Fatalf("the removable trees must be gone, %s: %v", catalog, err)
+ }
+ }
+}
+
+func TestSession_CreateFailureRollsBack(t *testing.T) {
+ ctx := context.Background()
+ m, backends, _ := newTestManager(t)
+ backends[commonv1.Catalog_CATALOG_TRACE].(*fakeBackend).fail = true
+ _, err := m.create(ctx, testSessionID, allCatalogs(), false)
+ if status.Code(err) != codes.Internal || !strings.Contains(err.Error(), "snapshot CATALOG_TRACE for export session "+testSessionID) ||
+ !strings.Contains(err.Error(), "injected snapshot failure") {
+ t.Fatalf("want Internal naming the failed catalog, got %v", err)
+ }
+ if got := sessionIDs(t, m); len(got) != 0 {
+ t.Fatalf("partial session must be removed on failure: %v", got)
+ }
+}
+
+func TestSession_CreateFailureAfterPreemptNamesPreempted(t *testing.T) {
+ ctx := context.Background()
+ m, backends, _ := newTestManager(t)
+ mustCreate(t, m, testSessionID, catalogSet(commonv1.Catalog_CATALOG_STREAM))
+ backends[commonv1.Catalog_CATALOG_TRACE].(*fakeBackend).fail = true
+ _, err := m.create(ctx, "bbbb", allCatalogs(), true)
+ if status.Code(err) != codes.Internal || !strings.Contains(err.Error(), "preempted sessions ["+testSessionID+"] were already removed") {
+ t.Fatalf("want Internal naming the preempted session, got %v", err)
+ }
+ if got := sessionIDs(t, m); len(got) != 0 {
+ t.Fatalf("the occupant is preempted and the failed create rolled back: %v", got)
+ }
+}
+
+func TestSession_LeaseCommitFailureRollsBack(t *testing.T) {
+ m, backends, _ := newTestManager(t)
+ // A directory squatting on the lease path makes the atomic rename of the lease fail.
+ backends[commonv1.Catalog_CATALOG_MEASURE].(*fakeBackend).onSnapshot = func(root string, _ int32) {
+ if err := os.MkdirAll(filepath.Join(root, LeaseFileName), 0o755); err != nil {
+ t.Error(err)
+ }
+ }
+ _, err := m.create(context.Background(), testSessionID, allCatalogs(), false)
+ if status.Code(err) != codes.Internal || !strings.Contains(err.Error(), "commit lease of export session "+testSessionID) {
+ t.Fatalf("want Internal about the lease commit, got %v", err)
+ }
+ if got := sessionIDs(t, m); len(got) != 0 {
+ t.Fatalf("a failed lease commit must remove every catalog of the session: %v", got)
+ }
+}
+
+func TestSession_RejectsUnsafeIDs(t *testing.T) {
+ m, _, _ := newTestManager(t)
+ for _, bad := range []string{"/../../data", "..", "ABC"} {
+ if _, err := m.create(context.Background(), bad, allCatalogs(), false); status.Code(err) != codes.InvalidArgument {
+ t.Fatalf("create(%q) = %v, want InvalidArgument", bad, err)
+ }
+ if _, err := m.release(bad); status.Code(err) != codes.InvalidArgument {
+ t.Fatalf("release(%q) = %v, want InvalidArgument", bad, err)
+ }
+ if _, err := m.locateAndHeartbeat(bad); status.Code(err) != codes.InvalidArgument {
+ t.Fatalf("locate(%q) = %v, want InvalidArgument", bad, err)
+ }
+ }
+ // The live data directory must be untouched by the rejected release above.
+ if _, err := os.Stat(filepath.Join(m.backends[commonv1.Catalog_CATALOG_STREAM].GetDataPath(), "sw_record")); err != nil {
+ t.Fatalf("live data was removed: %v", err)
+ }
+ if _, err := m.sessionDir(commonv1.Catalog_CATALOG_UNSPECIFIED, testSessionID); status.Code(err) != codes.InvalidArgument {
+ t.Fatalf("a catalog without backend must be InvalidArgument, got %v", err)
+ }
+}
+
+func TestSession_HeartbeatDoesNotSnapshot(t *testing.T) {
+ m, backends, _ := newTestManager(t)
+ mustCreate(t, m, testSessionID, allCatalogs())
+ before := backends[commonv1.Catalog_CATALOG_STREAM].(*fakeBackend).snapshots.Load()
+ if _, err := m.locateAndHeartbeat(testSessionID); err != nil {
+ t.Fatal(err)
+ }
+ if after := backends[commonv1.Catalog_CATALOG_STREAM].(*fakeBackend).snapshots.Load(); after != before {
+ t.Fatal("renewing must not take another snapshot")
+ }
+}
+
+func TestSession_RootsAreTheSessionDirs(t *testing.T) {
+ m, _, _ := newTestManager(t)
+ created := mustCreate(t, m, "aabb", allCatalogs())
+ roots, err := m.locateAndHeartbeat("aabb")
+ if err != nil {
+ t.Fatal(err)
+ }
+ for catalog := range allCatalogs() {
+ dir := mustSessionDir(t, m, catalog, "aabb")
+ if created.roots[catalog] != dir || roots[catalog] != dir {
+ t.Fatalf("%s root must be the session dir %s, got create %s, renew %s", catalog, dir, created.roots[catalog], roots[catalog])
+ }
+ }
+}
+
+func TestSession_ProbeAllCatalogsMatchesOnDiskLease(t *testing.T) {
+ m, _, _ := newTestManager(t)
+ mustCreate(t, m, "aabb", allCatalogs())
+ got, err := m.probe()
+ if err != nil || got == nil {
+ t.Fatalf("probe() = %+v, %v", got, err)
+ }
+ if !slices.Equal(got.Catalogs, slices.Sorted(maps.Keys(allCatalogs()))) {
+ t.Fatalf("probe must list every catalog that holds the session, sorted: %v", got.Catalogs)
+ }
+ streamDir := mustSessionDir(t, m, commonv1.Catalog_CATALOG_STREAM, "aabb")
+ diskLease, err := readLease(streamDir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got.StartedAt != diskLease.StartedAt || got.ExpiresAt != diskLease.ExpiresAt || got.LastHeartbeatAt != diskLease.LastHeartbeatAt {
+ t.Fatalf("probe times %+v must match the on-disk lease %+v", got, diskLease)
+ }
+ // Catalogs come from the directories, not from the lease: dropping one directory drops it.
+ if rmErr := os.RemoveAll(filepath.Dir(streamDir)); rmErr != nil {
+ t.Fatal(rmErr)
+ }
+ got, err = m.probe()
+ if err != nil || got == nil || slices.Contains(got.Catalogs, commonv1.Catalog_CATALOG_STREAM) || len(got.Catalogs) != 3 {
+ t.Fatalf("probe after removing the stream directory = %+v, %v", got, err)
+ }
+}
+
+// One session id covers the cluster, so a node holding two ids is a half-won create or an
+// interrupted release: probe reports the one started last and leaves the other to the
+// sweeper, while an empty node reports nothing at all.
+func TestSession_ProbeReportsNewestOfSeveralSessions(t *testing.T) {
+ m, _, clock := newTestManager(t)
+ if lease, err := m.probe(); err != nil || lease != nil {
+ t.Fatalf("an empty node must report no session, got %+v, %v", lease, err)
+ }
+ mustCreate(t, m, "bbbb", allCatalogs())
+ // create never leaves two sessions behind, so the second one is planted on disk the way
+ // an interrupted release or a crashed preempt would leave it.
+ clock.advance(time.Minute)
+ planted := mustSessionDir(t, m, commonv1.Catalog_CATALOG_STREAM, "aaaa")
+ if err := os.MkdirAll(planted, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ now := clock.now().UnixNano()
+ if err := writeLease(planted, Lease{SessionID: "aaaa", StartedAt: now, ExpiresAt: clock.now().Add(defaultLease).UnixNano(), LastHeartbeatAt: now}); err != nil {
+ t.Fatal(err)
+ }
+ if got := sessionIDs(t, m); !slices.Equal(got, []string{"aaaa", "bbbb"}) {
+ t.Fatalf("both session directories must exist, got %v", got)
+ }
+ lease, err := m.probe()
+ if err != nil || lease.GetSessionId() != "aaaa" || !slices.Equal(lease.GetCatalogs(), []commonv1.Catalog{commonv1.Catalog_CATALOG_STREAM}) {
+ t.Fatalf("probe must report the session started last, got %+v, %v", lease, err)
+ }
+}
+
+func TestSession_ProbeReportsNewestWhenItSortsLast(t *testing.T) {
+ m, _, clock := newTestManager(t)
+ mustCreate(t, m, "aaaa", allCatalogs())
+ clock.advance(time.Minute)
+ planted := mustSessionDir(t, m, commonv1.Catalog_CATALOG_MEASURE, "bbbb")
+ if err := os.MkdirAll(planted, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ now := clock.now().UnixNano()
+ if err := writeLease(planted, Lease{SessionID: "bbbb", StartedAt: now, ExpiresAt: clock.now().Add(defaultLease).UnixNano(), LastHeartbeatAt: now}); err != nil {
+ t.Fatal(err)
+ }
+ lease, err := m.probe()
+ if err != nil || lease.GetSessionId() != "bbbb" || lease.GetStartedAt() != now {
+ t.Fatalf("probe must report the session started last even when its id sorts last, got %+v, %v", lease, err)
+ }
+}
+
+func TestSession_ProbeReportsLeaselessDirectoryWithZeroTimes(t *testing.T) {
+ m, _, _ := newTestManager(t)
+ orphan := mustSessionDir(t, m, commonv1.Catalog_CATALOG_MEASURE, "dead")
+ if err := os.MkdirAll(orphan, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ got, err := m.probe()
+ if err != nil || got == nil {
+ t.Fatalf("probe() = %+v, %v", got, err)
+ }
+ if got.SessionId != "dead" || got.StartedAt != 0 || got.ExpiresAt != 0 || got.LastHeartbeatAt != 0 ||
+ !slices.Equal(got.Catalogs, []commonv1.Catalog{commonv1.Catalog_CATALOG_MEASURE}) {
+ t.Fatalf("a directory without lease must be reported with zero times and its catalog: %+v", got)
+ }
+}
+
+func TestSession_ExistingIgnoresMalformedExportDirs(t *testing.T) {
+ m, backends, _ := newTestManager(t)
+ exportDir := backends[commonv1.Catalog_CATALOG_STREAM].GetExportSnapshotDir()
+ for _, name := range []string{"UPPER", "has-dash", "aaaa.removing", "export-aaaa", "20260928000000-00000001"} {
+ if err := os.MkdirAll(filepath.Join(exportDir, name), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ }
+ writeFile(t, filepath.Join(exportDir, "bbbb"), []byte("a file, not a session"))
+ sessions, err := m.existing()
+ if err != nil || len(sessions) != 0 {
+ t.Fatalf("existing() must skip malformed dirs, got %d sessions, err=%v", len(sessions), err)
+ }
+ if lease, probeErr := m.probe(); probeErr != nil || lease != nil {
+ t.Fatalf("probe() must return no session when only malformed dirs exist, got %+v, %v", lease, probeErr)
+ }
+ mustCreate(t, m, testSessionID, catalogSet(commonv1.Catalog_CATALOG_STREAM))
+}
+
+func TestSession_ConcurrentCreateAndRelease(t *testing.T) {
+ ctx := context.Background()
+ m, _, _ := newTestManager(t)
+ ids := []string{"aa01", "aa02", "aa03", "aa04", "aa05", "aa06", "aa07", "aa08"}
+ var wg sync.WaitGroup
+ errs := make([]error, len(ids))
+ for i, id := range ids {
+ wg.Add(1)
+ //panicdiag:allow-rawgo test goroutine racing create/release; failures are collected in errs
+ go func(idx int, sessionID string) {
+ defer wg.Done()
+ if _, err := m.create(ctx, sessionID, allCatalogs(), true); err != nil {
+ errs[idx] = err
+ return
+ }
+ if idx%2 == 0 {
+ _, errs[idx] = m.release(sessionID)
+ }
+ }(i, id)
+ }
+ wg.Wait()
+ for i, err := range errs {
+ if err != nil {
+ t.Fatalf("%s: serialized create/release must not fail: %v", ids[i], err)
+ }
+ }
+ if got := sessionIDs(t, m); len(got) > 1 {
+ t.Fatalf("at most one session may exist after concurrent create/release, got %v", got)
+ }
+}
diff --git a/bydbctl/cmd/cli/main.go b/bydbctl/cmd/cli/main.go
index 557ef1464..8c03701a4 100644
--- a/bydbctl/cmd/cli/main.go
+++ b/bydbctl/cmd/cli/main.go
@@ -19,15 +19,21 @@
package main
import (
+ "errors"
"fmt"
"os"
"github.com/apache/skywalking-banyandb/bydbctl/internal/cmd"
+ "github.com/apache/skywalking-banyandb/pkg/transfer/exporter"
)
func main() {
if err := cmd.Execute(); err != nil {
_, _ = fmt.Fprintln(os.Stderr, err)
+ var exitErr *exporter.ExitError
+ if errors.As(err, &exitErr) {
+ os.Exit(exitErr.Code)
+ }
os.Exit(1)
}
}
diff --git a/bydbctl/internal/cmd/data.go b/bydbctl/internal/cmd/data.go
new file mode 100644
index 000000000..c145625b9
--- /dev/null
+++ b/bydbctl/internal/cmd/data.go
@@ -0,0 +1,304 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package cmd
+
+import (
+ "context"
+ "fmt"
+ "strings"
+ "time"
+
+ "github.com/spf13/cobra"
+ "github.com/spf13/viper"
+
+ transferv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/transfer/v1"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+ "github.com/apache/skywalking-banyandb/pkg/transfer"
+ "github.com/apache/skywalking-banyandb/pkg/transfer/exporter"
+)
+
+func newDataCmd() *cobra.Command {
+ dataCmd := &cobra.Command{
+ Use: "data",
+ Short: "Export data through the liaison gRPC endpoint",
+ SilenceUsage: true,
+ SilenceErrors: true,
+ }
+ dataCmd.AddCommand(newDataExportCmd(), newReleaseSessionCmd())
+ return dataCmd
+}
+
+// newReleaseSessionCmd is `data release-session`: it releases an export session snapshot on
+// every data node.
+func newReleaseSessionCmd() *cobra.Command {
+ var id string
+ f := &connectionFlags{}
+ c := &cobra.Command{
+ Use: "release-session",
+ Short: "Release an export session snapshot on every data node",
+ SilenceUsage: true,
+ SilenceErrors: true,
+ Args: cobra.NoArgs,
+ RunE: func(cmd *cobra.Command, _ []string) error {
+ // The request's own validation rules make a malformed --id a usage error here
+ // rather than an INVALID_ARGUMENT round trip; the rule admits the empty id
+ // ACTION_LIST ignores, so emptiness is checked explicitly.
+ if id == "" {
+ return exporter.Exit(exporter.ExitUsage, "--id must be a session id (1-64 lowercase hex characters)")
+ }
+ req := &transferv1.SessionsRequest{Action: transferv1.SessionsRequest_ACTION_RELEASE, SessionId: id}
+ if err := req.ValidateAll(); err != nil {
+ return exporter.Exit(exporter.ExitUsage, "--id %q is not a session id (1-64 lowercase hex characters): %w", id, err)
+ }
+ _, nodes, err := resolveConnection(cmd, f)
+ if err != nil {
+ return err
+ }
+ ctx, cancel := context.WithTimeout(cmd.Context(), 3*time.Minute)
+ defer cancel()
+ conn, cluster, err := exporter.Preflight(ctx, nodes, connectOptions())
+ if err != nil {
+ return err
+ }
+ defer func() { _ = conn.Close() }()
+ result, err := exporter.ReleaseSession(ctx, conn, id)
+ if err != nil {
+ return exporter.Exit(exporter.ExitCodeFor(err), "%w", err)
+ }
+ var failed []string
+ for _, frame := range result.Failed {
+ failed = append(failed, frame.GetNodeId())
+ fmt.Fprintln(cmd.ErrOrStderr(), "release failed on "+frame.GetNodeId()+": "+frame.GetError())
+ }
+ if len(failed) > 0 {
+ if len(result.Released) == 0 {
+ return exporter.Exit(exporter.ExitRuntime, "session %s could not be released on %v (via %s); retry later", id, failed, cluster.Liaison)
+ }
+ return exporter.Exit(exporter.ExitRuntime, "session %s released except on %v (via %s); retry later", id, failed, cluster.Liaison)
+ }
+ if len(result.Released) == 0 {
+ return exporter.Exit(exporter.ExitPreflight, "export session %s is not held on any data node", id)
+ }
+ if len(result.NotHeld) > 0 {
+ fmt.Fprintf(cmd.ErrOrStderr(), "export session %s was not held on %v\n", id, result.NotHeld)
+ }
+ fmt.Fprintf(cmd.OutOrStdout(), "export session %s released\n", id)
+ return nil
+ },
+ }
+ c.Flags().StringVar(&id, "id", "", "export session id (from the first Plan frame)")
+ _ = c.MarkFlagRequired("id")
+ bindConnectionFlags(c, f)
+ return c
+}
+
+// isDataCommand reports whether c is the data command or one of its subcommands, which do
+// not use the bydbctl config file.
+func isDataCommand(c *cobra.Command) bool {
+ for ; c != nil && c.HasParent(); c = c.Parent() {
+ if c.Name() == "data" && !c.Parent().HasParent() {
+ return true
+ }
+ }
+ return false
+}
+
+// rejectRootScopeFlags refuses an explicit root -a/--addr, -g/--group or --config flag: data
+// commands take liaison gRPC addresses through --nodes, their scope through --selector and
+// their settings through --plan; they never read the config file. A group set in the
+// environment is simply ignored.
+func rejectRootScopeFlags() error {
+ if explicitRootFlags["addr"] {
+ return exporter.Exit(exporter.ExitUsage, "-a/--addr is the liaison HTTP endpoint; data commands take liaison gRPC addresses via --nodes")
+ }
+ if explicitRootFlags["group"] {
+ return exporter.Exit(exporter.ExitUsage, "-g/--group does not apply to data commands; the scope comes from --selector")
+ }
+ if explicitRootFlags["config"] {
+ return exporter.Exit(exporter.ExitUsage, "--config does not apply to data commands; give their settings with flags, --plan or BYDBCTL_* variables")
+ }
+ return nil
+}
+
+// connectOptions assembles the dial options shared by every data command.
+func connectOptions() exporter.ConnectOptions {
+ return exporter.ConnectOptions{
+ EnableTLS: enableTLS,
+ Insecure: insecure,
+ Cert: cert,
+ Username: viper.GetString("username"),
+ Password: viper.GetString("password"),
+ Timeout: 10 * time.Second,
+ }
+}
+
+// resolveString applies the precedence for one scalar setting:
+// explicit flag > plan.yaml > BYDBCTL_* environment.
+func resolveString(cmd *cobra.Command, flag, flagValue, planValue, viperKey, def string) string {
+ if cmd.Flags().Changed(flag) {
+ return flagValue
+ }
+ if planValue != "" {
+ return planValue
+ }
+ if v := viper.GetString(viperKey); v != "" {
+ return v
+ }
+ return def
+}
+
+// connectionFlags are the connection flags every data command takes: the plan file itself,
+// the liaison addresses (plan.yaml connection.nodes) and the logging flags, which plan.yaml
+// does not cover (the TLS flags are the root ones).
+type connectionFlags struct {
+ planFile string
+ loggingLevel string
+ loggingEnv string
+ nodes []string
+}
+
+// bindConnectionFlags registers --plan, --nodes, the logging flags and the TLS flags.
+func bindConnectionFlags(c *cobra.Command, f *connectionFlags) {
+ c.Flags().StringVar(&f.planFile, "plan", "", "plan.yaml describing connection, scope and parallelism")
+ c.Flags().StringSliceVar(&f.nodes, "nodes", nil, "liaison gRPC addresses (host:17912), tried in order")
+ c.Flags().StringVar(&f.loggingLevel, "logging-level", "info", "log level")
+ c.Flags().StringVar(&f.loggingEnv, "logging-env", "prod", "log environment: prod|dev")
+ bindTLSRelatedFlag(c)
+}
+
+// resolveConnection initializes logging, rejects the root -a/-g flags, reads plan.yaml when
+// given, resolves the liaison address list and lets each connection.nodesTLS setting take
+// effect unless its flag was given explicitly.
+func resolveConnection(cmd *cobra.Command, f *connectionFlags) (*transfer.Plan, []string, error) {
+ if err := logger.Init(logger.Logging{Env: f.loggingEnv, Level: f.loggingLevel}); err != nil {
+ return nil, nil, exporter.Exit(exporter.ExitUsage, "init logger: %w", err)
+ }
+ if err := rejectRootScopeFlags(); err != nil {
+ return nil, nil, err
+ }
+ plan := &transfer.Plan{}
+ if f.planFile != "" {
+ var err error
+ if plan, err = transfer.LoadPlanFile(f.planFile); err != nil {
+ return nil, nil, exporter.Exit(exporter.ExitUsage, "%w", err)
+ }
+ }
+ nodes := resolveNodes(cmd, f.nodes, plan.Connection.Nodes)
+ if len(nodes) == 0 {
+ return nil, nil, exporter.Exit(exporter.ExitUsage, "no liaison address: give --nodes or connection.nodes in plan.yaml")
+ }
+ mergeNodesTLS(cmd, plan.Connection.NodesTLS)
+ return plan, nodes, nil
+}
+
+// planFlags are every plan.yaml setting a data export takes: the connection plus the scope
+// (--selector) and the node parallelism.
+type planFlags struct {
+ parallelism string
+ selectors []string
+ connection connectionFlags
+}
+
+// bindPlanFlags registers the connection flags plus --selector and --parallelism.
+func bindPlanFlags(c *cobra.Command, f *planFlags) {
+ bindConnectionFlags(c, &f.connection)
+ c.Flags().StringArrayVar(&f.selectors, "selector", nil, "scope, repeatable: catalog=<stream|measure|trace|property>[,groups=<g1>;<g2>]")
+ c.Flags().StringVar(&f.parallelism, "parallelism", "max", "number of data nodes exported concurrently: max or an integer >= 1")
+}
+
+// resolvedPlan is a plan whose every setting went through the flag > plan.yaml > BYDBCTL_*
+// environment precedence.
+type resolvedPlan struct {
+ parallelismRaw string
+ nodes []string
+ selectors []*transferv1.Selector
+ parallelism int
+}
+
+// resolvePlan resolves the connection, the parallelism and the selectors of a data export.
+func resolvePlan(cmd *cobra.Command, f *planFlags) (*resolvedPlan, error) {
+ plan, nodes, err := resolveConnection(cmd, &f.connection)
+ if err != nil {
+ return nil, err
+ }
+ out := &resolvedPlan{nodes: nodes}
+ out.parallelismRaw = resolveString(cmd, "parallelism", f.parallelism, plan.Export.Parallelism.String(), "parallelism", "max")
+ if out.parallelism, err = transfer.ParseParallelism(out.parallelismRaw); err != nil {
+ source := "BYDBCTL_PARALLELISM"
+ switch {
+ case cmd.Flags().Changed("parallelism"):
+ source = "--parallelism"
+ case plan.Export.Parallelism != "":
+ source = "plan.yaml export.parallelism"
+ }
+ return nil, exporter.Exit(exporter.ExitUsage, "%s: %w", source, err)
+ }
+ cfgs, source := plan.Export.Selectors, "plan.yaml export.selectors"
+ if cmd.Flags().Changed("selector") {
+ cfgs, source = nil, "--selector"
+ for _, raw := range f.selectors {
+ s, parseErr := transfer.ParseSelector(raw)
+ if parseErr != nil {
+ return nil, exporter.Exit(exporter.ExitUsage, "--selector: %w", parseErr)
+ }
+ cfgs = append(cfgs, s)
+ }
+ }
+ for i := range cfgs {
+ s, protoErr := cfgs[i].ToProto()
+ if protoErr != nil {
+ return nil, exporter.Exit(exporter.ExitUsage, "%s[%d]: %w", source, i, protoErr)
+ }
+ out.selectors = append(out.selectors, s)
+ }
+ return out, nil
+}
+
+// mergeNodesTLS applies plan.yaml connection.nodesTLS flag by flag: an explicit --enable-tls,
+// --insecure or --cert keeps its value, the others take the file's.
+func mergeNodesTLS(cmd *cobra.Command, tls transfer.TLSConfig) {
+ if !cmd.Flags().Changed("enable-tls") {
+ enableTLS = tls.Enable
+ }
+ if !cmd.Flags().Changed("insecure") {
+ insecure = tls.Insecure
+ }
+ if !cmd.Flags().Changed("cert") {
+ cert = tls.Cert
+ }
+}
+
+// resolveNodes applies the same precedence to the liaison address list. The environment
+// value is comma separated.
+func resolveNodes(cmd *cobra.Command, flagValue, planValue []string) []string {
+ if cmd.Flags().Changed("nodes") {
+ return flagValue
+ }
+ if len(planValue) > 0 {
+ return planValue
+ }
+ var out []string
+ for _, item := range viper.GetStringSlice("nodes") {
+ for _, n := range strings.Split(item, ",") {
+ if n = strings.TrimSpace(n); n != "" {
+ out = append(out, n)
+ }
+ }
+ }
+ return out
+}
diff --git a/bydbctl/internal/cmd/data_export.go b/bydbctl/internal/cmd/data_export.go
new file mode 100644
index 000000000..f6c9bec7f
--- /dev/null
+++ b/bydbctl/internal/cmd/data_export.go
@@ -0,0 +1,116 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package cmd
+
+import (
+ "context"
+ "fmt"
+ "strings"
+ "time"
+
+ "github.com/spf13/cobra"
+
+ "github.com/apache/skywalking-banyandb/pkg/transfer"
+ "github.com/apache/skywalking-banyandb/pkg/transfer/exporter"
+)
+
+type dataExportFlags struct {
+ outputFormat string
+ plan planFlags
+ dryRun bool
+ strictCoverage bool
+}
+
+func newDataExportCmd() *cobra.Command {
+ f := &dataExportFlags{}
+ c := &cobra.Command{
+ Use: "export",
+ Short: "Export data from a BanyanDB cluster (currently the --dry-run inventory only)",
+ SilenceUsage: true,
+ SilenceErrors: true,
+ Args: cobra.NoArgs,
+ }
+ bindPlanFlags(c, &f.plan)
+ c.Flags().BoolVar(&f.dryRun, "dry-run", false,
+ "inventory only: list units and estimated sizes without creating snapshots or writing files (required: the real export is not implemented yet)")
+ c.Flags().StringVarP(&f.outputFormat, "output-format", "o", "table", "dry-run rendering: table|yaml|json")
+ c.Flags().BoolVar(&f.strictCoverage, "strict-coverage", false, "treat a data node that returned no inventory as an error (exit 2)")
+
+ var plan *resolvedPlan
+ c.PreRunE = func(cmd *cobra.Command, _ []string) error {
+ var err error
+ if plan, err = resolvePlan(cmd, &f.plan); err != nil {
+ return err
+ }
+ if err = exporter.ValidateOutputFormat(f.outputFormat); err != nil {
+ return exporter.Exit(exporter.ExitUsage, "--output-format: %w", err)
+ }
+ if !f.dryRun {
+ return exporter.Exit(exporter.ExitUsage, "data export without --dry-run is not implemented yet; run with --dry-run to inventory the cluster")
+ }
+ return nil
+ }
+ c.RunE = func(cmd *cobra.Command, _ []string) error {
+ ctx, cancel := context.WithTimeout(cmd.Context(), 10*time.Minute)
+ defer cancel()
+ conn, cluster, err := exporter.Preflight(ctx, plan.nodes, connectOptions())
+ if err != nil {
+ return err
+ }
+ defer func() { _ = conn.Close() }()
+ errOut := cmd.ErrOrStderr()
+ fmt.Fprintf(errOut, "liaison: %s (standalone=%v), data nodes: %d\n", cluster.Liaison, cluster.Standalone, len(cluster.DataNodes))
+
+ groups, err := exporter.FetchGroups(ctx, conn)
+ if err != nil {
+ return err
+ }
+ if err = exporter.ValidateSelectors(groups, plan.selectors); err != nil {
+ return exporter.Exit(exporter.ExitUsage, "%w", err)
+ }
+ result, err := exporter.RunPlan(ctx, conn, plan.selectors)
+ if err != nil {
+ return exporter.Exit(exporter.ExitCodeFor(err), "%w", err)
+ }
+ report := exporter.BuildReport(cluster, result)
+ for _, w := range exporter.StageWarnings(groups, result) {
+ fmt.Fprintln(errOut, "WARN "+w)
+ }
+
+ if len(report.AnsweredNodes) > 0 {
+ effective, clamped := transfer.ClampParallelism(plan.parallelism, len(report.AnsweredNodes))
+ if clamped {
+ fmt.Fprintf(errOut, "parallelism: %s -> %d (clamped by node count)\n", plan.parallelismRaw, effective)
+ } else {
+ fmt.Fprintf(errOut, "parallelism: %d\n", effective)
+ }
+ }
+ if err = exporter.Render(cmd.OutOrStdout(), f.outputFormat, report); err != nil {
+ return exporter.Exit(exporter.ExitRuntime, "render report: %w", err)
+ }
+ if len(report.MissingNodes) > 0 {
+ msg := fmt.Sprintf("%d data node(s) returned no inventory: %s", len(report.MissingNodes), strings.Join(report.MissingNodes, ", "))
+ if f.strictCoverage {
+ return exporter.Exit(exporter.ExitPreflight, "coverage gap: %s", msg)
+ }
+ fmt.Fprintln(errOut, "WARN "+msg)
+ }
+ return nil
+ }
+ return c
+}
diff --git a/bydbctl/internal/cmd/data_export_test.go b/bydbctl/internal/cmd/data_export_test.go
new file mode 100644
index 000000000..3d377517f
--- /dev/null
+++ b/bydbctl/internal/cmd/data_export_test.go
@@ -0,0 +1,323 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package cmd_test
+
+import (
+ "bytes"
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "io/fs"
+ "os"
+ "path/filepath"
+ "strings"
+ "time"
+
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+ "github.com/spf13/cobra"
+ grpclib "google.golang.org/grpc"
+ "google.golang.org/grpc/credentials/insecure"
+
+ transferv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/transfer/v1"
+ "github.com/apache/skywalking-banyandb/bydbctl/internal/cmd"
+ "github.com/apache/skywalking-banyandb/pkg/test"
+ "github.com/apache/skywalking-banyandb/pkg/test/flags"
+ "github.com/apache/skywalking-banyandb/pkg/test/setup"
+ "github.com/apache/skywalking-banyandb/pkg/timestamp"
+ "github.com/apache/skywalking-banyandb/pkg/transfer/exporter"
+ casesstreamdata "github.com/apache/skywalking-banyandb/test/cases/stream/data"
+)
+
+// dataTree fingerprints the part files and segment metadata below root. The inverted index
+// directories (sidx/, idx/) are skipped: their writers persist and merge on their own
+// schedule, so their bytes move even when nobody reads them; parts are immutable.
+func dataTree(root string) map[string]string {
+ out := map[string]string{}
+ err := filepath.WalkDir(root, func(p string, d fs.DirEntry, walkErr error) error {
+ if walkErr != nil {
+ return walkErr
+ }
+ if d.IsDir() {
+ if d.Name() == "sidx" || d.Name() == "idx" {
+ return filepath.SkipDir
+ }
+ return nil
+ }
+ if filepath.Ext(d.Name()) == ".tmp" {
+ return nil
+ }
+ body, readErr := os.ReadFile(p)
+ if readErr != nil {
+ return readErr
+ }
+ sum := sha256.Sum256(body)
+ rel, _ := filepath.Rel(root, p)
+ out[rel] = hex.EncodeToString(sum[:])
+ return nil
+ })
+ ExpectWithOffset(1, err).NotTo(HaveOccurred())
+ return out
+}
+
+var _ = Describe("Data Export Dry Run", func() {
+ var grpcAddr, httpAddr, dataDir, configFile string
+
+ BeforeEach(func() {
+ var deferFunc func()
+ var err error
+ dataDir, deferFunc, err = test.NewSpace()
+ Expect(err).NotTo(HaveOccurred())
+ ports, err := test.AllocateFreePorts(5)
+ Expect(err).NotTo(HaveOccurred())
+ var closeFn func()
+ grpcAddr, httpAddr, closeFn = setup.ClosableStandalone(nil, dataDir, ports, "--stream-flush-timeout=500ms")
+ // Stop the server before its directories go away, then forget this test's flags.
+ DeferCleanup(func() {
+ closeFn()
+ deferFunc()
+ cmd.ResetFlags()
+ })
+ httpAddr = httpSchema + httpAddr
+ // Data commands never read the config file; point HOME at an empty directory so a
+ // test that checks this cannot be fooled by the developer's own file.
+ home := filepath.Join(dataDir, "home")
+ Expect(os.MkdirAll(home, 0o700)).To(Succeed())
+ savedHome := os.Getenv("HOME")
+ Expect(os.Setenv("HOME", home)).To(Succeed())
+ DeferCleanup(func() { _ = os.Setenv("HOME", savedHome) })
+ configFile = filepath.Join(home, ".bydbctl.yaml")
+ // The specs choose the BYDBCTL_* fallbacks themselves; none may come from the caller.
+ for _, key := range []string{"BYDBCTL_NODES", "BYDBCTL_PARALLELISM"} {
+ if saved, ok := os.LookupEnv(key); ok {
+ Expect(os.Unsetenv(key)).To(Succeed())
+ DeferCleanup(func() { _ = os.Setenv(key, saved) })
+ }
+ }
+ conn, err := grpclib.NewClient(grpcAddr, grpclib.WithTransportCredentials(insecure.NewCredentials()))
+ Expect(err).NotTo(HaveOccurred())
+ DeferCleanup(func() { _ = conn.Close() })
+ casesstreamdata.Write(conn, "sw", timestamp.NowMilli(), 500*time.Millisecond)
+ })
+ // run executes one bydbctl invocation on a fresh root command.
+ run := func(args ...string) (string, string, error) {
+ rootCmd := &cobra.Command{Use: "root"}
+ cmd.RootCmdFlags(rootCmd)
+ cmd.SetTestArgs(args)
+ rootCmd.SetArgs(args)
+ var out, errBuf bytes.Buffer
+ rootCmd.SetOut(&out)
+ rootCmd.SetErr(&errBuf)
+ err := rootCmd.Execute()
+ return out.String(), errBuf.String(), err
+ }
+
+ It("renders the inventory table and leaves the data directory untouched", func() {
+ liveDir := filepath.Join(dataDir, "stream", "data")
+ Eventually(func(g Gomega) {
+ out, stderr, err := run("data", "export", "--dry-run", "--nodes", grpcAddr, "--selector", "catalog=stream")
+ g.Expect(err).NotTo(HaveOccurred(), stderr)
+ g.Expect(out).To(ContainSubstring("NODE"))
+ g.Expect(out).To(MatchRegexp(`stream\s+default\s+hot`))
+ g.Expect(out).To(ContainSubstring("SNAPSHOT"))
+ g.Expect(stderr).To(ContainSubstring("standalone=true"))
+ g.Expect(stderr).To(MatchRegexp(`parallelism: (max -> )?1`))
+ }, flags.EventuallyTimeout, time.Second).Should(Succeed())
+ var before map[string]string
+ Eventually(func() bool {
+ first := dataTree(liveDir)
+ time.Sleep(time.Second)
+ before = dataTree(liveDir)
+ return len(first) == len(before) && len(first) > 0
+ }, flags.EventuallyTimeout, time.Second).Should(BeTrue(), "the data directory never went quiet")
+ _, _, err := run("data", "export", "--dry-run", "--nodes", grpcAddr)
+ Expect(err).NotTo(HaveOccurred())
+ Expect(dataTree(liveDir)).To(Equal(before))
+ })
+
+ It("emits machine-readable JSON on stdout only", func() {
+ out, _, err := run("data", "export", "--dry-run", "--nodes", grpcAddr, "-o", "json", "--parallelism", "4")
+ Expect(err).NotTo(HaveOccurred())
+ var rep map[string]any
+ Expect(json.Unmarshal([]byte(out), &rep)).To(Succeed(), out)
+ Expect(rep).To(HaveKey("rows"))
+ Expect(rep).To(HaveKeyWithValue("standalone", true))
+ })
+
+ It("takes nodes from plan.yaml and lets --nodes override it", func() {
+ planFile := filepath.Join(dataDir, "plan.yaml")
+ Expect(os.WriteFile(planFile, []byte("connection:\n nodes: [127.0.0.1:1]\nexport:\n selectors:\n - catalog: stream\n"), 0o600)).To(Succeed())
+ _, _, err := run("data", "export", "--dry-run", "--plan", planFile)
+ var exitErr *exporter.ExitError
+ Expect(errors.As(err, &exitErr)).To(BeTrue(), "unreachable plan.yaml nodes must exit 2: %v", err)
+ Expect(exitErr.Code).To(Equal(exporter.ExitPreflight))
+ _, _, err = run("data", "export", "--dry-run", "--plan", planFile, "--nodes", grpcAddr)
+ Expect(err).NotTo(HaveOccurred())
+ })
+
+ It("rejects the root -a flag, an explicit -g flag, unknown selectors and bad parallelism", func() {
+ _, _, err := run("data", "export", "--dry-run", "-a", httpAddr, "--nodes", grpcAddr)
+ Expect(err).To(MatchError(ContainSubstring("--nodes")))
+ _, _, err = run("data", "export", "--dry-run", "-g", "default", "--nodes", grpcAddr)
+ Expect(err).To(MatchError(ContainSubstring("--selector")))
+ _, _, err = run("data", "export", "--dry-run", "--nodes", grpcAddr, "--selector", "catalog=stream,groups=nope")
+ Expect(err).To(MatchError(ContainSubstring("does not exist")))
+ _, _, err = run("data", "export", "--dry-run", "--nodes", grpcAddr, "--parallelism", "0")
+ Expect(err).To(MatchError(ContainSubstring("--parallelism")))
+ planFile := filepath.Join(dataDir, "plan.yaml")
+ Expect(os.WriteFile(planFile, []byte("export:\n parallelism: 0\n"), 0o600)).To(Succeed())
+ _, _, err = run("data", "export", "--dry-run", "--nodes", grpcAddr, "--plan", planFile)
+ Expect(err).To(MatchError(ContainSubstring("export.parallelism")))
+ // A clean run right after the rejected ones: none of their flags carries over.
+ _, _, err = run("data", "export", "--dry-run", "--nodes", grpcAddr)
+ Expect(err).NotTo(HaveOccurred())
+ })
+
+ It("ignores a group persisted by `bydbctl use` but rejects the explicit flag", func() {
+ Expect(os.WriteFile(configFile, []byte("group: default\n"), 0o600)).To(Succeed())
+ _, _, err := run("data", "export", "--dry-run", "--nodes", grpcAddr)
+ Expect(err).NotTo(HaveOccurred(), "a configured group must not break data commands")
+ _, _, err = run("data", "export", "--dry-run", "-g", "default", "--nodes", grpcAddr)
+ Expect(err).To(MatchError(ContainSubstring("-g/--group")))
+ })
+
+ It("validates -o and --id before connecting", func() {
+ // 127.0.0.1:1 never answers: reaching the preflight would fail with exit 2, not 1.
+ _, _, err := run("data", "export", "--dry-run", "--nodes", "127.0.0.1:1", "-o", "xml")
+ var exitErr *exporter.ExitError
+ Expect(errors.As(err, &exitErr)).To(BeTrue(), "%v", err)
+ Expect(exitErr.Code).To(Equal(exporter.ExitUsage))
+ Expect(err).To(MatchError(ContainSubstring("--output-format")))
+ // The request rule admits an empty id (ACTION_LIST ignores it); release-session must not.
+ for _, id := range []string{"FEED-FEED", ""} {
+ _, _, err = run("data", "release-session", "--id", id, "--nodes", "127.0.0.1:1")
+ Expect(errors.As(err, &exitErr)).To(BeTrue(), "%v", err)
+ Expect(exitErr.Code).To(Equal(exporter.ExitUsage), "%q", id)
+ Expect(err).To(MatchError(ContainSubstring("--id")), "%q", id)
+ }
+ })
+
+ It("refuses a real export and the flags that only a real export reads", func() {
+ _, _, err := run("data", "export", "--nodes", grpcAddr)
+ Expect(err).To(MatchError(ContainSubstring("--dry-run")))
+ var exitErr *exporter.ExitError
+ Expect(errors.As(err, &exitErr)).To(BeTrue(), "%v", err)
+ Expect(exitErr.Code).To(Equal(exporter.ExitUsage))
+ for _, flag := range []string{"--output=" + filepath.Join(dataDir, "out"), "--preempt", "--format=csv", "--include-schema=false"} {
+ _, _, err = run("data", "export", "--dry-run", flag, "--nodes", grpcAddr)
+ Expect(err).To(MatchError(ContainSubstring("unknown flag")), flag)
+ }
+ })
+
+ It("lets --selector override the plan.yaml selectors and checks the plan.yaml ones", func() {
+ planFile := filepath.Join(dataDir, "plan.yaml")
+ Expect(os.WriteFile(planFile, []byte("export:\n selectors:\n - catalog: stream\n groups: [nope]\n"), 0o600)).To(Succeed())
+ _, _, err := run("data", "export", "--dry-run", "--nodes", grpcAddr, "--plan", planFile)
+ Expect(err).To(MatchError(ContainSubstring("does not exist")))
+ _, _, err = run("data", "export", "--dry-run", "--nodes", grpcAddr, "--plan", planFile, "--selector", "catalog=stream")
+ Expect(err).NotTo(HaveOccurred(), "--selector replaces the plan.yaml selectors")
+ for body, want := range map[string]string{
+ "export:\n selectors:\n - catalog: stream\n groups: [\"\"]\n": "plan.yaml export.selectors[0]: empty group name",
+ "export:\n selectors:\n - catalog: stream\n - catalog: stream\n groups: [a, a]\n": "plan.yaml export.selectors[1]: group \"a\" given twice",
+ } {
+ Expect(os.WriteFile(planFile, []byte(body), 0o600)).To(Succeed())
+ _, _, err = run("data", "export", "--dry-run", "--nodes", grpcAddr, "--plan", planFile)
+ var exitErr *exporter.ExitError
+ Expect(errors.As(err, &exitErr)).To(BeTrue(), "%v", err)
+ Expect(exitErr.Code).To(Equal(exporter.ExitUsage))
+ Expect(err).To(MatchError(ContainSubstring(want)))
+ }
+ })
+
+ It("ignores the config file and takes nodes from BYDBCTL_NODES below plan.yaml", func() {
+ // No address anywhere: there is no default liaison, and no config file is created.
+ _, stderr, err := run("data", "export", "--dry-run")
+ var exitErr *exporter.ExitError
+ Expect(errors.As(err, &exitErr)).To(BeTrue(), "%v", err)
+ Expect(exitErr.Code).To(Equal(exporter.ExitUsage))
+ Expect(err).To(MatchError(ContainSubstring("no liaison address")))
+ Expect(configFile).NotTo(BeAnExistingFile(), "a data command must not create the config file")
+ Expect(stderr).NotTo(ContainSubstring("Using config file"))
+ // Nodes in $HOME/.bydbctl.yaml are not used either.
+ Expect(os.WriteFile(configFile, []byte("nodes:\n - "+grpcAddr+"\n"), 0o600)).To(Succeed())
+ _, _, err = run("data", "export", "--dry-run")
+ Expect(err).To(MatchError(ContainSubstring("no liaison address")), "config nodes must be ignored")
+ // An explicit --config is refused rather than silently ignored.
+ _, _, err = run("--config", configFile, "data", "export", "--dry-run", "--nodes", grpcAddr)
+ Expect(errors.As(err, &exitErr)).To(BeTrue(), "%v", err)
+ Expect(exitErr.Code).To(Equal(exporter.ExitUsage))
+ Expect(err).To(MatchError(ContainSubstring("--config does not apply to data commands")))
+ Expect(os.Setenv("BYDBCTL_NODES", grpcAddr)).To(Succeed())
+ DeferCleanup(func() { _ = os.Unsetenv("BYDBCTL_NODES") })
+ _, _, err = run("data", "export", "--dry-run")
+ Expect(err).NotTo(HaveOccurred(), "BYDBCTL_NODES must be used")
+ // plan.yaml beats the environment: its unreachable node is the only candidate.
+ planFile := filepath.Join(dataDir, "plan.yaml")
+ Expect(os.WriteFile(planFile, []byte("connection:\n nodes: [127.0.0.1:1]\n"), 0o600)).To(Succeed())
+ _, _, err = run("data", "export", "--dry-run", "--plan", planFile)
+ Expect(errors.As(err, &exitErr)).To(BeTrue(), "%v", err)
+ Expect(exitErr.Code).To(Equal(exporter.ExitPreflight))
+ })
+
+ It("names the source of an invalid parallelism", func() {
+ Expect(os.Setenv("BYDBCTL_PARALLELISM", "0")).To(Succeed())
+ DeferCleanup(func() { _ = os.Unsetenv("BYDBCTL_PARALLELISM") })
+ _, _, err := run("data", "export", "--dry-run", "--nodes", grpcAddr)
+ Expect(err).To(MatchError(ContainSubstring("BYDBCTL_PARALLELISM")))
+ _, _, err = run("data", "export", "--dry-run", "--nodes", grpcAddr, "--parallelism", "2")
+ Expect(err).NotTo(HaveOccurred(), "the flag beats the environment")
+ })
+
+ It("releases an export session created through the liaison", func() {
+ conn, err := grpclib.NewClient(grpcAddr, grpclib.WithTransportCredentials(insecure.NewCredentials()))
+ Expect(err).NotTo(HaveOccurred())
+ defer func() { _ = conn.Close() }()
+ create := &transferv1.PlanRequest{Session: &transferv1.PlanRequest_Create{Create: &transferv1.CreateSession{}}}
+ stream, err := transferv1.NewExportServiceClient(conn).Plan(context.Background(), create)
+ Expect(err).NotTo(HaveOccurred())
+ first, err := stream.Recv()
+ Expect(err).NotTo(HaveOccurred())
+ id := first.GetCreated().GetSessionId()
+ Expect(id).NotTo(BeEmpty())
+ for {
+ if _, recvErr := stream.Recv(); recvErr != nil {
+ break
+ }
+ }
+ lease := filepath.Join(dataDir, "stream", "export-snapshots", id, ".lease")
+ Expect(lease).To(BeAnExistingFile())
+
+ out, _, err := run("data", "release-session", "--id", id, "--nodes", grpcAddr)
+ Expect(err).NotTo(HaveOccurred())
+ Expect(out).To(Equal("export session " + id + " released\n"))
+ Expect(lease).NotTo(BeAnExistingFile())
+ // Releasing again, or an id that never existed, deletes nothing: no data node holds it,
+ // so the command says so and exits 2.
+ for _, gone := range []string{id, strings.Repeat("ab", 8)} {
+ _, _, err = run("data", "release-session", "--id", gone, "--nodes", grpcAddr)
+ var exitErr *exporter.ExitError
+ Expect(errors.As(err, &exitErr)).To(BeTrue(), "want an exit error, got %v", err)
+ Expect(exitErr.Code).To(Equal(exporter.ExitPreflight))
+ Expect(err).To(MatchError(ContainSubstring("export session " + gone + " is not held on any data node")))
+ }
+ _, _, err = run("data", "release-session", "extra", "--id", id, "--nodes", grpcAddr)
+ Expect(err).To(MatchError(ContainSubstring("unknown command")))
+ })
+})
diff --git a/bydbctl/internal/cmd/data_test.go b/bydbctl/internal/cmd/data_test.go
new file mode 100644
index 000000000..4f76feaa7
--- /dev/null
+++ b/bydbctl/internal/cmd/data_test.go
@@ -0,0 +1,228 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package cmd
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "net"
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/spf13/cobra"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+ "google.golang.org/grpc"
+ "google.golang.org/grpc/health"
+ "google.golang.org/grpc/health/grpc_health_v1"
+
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ transferv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/transfer/v1"
+ "github.com/apache/skywalking-banyandb/pkg/transfer"
+ "github.com/apache/skywalking-banyandb/pkg/transfer/exporter"
+)
+
+func TestMergeNodesTLS_KeepsExplicitFlags(t *testing.T) {
+ t.Cleanup(ResetFlags)
+ fromPlan := transfer.TLSConfig{Enable: true, Insecure: true, Cert: "/etc/plan.crt"}
+
+ c := &cobra.Command{Use: "x"}
+ bindTLSRelatedFlag(c)
+ require.NoError(t, c.ParseFlags([]string{"--cert", "/etc/flag.crt"}))
+ mergeNodesTLS(c, fromPlan)
+ assert.True(t, enableTLS, "enable-tls was not given: plan.yaml decides")
+ assert.True(t, insecure, "insecure was not given: plan.yaml decides")
+ assert.Equal(t, "/etc/flag.crt", cert, "an explicit --cert must not be clobbered by plan.yaml")
+
+ ResetFlags()
+ c = &cobra.Command{Use: "x"}
+ bindTLSRelatedFlag(c)
+ require.NoError(t, c.ParseFlags([]string{"--insecure=false", "--enable-tls=false"}))
+ mergeNodesTLS(c, fromPlan)
+ assert.False(t, enableTLS, "an explicit --enable-tls=false wins over plan.yaml")
+ assert.False(t, insecure, "an explicit --insecure=false wins over plan.yaml")
+ assert.Equal(t, "/etc/plan.crt", cert)
+}
+
+// releaseLiaison is a liaison with data nodes data-a and data-b whose release answers frames.
+type releaseLiaison struct {
+ databasev1.UnimplementedNodeQueryServiceServer
+ databasev1.UnimplementedClusterStateServiceServer
+ transferv1.UnimplementedExportServiceServer
+ frames []*transferv1.SessionsResponse
+}
+
+func (releaseLiaison) GetCurrentNode(context.Context, *databasev1.GetCurrentNodeRequest) (*databasev1.GetCurrentNodeResponse, error) {
+ return &databasev1.GetCurrentNodeResponse{Node: &databasev1.Node{
+ Metadata: &commonv1.Metadata{Name: "liaison-1"}, Roles: []databasev1.Role{databasev1.Role_ROLE_LIAISON},
+ }}, nil
+}
+
+func (releaseLiaison) GetClusterState(context.Context, *databasev1.GetClusterStateRequest) (*databasev1.GetClusterStateResponse, error) {
+ var registered []*databasev1.Node
+ for _, n := range []string{"data-a", "data-b"} {
+ registered = append(registered, &databasev1.Node{Metadata: &commonv1.Metadata{Name: n}, Roles: []databasev1.Role{databasev1.Role_ROLE_DATA}})
+ }
+ return &databasev1.GetClusterStateResponse{RouteTables: map[string]*databasev1.RouteTable{"tire2": {Registered: registered}}}, nil
+}
+
+func (l releaseLiaison) Sessions(_ *transferv1.SessionsRequest, stream transferv1.ExportService_SessionsServer) error {
+ for _, f := range l.frames {
+ if err := stream.Send(f); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func releaseFrame(node, outcome string) *transferv1.SessionsResponse {
+ f := &transferv1.SessionsResponse{NodeId: node}
+ switch outcome {
+ case "done":
+ f.Outcome = &transferv1.SessionsResponse_Done{Done: &transferv1.Ack{}}
+ case "none":
+ f.Outcome = &transferv1.SessionsResponse_None{None: &transferv1.Ack{}}
+ default:
+ f.Outcome = &transferv1.SessionsResponse_Error{Error: outcome}
+ }
+ return f
+}
+
+// runData executes `bydbctl <args>` on a fresh root, the way main does, so initConfig sees a
+// data command. HOME points at an empty directory, which must still hold no config file
+// afterwards: data commands neither read nor create it.
+func runData(t *testing.T, args ...string) (string, string, error) {
+ t.Helper()
+ home := t.TempDir()
+ t.Setenv("HOME", home)
+ out, errOut, err := executeOnFreshRoot(t, args...)
+ _, statErr := os.Stat(filepath.Join(home, ".bydbctl.yaml"))
+ require.True(t, os.IsNotExist(statErr), "a data command must not create the config file: %v", statErr)
+ return out, errOut, err
+}
+
+// executeOnFreshRoot runs args on a new root command and restores the package root afterwards.
+func executeOnFreshRoot(t *testing.T, args ...string) (string, string, error) {
+ t.Helper()
+ ResetFlags()
+ savedRoot := activeRoot
+ t.Cleanup(func() {
+ activeRoot = savedRoot
+ SetTestArgs(nil)
+ ResetFlags()
+ })
+ root := &cobra.Command{Use: "root"}
+ RootCmdFlags(root)
+ SetTestArgs(args)
+ root.SetArgs(args)
+ var out, errOut bytes.Buffer
+ root.SetOut(&out)
+ root.SetErr(&errOut)
+ err := root.ExecuteContext(context.Background())
+ return out.String(), errOut.String(), err
+}
+
+// serveRelease starts a liaison whose release answers frames and returns its address.
+func serveRelease(t *testing.T, frames ...*transferv1.SessionsResponse) string {
+ t.Helper()
+ lis, err := net.Listen("tcp", "127.0.0.1:0")
+ require.NoError(t, err)
+ srv := grpc.NewServer()
+ fake := releaseLiaison{frames: frames}
+ databasev1.RegisterNodeQueryServiceServer(srv, fake)
+ databasev1.RegisterClusterStateServiceServer(srv, fake)
+ transferv1.RegisterExportServiceServer(srv, fake)
+ grpc_health_v1.RegisterHealthServer(srv, health.NewServer())
+ //panicdiag:allow-rawgo test server
+ go func() { _ = srv.Serve(lis) }()
+ t.Cleanup(srv.Stop)
+ return lis.Addr().String()
+}
+
+// runRelease runs `data release-session --id abcd` against a liaison answering frames.
+func runRelease(t *testing.T, frames ...*transferv1.SessionsResponse) (string, string, error) {
+ t.Helper()
+ return runData(t, "data", "release-session", "--id", "abcd", "--nodes", serveRelease(t, frames...))
+}
+
+// Data commands never look up the home directory, so they run with HOME unset.
+func TestReleaseSession_RunsWithoutHome(t *testing.T) {
+ addr := serveRelease(t, releaseFrame("data-a", "done"))
+ t.Setenv("HOME", "")
+ out, _, err := executeOnFreshRoot(t, "data", "release-session", "--id", "abcd", "--nodes", addr)
+ require.NoError(t, err)
+ assert.Equal(t, "export session abcd released\n", out)
+}
+
+func TestReleaseSession_ReportsPartialFailure(t *testing.T) {
+ out, errOut, err := runRelease(t, releaseFrame("data-a", "done"), releaseFrame("data-b", "disk busy"))
+ var exitErr *exporter.ExitError
+ require.True(t, errors.As(err, &exitErr), "want an exit error, got %v", err)
+ assert.Equal(t, exporter.ExitRuntime, exitErr.Code, "a partial release must exit non-zero")
+ assert.Contains(t, err.Error(), "released except on [data-b]")
+ assert.Equal(t, "release failed on data-b: disk busy\n", errOut, "one line per failed node")
+ assert.Empty(t, out, "a partial release must not claim success")
+}
+
+func TestReleaseSession_ReportsTotalFailure(t *testing.T) {
+ out, _, err := runRelease(t, releaseFrame("data-a", "disk busy"), releaseFrame("data-b", "none"))
+ var exitErr *exporter.ExitError
+ require.True(t, errors.As(err, &exitErr), "want an exit error, got %v", err)
+ assert.Equal(t, exporter.ExitRuntime, exitErr.Code)
+ assert.Contains(t, err.Error(), "session abcd could not be released on [data-a]")
+ assert.NotContains(t, err.Error(), "released except on", "nothing was released")
+ assert.Empty(t, out)
+}
+
+func TestReleaseSession_UnknownSessionIsAnError(t *testing.T) {
+ out, _, err := runRelease(t, releaseFrame("data-a", "none"), releaseFrame("data-b", "none"))
+ var exitErr *exporter.ExitError
+ require.True(t, errors.As(err, &exitErr), "want an exit error, got %v", err)
+ assert.Equal(t, exporter.ExitPreflight, exitErr.Code, "a session no node holds is a preflight rejection")
+ assert.Contains(t, err.Error(), "export session abcd is not held on any data node")
+ assert.Empty(t, out, "nothing was released")
+}
+
+func TestReleaseSession_NotesNodesThatDidNotHoldIt(t *testing.T) {
+ out, errOut, err := runRelease(t, releaseFrame("data-a", "done"), releaseFrame("data-b", "none"))
+ require.NoError(t, err)
+ assert.Equal(t, "export session abcd released\n", out)
+ assert.Equal(t, "export session abcd was not held on [data-b]\n", errOut)
+}
+
+func TestDataExport_RejectsPositionalArgs(t *testing.T) {
+ _, _, err := runData(t, "data", "export", "extra", "--dry-run", "--nodes", "127.0.0.1:1")
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), `unknown command "extra"`)
+}
+
+func TestDataExport_SelectorRejectsDuplicateGroup(t *testing.T) {
+ ResetFlags()
+ t.Cleanup(ResetFlags)
+ c := newDataExportCmd()
+ // 127.0.0.1:1 never answers: the selector must be rejected before the preflight.
+ require.NoError(t, c.ParseFlags([]string{"--dry-run", "--nodes", "127.0.0.1:1", "--selector", "catalog=stream,groups=a;a"}))
+ err := c.PreRunE(c, nil)
+ var exitErr *exporter.ExitError
+ require.True(t, errors.As(err, &exitErr), "want an exit error, got %v", err)
+ assert.Equal(t, exporter.ExitUsage, exitErr.Code)
+ assert.Contains(t, err.Error(), `--selector[0]: group "a" given twice`)
+}
diff --git a/bydbctl/internal/cmd/root.go b/bydbctl/internal/cmd/root.go
index 29772c56b..a3be5e913 100644
--- a/bydbctl/internal/cmd/root.go
+++ b/bydbctl/internal/cmd/root.go
@@ -24,6 +24,7 @@ import (
"os"
"github.com/spf13/cobra"
+ "github.com/spf13/pflag"
"github.com/spf13/viper"
"github.com/apache/skywalking-banyandb/pkg/config"
@@ -51,14 +52,30 @@ var (
Version: version.Build(),
Short: "bydbctl is the command line tool of BanyanDB",
}
+ // activeRoot is the root command whose flags initConfig binds; tests build their own.
+ activeRoot = rootCmd
+ // explicitRootFlags records which persistent root flags were given on the command line
+ // of the current invocation. It is captured before config.BindFlags copies viper values
+ // into the flag set, because after that point Changed() is true for every flag that has
+ // a viper default (e.g. addr). The data commands use it to reject -a/--addr.
+ explicitRootFlags = map[string]bool{}
)
-// ResetFlags resets the flags.
+// ResetFlags resets the flags, including the config file path and the test argument
+// vector, so one test's invocation cannot leak into the next.
func ResetFlags() {
filePath = ""
name = ""
start = ""
end = ""
+ cfgFile = ""
+ enableTLS = false
+ insecure = false
+ cert = ""
+ username = ""
+ password = ""
+ testArgs = nil
+ explicitRootFlags = map[string]bool{}
}
// Execute executes the root command.
@@ -80,7 +97,40 @@ func RootCmdFlags(command *cobra.Command) {
_ = viper.BindPFlag("password", command.PersistentFlags().Lookup("password"))
command.AddCommand(newGroupCmd(), newUseCmd(), newStreamCmd(), newMeasureCmd(), newTopnCmd(),
- newIndexRuleCmd(), newIndexRuleBindingCmd(), newPropertyCmd(), newTraceCmd(), newHealthCheckCmd(), newAnalyzeCmd(), newAgentCmd(), newAgentToolBridgeCmd())
+ newIndexRuleCmd(), newIndexRuleBindingCmd(), newPropertyCmd(), newTraceCmd(), newHealthCheckCmd(), newAnalyzeCmd(), newAgentCmd(), newAgentToolBridgeCmd(),
+ newDataCmd())
+ activeRoot = command
+}
+
+// testArgs lets tests that drive a private root command through SetArgs tell initConfig
+// which command line to inspect; production always reads os.Args.
+var testArgs []string
+
+// SetTestArgs records the argument vector a test is about to pass to SetArgs.
+func SetTestArgs(args []string) { testArgs = args }
+
+func activeRootArgs() []string {
+ if testArgs != nil {
+ return testArgs
+ }
+ return os.Args[1:]
+}
+
+// snapshotExplicitRootFlags records the persistent root flags the executing command saw
+// on the command line. Cobra parses flags before OnInitialize runs and merges the root's
+// persistent flags into the executed command's flag set, so that set is where Changed()
+// still reflects the command line only.
+func snapshotExplicitRootFlags(root *cobra.Command, args []string) {
+ explicitRootFlags = map[string]bool{}
+ command, _, err := root.Find(args)
+ if err != nil {
+ return
+ }
+ root.PersistentFlags().VisitAll(func(f *pflag.Flag) {
+ if merged := command.Flags().Lookup(f.Name); merged != nil && merged.Changed {
+ explicitRootFlags[f.Name] = true
+ }
+ })
}
func init() {
@@ -88,11 +138,29 @@ func init() {
RootCmdFlags(rootCmd)
}
+// bindEnvAndFlags makes viper fall back from the root's persistent flags to BYDBCTL_* env.
+func bindEnvAndFlags() {
+ viper.SetEnvPrefix(envPrefix)
+ viper.AutomaticEnv()
+ // Bind the package-level root's flags, as before: the explicit-flag snapshot in initConfig
+ // is the only place the executing (possibly test-built) root is consulted, so the viper
+ // fallbacks keep flowing through viper.GetString rather than being copied into the flag
+ // variables of a command built for a single test.
+ cobra.CheckErr(config.BindFlags(rootCmd.PersistentFlags(), viper.GetViper(), envPrefix))
+}
+
func initConfig() {
- command, _, findErr := rootCmd.Find(os.Args[1:])
+ snapshotExplicitRootFlags(activeRoot, activeRootArgs())
+ command, _, findErr := activeRoot.Find(activeRootArgs())
if findErr == nil && command.Name() == "agent-tool-bridge" {
return
}
+ // The data commands take their settings from flags, plan.yaml and BYDBCTL_* only: they
+ // neither read nor create the config file, so they never look up the home directory.
+ if findErr == nil && isDataCommand(command) {
+ bindEnvAndFlags()
+ return
+ }
if cfgFile != "" {
if cfgFile == "-" {
return
@@ -109,13 +177,7 @@ func initConfig() {
viper.SetConfigType("yaml")
viper.SetConfigName(".bydbctl")
}
-
- viper.SetEnvPrefix(envPrefix)
- viper.AutomaticEnv()
- err := config.BindFlags(rootCmd.PersistentFlags(), viper.GetViper(), envPrefix)
- if err != nil {
- cobra.CheckErr(err)
- }
+ bindEnvAndFlags()
readCfg := func() error {
if err := viper.ReadInConfig(); err != nil {
diff --git a/docs/api-reference.md b/docs/api-reference.md
index ad83512bc..6ff57b7ba 100644
--- a/docs/api-reference.md
+++ b/docs/api-reference.md
@@ -495,6 +495,32 @@
- [TraceService](#banyandb-trace-v1-TraceService)
+- [banyandb/transfer/v1/export.proto](#banyandb_transfer_v1_export-proto)
+ - [Ack](#banyandb-transfer-v1-Ack)
+ - [CreateSession](#banyandb-transfer-v1-CreateSession)
+ - [PartStat](#banyandb-transfer-v1-PartStat)
+ - [PlanRequest](#banyandb-transfer-v1-PlanRequest)
+ - [PlanResponse](#banyandb-transfer-v1-PlanResponse)
+ - [PlanSummary](#banyandb-transfer-v1-PlanSummary)
+ - [PropertyInventory](#banyandb-transfer-v1-PropertyInventory)
+ - [PropertyShardStat](#banyandb-transfer-v1-PropertyShardStat)
+ - [ReadSession](#banyandb-transfer-v1-ReadSession)
+ - [SegmentInventory](#banyandb-transfer-v1-SegmentInventory)
+ - [SegmentUnit](#banyandb-transfer-v1-SegmentUnit)
+ - [Selector](#banyandb-transfer-v1-Selector)
+ - [SessionCreated](#banyandb-transfer-v1-SessionCreated)
+ - [SessionLease](#banyandb-transfer-v1-SessionLease)
+ - [SessionsRequest](#banyandb-transfer-v1-SessionsRequest)
+ - [SessionsResponse](#banyandb-transfer-v1-SessionsResponse)
+ - [ShardStat](#banyandb-transfer-v1-ShardStat)
+ - [SidxStat](#banyandb-transfer-v1-SidxStat)
+ - [UnitFrame](#banyandb-transfer-v1-UnitFrame)
+ - [UnitInventory](#banyandb-transfer-v1-UnitInventory)
+
+ - [SessionsRequest.Action](#banyandb-transfer-v1-SessionsRequest-Action)
+
+ - [ExportService](#banyandb-transfer-v1-ExportService)
+
- [Scalar Value Types](#scalar-value-types)
@@ -7599,6 +7625,414 @@ TagFamilySpec defines the specification of a tag family.
+<a name="banyandb_transfer_v1_export-proto"></a>
+<p align="right"><a href="#top">Top</a></p>
+
+## banyandb/transfer/v1/export.proto
+
+
+
+<a name="banyandb-transfer-v1-Ack"></a>
+
+### Ack
+Ack is an outcome that carries no data.
+
+
+
+
+
+
+<a name="banyandb-transfer-v1-CreateSession"></a>
+
+### CreateSession
+CreateSession is the first Plan of an export that will transfer data. Every data node
+snapshots the selected catalogs into a new session and inventories that
+snapshot. One session id covers the whole cluster: the client leaves id empty, the liaison
+generates it, announces it in the first frame and forwards it, so on a data node id is
+always set.
+
+
+| Field | Type | Label | Description |
+| ----- | ---- | ----- | ----------- |
+| id | [string](#string) | | |
+| preempt | [bool](#bool) | | Remove every session the node holds, whatever its lease says, before creating this one; the removed ids come back in preempted_session_ids. Without it any other session on the node answers ALREADY_EXISTS naming the occupants. |
+
+
+
+
+
+
+<a name="banyandb-transfer-v1-PartStat"></a>
+
+### PartStat
+PartStat is the per-part view of the ShardStat aggregates. Timestamps are Unix nanoseconds.
+
+
+| Field | Type | Label | Description |
+| ----- | ---- | ----- | ----------- |
+| id | [uint64](#uint64) | | |
+| min_timestamp | [int64](#int64) | | |
+| max_timestamp | [int64](#int64) | | |
+| total_count | [uint64](#uint64) | | |
+
+
+
+
+
+
+<a name="banyandb-transfer-v1-PlanRequest"></a>
+
+### PlanRequest
+PlanRequest selects what to enumerate and which view to read. Without a session it reads
+the live directories and creates nothing on any node.
+
+
+| Field | Type | Label | Description |
+| ----- | ---- | ----- | ----------- |
+| selectors | [Selector](#banyandb-transfer-v1-Selector) | repeated | |
+| create | [CreateSession](#banyandb-transfer-v1-CreateSession) | | |
+| read | [ReadSession](#banyandb-transfer-v1-ReadSession) | | |
+
+
+
+
+
+
+<a name="banyandb-transfer-v1-PlanResponse"></a>
+
+### PlanResponse
+PlanResponse is one frame of the Plan stream. The liaison stream is: with CreateSession a
+`created` frame first; then `units` frames, each carrying the units of exactly one
... 6754 lines suppressed ...