You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ha...@apache.org on 2026/03/30 06:16:06 UTC
(skywalking-banyandb) branch main updated: Add new lifecycle service for FODC (#1026)
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
The following commit(s) were added to refs/heads/main by this push:
new ee65a7353 Add new lifecycle service for FODC (#1026)
ee65a7353 is described below
commit ee65a73536b58693ef219d36f1a1194a58b6cc49
Author: mrproliu <74...@qq.com>
AuthorDate: Mon Mar 30 14:16:00 2026 +0800
Add new lifecycle service for FODC (#1026)
* Add new lifecycle service for FODC
---------
Co-authored-by: Gao Hongtao <ha...@gmail.com>
---
api/proto/banyandb/fodc/v1/rpc.proto | 45 +++
banyand/queue/local.go | 4 +
banyand/queue/queue.go | 2 +
banyand/queue/sub/group_lifecycle.go | 69 ++++
banyand/queue/sub/group_lifecycle_test.go | 210 ++++++++++++
banyand/queue/sub/server.go | 22 +-
docs/api-reference.md | 127 +++++++
docs/operation/fodc/apis.md | 55 +++
fodc/agent/internal/cmd/root.go | 18 +-
fodc/agent/internal/lifecycle/collector.go | 188 ++++++++++
fodc/agent/internal/lifecycle/collector_test.go | 182 ++++++++++
fodc/agent/internal/proxy/client.go | 184 +++++++++-
fodc/agent/internal/proxy/client_test.go | 136 +++++---
fodc/agent/testhelper/flightrecorder.go | 38 ++-
fodc/proxy/cmd/proxy/main.go | 7 +-
fodc/proxy/internal/api/server.go | 23 ++
fodc/proxy/internal/api/server_test.go | 96 +++++-
fodc/proxy/internal/grpc/server_test.go | 8 +-
fodc/proxy/internal/grpc/service.go | 125 +++++++
fodc/proxy/internal/grpc/service_test.go | 295 +++++++++++++++-
.../integration/agent_reconnection_test.go | 6 +-
.../agent_registration_metrics_flow_test.go | 5 +-
.../cluster_topology_integration_test.go | 6 +-
.../internal/integration/failure_scenarios_test.go | 7 +-
.../integration/full_proxy_workflow_test.go | 7 +-
.../high_availability_scalability_test.go | 6 +-
...ation_test.go => lifecycle_integration_test.go} | 150 ++++----
.../integration/metrics_time_window_test.go | 6 +-
.../integration/multiple_agents_roles_test.go | 7 +-
.../integration/prometheus_integration_test.go | 5 +-
fodc/proxy/internal/lifecycle/manager.go | 245 +++++++++++++
fodc/proxy/internal/lifecycle/manager_test.go | 379 +++++++++++++++++++++
pkg/cmdsetup/liaison.go | 1 +
33 files changed, 2483 insertions(+), 181 deletions(-)
diff --git a/api/proto/banyandb/fodc/v1/rpc.proto b/api/proto/banyandb/fodc/v1/rpc.proto
index 273b50fb2..dd7a5c67a 100644
--- a/api/proto/banyandb/fodc/v1/rpc.proto
+++ b/api/proto/banyandb/fodc/v1/rpc.proto
@@ -19,7 +19,9 @@ syntax = "proto3";
package banyandb.fodc.v1;
+import "banyandb/common/v1/common.proto";
import "banyandb/database/v1/database.proto";
+import "banyandb/database/v1/rpc.proto";
import "google/protobuf/timestamp.proto";
option go_package = "github.com/apache/skywalking-banyandb/api/proto/banyandb/fodc/v1";
@@ -34,6 +36,9 @@ service FODCService {
// Bi-directional stream for cluster topology
rpc StreamClusterTopology(stream StreamClusterTopologyRequest) returns (stream StreamClusterTopologyResponse);
+
+ // Bi-directional stream for lifecycle data
+ rpc StreamLifecycle(stream StreamLifecycleRequest) returns (stream StreamLifecycleResponse);
}
message RegisterAgentRequest {
@@ -88,3 +93,43 @@ message StreamClusterTopologyResponse {
// Request topology from agent
bool request_topology = 1;
}
+
+message StreamLifecycleRequest {
+ google.protobuf.Timestamp timestamp = 1;
+ string pod_name = 2;
+ LifecycleData lifecycle_data = 3;
+}
+
+message StreamLifecycleResponse {
+ bool request_lifecycle = 1;
+}
+
+message LifecycleData {
+ repeated LifecycleReport reports = 1;
+ repeated GroupLifecycleInfo groups = 2;
+}
+
+message GroupLifecycleInfo {
+ string name = 1;
+ string catalog = 2;
+ banyandb.common.v1.ResourceOpts resource_opts = 3;
+ repeated banyandb.database.v1.DataInfo data_info = 4;
+}
+
+message LifecycleReport {
+ string filename = 1;
+ string report_json = 2;
+}
+
+// GroupLifecycleService provides group lifecycle information from liaison nodes.
+// Registered on the internal gRPC server so FODC agent can access it.
+service GroupLifecycleService {
+ // InspectAll lists all groups and returns their full lifecycle info in one call.
+ rpc InspectAll(InspectAllRequest) returns (InspectAllResponse);
+}
+
+message InspectAllRequest {}
+
+message InspectAllResponse {
+ repeated GroupLifecycleInfo groups = 1;
+}
diff --git a/banyand/queue/local.go b/banyand/queue/local.go
index a83343b89..e59269785 100644
--- a/banyand/queue/local.go
+++ b/banyand/queue/local.go
@@ -26,6 +26,7 @@ import (
"github.com/apache/skywalking-banyandb/api/common"
databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
"github.com/apache/skywalking-banyandb/banyand/liaison/grpc/route"
+ "github.com/apache/skywalking-banyandb/banyand/metadata"
"github.com/apache/skywalking-banyandb/banyand/metadata/schema"
"github.com/apache/skywalking-banyandb/pkg/bus"
"github.com/apache/skywalking-banyandb/pkg/run"
@@ -101,6 +102,9 @@ func (*local) GetPort() *uint32 {
func (*local) SetRouteProviders(_ map[string]route.TableProvider) {
}
+func (*local) SetMetadataRepo(_ metadata.Repo) {
+}
+
func (*local) Register(bus.Topic, schema.EventHandler) {
}
diff --git a/banyand/queue/queue.go b/banyand/queue/queue.go
index c51dc294a..76494a0ae 100644
--- a/banyand/queue/queue.go
+++ b/banyand/queue/queue.go
@@ -25,6 +25,7 @@ import (
"github.com/apache/skywalking-banyandb/api/common"
clusterv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/cluster/v1"
"github.com/apache/skywalking-banyandb/banyand/liaison/grpc/route"
+ "github.com/apache/skywalking-banyandb/banyand/metadata"
"github.com/apache/skywalking-banyandb/banyand/metadata/schema"
"github.com/apache/skywalking-banyandb/pkg/bus"
"github.com/apache/skywalking-banyandb/pkg/fs"
@@ -63,6 +64,7 @@ type Server interface {
RegisterChunkedSyncHandler(topic bus.Topic, handler ChunkedSyncHandler)
GetPort() *uint32
SetRouteProviders(providers map[string]route.TableProvider)
+ SetMetadataRepo(repo metadata.Repo)
}
// BatchPublisher is the interface for publishing data in batch.
diff --git a/banyand/queue/sub/group_lifecycle.go b/banyand/queue/sub/group_lifecycle.go
new file mode 100644
index 000000000..1fc8b30c0
--- /dev/null
+++ b/banyand/queue/sub/group_lifecycle.go
@@ -0,0 +1,69 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package sub
+
+import (
+ "context"
+ "fmt"
+
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ fodcv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/fodc/v1"
+)
+
+// InspectAll lists all groups and returns their full lifecycle info.
+func (s *server) InspectAll(ctx context.Context, _ *fodcv1.InspectAllRequest) (*fodcv1.InspectAllResponse, error) {
+ if s.metadataRepo == nil {
+ return nil, fmt.Errorf("metadata repository not available")
+ }
+ groups, err := s.metadataRepo.GroupRegistry().ListGroup(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("failed to list groups: %w", err)
+ }
+ result := make([]*fodcv1.GroupLifecycleInfo, 0, len(groups))
+ for _, group := range groups {
+ if group == nil || group.Metadata == nil {
+ continue
+ }
+ info := s.inspectGroup(ctx, group)
+ result = append(result, info)
+ }
+ return &fodcv1.InspectAllResponse{Groups: result}, nil
+}
+
+func (s *server) inspectGroup(ctx context.Context, group *commonv1.Group) *fodcv1.GroupLifecycleInfo {
+ groupName := group.Metadata.Name
+ info := &fodcv1.GroupLifecycleInfo{
+ Name: groupName,
+ Catalog: catalogToString(group.Catalog),
+ ResourceOpts: group.ResourceOpts,
+ }
+ dataInfo, err := s.metadataRepo.CollectDataInfo(ctx, groupName)
+ if err != nil {
+ s.log.Warn().Err(err).Str("group", groupName).Msg("Failed to collect data info")
+ } else {
+ info.DataInfo = dataInfo
+ }
+ return info
+}
+
+func catalogToString(catalog commonv1.Catalog) string {
+ if name, ok := commonv1.Catalog_name[int32(catalog)]; ok {
+ return name
+ }
+ return fmt.Sprintf("UNKNOWN_%d", catalog)
+}
diff --git a/banyand/queue/sub/group_lifecycle_test.go b/banyand/queue/sub/group_lifecycle_test.go
new file mode 100644
index 000000000..2a88ef995
--- /dev/null
+++ b/banyand/queue/sub/group_lifecycle_test.go
@@ -0,0 +1,210 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package sub
+
+import (
+ "context"
+ "fmt"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ fodcv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/fodc/v1"
+ "github.com/apache/skywalking-banyandb/banyand/metadata"
+ "github.com/apache/skywalking-banyandb/banyand/metadata/schema"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+)
+
+type mockGroupRegistry struct {
+ schema.Group
+ err error
+ groups []*commonv1.Group
+}
+
+func (m *mockGroupRegistry) ListGroup(_ context.Context) ([]*commonv1.Group, error) {
+ return m.groups, m.err
+}
+
+type mockMetadataRepo struct {
+ metadata.Repo
+ groupRegistry *mockGroupRegistry
+ dataInfoMap map[string][]*databasev1.DataInfo
+ dataInfoErr map[string]error
+}
+
+func (m *mockMetadataRepo) GroupRegistry() schema.Group {
+ return m.groupRegistry
+}
+
+func (m *mockMetadataRepo) CollectDataInfo(_ context.Context, group string) ([]*databasev1.DataInfo, error) {
+ if m.dataInfoErr != nil {
+ if err, ok := m.dataInfoErr[group]; ok {
+ return nil, err
+ }
+ }
+ if m.dataInfoMap != nil {
+ return m.dataInfoMap[group], nil
+ }
+ return nil, nil
+}
+
+func TestCatalogToString(t *testing.T) {
+ tests := []struct {
+ name string
+ expected string
+ catalog commonv1.Catalog
+ }{
+ {
+ name: "unspecified",
+ catalog: commonv1.Catalog_CATALOG_UNSPECIFIED,
+ expected: "CATALOG_UNSPECIFIED",
+ },
+ {
+ name: "stream",
+ catalog: commonv1.Catalog_CATALOG_STREAM,
+ expected: "CATALOG_STREAM",
+ },
+ {
+ name: "measure",
+ catalog: commonv1.Catalog_CATALOG_MEASURE,
+ expected: "CATALOG_MEASURE",
+ },
+ {
+ name: "unknown value",
+ catalog: commonv1.Catalog(999),
+ expected: "UNKNOWN_999",
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ result := catalogToString(tt.catalog)
+ assert.Equal(t, tt.expected, result)
+ })
+ }
+}
+
+func TestInspectAll_NilMetadataRepo(t *testing.T) {
+ s := &server{
+ log: logger.GetLogger("test"),
+ }
+ resp, err := s.InspectAll(context.Background(), &fodcv1.InspectAllRequest{})
+ require.Error(t, err)
+ assert.Nil(t, resp)
+ assert.Contains(t, err.Error(), "metadata repository not available")
+}
+
+func TestInspectAll_EmptyGroups(t *testing.T) {
+ repo := &mockMetadataRepo{
+ groupRegistry: &mockGroupRegistry{
+ groups: []*commonv1.Group{},
+ },
+ }
+ s := &server{
+ log: logger.GetLogger("test"),
+ metadataRepo: repo,
+ }
+ resp, err := s.InspectAll(context.Background(), &fodcv1.InspectAllRequest{})
+ require.NoError(t, err)
+ assert.NotNil(t, resp)
+ assert.Empty(t, resp.Groups)
+}
+
+func TestInspectAll_SkipsNilGroups(t *testing.T) {
+ repo := &mockMetadataRepo{
+ groupRegistry: &mockGroupRegistry{
+ groups: []*commonv1.Group{
+ nil,
+ {Metadata: nil},
+ {
+ Metadata: &commonv1.Metadata{Name: "valid_group"},
+ Catalog: commonv1.Catalog_CATALOG_STREAM,
+ },
+ },
+ },
+ dataInfoMap: map[string][]*databasev1.DataInfo{
+ "valid_group": {},
+ },
+ }
+ s := &server{
+ log: logger.GetLogger("test"),
+ metadataRepo: repo,
+ }
+ resp, err := s.InspectAll(context.Background(), &fodcv1.InspectAllRequest{})
+ require.NoError(t, err)
+ require.Len(t, resp.Groups, 1)
+ assert.Equal(t, "valid_group", resp.Groups[0].Name)
+ assert.Equal(t, "CATALOG_STREAM", resp.Groups[0].Catalog)
+}
+
+func TestInspectAll_MultipleGroups(t *testing.T) {
+ resourceOpts := &commonv1.ResourceOpts{
+ ShardNum: 2,
+ SegmentInterval: &commonv1.IntervalRule{
+ Unit: commonv1.IntervalRule_UNIT_DAY,
+ Num: 1,
+ },
+ Ttl: &commonv1.IntervalRule{
+ Unit: commonv1.IntervalRule_UNIT_DAY,
+ Num: 7,
+ },
+ }
+ dataInfo := []*databasev1.DataInfo{
+ {DataSizeBytes: 1024},
+ }
+ repo := &mockMetadataRepo{
+ groupRegistry: &mockGroupRegistry{
+ groups: []*commonv1.Group{
+ {
+ Metadata: &commonv1.Metadata{Name: "sw_metric"},
+ Catalog: commonv1.Catalog_CATALOG_MEASURE,
+ ResourceOpts: resourceOpts,
+ },
+ {
+ Metadata: &commonv1.Metadata{Name: "sw_record"},
+ Catalog: commonv1.Catalog_CATALOG_STREAM,
+ },
+ },
+ },
+ dataInfoMap: map[string][]*databasev1.DataInfo{
+ "sw_metric": dataInfo,
+ "sw_record": {},
+ },
+ dataInfoErr: map[string]error{
+ "sw_record": fmt.Errorf("collect error"),
+ },
+ }
+ s := &server{
+ log: logger.GetLogger("test"),
+ metadataRepo: repo,
+ }
+ resp, err := s.InspectAll(context.Background(), &fodcv1.InspectAllRequest{})
+ require.NoError(t, err)
+ require.Len(t, resp.Groups, 2)
+
+ assert.Equal(t, "sw_metric", resp.Groups[0].Name)
+ assert.Equal(t, "CATALOG_MEASURE", resp.Groups[0].Catalog)
+ assert.Equal(t, resourceOpts, resp.Groups[0].ResourceOpts)
+ assert.Equal(t, dataInfo, resp.Groups[0].DataInfo)
+
+ assert.Equal(t, "sw_record", resp.Groups[1].Name)
+ assert.Equal(t, "CATALOG_STREAM", resp.Groups[1].Catalog)
+ assert.Nil(t, resp.Groups[1].DataInfo)
+}
diff --git a/banyand/queue/sub/server.go b/banyand/queue/sub/server.go
index 7f46ff08a..ffe482229 100644
--- a/banyand/queue/sub/server.go
+++ b/banyand/queue/sub/server.go
@@ -42,10 +42,12 @@ import (
"github.com/apache/skywalking-banyandb/api/common"
clusterv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/cluster/v1"
databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ fodcv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/fodc/v1"
measurev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/measure/v1"
streamv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/stream/v1"
tracev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/trace/v1"
"github.com/apache/skywalking-banyandb/banyand/liaison/grpc/route"
+ "github.com/apache/skywalking-banyandb/banyand/metadata"
"github.com/apache/skywalking-banyandb/banyand/observability"
"github.com/apache/skywalking-banyandb/banyand/queue"
"github.com/apache/skywalking-banyandb/pkg/bus"
@@ -77,10 +79,11 @@ type server struct {
databasev1.UnimplementedNodeQueryServiceServer
databasev1.UnimplementedClusterStateServiceServer
clusterv1.UnimplementedServiceServer
+ fodcv1.UnimplementedGroupLifecycleServiceServer
omr observability.MetricsRegistry
creds credentials.TransportCredentials
- curNode *databasev1.Node
- metrics *metrics
+ metadataRepo metadata.Repo
+ clientCloser context.CancelFunc
ser *grpclib.Server
listeners map[bus.Topic][]bus.MessageListener
topicMap map[string]bus.Topic
@@ -88,14 +91,15 @@ type server struct {
log *logger.Logger
httpSrv *http.Server
tlsReloader *pkgtls.Reloader
- clientCloser context.CancelFunc
+ metrics *metrics
routeTableProvider map[string]route.TableProvider
+ curNode *databasev1.Node
+ flagNamePrefix string
certFile string
addr string
- keyFile string
- flagNamePrefix string
httpAddr string
host string
+ keyFile string
chunkBufferTimeout time.Duration
maxRecvMsgSize run.Bytes
listenersLock sync.RWMutex
@@ -285,6 +289,9 @@ func (s *server) Serve() run.StopNotify {
streamv1.RegisterStreamServiceServer(s.ser, &streamService{ser: s})
measurev1.RegisterMeasureServiceServer(s.ser, &measureService{ser: s})
tracev1.RegisterTraceServiceServer(s.ser, &traceService{ser: s})
+ if s.metadataRepo != nil {
+ fodcv1.RegisterGroupLifecycleServiceServer(s.ser, s)
+ }
var ctx context.Context
ctx, s.clientCloser = context.WithCancel(context.Background())
@@ -391,6 +398,11 @@ func (s *server) SetRouteProviders(providers map[string]route.TableProvider) {
s.routeTableProviderMu.Unlock()
}
+// SetMetadataRepo sets the metadata repository for the internal gRPC server.
+func (s *server) SetMetadataRepo(repo metadata.Repo) {
+ s.metadataRepo = repo
+}
+
type metrics struct {
totalStarted meter.Counter
totalFinished meter.Counter
diff --git a/docs/api-reference.md b/docs/api-reference.md
index d75e4fd19..44696cff7 100644
--- a/docs/api-reference.md
+++ b/docs/api-reference.md
@@ -306,6 +306,11 @@
- [banyandb/fodc/v1/rpc.proto](#banyandb_fodc_v1_rpc-proto)
- [Call](#banyandb-fodc-v1-Call)
+ - [GroupLifecycleInfo](#banyandb-fodc-v1-GroupLifecycleInfo)
+ - [InspectAllRequest](#banyandb-fodc-v1-InspectAllRequest)
+ - [InspectAllResponse](#banyandb-fodc-v1-InspectAllResponse)
+ - [LifecycleData](#banyandb-fodc-v1-LifecycleData)
+ - [LifecycleReport](#banyandb-fodc-v1-LifecycleReport)
- [Metric](#banyandb-fodc-v1-Metric)
- [Metric.LabelsEntry](#banyandb-fodc-v1-Metric-LabelsEntry)
- [RegisterAgentRequest](#banyandb-fodc-v1-RegisterAgentRequest)
@@ -313,11 +318,14 @@
- [RegisterAgentResponse](#banyandb-fodc-v1-RegisterAgentResponse)
- [StreamClusterTopologyRequest](#banyandb-fodc-v1-StreamClusterTopologyRequest)
- [StreamClusterTopologyResponse](#banyandb-fodc-v1-StreamClusterTopologyResponse)
+ - [StreamLifecycleRequest](#banyandb-fodc-v1-StreamLifecycleRequest)
+ - [StreamLifecycleResponse](#banyandb-fodc-v1-StreamLifecycleResponse)
- [StreamMetricsRequest](#banyandb-fodc-v1-StreamMetricsRequest)
- [StreamMetricsResponse](#banyandb-fodc-v1-StreamMetricsResponse)
- [Topology](#banyandb-fodc-v1-Topology)
- [FODCService](#banyandb-fodc-v1-FODCService)
+ - [GroupLifecycleService](#banyandb-fodc-v1-GroupLifecycleService)
- [banyandb/measure/v1/write.proto](#banyandb_measure_v1_write-proto)
- [DataPointSpec](#banyandb-measure-v1-DataPointSpec)
@@ -4752,6 +4760,81 @@ Phase represents the current phase of the deletion task.
+<a name="banyandb-fodc-v1-GroupLifecycleInfo"></a>
+
+### GroupLifecycleInfo
+
+
+
+| Field | Type | Label | Description |
+| ----- | ---- | ----- | ----------- |
+| name | [string](#string) | | |
+| catalog | [string](#string) | | |
+| resource_opts | [banyandb.common.v1.ResourceOpts](#banyandb-common-v1-ResourceOpts) | | |
+| data_info | [banyandb.database.v1.DataInfo](#banyandb-database-v1-DataInfo) | repeated | |
+
+
+
+
+
+
+<a name="banyandb-fodc-v1-InspectAllRequest"></a>
+
+### InspectAllRequest
+
+
+
+
+
+
+
+<a name="banyandb-fodc-v1-InspectAllResponse"></a>
+
+### InspectAllResponse
+
+
+
+| Field | Type | Label | Description |
+| ----- | ---- | ----- | ----------- |
+| groups | [GroupLifecycleInfo](#banyandb-fodc-v1-GroupLifecycleInfo) | repeated | |
+
+
+
+
+
+
+<a name="banyandb-fodc-v1-LifecycleData"></a>
+
+### LifecycleData
+
+
+
+| Field | Type | Label | Description |
+| ----- | ---- | ----- | ----------- |
+| reports | [LifecycleReport](#banyandb-fodc-v1-LifecycleReport) | repeated | |
+| groups | [GroupLifecycleInfo](#banyandb-fodc-v1-GroupLifecycleInfo) | repeated | |
+
+
+
+
+
+
+<a name="banyandb-fodc-v1-LifecycleReport"></a>
+
+### LifecycleReport
+
+
+
+| Field | Type | Label | Description |
+| ----- | ---- | ----- | ----------- |
+| filename | [string](#string) | | |
+| report_json | [string](#string) | | |
+
+
+
+
+
+
<a name="banyandb-fodc-v1-Metric"></a>
### Metric
@@ -4870,6 +4953,38 @@ Phase represents the current phase of the deletion task.
+<a name="banyandb-fodc-v1-StreamLifecycleRequest"></a>
+
+### StreamLifecycleRequest
+
+
+
+| Field | Type | Label | Description |
+| ----- | ---- | ----- | ----------- |
+| timestamp | [google.protobuf.Timestamp](#google-protobuf-Timestamp) | | |
+| pod_name | [string](#string) | | |
+| lifecycle_data | [LifecycleData](#banyandb-fodc-v1-LifecycleData) | | |
+
+
+
+
+
+
+<a name="banyandb-fodc-v1-StreamLifecycleResponse"></a>
+
+### StreamLifecycleResponse
+
+
+
+| Field | Type | Label | Description |
+| ----- | ---- | ----- | ----------- |
+| request_lifecycle | [bool](#bool) | | |
+
+
+
+
+
+
<a name="banyandb-fodc-v1-StreamMetricsRequest"></a>
### StreamMetricsRequest
@@ -4934,6 +5049,18 @@ Phase represents the current phase of the deletion task.
| RegisterAgent | [RegisterAgentRequest](#banyandb-fodc-v1-RegisterAgentRequest) stream | [RegisterAgentResponse](#banyandb-fodc-v1-RegisterAgentResponse) stream | Bi-directional stream for agent registration and heartbeat |
| StreamMetrics | [StreamMetricsRequest](#banyandb-fodc-v1-StreamMetricsRequest) stream | [StreamMetricsResponse](#banyandb-fodc-v1-StreamMetricsResponse) stream | Bi-directional stream for metrics Agent sends StreamMetricsRequest (metrics data), Proxy sends StreamMetricsResponse (metrics requests) |
| StreamClusterTopology | [StreamClusterTopologyRequest](#banyandb-fodc-v1-StreamClusterTopologyRequest) stream | [StreamClusterTopologyResponse](#banyandb-fodc-v1-StreamClusterTopologyResponse) stream | Bi-directional stream for cluster topology |
+| StreamLifecycle | [StreamLifecycleRequest](#banyandb-fodc-v1-StreamLifecycleRequest) stream | [StreamLifecycleResponse](#banyandb-fodc-v1-StreamLifecycleResponse) stream | Bi-directional stream for lifecycle data |
+
+
+<a name="banyandb-fodc-v1-GroupLifecycleService"></a>
+
+### GroupLifecycleService
+GroupLifecycleService provides group lifecycle information from liaison nodes.
+Registered on the internal gRPC server so FODC agent can access it.
+
+| Method Name | Request Type | Response Type | Description |
+| ----------- | ------------ | ------------- | ------------|
+| InspectAll | [InspectAllRequest](#banyandb-fodc-v1-InspectAllRequest) | [InspectAllResponse](#banyandb-fodc-v1-InspectAllResponse) | InspectAll lists all groups and returns their full lifecycle info in one call. |
diff --git a/docs/operation/fodc/apis.md b/docs/operation/fodc/apis.md
index a7d45deb9..22f29f993 100644
--- a/docs/operation/fodc/apis.md
+++ b/docs/operation/fodc/apis.md
@@ -156,6 +156,58 @@ The proxy triggers a topology collection across all registered agents and merges
- `status` / `last_heartbeat`: online status from the agent registry.
- `calls` entries describe the node-to-node call graph reported by agents.
+### GET /cluster/lifecycle
+
+Requests lifecycle data from all connected agents and returns aggregated lifecycle reports and group information as JSON.
+The proxy triggers lifecycle data collection from all agents that support the lifecycle stream.
+
+**Response**
+
+- Content-Type: `application/json`
+- Body shape:
+
+```json
+{
+ "groups": [
+ {
+ "name": "sw_metric",
+ "catalog": "CATALOG_MEASURE",
+ "resource_opts": {
+ "shard_num": 2,
+ "segment_interval": {"unit": "UNIT_DAY", "num": 1},
+ "ttl": {"unit": "UNIT_DAY", "num": 7}
+ },
+ "data_info": []
+ }
+ ],
+ "lifecycle_statuses": [
+ {
+ "pod_name": "banyandb-data-0",
+ "reports": [
+ {
+ "filename": "2026-03-26.json",
+ "report_json": "{\"status\":\"ok\"}"
+ }
+ ]
+ }
+ ]
+}
+```
+
+**Field notes**
+
+- `groups`: Group lifecycle information collected from the first agent that provides it (typically the liaison node).
+ Each entry contains the group name, catalog type, resource options, and data info.
+- `lifecycle_statuses`: Per-pod lifecycle reports. Each entry contains the pod name and an array of report files
+ (JSON files read from the agent's lifecycle report directory).
+- Agents that do not support the lifecycle stream are silently skipped.
+
+**Example**
+
+```text
+GET http://localhost:17913/cluster/lifecycle
+```
+
## FODC Agent CLI Flags
The agent binary is `fodc` (see `fodc/agent/cmd/agent`).
@@ -173,6 +225,9 @@ The agent binary is `fodc` (see `fodc/agent/cmd/agent`).
| `--reconnect-interval` | `5s` | Backoff between reconnection attempts to the proxy. |
| `--cluster-state-ports` | _empty_ | gRPC ports for BanyanDB cluster state polling; enables topology collection. |
| `--cluster-state-poll-interval` | `30s` | Interval for polling cluster state from BanyanDB nodes. |
+| `--lifecycle-port` | `18912` | gRPC port for lifecycle InspectAll service. Set to 0 to disable lifecycle collection. |
+| `--lifecycle-report-dir` | `/tmp/lifecycle-reports` | Directory where lifecycle sidecar writes report files. |
+| `--lifecycle-cache-ttl` | `10m` | TTL for cached lifecycle data. After expiry, the next collection call refreshes the cache. |
**Behavior notes**
diff --git a/fodc/agent/internal/cmd/root.go b/fodc/agent/internal/cmd/root.go
index fa7a09167..d0ded635b 100644
--- a/fodc/agent/internal/cmd/root.go
+++ b/fodc/agent/internal/cmd/root.go
@@ -36,6 +36,7 @@ import (
"github.com/apache/skywalking-banyandb/fodc/agent/internal/exporter"
"github.com/apache/skywalking-banyandb/fodc/agent/internal/flightrecorder"
"github.com/apache/skywalking-banyandb/fodc/agent/internal/ktm"
+ "github.com/apache/skywalking-banyandb/fodc/agent/internal/lifecycle"
fodcmetrics "github.com/apache/skywalking-banyandb/fodc/agent/internal/metrics"
"github.com/apache/skywalking-banyandb/fodc/agent/internal/proxy"
"github.com/apache/skywalking-banyandb/fodc/agent/internal/server"
@@ -70,6 +71,9 @@ var (
reconnectInterval time.Duration
clusterStatePorts []string
clusterStatePollInterval time.Duration
+ lifecyclePort int
+ lifecycleReportDir string
+ lifecycleCacheTTL time.Duration
rootCmd = &cobra.Command{
Use: "fodc",
Short: "First Occurrence Data Collection (FODC) agent",
@@ -111,6 +115,12 @@ func init() {
"Ports of the BanyanDB node's gRPC endpoints to poll cluster state from. If empty, cluster state polling is disabled.")
rootCmd.Flags().DurationVar(&clusterStatePollInterval, "cluster-state-poll-interval", defaultClusterStatePollInterval,
"Interval at which to poll cluster state from the BanyanDB nodes")
+ rootCmd.Flags().IntVar(&lifecyclePort, "lifecycle-port", 18912,
+ "gRPC port for lifecycle InspectAll service. Set to 0 to disable lifecycle collection")
+ rootCmd.Flags().StringVar(&lifecycleReportDir, "lifecycle-report-dir", lifecycle.DefaultReportDir,
+ "Directory where lifecycle sidecar writes report files")
+ rootCmd.Flags().DurationVar(&lifecycleCacheTTL, "lifecycle-cache-ttl", 10*time.Minute,
+ "TTL for cached lifecycle data. After expiry, the next collection call refreshes the cache")
}
func calculateCapacity(log *logger.Logger) int64 {
@@ -341,8 +351,14 @@ func startProxyClient(ctx context.Context, log *logger.Logger, fr *flightrecorde
log.Info().Msg("Proxy client not started (missing: --proxy-addr, --pod-name, and --node-role)")
return nil
}
+ var lifecycleCollector *lifecycle.Collector
+ if lifecyclePort > 0 {
+ grpcAddr := fmt.Sprintf("localhost:%d", lifecyclePort)
+ lifecycleCollector = lifecycle.NewCollector(log, grpcAddr, lifecycleReportDir, lifecycleCacheTTL)
+ log.Info().Str("grpc_addr", grpcAddr).Msg("Lifecycle collector initialized")
+ }
client := proxy.NewClient(proxyAddr, nodeRole, podName, containerNames, nodeLabels,
- heartbeatInterval, reconnectInterval, fr, collector, log)
+ heartbeatInterval, reconnectInterval, fr, collector, lifecycleCollector, log)
go func() {
if startErr := client.Start(ctx); startErr != nil {
log.Error().Err(startErr).Msg("Proxy client error")
diff --git a/fodc/agent/internal/lifecycle/collector.go b/fodc/agent/internal/lifecycle/collector.go
new file mode 100644
index 000000000..6b9c559ff
--- /dev/null
+++ b/fodc/agent/internal/lifecycle/collector.go
@@ -0,0 +1,188 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Package lifecycle provides lifecycle data collection functionality for FODC agent.
+package lifecycle
+
+import (
+ "context"
+ "os"
+ "path/filepath"
+ "sort"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "google.golang.org/grpc"
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/credentials/insecure"
+ "google.golang.org/grpc/status"
+
+ fodcv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/fodc/v1"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+)
+
+const (
+ // DefaultReportDir is the default directory for lifecycle report files.
+ DefaultReportDir = "/tmp/lifecycle-reports"
+ maxReportFiles = 5
+ maxReportFileSize = 5 * 1024 * 1024 // 5MB
+)
+
+const grpcTimeout = 10 * time.Second
+
+// Collector collects lifecycle data from local files.
+type Collector struct {
+ lastCollectTime time.Time
+ log *logger.Logger
+ nowFunc func() time.Time
+ currentData *fodcv1.LifecycleData
+ grpcAddr string
+ reportDir string
+ cacheTTL time.Duration
+ mu sync.RWMutex
+ grpcUnimplemented atomic.Bool
+}
+
+// NewCollector creates a new lifecycle data collector.
+func NewCollector(log *logger.Logger, grpcAddr, reportDir string, cacheTTL time.Duration) *Collector {
+ if reportDir == "" {
+ reportDir = DefaultReportDir
+ }
+ return &Collector{
+ log: log,
+ grpcAddr: grpcAddr,
+ reportDir: reportDir,
+ cacheTTL: cacheTTL,
+ nowFunc: time.Now,
+ }
+}
+
+// Collect collects lifecycle data from local files.
+// Data is cached for cacheTTL duration; after expiry the next call refreshes the cache.
+func (c *Collector) Collect(ctx context.Context) (*fodcv1.LifecycleData, error) {
+ c.mu.RLock()
+ if c.currentData != nil && c.cacheTTL > 0 && c.nowFunc().Sub(c.lastCollectTime) < c.cacheTTL {
+ data := c.currentData
+ c.mu.RUnlock()
+ return data, nil
+ }
+ c.mu.RUnlock()
+
+ data := &fodcv1.LifecycleData{
+ Reports: c.readReportFiles(),
+ Groups: c.collectGroups(ctx),
+ }
+ c.mu.Lock()
+ c.currentData = data
+ c.lastCollectTime = c.nowFunc()
+ c.mu.Unlock()
+ return data, nil
+}
+
+func (c *Collector) collectGroups(ctx context.Context) []*fodcv1.GroupLifecycleInfo {
+ if c.grpcAddr == "" || c.grpcUnimplemented.Load() {
+ return nil
+ }
+ conn, err := grpc.NewClient(c.grpcAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
+ if err != nil {
+ if c.log != nil {
+ c.log.Warn().Err(err).Str("addr", c.grpcAddr).Msg("Failed to create gRPC connection for InspectAll")
+ }
+ return nil
+ }
+ defer func() {
+ _ = conn.Close()
+ }()
+
+ client := fodcv1.NewGroupLifecycleServiceClient(conn)
+ reqCtx, cancel := context.WithTimeout(ctx, grpcTimeout)
+ defer cancel()
+ resp, err := client.InspectAll(reqCtx, &fodcv1.InspectAllRequest{})
+ if err != nil {
+ if status.Code(err) == codes.Unimplemented {
+ c.grpcUnimplemented.Store(true)
+ if c.log != nil {
+ c.log.Info().Str("addr", c.grpcAddr).Msg("GroupLifecycleService.InspectAll is not implemented, skipping future calls")
+ }
+ return nil
+ }
+ if c.log != nil {
+ c.log.Warn().Err(err).Str("addr", c.grpcAddr).Msg("InspectAll failed, skipping groups")
+ }
+ return nil
+ }
+ return resp.GetGroups()
+}
+
+func (c *Collector) readReportFiles() []*fodcv1.LifecycleReport {
+ entries, err := os.ReadDir(c.reportDir)
+ if err != nil {
+ if !os.IsNotExist(err) && c.log != nil {
+ c.log.Warn().Err(err).Msg("Error reading lifecycle reports directory")
+ }
+ return nil
+ }
+
+ var jsonFiles []os.DirEntry
+ for _, entry := range entries {
+ if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".json") {
+ jsonFiles = append(jsonFiles, entry)
+ }
+ }
+
+ sort.Slice(jsonFiles, func(i, j int) bool {
+ return jsonFiles[i].Name() > jsonFiles[j].Name()
+ })
+
+ if len(jsonFiles) > maxReportFiles {
+ jsonFiles = jsonFiles[:maxReportFiles]
+ }
+
+ reports := make([]*fodcv1.LifecycleReport, 0, len(jsonFiles))
+ for _, entry := range jsonFiles {
+ info, statErr := entry.Info()
+ if statErr != nil {
+ if c.log != nil {
+ c.log.Warn().Err(statErr).Str("file", entry.Name()).Msg("Error getting report file info")
+ }
+ continue
+ }
+ if info.Size() > maxReportFileSize {
+ if c.log != nil {
+ c.log.Warn().Str("file", entry.Name()).Int64("size", info.Size()).Int64("max", maxReportFileSize).
+ Msg("Report file exceeds max size, skipping")
+ }
+ continue
+ }
+ filePath := filepath.Join(c.reportDir, entry.Name())
+ data, readErr := os.ReadFile(filePath)
+ if readErr != nil {
+ if c.log != nil {
+ c.log.Warn().Err(readErr).Str("file", entry.Name()).Msg("Error reading report file")
+ }
+ continue
+ }
+ reports = append(reports, &fodcv1.LifecycleReport{
+ Filename: entry.Name(),
+ ReportJson: string(data),
+ })
+ }
+
+ return reports
+}
diff --git a/fodc/agent/internal/lifecycle/collector_test.go b/fodc/agent/internal/lifecycle/collector_test.go
new file mode 100644
index 000000000..0be13c09a
--- /dev/null
+++ b/fodc/agent/internal/lifecycle/collector_test.go
@@ -0,0 +1,182 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package lifecycle
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+)
+
+func initTestLogger(t *testing.T) *logger.Logger {
+ t.Helper()
+ err := logger.Init(logger.Logging{Env: "dev", Level: "debug"})
+ require.NoError(t, err)
+ return logger.GetLogger("test", "lifecycle")
+}
+
+func TestNewCollector(t *testing.T) {
+ log := initTestLogger(t)
+ collector := NewCollector(log, "", "", 10*time.Minute)
+ require.NotNil(t, collector)
+ assert.Nil(t, collector.currentData)
+ assert.Equal(t, 10*time.Minute, collector.cacheTTL)
+}
+
+func TestCollector_ReadReportFiles_EmptyDir(t *testing.T) {
+ log := initTestLogger(t)
+ collector := NewCollector(log, "", t.TempDir(), 0)
+ reports := collector.readReportFiles()
+ assert.Empty(t, reports)
+}
+
+func TestCollector_Collect(t *testing.T) {
+ log := initTestLogger(t)
+ collector := NewCollector(log, "", "", 0)
+ ctx := t.Context()
+ data, err := collector.Collect(ctx)
+ require.NoError(t, err)
+ require.NotNil(t, data)
+ assert.NotNil(t, collector.currentData)
+}
+
+func TestCollector_CacheTTL(t *testing.T) {
+ log := initTestLogger(t)
+ collector := NewCollector(log, "", "", 5*time.Minute)
+ ctx := t.Context()
+
+ now := time.Now()
+ collector.nowFunc = func() time.Time { return now }
+
+ // First call should collect and cache
+ data1, err := collector.Collect(ctx)
+ require.NoError(t, err)
+ require.NotNil(t, data1)
+
+ // Second call within TTL should return cached data
+ now = now.Add(2 * time.Minute)
+ data2, err := collector.Collect(ctx)
+ require.NoError(t, err)
+ assert.Same(t, data1, data2)
+
+ // After TTL expires, should re-collect
+ now = now.Add(4 * time.Minute) // total 6 minutes > 5 minute TTL
+ data3, err := collector.Collect(ctx)
+ require.NoError(t, err)
+ require.NotNil(t, data3)
+ assert.NotSame(t, data1, data3)
+}
+
+func TestCollector_GrpcUnimplemented_SkipsFutureCalls(t *testing.T) {
+ log := initTestLogger(t)
+ collector := NewCollector(log, "localhost:0", "", 0)
+
+ assert.False(t, collector.grpcUnimplemented.Load())
+
+ groups := collector.collectGroups(t.Context())
+ assert.Nil(t, groups)
+ assert.False(t, collector.grpcUnimplemented.Load())
+
+ collector.grpcUnimplemented.Store(true)
+ groups = collector.collectGroups(t.Context())
+ assert.Nil(t, groups)
+ assert.True(t, collector.grpcUnimplemented.Load())
+}
+
+func TestCollector_ReadReportFiles_MaxFiles(t *testing.T) {
+ log := initTestLogger(t)
+ dir := t.TempDir()
+ collector := NewCollector(log, "", dir, 0)
+
+ // Create 7 files with timestamp-like names so sort order is predictable
+ for idx, name := range []string{
+ "2026-03-20.json",
+ "2026-03-21.json",
+ "2026-03-22.json",
+ "2026-03-23.json",
+ "2026-03-24.json",
+ "2026-03-25.json",
+ "2026-03-26.json",
+ } {
+ require.NoError(t, os.WriteFile(
+ filepath.Join(dir, name),
+ []byte(`{"idx":`+string(rune('0'+idx))+`}`),
+ 0o600,
+ ))
+ }
+
+ reports := collector.readReportFiles()
+ require.Equal(t, maxReportFiles, len(reports))
+
+ // Should be the 5 most recent (sorted descending by name)
+ assert.Equal(t, "2026-03-26.json", reports[0].Filename)
+ assert.Equal(t, "2026-03-25.json", reports[1].Filename)
+ assert.Equal(t, "2026-03-24.json", reports[2].Filename)
+ assert.Equal(t, "2026-03-23.json", reports[3].Filename)
+ assert.Equal(t, "2026-03-22.json", reports[4].Filename)
+}
+
+func TestCollector_ReadReportFiles_SkipsNonJSON(t *testing.T) {
+ log := initTestLogger(t)
+ dir := t.TempDir()
+ collector := NewCollector(log, "", dir, 0)
+
+ require.NoError(t, os.WriteFile(filepath.Join(dir, "2026-03-26.json"), []byte(`{}`), 0o600))
+ require.NoError(t, os.WriteFile(filepath.Join(dir, "readme.txt"), []byte("text"), 0o600))
+ require.NoError(t, os.WriteFile(filepath.Join(dir, "data.csv"), []byte("a,b"), 0o600))
+
+ reports := collector.readReportFiles()
+ require.Equal(t, 1, len(reports))
+ assert.Equal(t, "2026-03-26.json", reports[0].Filename)
+}
+
+func TestCollector_ReadReportFiles_OversizedFile(t *testing.T) {
+ log := initTestLogger(t)
+ dir := t.TempDir()
+ collector := NewCollector(log, "", dir, 0)
+
+ require.NoError(t, os.WriteFile(filepath.Join(dir, "2026-03-26.json"), []byte(`{"ok":true}`), 0o600))
+ oversized := make([]byte, maxReportFileSize+1)
+ require.NoError(t, os.WriteFile(filepath.Join(dir, "2026-03-25.json"), oversized, 0o600))
+
+ reports := collector.readReportFiles()
+ require.Equal(t, 1, len(reports))
+ assert.Equal(t, "2026-03-26.json", reports[0].Filename)
+}
+
+func TestCollector_ZeroTTL_AlwaysRefreshes(t *testing.T) {
+ log := initTestLogger(t)
+ collector := NewCollector(log, "", "", 0)
+ ctx := t.Context()
+
+ data1, err := collector.Collect(ctx)
+ require.NoError(t, err)
+ require.NotNil(t, data1)
+
+ // With zero TTL, every call should re-collect
+ data2, err := collector.Collect(ctx)
+ require.NoError(t, err)
+ require.NotNil(t, data2)
+ assert.NotSame(t, data1, data2)
+}
diff --git a/fodc/agent/internal/proxy/client.go b/fodc/agent/internal/proxy/client.go
index 29eac781a..cf7b44203 100644
--- a/fodc/agent/internal/proxy/client.go
+++ b/fodc/agent/internal/proxy/client.go
@@ -36,6 +36,7 @@ import (
fodcv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/fodc/v1"
"github.com/apache/skywalking-banyandb/fodc/agent/internal/cluster"
"github.com/apache/skywalking-banyandb/fodc/agent/internal/flightrecorder"
+ "github.com/apache/skywalking-banyandb/fodc/agent/internal/lifecycle"
"github.com/apache/skywalking-banyandb/fodc/agent/internal/metrics"
"github.com/apache/skywalking-banyandb/pkg/logger"
)
@@ -53,12 +54,14 @@ type Client struct {
flightRecorder *flightrecorder.FlightRecorder
connManager *connManager
clusterCollector *cluster.Collector
+ lifecycleCollector *lifecycle.Collector
stopCh chan struct{}
reconnectCh chan struct{}
labels map[string]string
metricsStream fodcv1.FODCService_StreamMetricsClient
registrationStream fodcv1.FODCService_RegisterAgentClient
clusterStateStream fodcv1.FODCService_StreamClusterTopologyClient
+ lifecycleStream fodcv1.FODCService_StreamLifecycleClient
client fodcv1.FODCServiceClient
proxyAddr string
@@ -85,23 +88,25 @@ func NewClient(
reconnectInterval time.Duration,
flightRecorder *flightrecorder.FlightRecorder,
clusterCollector *cluster.Collector,
+ lifecycleCollector *lifecycle.Collector,
logger *logger.Logger,
) *Client {
connMgr := newConnManager(proxyAddr, reconnectInterval, logger)
client := &Client{
- connManager: connMgr,
- proxyAddr: proxyAddr,
- nodeRole: nodeRole,
- podName: podName,
- containerNames: containerNames,
- labels: labels,
- heartbeatInterval: heartbeatInterval,
- reconnectInterval: reconnectInterval,
- flightRecorder: flightRecorder,
- clusterCollector: clusterCollector,
- logger: logger,
- stopCh: make(chan struct{}),
- reconnectCh: make(chan struct{}, 1),
+ connManager: connMgr,
+ proxyAddr: proxyAddr,
+ nodeRole: nodeRole,
+ podName: podName,
+ containerNames: containerNames,
+ labels: labels,
+ heartbeatInterval: heartbeatInterval,
+ reconnectInterval: reconnectInterval,
+ flightRecorder: flightRecorder,
+ clusterCollector: clusterCollector,
+ lifecycleCollector: lifecycleCollector,
+ logger: logger,
+ stopCh: make(chan struct{}),
+ reconnectCh: make(chan struct{}, 1),
}
connMgr.setHeartbeatChecker(client.SendHeartbeat)
return client
@@ -319,6 +324,125 @@ func (c *Client) sendClusterTopology() error {
return nil
}
+// StartLifecycleStream establishes bi-directional lifecycle stream with Proxy.
+func (c *Client) StartLifecycleStream(ctx context.Context) error {
+ c.streamsMu.Lock()
+ if c.client == nil {
+ c.streamsMu.Unlock()
+ return fmt.Errorf("client not connected, call Connect() first")
+ }
+ client := c.client
+ agentID := c.agentID
+ c.streamsMu.Unlock()
+
+ if agentID == "" {
+ return fmt.Errorf("agent ID not available, register agent first")
+ }
+
+ md := metadata.New(map[string]string{"agent_id": agentID})
+ ctxWithMetadata := metadata.NewOutgoingContext(ctx, md)
+
+ stream, streamErr := client.StreamLifecycle(ctxWithMetadata)
+ if streamErr != nil {
+ return fmt.Errorf("failed to create lifecycle stream: %w", streamErr)
+ }
+
+ c.streamsMu.Lock()
+ c.lifecycleStream = stream
+ c.streamsMu.Unlock()
+
+ go c.handleLifecycleStream(ctx, stream)
+
+ c.logger.Info().
+ Str("agent_id", agentID).
+ Msg("Lifecycle stream established with Proxy")
+
+ return nil
+}
+
+// sendLifecycleData sends lifecycle data to Proxy.
+func (c *Client) sendLifecycleData(ctx context.Context) error {
+ c.streamsMu.RLock()
+ if c.disconnected || c.lifecycleStream == nil {
+ c.streamsMu.RUnlock()
+ return fmt.Errorf("lifecycle stream not established")
+ }
+ lifecycleStream := c.lifecycleStream
+ collector := c.lifecycleCollector
+ c.streamsMu.RUnlock()
+ if collector == nil {
+ c.logger.Warn().Msg("Lifecycle collector is nil, skipping send")
+ return nil
+ }
+ data, collectErr := collector.Collect(ctx)
+ if collectErr != nil {
+ return fmt.Errorf("failed to collect lifecycle data: %w", collectErr)
+ }
+ if len(data.Reports) == 0 && len(data.Groups) == 0 {
+ c.logger.Debug().Msg("No lifecycle data available to send")
+ }
+ req := &fodcv1.StreamLifecycleRequest{
+ Timestamp: timestamppb.Now(),
+ PodName: c.podName,
+ LifecycleData: data,
+ }
+ c.streamsMu.Lock()
+ if c.lifecycleStream != lifecycleStream {
+ c.streamsMu.Unlock()
+ return fmt.Errorf("lifecycle stream changed during send")
+ }
+ sendErr := lifecycleStream.Send(req)
+ c.streamsMu.Unlock()
+ if sendErr != nil {
+ return fmt.Errorf("failed to send lifecycle data: %w", sendErr)
+ }
+ c.logger.Info().
+ Int("reports_count", len(data.Reports)).
+ Int("groups_count", len(data.Groups)).
+ Msg("Successfully sent lifecycle data to proxy")
+ return nil
+}
+
+// handleLifecycleStream handles the lifecycle stream.
+func (c *Client) handleLifecycleStream(ctx context.Context, stream fodcv1.FODCService_StreamLifecycleClient) {
+ c.streamsMu.RLock()
+ stopCh := c.stopCh
+ c.streamsMu.RUnlock()
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-stopCh:
+ return
+ default:
+ }
+ resp, recvErr := stream.Recv()
+ if errors.Is(recvErr, io.EOF) {
+ c.logger.Warn().Msg("Lifecycle stream closed by Proxy, reconnecting...")
+ go c.reconnect(ctx)
+ return
+ }
+ if recvErr != nil {
+ c.streamsMu.RLock()
+ disconnected := c.disconnected
+ c.streamsMu.RUnlock()
+ if disconnected {
+ c.logger.Debug().Err(recvErr).Msg("Lifecycle stream closed")
+ return
+ }
+ c.logger.Error().Err(recvErr).Msg("Error receiving from lifecycle stream, reconnecting...")
+ go c.reconnect(ctx)
+ return
+ }
+ if resp != nil && resp.RequestLifecycle {
+ c.logger.Debug().Msg("Received lifecycle data request from proxy")
+ if sendErr := c.sendLifecycleData(ctx); sendErr != nil {
+ c.logger.Error().Err(sendErr).Msg("Failed to send lifecycle data to proxy")
+ }
+ }
+ }
+}
+
// handleClusterStateStream handles the cluster state stream.
func (c *Client) handleClusterStateStream(ctx context.Context, stream fodcv1.FODCService_StreamClusterTopologyClient) {
c.streamsMu.RLock()
@@ -645,6 +769,13 @@ func (c *Client) cleanupStreams() {
}
c.clusterStateStream = nil
}
+
+ if c.lifecycleStream != nil {
+ if closeErr := c.lifecycleStream.CloseSend(); closeErr != nil {
+ c.logger.Warn().Err(closeErr).Msg("Error closing lifecycle stream")
+ }
+ c.lifecycleStream = nil
+ }
c.client = nil
c.streamsMu.Unlock()
}
@@ -691,6 +822,13 @@ func (c *Client) Disconnect() error {
}
c.clusterStateStream = nil
}
+
+ if c.lifecycleStream != nil {
+ if closeErr := c.lifecycleStream.CloseSend(); closeErr != nil {
+ c.logger.Warn().Err(closeErr).Msg("Error closing lifecycle stream")
+ }
+ c.lifecycleStream = nil
+ }
c.streamsMu.Unlock()
c.connManager.stop()
@@ -748,6 +886,15 @@ func (c *Client) Start(ctx context.Context) error {
continue
}
+ if c.lifecycleCollector != nil {
+ if lifecycleErr := c.StartLifecycleStream(ctx); lifecycleErr != nil {
+ c.logger.Error().Err(lifecycleErr).Msg("Failed to start lifecycle stream, reconnecting...")
+ c.cleanupStreams()
+ time.Sleep(c.reconnectInterval)
+ continue
+ }
+ }
+
c.logger.Info().Msg("Proxy client started successfully")
return nil
}
@@ -891,6 +1038,10 @@ func (c *Client) reconnect(ctx context.Context) {
_ = c.clusterStateStream.CloseSend()
c.clusterStateStream = nil
}
+ if c.lifecycleStream != nil {
+ _ = c.lifecycleStream.CloseSend()
+ c.lifecycleStream = nil
+ }
c.streamsMu.Unlock()
connResultCh := c.connManager.RequestConnect(ctx)
@@ -931,6 +1082,13 @@ func (c *Client) reconnect(ctx context.Context) {
return
}
+ if c.lifecycleCollector != nil {
+ if lifecycleErr := c.StartLifecycleStream(ctx); lifecycleErr != nil {
+ c.logger.Error().Err(lifecycleErr).Msg("Failed to restart lifecycle stream")
+ return
+ }
+ }
+
c.logger.Info().Msg("Successfully reconnected to Proxy")
}
diff --git a/fodc/agent/internal/proxy/client_test.go b/fodc/agent/internal/proxy/client_test.go
index 650beeb2f..b37229ef5 100644
--- a/fodc/agent/internal/proxy/client_test.go
+++ b/fodc/agent/internal/proxy/client_test.go
@@ -80,6 +80,10 @@ func (m *mockFODCServiceClient) StreamClusterTopology(ctx context.Context, opts
return nil, errors.New("not implemented")
}
+func (m *mockFODCServiceClient) StreamLifecycle(_ context.Context, _ ...grpc.CallOption) (fodcv1.FODCService_StreamLifecycleClient, error) {
+ return nil, errors.New("not implemented")
+}
+
// mockRegisterAgentClient implements fodcv1.FODCService_RegisterAgentClient for testing.
type mockRegisterAgentClient struct {
ctx context.Context
@@ -358,6 +362,7 @@ func TestNewProxyClient(t *testing.T) {
10*time.Second,
fr,
nil,
+ nil,
testLogger,
)
@@ -377,7 +382,7 @@ func TestNewProxyClient(t *testing.T) {
func TestProxyClient_Connect_Success(t *testing.T) {
testLogger := initTestLogger(t)
fr := flightrecorder.NewFlightRecorder(1000000)
- pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, testLogger)
+ pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, nil, testLogger)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@@ -405,7 +410,7 @@ func TestProxyClient_Connect_Success(t *testing.T) {
func TestProxyClient_Connect_AlreadyConnected(t *testing.T) {
testLogger := initTestLogger(t)
fr := flightrecorder.NewFlightRecorder(1000000)
- pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, testLogger)
+ pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, nil, testLogger)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@@ -426,7 +431,7 @@ func TestProxyClient_Connect_AlreadyConnected(t *testing.T) {
func TestProxyClient_Connect_Reconnection(t *testing.T) {
testLogger := initTestLogger(t)
fr := flightrecorder.NewFlightRecorder(1000000)
- pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, testLogger)
+ pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, nil, testLogger)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@@ -462,7 +467,7 @@ func TestProxyClient_Connect_Reconnection(t *testing.T) {
func TestProxyClient_StartRegistrationStream_NotConnected(t *testing.T) {
testLogger := initTestLogger(t)
fr := flightrecorder.NewFlightRecorder(1000000)
- pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, testLogger)
+ pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, nil, testLogger)
ctx := context.Background()
err := pc.StartRegistrationStream(ctx)
@@ -476,7 +481,7 @@ func TestProxyClient_StartRegistrationStream_Success(t *testing.T) {
fr := flightrecorder.NewFlightRecorder(1000000)
pc := NewProxyClient(
"localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, map[string]string{"env": "test"},
- 5*time.Second, 10*time.Second, fr, nil, testLogger)
+ 5*time.Second, 10*time.Second, fr, nil, nil, testLogger)
ctx := context.Background()
mockClient := &mockFODCServiceClient{}
@@ -510,7 +515,7 @@ func TestProxyClient_StartRegistrationStream_Success(t *testing.T) {
func TestProxyClient_StartRegistrationStream_StreamError(t *testing.T) {
testLogger := initTestLogger(t)
fr := flightrecorder.NewFlightRecorder(1000000)
- pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, testLogger)
+ pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, nil, testLogger)
ctx := context.Background()
mockClient := &mockFODCServiceClient{}
@@ -531,7 +536,7 @@ func TestProxyClient_StartRegistrationStream_StreamError(t *testing.T) {
func TestProxyClient_StartRegistrationStream_SendError(t *testing.T) {
testLogger := initTestLogger(t)
fr := flightrecorder.NewFlightRecorder(1000000)
- pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, testLogger)
+ pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, nil, testLogger)
ctx := context.Background()
mockClient := &mockFODCServiceClient{}
@@ -555,7 +560,7 @@ func TestProxyClient_StartRegistrationStream_SendError(t *testing.T) {
func TestProxyClient_StartRegistrationStream_RecvError(t *testing.T) {
testLogger := initTestLogger(t)
fr := flightrecorder.NewFlightRecorder(1000000)
- pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, testLogger)
+ pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, nil, testLogger)
ctx := context.Background()
mockClient := &mockFODCServiceClient{}
@@ -579,7 +584,7 @@ func TestProxyClient_StartRegistrationStream_RecvError(t *testing.T) {
func TestProxyClient_StartRegistrationStream_RegistrationFailed(t *testing.T) {
testLogger := initTestLogger(t)
fr := flightrecorder.NewFlightRecorder(1000000)
- pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, testLogger)
+ pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, nil, testLogger)
ctx := context.Background()
mockClient := &mockFODCServiceClient{}
@@ -608,7 +613,7 @@ func TestProxyClient_StartRegistrationStream_RegistrationFailed(t *testing.T) {
func TestProxyClient_StartRegistrationStream_MissingAgentID(t *testing.T) {
testLogger := initTestLogger(t)
fr := flightrecorder.NewFlightRecorder(1000000)
- pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, testLogger)
+ pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, nil, testLogger)
ctx := context.Background()
mockClient := &mockFODCServiceClient{}
@@ -637,7 +642,7 @@ func TestProxyClient_StartRegistrationStream_MissingAgentID(t *testing.T) {
func TestProxyClient_StartMetricsStream_NotConnected(t *testing.T) {
testLogger := initTestLogger(t)
fr := flightrecorder.NewFlightRecorder(1000000)
- pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, testLogger)
+ pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, nil, testLogger)
ctx := context.Background()
err := pc.StartMetricsStream(ctx)
@@ -649,7 +654,7 @@ func TestProxyClient_StartMetricsStream_NotConnected(t *testing.T) {
func TestProxyClient_StartMetricsStream_NoAgentID(t *testing.T) {
testLogger := initTestLogger(t)
fr := flightrecorder.NewFlightRecorder(1000000)
- pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, testLogger)
+ pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, nil, testLogger)
mockClient := &mockFODCServiceClient{}
pc.streamsMu.Lock()
@@ -666,7 +671,7 @@ func TestProxyClient_StartMetricsStream_NoAgentID(t *testing.T) {
func TestProxyClient_StartMetricsStream_Success(t *testing.T) {
testLogger := initTestLogger(t)
fr := flightrecorder.NewFlightRecorder(1000000)
- pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, testLogger)
+ pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, nil, testLogger)
ctx := context.Background()
mockClient := &mockFODCServiceClient{}
@@ -699,7 +704,7 @@ func TestProxyClient_StartMetricsStream_Success(t *testing.T) {
func TestProxyClient_StartMetricsStream_StreamError(t *testing.T) {
testLogger := initTestLogger(t)
fr := flightrecorder.NewFlightRecorder(1000000)
- pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, testLogger)
+ pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, nil, testLogger)
ctx := context.Background()
mockClient := &mockFODCServiceClient{}
@@ -721,7 +726,7 @@ func TestProxyClient_StartMetricsStream_StreamError(t *testing.T) {
func TestProxyClient_StartClusterStateStream_NotConnected(t *testing.T) {
testLogger := initTestLogger(t)
fr := flightrecorder.NewFlightRecorder(1000000)
- pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, testLogger)
+ pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, nil, testLogger)
ctx := context.Background()
err := pc.StartClusterStateStream(ctx)
@@ -733,7 +738,7 @@ func TestProxyClient_StartClusterStateStream_NotConnected(t *testing.T) {
func TestProxyClient_StartClusterStateStream_NoAgentID(t *testing.T) {
testLogger := initTestLogger(t)
fr := flightrecorder.NewFlightRecorder(1000000)
- pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, testLogger)
+ pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, nil, testLogger)
mockClient := &mockFODCServiceClient{}
pc.streamsMu.Lock()
@@ -750,7 +755,7 @@ func TestProxyClient_StartClusterStateStream_NoAgentID(t *testing.T) {
func TestProxyClient_StartClusterStateStream_Success(t *testing.T) {
testLogger := initTestLogger(t)
fr := flightrecorder.NewFlightRecorder(1000000)
- pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, testLogger)
+ pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, nil, testLogger)
ctx := context.Background()
mockClient := &mockFODCServiceClient{}
@@ -783,7 +788,7 @@ func TestProxyClient_StartClusterStateStream_Success(t *testing.T) {
func TestProxyClient_StartClusterStateStream_StreamError(t *testing.T) {
testLogger := initTestLogger(t)
fr := flightrecorder.NewFlightRecorder(1000000)
- pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, testLogger)
+ pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, nil, testLogger)
ctx := context.Background()
mockClient := &mockFODCServiceClient{}
@@ -805,7 +810,7 @@ func TestProxyClient_StartClusterStateStream_StreamError(t *testing.T) {
func TestProxyClient_SendClusterTopology_NoStream(t *testing.T) {
testLogger := initTestLogger(t)
fr := flightrecorder.NewFlightRecorder(1000000)
- pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, testLogger)
+ pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, nil, testLogger)
err := pc.sendClusterTopology()
@@ -836,7 +841,7 @@ func TestProxyClient_SendClusterTopology_Success(t *testing.T) {
}
collector.SetClusterTopology(testTopology)
- pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, collector, testLogger)
+ pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, collector, nil, testLogger)
ctx := context.Background()
mockStream := newMockStreamClusterStateClient(ctx)
@@ -860,7 +865,7 @@ func TestProxyClient_SendClusterTopology_Success(t *testing.T) {
func TestProxyClient_SendHeartbeat_NoStream(t *testing.T) {
testLogger := initTestLogger(t)
fr := flightrecorder.NewFlightRecorder(1000000)
- pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, testLogger)
+ pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, nil, testLogger)
ctx := context.Background()
err := pc.SendHeartbeat(ctx)
@@ -875,7 +880,7 @@ func TestProxyClient_SendHeartbeat_Success(t *testing.T) {
pc := NewProxyClient(
"localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"},
map[string]string{"env": "test"},
- 5*time.Second, 10*time.Second, fr, nil, testLogger)
+ 5*time.Second, 10*time.Second, fr, nil, nil, testLogger)
ctx := context.Background()
mockStream := newMockRegisterAgentClient(ctx)
@@ -896,7 +901,7 @@ func TestProxyClient_SendHeartbeat_Success(t *testing.T) {
func TestProxyClient_SendHeartbeat_SendError(t *testing.T) {
testLogger := initTestLogger(t)
fr := flightrecorder.NewFlightRecorder(1000000)
- pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, testLogger)
+ pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, nil, testLogger)
ctx := context.Background()
mockStream := newMockRegisterAgentClient(ctx)
@@ -915,7 +920,7 @@ func TestProxyClient_SendHeartbeat_SendError(t *testing.T) {
func TestProxyClient_RetrieveAndSendMetrics_NoStream(t *testing.T) {
testLogger := initTestLogger(t)
fr := flightrecorder.NewFlightRecorder(1000000)
- pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, testLogger)
+ pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, nil, testLogger)
ctx := context.Background()
err := pc.RetrieveAndSendMetrics(ctx, nil)
@@ -927,7 +932,7 @@ func TestProxyClient_RetrieveAndSendMetrics_NoStream(t *testing.T) {
func TestProxyClient_RetrieveAndSendMetrics_NoDatasources(t *testing.T) {
testLogger := initTestLogger(t)
fr := flightrecorder.NewFlightRecorder(1000000)
- pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, testLogger)
+ pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, nil, testLogger)
ctx := context.Background()
mockStream := newMockStreamMetricsClient(ctx)
@@ -949,7 +954,7 @@ func TestProxyClient_RetrieveAndSendMetrics_NoDatasources(t *testing.T) {
func TestProxyClient_RetrieveAndSendMetrics_LatestMetrics(t *testing.T) {
testLogger := initTestLogger(t)
fr := flightrecorder.NewFlightRecorder(1000000)
- pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, testLogger)
+ pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, nil, testLogger)
// Add metrics to flight recorder
rawMetrics := []metrics.RawMetric{
@@ -991,7 +996,7 @@ func TestProxyClient_RetrieveAndSendMetrics_LatestMetrics(t *testing.T) {
func TestProxyClient_RetrieveAndSendMetrics_FilteredMetrics(t *testing.T) {
testLogger := initTestLogger(t)
fr := flightrecorder.NewFlightRecorder(1000000)
- pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, testLogger)
+ pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, nil, testLogger)
// Add metrics with timestamps
rawMetrics := []metrics.RawMetric{
@@ -1039,7 +1044,7 @@ func TestProxyClient_RetrieveAndSendMetrics_FilteredMetrics(t *testing.T) {
func TestProxyClient_RetrieveAndSendMetrics_FilteredNoTimestamps(t *testing.T) {
testLogger := initTestLogger(t)
fr := flightrecorder.NewFlightRecorder(1000000)
- pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, testLogger)
+ pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, nil, testLogger)
// Add metrics with timestamps far in the past
// Then filter for a future time window to ensure no matches
@@ -1083,7 +1088,7 @@ func TestProxyClient_RetrieveAndSendMetrics_FilteredNoTimestamps(t *testing.T) {
func TestProxyClient_Disconnect_Success(t *testing.T) {
testLogger := initTestLogger(t)
fr := flightrecorder.NewFlightRecorder(1000000)
- pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, testLogger)
+ pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, nil, testLogger)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@@ -1132,7 +1137,7 @@ func TestProxyClient_Disconnect_Success(t *testing.T) {
func TestProxyClient_Disconnect_Idempotent(t *testing.T) {
testLogger := initTestLogger(t)
fr := flightrecorder.NewFlightRecorder(1000000)
- pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, testLogger)
+ pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, nil, testLogger)
pc.streamsMu.Lock()
pc.disconnected = true
@@ -1149,7 +1154,7 @@ func TestProxyClient_Disconnect_Idempotent(t *testing.T) {
func TestProxyClient_handleRegistrationStream_EOF(t *testing.T) {
testLogger := initTestLogger(t)
fr := flightrecorder.NewFlightRecorder(1000000)
- pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, testLogger)
+ pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, nil, testLogger)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@@ -1165,7 +1170,7 @@ func TestProxyClient_handleRegistrationStream_EOF(t *testing.T) {
func TestProxyClient_handleRegistrationStream_ContextCanceled(t *testing.T) {
testLogger := initTestLogger(t)
fr := flightrecorder.NewFlightRecorder(1000000)
- pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, testLogger)
+ pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, nil, testLogger)
ctx, cancel := context.WithCancel(context.Background())
cancel()
@@ -1181,7 +1186,7 @@ func TestProxyClient_handleRegistrationStream_ContextCanceled(t *testing.T) {
func TestProxyClient_handleMetricsStream_EOF(t *testing.T) {
testLogger := initTestLogger(t)
fr := flightrecorder.NewFlightRecorder(1000000)
- pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, testLogger)
+ pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, nil, testLogger)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@@ -1197,7 +1202,7 @@ func TestProxyClient_handleMetricsStream_EOF(t *testing.T) {
func TestProxyClient_handleMetricsStream_RequestMetrics(t *testing.T) {
testLogger := initTestLogger(t)
fr := flightrecorder.NewFlightRecorder(1000000)
- pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, testLogger)
+ pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, nil, testLogger)
// Add metrics
rawMetrics := []metrics.RawMetric{
@@ -1243,7 +1248,7 @@ func TestProxyClient_handleMetricsStream_RequestMetrics(t *testing.T) {
func TestProxyClient_parseMetricKey_Simple(t *testing.T) {
testLogger := initTestLogger(t)
fr := flightrecorder.NewFlightRecorder(1000000)
- pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, testLogger)
+ pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, nil, testLogger)
key, err := pc.parseMetricKey("cpu_usage")
@@ -1255,7 +1260,7 @@ func TestProxyClient_parseMetricKey_Simple(t *testing.T) {
func TestProxyClient_parseMetricKey_WithLabels(t *testing.T) {
testLogger := initTestLogger(t)
fr := flightrecorder.NewFlightRecorder(1000000)
- pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, testLogger)
+ pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, nil, testLogger)
key, err := pc.parseMetricKey(`http_requests_total{method="GET",status="200"}`)
@@ -1267,7 +1272,7 @@ func TestProxyClient_parseMetricKey_WithLabels(t *testing.T) {
func TestProxyClient_parseMetricKey_InvalidFormat(t *testing.T) {
testLogger := initTestLogger(t)
fr := flightrecorder.NewFlightRecorder(1000000)
- pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, testLogger)
+ pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, nil, testLogger)
key, err := pc.parseMetricKey("invalid{format")
@@ -1279,7 +1284,7 @@ func TestProxyClient_parseMetricKey_InvalidFormat(t *testing.T) {
func TestProxyClient_parseMetricKey_MalformedLabels(t *testing.T) {
testLogger := initTestLogger(t)
fr := flightrecorder.NewFlightRecorder(1000000)
- pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, testLogger)
+ pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, nil, testLogger)
// Test with malformed labels - should still parse what it can
key, err := pc.parseMetricKey(`metric{invalid,key="value"}`)
@@ -1293,7 +1298,7 @@ func TestProxyClient_parseMetricKey_MalformedLabels(t *testing.T) {
func TestProxyClient_sendLatestMetrics_WithUnfinalizedValue(t *testing.T) {
testLogger := initTestLogger(t)
fr := flightrecorder.NewFlightRecorder(1000000)
- pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, testLogger)
+ pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, nil, testLogger)
// Create a datasource with metrics
ds := flightrecorder.NewDatasource()
@@ -1327,7 +1332,7 @@ func TestProxyClient_sendLatestMetrics_WithUnfinalizedValue(t *testing.T) {
func TestProxyClient_sendFilteredMetrics_TimeWindow(t *testing.T) {
testLogger := initTestLogger(t)
fr := flightrecorder.NewFlightRecorder(1000000)
- pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, testLogger)
+ pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, nil, testLogger)
// Create a datasource with metrics and timestamps
ds := flightrecorder.NewDatasource()
@@ -1376,7 +1381,7 @@ func TestProxyClient_sendFilteredMetrics_TimeWindow(t *testing.T) {
func TestProxyClient_sendFilteredMetrics_NoMatchingTimeWindow(t *testing.T) {
testLogger := initTestLogger(t)
fr := flightrecorder.NewFlightRecorder(1000000)
- pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, testLogger)
+ pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 10*time.Second, fr, nil, nil, testLogger)
// Create a datasource with metrics and timestamps
ds := flightrecorder.NewDatasource()
@@ -1422,7 +1427,7 @@ func TestProxyClient_sendFilteredMetrics_NoMatchingTimeWindow(t *testing.T) {
func TestProxyClient_Start_ContextCancellation(t *testing.T) {
testLogger := initTestLogger(t)
fr := flightrecorder.NewFlightRecorder(1000000)
- pc := NewProxyClient("invalid-address:99999", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 100*time.Millisecond, fr, nil, testLogger)
+ pc := NewProxyClient("invalid-address:99999", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 100*time.Millisecond, fr, nil, nil, testLogger)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@@ -1439,7 +1444,7 @@ func TestProxyClient_Start_ContextCancellation(t *testing.T) {
func TestProxyClient_Start_StopChannel(t *testing.T) {
testLogger := initTestLogger(t)
fr := flightrecorder.NewFlightRecorder(1000000)
- pc := NewProxyClient("invalid-address:99999", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 100*time.Millisecond, fr, nil, testLogger)
+ pc := NewProxyClient("invalid-address:99999", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 5*time.Second, 100*time.Millisecond, fr, nil, nil, testLogger)
ctx := context.Background()
@@ -1454,7 +1459,7 @@ func TestProxyClient_Start_StopChannel(t *testing.T) {
func TestProxyClient_startHeartbeat_Success(t *testing.T) {
testLogger := initTestLogger(t)
fr := flightrecorder.NewFlightRecorder(1000000)
- pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 100*time.Millisecond, 10*time.Second, fr, nil, testLogger)
+ pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 100*time.Millisecond, 10*time.Second, fr, nil, nil, testLogger)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@@ -1487,7 +1492,7 @@ func TestProxyClient_startHeartbeat_Success(t *testing.T) {
func TestProxyClient_startHeartbeat_ReplacesExistingTicker(t *testing.T) {
testLogger := initTestLogger(t)
fr := flightrecorder.NewFlightRecorder(1000000)
- pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 100*time.Millisecond, 10*time.Second, fr, nil, testLogger)
+ pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil, 100*time.Millisecond, 10*time.Second, fr, nil, nil, testLogger)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@@ -1511,3 +1516,44 @@ func TestProxyClient_startHeartbeat_ReplacesExistingTicker(t *testing.T) {
// Wait a bit for goroutine to exit
time.Sleep(50 * time.Millisecond)
}
+
+func TestProxyClient_reconnect_NoDeadlockWithActiveHeartbeat(t *testing.T) {
+ testLogger := initTestLogger(t)
+ fr := flightrecorder.NewFlightRecorder(1000000)
+ pc := NewProxyClient("localhost:8080", "datanode-hot", "192.168.1.1", []string{"data"}, nil,
+ 100*time.Millisecond, 10*time.Second, fr, nil, nil, testLogger)
+
+ // Use a short-lived context: reconnect now retries with backoff, so we cancel
+ // the context after enough time for cleanupStreams() to prove no deadlock.
+ ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+ defer cancel()
+
+ mockStream := newMockRegisterAgentClient(ctx)
+ pc.streamsMu.Lock()
+ pc.registrationStream = mockStream
+ pc.streamsMu.Unlock()
+
+ // Start heartbeat so that a goroutine is actively waiting on stopCh.
+ // Before the fix, reconnect() would call heartbeatWg.Wait() before
+ // closing stopCh, causing a deadlock because the heartbeat goroutine
+ // would never exit.
+ pc.startHeartbeat(ctx)
+
+ // Start connManager so RequestConnect doesn't block forever
+ pc.connManager.start(ctx)
+ defer pc.connManager.stop()
+
+ // Run reconnect in a goroutine and detect deadlock via timeout
+ done := make(chan struct{})
+ go func() {
+ pc.reconnect(ctx)
+ close(done)
+ }()
+
+ select {
+ case <-done:
+ // reconnect completed without deadlock (exited via context cancellation)
+ case <-time.After(5 * time.Second):
+ t.Fatal("reconnect deadlocked: heartbeatWg.Wait() blocked because stopCh was not closed before waiting")
+ }
+}
diff --git a/fodc/agent/testhelper/flightrecorder.go b/fodc/agent/testhelper/flightrecorder.go
index 931f53e2a..4b3390cf2 100644
--- a/fodc/agent/testhelper/flightrecorder.go
+++ b/fodc/agent/testhelper/flightrecorder.go
@@ -27,6 +27,7 @@ import (
fodcv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/fodc/v1"
agentcluster "github.com/apache/skywalking-banyandb/fodc/agent/internal/cluster"
"github.com/apache/skywalking-banyandb/fodc/agent/internal/flightrecorder"
+ agentlifecycle "github.com/apache/skywalking-banyandb/fodc/agent/internal/lifecycle"
agentmetrics "github.com/apache/skywalking-banyandb/fodc/agent/internal/metrics"
"github.com/apache/skywalking-banyandb/fodc/agent/internal/proxy"
"github.com/apache/skywalking-banyandb/pkg/logger"
@@ -174,16 +175,18 @@ func NewProxyClientWithCollector(
reconnectInterval,
frTyped,
collector,
+ nil,
logger,
)
}
// ProxyClientWrapper wraps ProxyClient methods for testing.
type ProxyClientWrapper struct {
- client *proxy.Client
- collector *agentcluster.Collector
- ctx context.Context
- connMgrActive bool
+ client *proxy.Client
+ collector *agentcluster.Collector
+ lifecycleCollector *agentlifecycle.Collector
+ ctx context.Context
+ connMgrActive bool
}
// StartConnManager starts the connection manager.
@@ -241,6 +244,11 @@ func (w *ProxyClientWrapper) SetClusterTopology(nodes []*databasev1.Node, calls
return nil
}
+// StartLifecycleStream establishes bi-directional lifecycle stream with Proxy.
+func (w *ProxyClientWrapper) StartLifecycleStream(ctx context.Context) error {
+ return w.client.StartLifecycleStream(ctx)
+}
+
// NewProxyClientWrapper creates a wrapped ProxyClient for testing.
func NewProxyClientWrapper(
proxyAddr string,
@@ -252,9 +260,15 @@ func NewProxyClientWrapper(
reconnectInterval time.Duration,
flightRecorder interface{},
logger *logger.Logger,
+ reportDir string,
) *ProxyClientWrapper {
- collector := &agentcluster.Collector{}
- client := NewProxyClientWithCollector(
+ frTyped, ok := flightRecorder.(*flightrecorder.FlightRecorder)
+ if !ok {
+ return nil
+ }
+ clusterCollector := &agentcluster.Collector{}
+ lifecycleCollector := agentlifecycle.NewCollector(nil, "", reportDir, 0)
+ client := proxy.NewClient(
proxyAddr,
nodeRole,
podName,
@@ -262,12 +276,14 @@ func NewProxyClientWrapper(
labels,
heartbeatInterval,
reconnectInterval,
- flightRecorder,
- collector,
+ frTyped,
+ clusterCollector,
+ lifecycleCollector,
logger,
)
- if client == nil {
- return nil
+ return &ProxyClientWrapper{
+ client: client,
+ collector: clusterCollector,
+ lifecycleCollector: lifecycleCollector,
}
- return &ProxyClientWrapper{client: client, collector: collector}
}
diff --git a/fodc/proxy/cmd/proxy/main.go b/fodc/proxy/cmd/proxy/main.go
index 7bbd328b6..5a921f390 100644
--- a/fodc/proxy/cmd/proxy/main.go
+++ b/fodc/proxy/cmd/proxy/main.go
@@ -30,6 +30,7 @@ import (
"github.com/apache/skywalking-banyandb/fodc/proxy/internal/api"
"github.com/apache/skywalking-banyandb/fodc/proxy/internal/cluster"
"github.com/apache/skywalking-banyandb/fodc/proxy/internal/grpc"
+ "github.com/apache/skywalking-banyandb/fodc/proxy/internal/lifecycle"
"github.com/apache/skywalking-banyandb/fodc/proxy/internal/metrics"
"github.com/apache/skywalking-banyandb/fodc/proxy/internal/registry"
"github.com/apache/skywalking-banyandb/pkg/logger"
@@ -123,12 +124,14 @@ func runProxy(_ *cobra.Command, _ []string) error {
defer agentRegistry.Stop()
clusterStateMgr := cluster.NewManager(agentRegistry, nil, log)
+ lifecycleMgr := lifecycle.NewManager(agentRegistry, nil, log)
metricsAggregator := metrics.NewAggregator(agentRegistry, nil, log)
- grpcService := grpc.NewFODCService(agentRegistry, metricsAggregator, clusterStateMgr, log, heartbeatInterval)
+ grpcService := grpc.NewFODCService(agentRegistry, metricsAggregator, clusterStateMgr, lifecycleMgr, log, heartbeatInterval)
metricsAggregator.SetGRPCService(grpcService)
clusterStateMgr.SetGRPCService(grpcService)
+ lifecycleMgr.SetGRPCService(grpcService)
grpcServer := grpc.NewServer(grpcService, grpcListenAddr, grpcMaxMsgSize, log)
if startErr := grpcServer.Start(); startErr != nil {
@@ -136,7 +139,7 @@ func runProxy(_ *cobra.Command, _ []string) error {
}
defer grpcServer.Stop()
- apiServer := api.NewServer(metricsAggregator, clusterStateMgr, agentRegistry, log)
+ apiServer := api.NewServer(metricsAggregator, clusterStateMgr, lifecycleMgr, agentRegistry, log)
if startErr := apiServer.Start(httpListenAddr, httpReadTimeout, httpWriteTimeout); startErr != nil {
return fmt.Errorf("failed to start HTTP API server: %w", startErr)
}
diff --git a/fodc/proxy/internal/api/server.go b/fodc/proxy/internal/api/server.go
index 7f3af5d5e..d823752ce 100644
--- a/fodc/proxy/internal/api/server.go
+++ b/fodc/proxy/internal/api/server.go
@@ -29,6 +29,7 @@ import (
"time"
"github.com/apache/skywalking-banyandb/fodc/proxy/internal/cluster"
+ "github.com/apache/skywalking-banyandb/fodc/proxy/internal/lifecycle"
"github.com/apache/skywalking-banyandb/fodc/proxy/internal/metrics"
"github.com/apache/skywalking-banyandb/fodc/proxy/internal/registry"
"github.com/apache/skywalking-banyandb/pkg/logger"
@@ -38,6 +39,7 @@ import (
type Server struct {
metricsAggregator *metrics.Aggregator
clusterStateCollector *cluster.Manager
+ lifecycleManager *lifecycle.Manager
registry *registry.AgentRegistry
server *http.Server
logger *logger.Logger
@@ -48,12 +50,14 @@ type Server struct {
func NewServer(
metricsAggregator *metrics.Aggregator,
clusterStateCollector *cluster.Manager,
+ lifecycleManager *lifecycle.Manager,
registry *registry.AgentRegistry,
logger *logger.Logger,
) *Server {
return &Server{
metricsAggregator: metricsAggregator,
clusterStateCollector: clusterStateCollector,
+ lifecycleManager: lifecycleManager,
registry: registry,
logger: logger,
startTime: time.Now(),
@@ -68,6 +72,7 @@ func (s *Server) Start(listenAddr string, readTimeout, writeTimeout time.Duratio
mux.HandleFunc("/metrics-windows", s.handleMetricsWindows)
mux.HandleFunc("/health", s.handleHealth)
mux.HandleFunc("/cluster/topology", s.handleClusterTopology)
+ mux.HandleFunc("/cluster/lifecycle", s.handleClusterLifecycle)
s.server = &http.Server{
Addr: listenAddr,
@@ -470,6 +475,24 @@ type timeSeriesMetric struct {
data []map[string]interface{}
}
+// handleClusterLifecycle handles GET /cluster/lifecycle endpoint.
+func (s *Server) handleClusterLifecycle(w http.ResponseWriter, r *http.Request) {
+ if r.Method != http.MethodGet {
+ http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+ return
+ }
+ if s.lifecycleManager == nil {
+ http.Error(w, "Lifecycle manager not available", http.StatusServiceUnavailable)
+ return
+ }
+ lifecycleData := s.lifecycleManager.CollectLifecycle(r.Context())
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(http.StatusOK)
+ if encodeErr := json.NewEncoder(w).Encode(lifecycleData); encodeErr != nil {
+ s.logger.Error().Err(encodeErr).Msg("Failed to encode lifecycle response")
+ }
+}
+
// handleClusterTopology handles GET /cluster/topology endpoint.
func (s *Server) handleClusterTopology(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
diff --git a/fodc/proxy/internal/api/server_test.go b/fodc/proxy/internal/api/server_test.go
index 113d6c857..e6182bb73 100644
--- a/fodc/proxy/internal/api/server_test.go
+++ b/fodc/proxy/internal/api/server_test.go
@@ -34,6 +34,7 @@ import (
databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
fodcv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/fodc/v1"
"github.com/apache/skywalking-banyandb/fodc/proxy/internal/cluster"
+ "github.com/apache/skywalking-banyandb/fodc/proxy/internal/lifecycle"
"github.com/apache/skywalking-banyandb/fodc/proxy/internal/metrics"
"github.com/apache/skywalking-banyandb/fodc/proxy/internal/registry"
"github.com/apache/skywalking-banyandb/pkg/logger"
@@ -88,7 +89,7 @@ func newTestServer(t *testing.T) (*Server, *registry.AgentRegistry, *mockCluster
mockRequester := &mockClusterDataRequester{}
clusterMgr := cluster.NewManager(testRegistry, mockRequester, testLogger)
mockRequester.clusterMgr = clusterMgr
- server := NewServer(aggregator, clusterMgr, testRegistry, testLogger)
+ server := NewServer(aggregator, clusterMgr, nil, testRegistry, testLogger)
return server, testRegistry, mockRequester
}
@@ -164,7 +165,7 @@ func TestHandleClusterTopology_NoClusterStateCollector(t *testing.T) {
testRegistry := registry.NewAgentRegistry(testLogger, 5*time.Second, 10*time.Second, 100)
mockSender := &mockRequestSender{}
aggregator := metrics.NewAggregator(testRegistry, mockSender, testLogger)
- server := NewServer(aggregator, nil, testRegistry, testLogger)
+ server := NewServer(aggregator, nil, nil, testRegistry, testLogger)
req := httptest.NewRequest(http.MethodGet, "/cluster/topology", nil)
resp := httptest.NewRecorder()
server.handleClusterTopology(resp, req)
@@ -179,7 +180,7 @@ func TestHandleClusterTopology_RequestClusterDataError(t *testing.T) {
aggregator := metrics.NewAggregator(testRegistry, mockSender, testLogger)
mockRequester := &mockClusterDataRequester{requestErr: assert.AnError}
clusterMgr := cluster.NewManager(testRegistry, mockRequester, testLogger)
- server := NewServer(aggregator, clusterMgr, testRegistry, testLogger)
+ server := NewServer(aggregator, clusterMgr, nil, testRegistry, testLogger)
ctx := context.Background()
identity := registry.AgentIdentity{
PodName: "test-pod",
@@ -201,7 +202,7 @@ func TestNewServer(t *testing.T) {
mockSender := &mockRequestSender{}
aggregator := metrics.NewAggregator(testRegistry, mockSender, testLogger)
- server := NewServer(aggregator, nil, testRegistry, testLogger)
+ server := NewServer(aggregator, nil, nil, testRegistry, testLogger)
assert.NotNil(t, server)
assert.Equal(t, aggregator, server.metricsAggregator)
@@ -675,6 +676,25 @@ func TestHandleMetricsWindows_WithFilters(t *testing.T) {
require.NoError(t, decodeErr)
}
+// mockLifecycleDataRequester is a mock implementation of lifecycle.RequestSender for testing.
+type mockLifecycleDataRequester struct {
+ requestErr error
+ lifecycleMgr *lifecycle.Manager
+ dataByAgent map[string]*fodcv1.LifecycleData
+ podByAgent map[string]string
+}
+
+func (m *mockLifecycleDataRequester) RequestLifecycleData(agentID string) error {
+ if m.requestErr != nil {
+ return m.requestErr
+ }
+ if data, exists := m.dataByAgent[agentID]; exists && m.lifecycleMgr != nil {
+ podName := m.podByAgent[agentID]
+ go m.lifecycleMgr.UpdateLifecycle(agentID, podName, data)
+ }
+ return nil
+}
+
func TestFormatPrometheusText_SortedMetrics(t *testing.T) {
server := newTestServerOnly(t)
@@ -693,3 +713,71 @@ func TestFormatPrometheusText_SortedMetrics(t *testing.T) {
assert.True(t, aIndex < mIndex && mIndex < zIndex, "Metrics should be sorted alphabetically")
}
+
+func TestHandleClusterLifecycle_Success(t *testing.T) {
+ initTestLogger(t)
+ testLogger := logger.GetLogger("test", "api")
+ testRegistry := registry.NewAgentRegistry(testLogger, 5*time.Second, 10*time.Second, 100)
+ mockSender := &mockRequestSender{}
+ aggregator := metrics.NewAggregator(testRegistry, mockSender, testLogger)
+
+ mockLifecycleRequester := &mockLifecycleDataRequester{
+ dataByAgent: make(map[string]*fodcv1.LifecycleData),
+ podByAgent: make(map[string]string),
+ }
+ lifecycleMgr := lifecycle.NewManager(testRegistry, mockLifecycleRequester, testLogger)
+ mockLifecycleRequester.lifecycleMgr = lifecycleMgr
+
+ ctx := context.Background()
+ identity := registry.AgentIdentity{
+ PodName: "test-pod-1",
+ Role: "datanode",
+ ContainerNames: []string{"banyandb"},
+ }
+ agentID, registerErr := testRegistry.RegisterAgent(ctx, identity)
+ require.NoError(t, registerErr)
+
+ mockLifecycleRequester.dataByAgent[agentID] = &fodcv1.LifecycleData{
+ Reports: []*fodcv1.LifecycleReport{
+ {Filename: "2026-03-26.json", ReportJson: `{"status":"ok"}`},
+ },
+ Groups: []*fodcv1.GroupLifecycleInfo{
+ {Name: "group1", Catalog: "measure"},
+ },
+ }
+ mockLifecycleRequester.podByAgent[agentID] = "test-pod-1"
+
+ server := NewServer(aggregator, nil, lifecycleMgr, testRegistry, testLogger)
+
+ req := httptest.NewRequest(http.MethodGet, "/cluster/lifecycle", nil)
+ resp := httptest.NewRecorder()
+ server.handleClusterLifecycle(resp, req)
+
+ assert.Equal(t, http.StatusOK, resp.Code)
+ assert.Equal(t, "application/json", resp.Header().Get("Content-Type"))
+
+ var result lifecycle.InspectionResult
+ decodeErr := json.NewDecoder(resp.Body).Decode(&result)
+ require.NoError(t, decodeErr)
+ assert.Equal(t, 1, len(result.Groups))
+ assert.Equal(t, "group1", result.Groups[0].Name)
+ assert.Equal(t, 1, len(result.LifecycleStatuses))
+ assert.Equal(t, "test-pod-1", result.LifecycleStatuses[0].PodName)
+ assert.Equal(t, 1, len(result.LifecycleStatuses[0].Reports))
+ assert.Equal(t, "2026-03-26.json", result.LifecycleStatuses[0].Reports[0].Filename)
+}
+
+func TestHandleClusterLifecycle_NoLifecycleManager(t *testing.T) {
+ initTestLogger(t)
+ testLogger := logger.GetLogger("test", "api")
+ testRegistry := registry.NewAgentRegistry(testLogger, 5*time.Second, 10*time.Second, 100)
+ mockSender := &mockRequestSender{}
+ aggregator := metrics.NewAggregator(testRegistry, mockSender, testLogger)
+ server := NewServer(aggregator, nil, nil, testRegistry, testLogger)
+
+ req := httptest.NewRequest(http.MethodGet, "/cluster/lifecycle", nil)
+ resp := httptest.NewRecorder()
+ server.handleClusterLifecycle(resp, req)
+
+ assert.Equal(t, http.StatusServiceUnavailable, resp.Code)
+}
diff --git a/fodc/proxy/internal/grpc/server_test.go b/fodc/proxy/internal/grpc/server_test.go
index 6190efa0b..e2f41d425 100644
--- a/fodc/proxy/internal/grpc/server_test.go
+++ b/fodc/proxy/internal/grpc/server_test.go
@@ -35,7 +35,7 @@ func TestNewServer(t *testing.T) {
testRegistry := registry.NewAgentRegistry(testLogger, 5*time.Second, 10*time.Second, 100)
mockSender := &mockRequestSender{}
aggregator := metrics.NewAggregator(testRegistry, mockSender, testLogger)
- service := NewFODCService(testRegistry, aggregator, nil, testLogger, 30*time.Second)
+ service := NewFODCService(testRegistry, aggregator, nil, nil, testLogger, 30*time.Second)
server := NewServer(service, "localhost:0", 1024*1024, testLogger)
@@ -51,7 +51,7 @@ func TestServer_StartStop(t *testing.T) {
testRegistry := registry.NewAgentRegistry(testLogger, 5*time.Second, 10*time.Second, 100)
mockSender := &mockRequestSender{}
aggregator := metrics.NewAggregator(testRegistry, mockSender, testLogger)
- service := NewFODCService(testRegistry, aggregator, nil, testLogger, 30*time.Second)
+ service := NewFODCService(testRegistry, aggregator, nil, nil, testLogger, 30*time.Second)
server := NewServer(service, "localhost:0", 1024*1024, testLogger)
@@ -69,7 +69,7 @@ func TestServer_Stop_NotStarted(t *testing.T) {
testRegistry := registry.NewAgentRegistry(testLogger, 5*time.Second, 10*time.Second, 100)
mockSender := &mockRequestSender{}
aggregator := metrics.NewAggregator(testRegistry, mockSender, testLogger)
- service := NewFODCService(testRegistry, aggregator, nil, testLogger, 30*time.Second)
+ service := NewFODCService(testRegistry, aggregator, nil, nil, testLogger, 30*time.Second)
server := NewServer(service, "localhost:0", 1024*1024, testLogger)
@@ -82,7 +82,7 @@ func TestServer_Start_InvalidAddress(t *testing.T) {
testRegistry := registry.NewAgentRegistry(testLogger, 5*time.Second, 10*time.Second, 100)
mockSender := &mockRequestSender{}
aggregator := metrics.NewAggregator(testRegistry, mockSender, testLogger)
- service := NewFODCService(testRegistry, aggregator, nil, testLogger, 30*time.Second)
+ service := NewFODCService(testRegistry, aggregator, nil, nil, testLogger, 30*time.Second)
server := NewServer(service, "invalid-address", 1024*1024, testLogger)
diff --git a/fodc/proxy/internal/grpc/service.go b/fodc/proxy/internal/grpc/service.go
index 04ea1c2e2..e33456443 100644
--- a/fodc/proxy/internal/grpc/service.go
+++ b/fodc/proxy/internal/grpc/service.go
@@ -34,6 +34,7 @@ import (
fodcv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/fodc/v1"
"github.com/apache/skywalking-banyandb/fodc/proxy/internal/cluster"
+ "github.com/apache/skywalking-banyandb/fodc/proxy/internal/lifecycle"
"github.com/apache/skywalking-banyandb/fodc/proxy/internal/metrics"
"github.com/apache/skywalking-banyandb/fodc/proxy/internal/registry"
"github.com/apache/skywalking-banyandb/pkg/logger"
@@ -43,6 +44,7 @@ import (
type agentConnection struct {
metricsStream fodcv1.FODCService_StreamMetricsServer
clusterStateStream fodcv1.FODCService_StreamClusterTopologyServer
+ lifecycleStream fodcv1.FODCService_StreamLifecycleServer
lastActivity time.Time
agentID string
mu sync.RWMutex
@@ -112,12 +114,43 @@ func (ac *agentConnection) sendClusterDataRequest() error {
return nil
}
+// setLifecycleStream sets the lifecycle stream.
+func (ac *agentConnection) setLifecycleStream(stream fodcv1.FODCService_StreamLifecycleServer) {
+ ac.mu.Lock()
+ defer ac.mu.Unlock()
+ ac.lifecycleStream = stream
+}
+
+// hasLifecycleStream reports whether the lifecycle stream is established.
+func (ac *agentConnection) hasLifecycleStream() bool {
+ ac.mu.RLock()
+ defer ac.mu.RUnlock()
+ return ac.lifecycleStream != nil
+}
+
+// sendLifecycleDataRequest sends a lifecycle data request to the agent via the lifecycle stream.
+func (ac *agentConnection) sendLifecycleDataRequest() error {
+ ac.mu.RLock()
+ defer ac.mu.RUnlock()
+ if ac.lifecycleStream == nil {
+ return fmt.Errorf("lifecycle stream not established for agent ID: %s", ac.agentID)
+ }
+ resp := &fodcv1.StreamLifecycleResponse{
+ RequestLifecycle: true,
+ }
+ if err := ac.lifecycleStream.Send(resp); err != nil {
+ return fmt.Errorf("failed to send lifecycle data request: %w", err)
+ }
+ return nil
+}
+
// FODCService implements the FODC gRPC service.
type FODCService struct {
fodcv1.UnimplementedFODCServiceServer
registry *registry.AgentRegistry
metricsAggregator *metrics.Aggregator
clusterStateManager *cluster.Manager
+ lifecycleManager *lifecycle.Manager
logger *logger.Logger
connections map[string]*agentConnection
connectionsMu sync.RWMutex
@@ -129,6 +162,7 @@ func NewFODCService(
registry *registry.AgentRegistry,
metricsAggregator *metrics.Aggregator,
clusterStateManager *cluster.Manager,
+ lifecycleManager *lifecycle.Manager,
logger *logger.Logger,
heartbeatInterval time.Duration,
) *FODCService {
@@ -136,6 +170,7 @@ func NewFODCService(
registry: registry,
metricsAggregator: metricsAggregator,
clusterStateManager: clusterStateManager,
+ lifecycleManager: lifecycleManager,
logger: logger,
connections: make(map[string]*agentConnection),
heartbeatInterval: heartbeatInterval,
@@ -153,6 +188,17 @@ func (s *FODCService) HasClusterStateStream(agentID string) bool {
return conn.hasClusterStateStream()
}
+// HasLifecycleStream reports whether the given agent currently has a lifecycle stream.
+func (s *FODCService) HasLifecycleStream(agentID string) bool {
+ s.connectionsMu.RLock()
+ defer s.connectionsMu.RUnlock()
+ conn, exists := s.connections[agentID]
+ if !exists || conn == nil {
+ return false
+ }
+ return conn.hasLifecycleStream()
+}
+
// RegisterAgent handles bi-directional agent registration stream.
func (s *FODCService) RegisterAgent(stream fodcv1.FODCService_RegisterAgentServer) error {
ctx, cancel := context.WithCancel(stream.Context())
@@ -481,3 +527,82 @@ func (s *FODCService) RequestClusterData(agentID string) error {
}
return agentConn.sendClusterDataRequest()
}
+
+// StreamLifecycle handles bi-directional lifecycle data streaming.
+func (s *FODCService) StreamLifecycle(stream fodcv1.FODCService_StreamLifecycleServer) error {
+ ctx := stream.Context()
+ agentID := s.getAgentIDFromContext(ctx)
+ if agentID == "" {
+ agentID = s.getAgentIDFromPeer(ctx)
+ if agentID != "" {
+ s.logger.Warn().
+ Str("agent_id", agentID).
+ Msg("Agent ID not found in metadata, using peer address fallback (this may be unreliable)")
+ }
+ }
+ if agentID == "" {
+ s.logger.Error().Msg("Agent ID not found in context metadata or peer address for lifecycle stream")
+ return status.Errorf(codes.Unauthenticated, "agent ID not found in context or peer address")
+ }
+
+ s.connectionsMu.Lock()
+ existingConn, exists := s.connections[agentID]
+ if exists {
+ existingConn.setLifecycleStream(stream)
+ existingConn.updateActivity()
+ } else {
+ agentConn := &agentConnection{
+ agentID: agentID,
+ lifecycleStream: stream,
+ lastActivity: time.Now(),
+ }
+ s.connections[agentID] = agentConn
+ }
+ s.connectionsMu.Unlock()
+
+ for {
+ req, err := stream.Recv()
+ if errors.Is(err, io.EOF) {
+ s.logger.Debug().Str("agent_id", agentID).Msg("Lifecycle stream closed by agent")
+ return nil
+ }
+ if err != nil {
+ if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+ s.logger.Debug().Err(err).Str("agent_id", agentID).Msg("Lifecycle stream closed")
+ } else if st, ok := status.FromError(err); ok {
+ code := st.Code()
+ if code == codes.Canceled || code == codes.DeadlineExceeded {
+ s.logger.Debug().Err(err).Str("agent_id", agentID).Msg("Lifecycle stream closed")
+ } else {
+ s.logger.Error().Err(err).Str("agent_id", agentID).Msg("Error receiving lifecycle data")
+ }
+ } else {
+ s.logger.Error().Err(err).Str("agent_id", agentID).Msg("Error receiving lifecycle data")
+ }
+ return err
+ }
+ if s.lifecycleManager != nil {
+ data := req.LifecycleData
+ if data == nil {
+ data = &fodcv1.LifecycleData{}
+ }
+ s.lifecycleManager.UpdateLifecycle(agentID, req.PodName, data)
+ s.logger.Debug().
+ Str("agent_id", agentID).
+ Str("pod_name", req.PodName).
+ Int("reports_count", len(data.Reports)).
+ Msg("Received lifecycle data from agent")
+ }
+ }
+}
+
+// RequestLifecycleData requests lifecycle data from an agent via the lifecycle stream.
+func (s *FODCService) RequestLifecycleData(agentID string) error {
+ s.connectionsMu.RLock()
+ agentConn, exists := s.connections[agentID]
+ s.connectionsMu.RUnlock()
+ if !exists {
+ return fmt.Errorf("agent connection not found for agent ID: %s", agentID)
+ }
+ return agentConn.sendLifecycleDataRequest()
+}
diff --git a/fodc/proxy/internal/grpc/service_test.go b/fodc/proxy/internal/grpc/service_test.go
index a47df96be..7e98a0b44 100644
--- a/fodc/proxy/internal/grpc/service_test.go
+++ b/fodc/proxy/internal/grpc/service_test.go
@@ -221,7 +221,7 @@ func newTestService(t *testing.T) (*FODCService, *registry.AgentRegistry) {
testRegistry := registry.NewAgentRegistry(testLogger, 5*time.Second, 10*time.Second, 100)
mockSender := &mockRequestSender{}
aggregator := metrics.NewAggregator(testRegistry, mockSender, testLogger)
- service := NewFODCService(testRegistry, aggregator, nil, testLogger, 30*time.Second)
+ service := NewFODCService(testRegistry, aggregator, nil, nil, testLogger, 30*time.Second)
return service, testRegistry
}
@@ -238,7 +238,7 @@ func TestNewFODCService(t *testing.T) {
mockSender := &mockRequestSender{}
aggregator := metrics.NewAggregator(testRegistry, mockSender, testLogger)
- service := NewFODCService(testRegistry, aggregator, nil, testLogger, 30*time.Second)
+ service := NewFODCService(testRegistry, aggregator, nil, nil, testLogger, 30*time.Second)
assert.NotNil(t, service)
assert.Equal(t, testRegistry, service.registry)
@@ -1078,3 +1078,294 @@ func TestRequestClusterData_SendError(t *testing.T) {
assert.Error(t, requestErr)
assert.Contains(t, requestErr.Error(), "send error")
}
+
+// mockStreamLifecycleServer implements fodcv1.FODCService_StreamLifecycleServer for testing.
+type mockStreamLifecycleServer struct {
+ ctx context.Context
+ recvErr error
+ sendErr error
+ sentResponses []*fodcv1.StreamLifecycleResponse
+ recvRequests []*fodcv1.StreamLifecycleRequest
+ mu sync.Mutex
+}
+
+func newMockStreamLifecycleServer(ctx context.Context) *mockStreamLifecycleServer {
+ return &mockStreamLifecycleServer{
+ ctx: ctx,
+ sentResponses: make([]*fodcv1.StreamLifecycleResponse, 0),
+ recvRequests: make([]*fodcv1.StreamLifecycleRequest, 0),
+ }
+}
+
+func (m *mockStreamLifecycleServer) Send(resp *fodcv1.StreamLifecycleResponse) error {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ if m.sendErr != nil {
+ return m.sendErr
+ }
+ m.sentResponses = append(m.sentResponses, resp)
+ return nil
+}
+
+func (m *mockStreamLifecycleServer) Recv() (*fodcv1.StreamLifecycleRequest, error) {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ if len(m.recvRequests) > 0 {
+ req := m.recvRequests[0]
+ m.recvRequests = m.recvRequests[1:]
+ return req, nil
+ }
+ if m.recvErr != nil {
+ return nil, m.recvErr
+ }
+ return nil, io.EOF
+}
+
+func (m *mockStreamLifecycleServer) SetRecvError(err error) {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ m.recvErr = err
+}
+
+func (m *mockStreamLifecycleServer) SetSendError(err error) {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ m.sendErr = err
+}
+
+func (m *mockStreamLifecycleServer) AddRequest(req *fodcv1.StreamLifecycleRequest) {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ m.recvRequests = append(m.recvRequests, req)
+}
+
+func (m *mockStreamLifecycleServer) GetSentResponses() []*fodcv1.StreamLifecycleResponse {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ result := make([]*fodcv1.StreamLifecycleResponse, len(m.sentResponses))
+ copy(result, m.sentResponses)
+ return result
+}
+
+func (m *mockStreamLifecycleServer) Context() context.Context {
+ return m.ctx
+}
+
+func (m *mockStreamLifecycleServer) SendMsg(_ interface{}) error {
+ return nil
+}
+
+func (m *mockStreamLifecycleServer) RecvMsg(_ interface{}) error {
+ return nil
+}
+
+func (m *mockStreamLifecycleServer) SetHeader(_ metadata.MD) error {
+ return nil
+}
+
+func (m *mockStreamLifecycleServer) SendHeader(_ metadata.MD) error {
+ return nil
+}
+
+func (m *mockStreamLifecycleServer) SetTrailer(_ metadata.MD) {
+}
+
+func TestStreamLifecycle_Success(t *testing.T) {
+ initTestLogger(t)
+ service, testRegistry := newTestService(t)
+ ctx := context.Background()
+ identity := registry.AgentIdentity{
+ PodName: "test-pod",
+ Role: "datanode",
+ ContainerNames: []string{"banyandb"},
+ }
+ agentID, registerErr := testRegistry.RegisterAgent(ctx, identity)
+ require.NoError(t, registerErr)
+ md := metadata.New(map[string]string{
+ "agent_id": agentID,
+ })
+ ctxWithMetadata := metadata.NewIncomingContext(ctx, md)
+ mockStream := newMockStreamLifecycleServer(ctxWithMetadata)
+ lifecycleReq := &fodcv1.StreamLifecycleRequest{
+ PodName: "test-pod",
+ Timestamp: timestamppb.Now(),
+ LifecycleData: &fodcv1.LifecycleData{
+ Reports: []*fodcv1.LifecycleReport{
+ {
+ Filename: "2026-03-26.json",
+ ReportJson: `{"status":"ok"}`,
+ },
+ },
+ },
+ }
+ mockStream.AddRequest(lifecycleReq)
+ mockStream.SetRecvError(io.EOF)
+ streamErr := service.StreamLifecycle(mockStream)
+ assert.NoError(t, streamErr)
+ service.connectionsMu.RLock()
+ conn, exists := service.connections[agentID]
+ service.connectionsMu.RUnlock()
+ assert.True(t, exists, "connection should remain (only RegisterAgent cleans up)")
+ assert.NotNil(t, conn)
+}
+
+func TestStreamLifecycle_NoAgentID(t *testing.T) {
+ initTestLogger(t)
+ service, _ := newTestService(t)
+ ctx := context.Background()
+ mockStream := newMockStreamLifecycleServer(ctx)
+ lifecycleReq := &fodcv1.StreamLifecycleRequest{
+ PodName: "test-pod",
+ Timestamp: timestamppb.Now(),
+ LifecycleData: &fodcv1.LifecycleData{
+ Reports: []*fodcv1.LifecycleReport{
+ {
+ Filename: "2026-03-26.json",
+ ReportJson: `{"status":"ok"}`,
+ },
+ },
+ },
+ }
+ mockStream.AddRequest(lifecycleReq)
+ mockStream.SetRecvError(io.EOF)
+ streamErr := service.StreamLifecycle(mockStream)
+ assert.Error(t, streamErr)
+ assert.Contains(t, streamErr.Error(), "agent ID not found")
+}
+
+func TestStreamLifecycle_RecvError(t *testing.T) {
+ initTestLogger(t)
+ service, testRegistry := newTestService(t)
+ ctx := context.Background()
+ identity := registry.AgentIdentity{
+ PodName: "test-pod",
+ Role: "datanode",
+ ContainerNames: []string{"banyandb"},
+ }
+ agentID, registerErr := testRegistry.RegisterAgent(ctx, identity)
+ require.NoError(t, registerErr)
+ md := metadata.New(map[string]string{
+ "agent_id": agentID,
+ })
+ ctxWithMetadata := metadata.NewIncomingContext(ctx, md)
+ mockStream := newMockStreamLifecycleServer(ctxWithMetadata)
+ testErr := errors.New("recv error")
+ mockStream.SetRecvError(testErr)
+ streamErr := service.StreamLifecycle(mockStream)
+ assert.Error(t, streamErr)
+}
+
+func TestRequestLifecycleData_Success(t *testing.T) {
+ initTestLogger(t)
+ service, testRegistry := newTestService(t)
+ ctx := context.Background()
+ identity := registry.AgentIdentity{
+ PodName: "test-pod",
+ Role: "datanode",
+ ContainerNames: []string{"banyandb"},
+ }
+ agentID, registerErr := testRegistry.RegisterAgent(ctx, identity)
+ require.NoError(t, registerErr)
+ mockStream := newMockStreamLifecycleServer(ctx)
+ service.connectionsMu.Lock()
+ service.connections[agentID] = &agentConnection{
+ agentID: agentID,
+ lifecycleStream: mockStream,
+ }
+ service.connectionsMu.Unlock()
+ requestErr := service.RequestLifecycleData(agentID)
+ assert.NoError(t, requestErr)
+ responses := mockStream.GetSentResponses()
+ require.Len(t, responses, 1)
+ assert.True(t, responses[0].RequestLifecycle)
+}
+
+func TestRequestLifecycleData_NoConnection(t *testing.T) {
+ initTestLogger(t)
+ service, _ := newTestService(t)
+ requestErr := service.RequestLifecycleData("non-existent-agent")
+ assert.Error(t, requestErr)
+ assert.Contains(t, requestErr.Error(), "connection not found")
+}
+
+func TestRequestLifecycleData_NoStream(t *testing.T) {
+ initTestLogger(t)
+ service, testRegistry := newTestService(t)
+ ctx := context.Background()
+ identity := registry.AgentIdentity{
+ PodName: "test-pod",
+ Role: "datanode",
+ ContainerNames: []string{"banyandb"},
+ }
+ agentID, registerErr := testRegistry.RegisterAgent(ctx, identity)
+ require.NoError(t, registerErr)
+ service.connectionsMu.Lock()
+ service.connections[agentID] = &agentConnection{
+ agentID: agentID,
+ lifecycleStream: nil,
+ }
+ service.connectionsMu.Unlock()
+ requestErr := service.RequestLifecycleData(agentID)
+ assert.Error(t, requestErr)
+ assert.Contains(t, requestErr.Error(), "lifecycle stream not established")
+}
+
+func TestRequestLifecycleData_SendError(t *testing.T) {
+ initTestLogger(t)
+ service, testRegistry := newTestService(t)
+ ctx := context.Background()
+ identity := registry.AgentIdentity{
+ PodName: "test-pod",
+ Role: "datanode",
+ ContainerNames: []string{"banyandb"},
+ }
+ agentID, registerErr := testRegistry.RegisterAgent(ctx, identity)
+ require.NoError(t, registerErr)
+ mockStream := newMockStreamLifecycleServer(ctx)
+ testErr := errors.New("send error")
+ mockStream.SetSendError(testErr)
+ service.connectionsMu.Lock()
+ service.connections[agentID] = &agentConnection{
+ agentID: agentID,
+ lifecycleStream: mockStream,
+ }
+ service.connectionsMu.Unlock()
+ requestErr := service.RequestLifecycleData(agentID)
+ assert.Error(t, requestErr)
+ assert.Contains(t, requestErr.Error(), "send error")
+}
+
+func TestHasLifecycleStream(t *testing.T) {
+ initTestLogger(t)
+ service, testRegistry := newTestService(t)
+ ctx := context.Background()
+ identity := registry.AgentIdentity{
+ PodName: "test-pod",
+ Role: "datanode",
+ ContainerNames: []string{"banyandb"},
+ }
+ agentID, registerErr := testRegistry.RegisterAgent(ctx, identity)
+ require.NoError(t, registerErr)
+
+ // No connection at all
+ assert.False(t, service.HasLifecycleStream(agentID))
+
+ // Connection exists but no lifecycle stream
+ service.connectionsMu.Lock()
+ service.connections[agentID] = &agentConnection{
+ agentID: agentID,
+ lifecycleStream: nil,
+ }
+ service.connectionsMu.Unlock()
+ assert.False(t, service.HasLifecycleStream(agentID))
+
+ // Connection exists with lifecycle stream
+ mockStream := newMockStreamLifecycleServer(ctx)
+ service.connectionsMu.Lock()
+ service.connections[agentID].setLifecycleStream(mockStream)
+ service.connectionsMu.Unlock()
+ assert.True(t, service.HasLifecycleStream(agentID))
+
+ // Non-existent agent
+ assert.False(t, service.HasLifecycleStream("non-existent-agent"))
+}
diff --git a/fodc/proxy/internal/integration/agent_reconnection_test.go b/fodc/proxy/internal/integration/agent_reconnection_test.go
index 171efd915..332bd3970 100644
--- a/fodc/proxy/internal/integration/agent_reconnection_test.go
+++ b/fodc/proxy/internal/integration/agent_reconnection_test.go
@@ -62,7 +62,7 @@ var _ = Describe("Test Case 3: Agent Reconnection", func() {
heartbeatInterval = 2 * time.Second
agentRegistry = registry.NewAgentRegistry(testLogger, heartbeatTimeout, cleanupTimeout, 100)
metricsAggregator = metricsproxy.NewAggregator(agentRegistry, nil, testLogger)
- grpcService = grpcproxy.NewFODCService(agentRegistry, metricsAggregator, nil, testLogger, heartbeatInterval)
+ grpcService = grpcproxy.NewFODCService(agentRegistry, metricsAggregator, nil, nil, testLogger, heartbeatInterval)
metricsAggregator.SetGRPCService(grpcService)
grpcListener, listenErr := net.Listen("tcp", "localhost:0")
@@ -78,7 +78,7 @@ var _ = Describe("Test Case 3: Agent Reconnection", func() {
Expect(httpListenErr).NotTo(HaveOccurred())
proxyHTTPAddr = httpListener.Addr().String()
_ = httpListener.Close()
- httpServer = api.NewServer(metricsAggregator, nil, agentRegistry, testLogger)
+ httpServer = api.NewServer(metricsAggregator, nil, nil, agentRegistry, testLogger)
Expect(httpServer.Start(proxyHTTPAddr, 10*time.Second, 10*time.Second)).To(Succeed())
Eventually(func() error {
@@ -103,6 +103,7 @@ var _ = Describe("Test Case 3: Agent Reconnection", func() {
1*time.Second,
flightRecorder,
testLogger,
+ "",
)
Expect(proxyClient).NotTo(BeNil())
@@ -251,6 +252,7 @@ var _ = Describe("Test Case 3: Agent Reconnection", func() {
1*time.Second,
flightRecorder,
testLogger,
+ "",
)
Expect(proxyClient).NotTo(BeNil())
diff --git a/fodc/proxy/internal/integration/agent_registration_metrics_flow_test.go b/fodc/proxy/internal/integration/agent_registration_metrics_flow_test.go
index 87ad76500..7a14e3345 100644
--- a/fodc/proxy/internal/integration/agent_registration_metrics_flow_test.go
+++ b/fodc/proxy/internal/integration/agent_registration_metrics_flow_test.go
@@ -62,7 +62,7 @@ var _ = Describe("Test Case 1: Agent Registration and Metrics Flow", func() {
heartbeatInterval := 2 * time.Second
agentRegistry = registry.NewAgentRegistry(testLogger, heartbeatTimeout, cleanupTimeout, 100)
metricsAggregator = metricsproxy.NewAggregator(agentRegistry, nil, testLogger)
- grpcService = grpcproxy.NewFODCService(agentRegistry, metricsAggregator, nil, testLogger, heartbeatInterval)
+ grpcService = grpcproxy.NewFODCService(agentRegistry, metricsAggregator, nil, nil, testLogger, heartbeatInterval)
metricsAggregator.SetGRPCService(grpcService)
// Start gRPC server on random port
@@ -81,7 +81,7 @@ var _ = Describe("Test Case 1: Agent Registration and Metrics Flow", func() {
Expect(httpListenErr).NotTo(HaveOccurred())
proxyHTTPAddr = httpListener.Addr().String()
_ = httpListener.Close() // Close listener so port is available for server
- httpServer = api.NewServer(metricsAggregator, nil, agentRegistry, testLogger)
+ httpServer = api.NewServer(metricsAggregator, nil, nil, agentRegistry, testLogger)
// Start HTTP server using the address from the listener
Expect(httpServer.Start(proxyHTTPAddr, 10*time.Second, 10*time.Second)).To(Succeed())
@@ -110,6 +110,7 @@ var _ = Describe("Test Case 1: Agent Registration and Metrics Flow", func() {
1*time.Second,
flightRecorder,
testLogger,
+ "",
)
Expect(proxyClient).NotTo(BeNil())
diff --git a/fodc/proxy/internal/integration/cluster_topology_integration_test.go b/fodc/proxy/internal/integration/cluster_topology_integration_test.go
index 3bf6dd4eb..01ee18b00 100644
--- a/fodc/proxy/internal/integration/cluster_topology_integration_test.go
+++ b/fodc/proxy/internal/integration/cluster_topology_integration_test.go
@@ -76,7 +76,7 @@ var _ = Describe("Cluster Topology Integration", func() {
clusterManager = cluster.NewManager(agentRegistry, nil, testLogger)
metricsAggregator = metricsproxy.NewAggregator(agentRegistry, nil, testLogger)
- grpcService = grpcproxy.NewFODCService(agentRegistry, metricsAggregator, clusterManager, testLogger, heartbeatInterval)
+ grpcService = grpcproxy.NewFODCService(agentRegistry, metricsAggregator, clusterManager, nil, testLogger, heartbeatInterval)
metricsAggregator.SetGRPCService(grpcService)
clusterManager.SetGRPCService(grpcService)
@@ -93,7 +93,7 @@ var _ = Describe("Cluster Topology Integration", func() {
Expect(httpListenErr).NotTo(HaveOccurred())
proxyHTTPAddr = httpListener.Addr().String()
_ = httpListener.Close()
- httpServer = api.NewServer(metricsAggregator, clusterManager, agentRegistry, testLogger)
+ httpServer = api.NewServer(metricsAggregator, clusterManager, nil, agentRegistry, testLogger)
Expect(httpServer.Start(proxyHTTPAddr, 10*time.Second, 10*time.Second)).To(Succeed())
Eventually(func() error {
@@ -153,6 +153,7 @@ var _ = Describe("Cluster Topology Integration", func() {
reconnectInterval,
flightRecorder1,
testLogger,
+ "",
)
Expect(proxyClient1).NotTo(BeNil())
proxyClient2 = testhelper.NewProxyClientWrapper(
@@ -165,6 +166,7 @@ var _ = Describe("Cluster Topology Integration", func() {
reconnectInterval,
flightRecorder2,
testLogger,
+ "",
)
Expect(proxyClient2).NotTo(BeNil())
diff --git a/fodc/proxy/internal/integration/failure_scenarios_test.go b/fodc/proxy/internal/integration/failure_scenarios_test.go
index a77b8bb59..c4176ec0c 100644
--- a/fodc/proxy/internal/integration/failure_scenarios_test.go
+++ b/fodc/proxy/internal/integration/failure_scenarios_test.go
@@ -68,7 +68,7 @@ func setupFailureFixture() *failureFixture {
reg := registry.NewAgentRegistry(testLogger, heartbeatTimeout, cleanupTimeout, 100)
agg := metricsproxy.NewAggregator(reg, nil, testLogger)
- svc := grpcproxy.NewFODCService(reg, agg, nil, testLogger, heartbeatInterval)
+ svc := grpcproxy.NewFODCService(reg, agg, nil, nil, testLogger, heartbeatInterval)
agg.SetGRPCService(svc)
grpcListener, listenErr := net.Listen("tcp", "localhost:0")
@@ -82,7 +82,7 @@ func setupFailureFixture() *failureFixture {
Expect(httpErr).NotTo(HaveOccurred())
httpAddr := httpListener.Addr().String()
_ = httpListener.Close()
- httpSrv := api.NewServer(agg, nil, reg, testLogger)
+ httpSrv := api.NewServer(agg, nil, nil, reg, testLogger)
Expect(httpSrv.Start(httpAddr, 10*time.Second, 10*time.Second)).To(Succeed())
Eventually(func() error {
@@ -140,6 +140,7 @@ func startAgentForFixture(f *failureFixture, podName string, role string) *agent
1*time.Second,
fr,
logger.GetLogger("test", fmt.Sprintf("agent-%s", podName)),
+ "",
)
Expect(client).NotTo(BeNil())
@@ -328,7 +329,7 @@ var _ = Describe("Failure Scenarios", func() {
Expect(agent.client.Disconnect()).To(Succeed())
fixture.grpcServer.Stop()
- fixture.service = grpcproxy.NewFODCService(fixture.registry, fixture.aggregator, nil, fixture.logger, 2*time.Second)
+ fixture.service = grpcproxy.NewFODCService(fixture.registry, fixture.aggregator, nil, nil, fixture.logger, 2*time.Second)
fixture.aggregator.SetGRPCService(fixture.service)
fixture.grpcServer = grpcproxy.NewServer(fixture.service, fixture.proxyGRPCAddr, 4194304, fixture.logger)
Expect(fixture.grpcServer.Start()).To(Succeed())
diff --git a/fodc/proxy/internal/integration/full_proxy_workflow_test.go b/fodc/proxy/internal/integration/full_proxy_workflow_test.go
index 49fbc9f1f..c2c735c5e 100644
--- a/fodc/proxy/internal/integration/full_proxy_workflow_test.go
+++ b/fodc/proxy/internal/integration/full_proxy_workflow_test.go
@@ -68,7 +68,7 @@ var _ = Describe("Full FODC Proxy Workflow", func() {
heartbeatInterval := 2 * time.Second
agentRegistry = registry.NewAgentRegistry(testLogger, heartbeatTimeout, cleanupTimeout, 100)
metricsAggregator = metricsproxy.NewAggregator(agentRegistry, nil, testLogger)
- grpcService = grpcproxy.NewFODCService(agentRegistry, metricsAggregator, nil, testLogger, heartbeatInterval)
+ grpcService = grpcproxy.NewFODCService(agentRegistry, metricsAggregator, nil, nil, testLogger, heartbeatInterval)
metricsAggregator.SetGRPCService(grpcService)
grpcListener, listenErr := net.Listen("tcp", "localhost:0")
@@ -84,7 +84,7 @@ var _ = Describe("Full FODC Proxy Workflow", func() {
Expect(httpListenErr).NotTo(HaveOccurred())
proxyHTTPAddr = httpListener.Addr().String()
_ = httpListener.Close()
- httpServer = api.NewServer(metricsAggregator, nil, agentRegistry, testLogger)
+ httpServer = api.NewServer(metricsAggregator, nil, nil, agentRegistry, testLogger)
Expect(httpServer.Start(proxyHTTPAddr, 10*time.Second, 10*time.Second)).To(Succeed())
Eventually(func() error {
@@ -111,6 +111,7 @@ var _ = Describe("Full FODC Proxy Workflow", func() {
1*time.Second,
flightRecorder1,
testLogger,
+ "",
)
Expect(proxyClient1).NotTo(BeNil())
@@ -124,6 +125,7 @@ var _ = Describe("Full FODC Proxy Workflow", func() {
1*time.Second,
flightRecorder2,
testLogger,
+ "",
)
Expect(proxyClient2).NotTo(BeNil())
@@ -137,6 +139,7 @@ var _ = Describe("Full FODC Proxy Workflow", func() {
1*time.Second,
flightRecorder3,
testLogger,
+ "",
)
Expect(proxyClient3).NotTo(BeNil())
diff --git a/fodc/proxy/internal/integration/high_availability_scalability_test.go b/fodc/proxy/internal/integration/high_availability_scalability_test.go
index 2f48daa38..4d5832524 100644
--- a/fodc/proxy/internal/integration/high_availability_scalability_test.go
+++ b/fodc/proxy/internal/integration/high_availability_scalability_test.go
@@ -75,7 +75,7 @@ var _ = Describe("High Availability and Scalability", func() {
agentRegistry = registry.NewAgentRegistry(testLogger, heartbeatTimeout, cleanupTimeout, 1000)
metricsAggregator = metricsproxy.NewAggregator(agentRegistry, nil, testLogger)
- grpcService = grpcproxy.NewFODCService(agentRegistry, metricsAggregator, nil, testLogger, heartbeatInterval)
+ grpcService = grpcproxy.NewFODCService(agentRegistry, metricsAggregator, nil, nil, testLogger, heartbeatInterval)
metricsAggregator.SetGRPCService(grpcService)
grpcListener, listenErr := net.Listen("tcp", "localhost:0")
@@ -91,7 +91,7 @@ var _ = Describe("High Availability and Scalability", func() {
Expect(httpListenErr).NotTo(HaveOccurred())
proxyHTTPAddr = httpListener.Addr().String()
_ = httpListener.Close()
- httpServer = api.NewServer(metricsAggregator, nil, agentRegistry, testLogger)
+ httpServer = api.NewServer(metricsAggregator, nil, nil, agentRegistry, testLogger)
Expect(httpServer.Start(proxyHTTPAddr, 30*time.Second, 30*time.Second)).To(Succeed())
Eventually(func() error {
@@ -125,6 +125,7 @@ var _ = Describe("High Availability and Scalability", func() {
1*time.Second,
fr,
logger.GetLogger("test", "agent"),
+ "",
)
Expect(client).NotTo(BeNil())
@@ -277,6 +278,7 @@ var _ = Describe("High Availability and Scalability", func() {
1*time.Second,
agent.flightRecorder,
logger.GetLogger("test", "agent"),
+ "",
)
Expect(newClient).NotTo(BeNil())
diff --git a/fodc/proxy/internal/integration/cluster_topology_integration_test.go b/fodc/proxy/internal/integration/lifecycle_integration_test.go
similarity index 62%
copy from fodc/proxy/internal/integration/cluster_topology_integration_test.go
copy to fodc/proxy/internal/integration/lifecycle_integration_test.go
index 3bf6dd4eb..9f147aa8f 100644
--- a/fodc/proxy/internal/integration/cluster_topology_integration_test.go
+++ b/fodc/proxy/internal/integration/lifecycle_integration_test.go
@@ -23,25 +23,24 @@ import (
"fmt"
"net"
"net/http"
+ "os"
+ "path/filepath"
"time"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
- commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
- databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
- fodcv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/fodc/v1"
"github.com/apache/skywalking-banyandb/fodc/agent/testhelper"
"github.com/apache/skywalking-banyandb/fodc/proxy/internal/api"
- "github.com/apache/skywalking-banyandb/fodc/proxy/internal/cluster"
grpcproxy "github.com/apache/skywalking-banyandb/fodc/proxy/internal/grpc"
+ proxylifecycle "github.com/apache/skywalking-banyandb/fodc/proxy/internal/lifecycle"
metricsproxy "github.com/apache/skywalking-banyandb/fodc/proxy/internal/metrics"
"github.com/apache/skywalking-banyandb/fodc/proxy/internal/registry"
"github.com/apache/skywalking-banyandb/pkg/logger"
"github.com/apache/skywalking-banyandb/pkg/test/flags"
)
-var _ = Describe("Cluster Topology Integration", func() {
+var _ = Describe("Lifecycle Integration", func() {
var (
proxyGRPCAddr string
proxyHTTPAddr string
@@ -50,7 +49,7 @@ var _ = Describe("Cluster Topology Integration", func() {
agentRegistry *registry.AgentRegistry
metricsAggregator *metricsproxy.Aggregator
grpcService *grpcproxy.FODCService
- clusterManager *cluster.Manager
+ lifecycleMgr *proxylifecycle.Manager
flightRecorder1 interface{}
flightRecorder2 interface{}
proxyClient1 *testhelper.ProxyClientWrapper
@@ -61,24 +60,24 @@ var _ = Describe("Cluster Topology Integration", func() {
agentCancel2 context.CancelFunc
httpClient *http.Client
testLogger *logger.Logger
+ reportDir string
)
BeforeEach(func() {
- testLogger = logger.GetLogger("test", "integration")
- httpClient = &http.Client{Timeout: 500 * time.Millisecond}
+ testLogger = logger.GetLogger("test", "lifecycle-integration")
+ httpClient = &http.Client{Timeout: 2 * time.Second}
+ reportDir = GinkgoT().TempDir()
heartbeatTimeout := 5 * time.Second
cleanupTimeout := 10 * time.Second
heartbeatInterval := 2 * time.Second
agentRegistry = registry.NewAgentRegistry(testLogger, heartbeatTimeout, cleanupTimeout, 100)
- // Create cluster manager for topology aggregation
- clusterManager = cluster.NewManager(agentRegistry, nil, testLogger)
-
+ lifecycleMgr = proxylifecycle.NewManager(agentRegistry, nil, testLogger)
metricsAggregator = metricsproxy.NewAggregator(agentRegistry, nil, testLogger)
- grpcService = grpcproxy.NewFODCService(agentRegistry, metricsAggregator, clusterManager, testLogger, heartbeatInterval)
+ grpcService = grpcproxy.NewFODCService(agentRegistry, metricsAggregator, nil, lifecycleMgr, testLogger, heartbeatInterval)
metricsAggregator.SetGRPCService(grpcService)
- clusterManager.SetGRPCService(grpcService)
+ lifecycleMgr.SetGRPCService(grpcService)
grpcListener, listenErr := net.Listen("tcp", "localhost:0")
Expect(listenErr).NotTo(HaveOccurred())
@@ -93,7 +92,7 @@ var _ = Describe("Cluster Topology Integration", func() {
Expect(httpListenErr).NotTo(HaveOccurred())
proxyHTTPAddr = httpListener.Addr().String()
_ = httpListener.Close()
- httpServer = api.NewServer(metricsAggregator, clusterManager, agentRegistry, testLogger)
+ httpServer = api.NewServer(metricsAggregator, nil, lifecycleMgr, agentRegistry, testLogger)
Expect(httpServer.Start(proxyHTTPAddr, 10*time.Second, 10*time.Second)).To(Succeed())
Eventually(func() error {
@@ -130,17 +129,14 @@ var _ = Describe("Cluster Topology Integration", func() {
}
})
- It("should aggregate topology from multiple agents", func() {
- // Create contexts for agents
+ setupTwoAgents := func() {
agentCtx1, agentCancel1 = context.WithCancel(context.Background())
agentCtx2, agentCancel2 = context.WithCancel(context.Background())
- // Create Flight Recorders for agents
capacitySize := int64(10 * 1024 * 1024)
flightRecorder1 = testhelper.NewFlightRecorder(capacitySize)
flightRecorder2 = testhelper.NewFlightRecorder(capacitySize)
- // Create proxy clients for two agents
heartbeatInterval := 2 * time.Second
reconnectInterval := 1 * time.Second
proxyClient1 = testhelper.NewProxyClientWrapper(
@@ -153,6 +149,7 @@ var _ = Describe("Cluster Topology Integration", func() {
reconnectInterval,
flightRecorder1,
testLogger,
+ reportDir,
)
Expect(proxyClient1).NotTo(BeNil())
proxyClient2 = testhelper.NewProxyClientWrapper(
@@ -165,84 +162,87 @@ var _ = Describe("Cluster Topology Integration", func() {
reconnectInterval,
flightRecorder2,
testLogger,
+ reportDir,
)
Expect(proxyClient2).NotTo(BeNil())
- // Start agent 1
Expect(proxyClient1.Connect(agentCtx1)).To(Succeed())
Expect(proxyClient1.StartRegistrationStream(agentCtx1)).To(Succeed())
- Expect(proxyClient1.StartClusterStateStream(agentCtx1)).To(Succeed())
+ Expect(proxyClient1.StartLifecycleStream(agentCtx1)).To(Succeed())
- // Start agent 2
Expect(proxyClient2.Connect(agentCtx2)).To(Succeed())
Expect(proxyClient2.StartRegistrationStream(agentCtx2)).To(Succeed())
- Expect(proxyClient2.StartClusterStateStream(agentCtx2)).To(Succeed())
+ Expect(proxyClient2.StartLifecycleStream(agentCtx2)).To(Succeed())
- // Wait for agents to register
Eventually(func() int {
return len(agentRegistry.ListAgents())
- }, "10s", "100ms").Should(Equal(2))
+ }, flags.EventuallyTimeout, 100*time.Millisecond).Should(Equal(2))
+ }
- // Simulate topology data from agents
+ waitForLifecycleStreams := func() {
agents := agentRegistry.ListAgents()
Expect(len(agents)).To(Equal(2))
-
- // Prepare topology data
- testTopology1 := &fodcv1.Topology{
- Nodes: []*databasev1.Node{
- {
- Metadata: &commonv1.Metadata{
- Name: "node-1",
- },
- },
- },
- }
-
- testTopology2 := &fodcv1.Topology{
- Nodes: []*databasev1.Node{
- {
- Metadata: &commonv1.Metadata{
- Name: "node-2",
- },
- },
- },
- }
+ Eventually(func(g Gomega) {
+ g.Expect(grpcService.HasLifecycleStream(agents[0].AgentID)).To(BeTrue())
+ g.Expect(grpcService.HasLifecycleStream(agents[1].AgentID)).To(BeTrue())
+ }, "5s", "100ms").Should(Succeed())
+ }
+
+ getLifecycleData := func() *proxylifecycle.InspectionResult {
+ resp, err := httpClient.Get(fmt.Sprintf("http://%s/cluster/lifecycle", proxyHTTPAddr))
+ Expect(err).NotTo(HaveOccurred())
+ defer resp.Body.Close()
+ Expect(resp.StatusCode).To(Equal(http.StatusOK))
+ var result proxylifecycle.InspectionResult
+ Expect(json.NewDecoder(resp.Body).Decode(&result)).To(Succeed())
+ return &result
+ }
+
+ writeReport := func(name, content string) {
+ Expect(os.WriteFile(filepath.Join(reportDir, name), []byte(content), 0o600)).To(Succeed())
+ }
+
+ It("should return empty lifecycle data when no report files exist", func() {
+ setupTwoAgents()
+ waitForLifecycleStreams()
Eventually(func(g Gomega) {
- g.Expect(grpcService.HasClusterStateStream(agents[0].AgentID)).To(BeTrue())
- g.Expect(grpcService.HasClusterStateStream(agents[1].AgentID)).To(BeTrue())
- }, "1s", "10ms").Should(Succeed())
+ data := getLifecycleData()
+ g.Expect(data.Groups).To(BeEmpty())
+ g.Expect(data.LifecycleStatuses).To(BeEmpty())
+ }, "5s", "100ms").Should(Succeed())
+ })
+
+ It("should aggregate lifecycle statuses from multiple agents via report files", func() {
+ setupTwoAgents()
+ waitForLifecycleStreams()
- Expect(proxyClient1.SetClusterTopology(testTopology1.Nodes, testTopology1.Calls)).To(Succeed())
- Expect(proxyClient2.SetClusterTopology(testTopology2.Nodes, testTopology2.Calls)).To(Succeed())
+ writeReport("2026-03-26.json", `{"status":"ok"}`)
- var topology cluster.TopologyMap
Eventually(func(g Gomega) {
- resp, err := httpClient.Get(fmt.Sprintf("http://%s/cluster/topology", proxyHTTPAddr))
- g.Expect(err).NotTo(HaveOccurred())
- defer resp.Body.Close()
- g.Expect(resp.StatusCode).To(Equal(http.StatusOK))
-
- var result cluster.TopologyMap
- g.Expect(json.NewDecoder(resp.Body).Decode(&result)).To(Succeed())
- g.Expect(len(result.Nodes)).To(BeNumerically(">=", 2))
- topology = result
+ result := getLifecycleData()
+ g.Expect(result.Groups).To(BeEmpty())
+ g.Expect(len(result.LifecycleStatuses)).To(Equal(2))
+ podNames := []string{result.LifecycleStatuses[0].PodName, result.LifecycleStatuses[1].PodName}
+ g.Expect(podNames).To(ContainElements("pod-1", "pod-2"))
}, "5s", "100ms").Should(Succeed())
+ })
+
+ It("should include report filenames collected from disk", func() {
+ setupTwoAgents()
+ waitForLifecycleStreams()
- // Check that both agents are present
- nodeNames := make(map[string]bool)
- for _, node := range topology.Nodes {
- if node.Metadata != nil {
- nodeNames[node.Metadata.Name] = true
+ writeReport("2026-03-25.json", `{"check":"pass"}`)
+ writeReport("2026-03-26.json", `{"check":"fail"}`)
+
+ Eventually(func(g Gomega) {
+ result := getLifecycleData()
+ g.Expect(len(result.LifecycleStatuses)).To(Equal(2))
+ for _, status := range result.LifecycleStatuses {
+ g.Expect(len(status.Reports)).To(Equal(2))
+ filenames := []string{status.Reports[0].Filename, status.Reports[1].Filename}
+ g.Expect(filenames).To(ContainElements("2026-03-25.json", "2026-03-26.json"))
}
- }
- Expect(nodeNames["node-1"]).To(BeTrue())
- Expect(nodeNames["node-2"]).To(BeTrue())
-
- // Clean up
- agentCancel1()
- agentCancel2()
- proxyClient1.Disconnect()
- proxyClient2.Disconnect()
+ }, "5s", "100ms").Should(Succeed())
})
})
diff --git a/fodc/proxy/internal/integration/metrics_time_window_test.go b/fodc/proxy/internal/integration/metrics_time_window_test.go
index f93cbccd8..e161a5cc0 100644
--- a/fodc/proxy/internal/integration/metrics_time_window_test.go
+++ b/fodc/proxy/internal/integration/metrics_time_window_test.go
@@ -76,7 +76,7 @@ var _ = Describe("Test Case 2: Metrics Time Window", func() {
heartbeatInterval := 2 * time.Second
agentRegistry = registry.NewAgentRegistry(testLogger, heartbeatTimeout, cleanupTimeout, 100)
metricsAggregator = metricsproxy.NewAggregator(agentRegistry, nil, testLogger)
- grpcService = grpcproxy.NewFODCService(agentRegistry, metricsAggregator, nil, testLogger, heartbeatInterval)
+ grpcService = grpcproxy.NewFODCService(agentRegistry, metricsAggregator, nil, nil, testLogger, heartbeatInterval)
metricsAggregator.SetGRPCService(grpcService)
grpcListener, listenErr := net.Listen("tcp", "localhost:0")
@@ -92,7 +92,7 @@ var _ = Describe("Test Case 2: Metrics Time Window", func() {
Expect(httpListenErr).NotTo(HaveOccurred())
proxyHTTPAddr = httpListener.Addr().String()
_ = httpListener.Close()
- httpServer = api.NewServer(metricsAggregator, nil, agentRegistry, testLogger)
+ httpServer = api.NewServer(metricsAggregator, nil, nil, agentRegistry, testLogger)
// Use longer timeouts to accommodate aggregator's timeout calculation:
// timeout = (endTime - startTime) + 5 seconds
// For time windows up to ~20 seconds, we need at least 25 seconds write timeout
@@ -121,6 +121,7 @@ var _ = Describe("Test Case 2: Metrics Time Window", func() {
1*time.Second,
flightRecorder1,
testLogger,
+ "",
)
Expect(proxyClient1).NotTo(BeNil())
@@ -134,6 +135,7 @@ var _ = Describe("Test Case 2: Metrics Time Window", func() {
1*time.Second,
flightRecorder2,
testLogger,
+ "",
)
Expect(proxyClient2).NotTo(BeNil())
diff --git a/fodc/proxy/internal/integration/multiple_agents_roles_test.go b/fodc/proxy/internal/integration/multiple_agents_roles_test.go
index 087bc904f..012040681 100644
--- a/fodc/proxy/internal/integration/multiple_agents_roles_test.go
+++ b/fodc/proxy/internal/integration/multiple_agents_roles_test.go
@@ -68,7 +68,7 @@ var _ = Describe("Test Case 4: Multiple Agents and Roles", func() {
heartbeatInterval := 2 * time.Second
agentRegistry = registry.NewAgentRegistry(testLogger, heartbeatTimeout, cleanupTimeout, 100)
metricsAggregator = metricsproxy.NewAggregator(agentRegistry, nil, testLogger)
- grpcService = grpcproxy.NewFODCService(agentRegistry, metricsAggregator, nil, testLogger, heartbeatInterval)
+ grpcService = grpcproxy.NewFODCService(agentRegistry, metricsAggregator, nil, nil, testLogger, heartbeatInterval)
metricsAggregator.SetGRPCService(grpcService)
grpcListener, listenErr := net.Listen("tcp", "localhost:0")
@@ -84,7 +84,7 @@ var _ = Describe("Test Case 4: Multiple Agents and Roles", func() {
Expect(httpListenErr).NotTo(HaveOccurred())
proxyHTTPAddr = httpListener.Addr().String()
_ = httpListener.Close()
- httpServer = api.NewServer(metricsAggregator, nil, agentRegistry, testLogger)
+ httpServer = api.NewServer(metricsAggregator, nil, nil, agentRegistry, testLogger)
Expect(httpServer.Start(proxyHTTPAddr, 10*time.Second, 10*time.Second)).To(Succeed())
Eventually(func() error {
@@ -111,6 +111,7 @@ var _ = Describe("Test Case 4: Multiple Agents and Roles", func() {
1*time.Second,
flightRecorder1,
testLogger,
+ "",
)
Expect(proxyClient1).NotTo(BeNil())
@@ -124,6 +125,7 @@ var _ = Describe("Test Case 4: Multiple Agents and Roles", func() {
1*time.Second,
flightRecorder2,
testLogger,
+ "",
)
Expect(proxyClient2).NotTo(BeNil())
@@ -137,6 +139,7 @@ var _ = Describe("Test Case 4: Multiple Agents and Roles", func() {
1*time.Second,
flightRecorder3,
testLogger,
+ "",
)
Expect(proxyClient3).NotTo(BeNil())
diff --git a/fodc/proxy/internal/integration/prometheus_integration_test.go b/fodc/proxy/internal/integration/prometheus_integration_test.go
index fcf4b4e54..61983c3ba 100644
--- a/fodc/proxy/internal/integration/prometheus_integration_test.go
+++ b/fodc/proxy/internal/integration/prometheus_integration_test.go
@@ -57,7 +57,7 @@ func newPromFixture() *promFixture {
reg := registry.NewAgentRegistry(testLogger, heartbeatTimeout, cleanupTimeout, 100)
agg := metricsproxy.NewAggregator(reg, nil, testLogger)
- svc := grpcproxy.NewFODCService(reg, agg, nil, testLogger, heartbeatInterval)
+ svc := grpcproxy.NewFODCService(reg, agg, nil, nil, testLogger, heartbeatInterval)
agg.SetGRPCService(svc)
grpcListener, err := net.Listen("tcp", "localhost:0")
@@ -71,7 +71,7 @@ func newPromFixture() *promFixture {
Expect(err).NotTo(HaveOccurred())
httpAddr := httpListener.Addr().String()
_ = httpListener.Close()
- httpSrv := api.NewServer(agg, nil, reg, testLogger)
+ httpSrv := api.NewServer(agg, nil, nil, reg, testLogger)
Expect(httpSrv.Start(httpAddr, 10*time.Second, 10*time.Second)).To(Succeed())
Eventually(func() error {
@@ -130,6 +130,7 @@ var _ = Describe("Prometheus Integration", func() {
1*time.Second,
fr,
fixture.testLogger,
+ "",
)
Expect(agent).NotTo(BeNil())
agent.StartConnManager(ctx)
diff --git a/fodc/proxy/internal/lifecycle/manager.go b/fodc/proxy/internal/lifecycle/manager.go
new file mode 100644
index 000000000..66cb9ce81
--- /dev/null
+++ b/fodc/proxy/internal/lifecycle/manager.go
@@ -0,0 +1,245 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Package lifecycle provides lifecycle state management for FODC proxy.
+package lifecycle
+
+import (
+ "context"
+ "sync"
+ "time"
+
+ fodcv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/fodc/v1"
+ "github.com/apache/skywalking-banyandb/fodc/proxy/internal/registry"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+)
+
+const defaultCollectionTimeout = 10 * time.Second
+
+// PodLifecycleStatus represents lifecycle status for a single pod.
+type PodLifecycleStatus struct {
+ PodName string `json:"pod_name,omitempty"`
+ Reports []*fodcv1.LifecycleReport `json:"reports,omitempty"`
+}
+
+// InspectionResult is the aggregated result from agents and liaison.
+type InspectionResult struct {
+ Groups []*fodcv1.GroupLifecycleInfo `json:"groups"`
+ LifecycleStatuses []*PodLifecycleStatus `json:"lifecycle_statuses"`
+}
+
+func emptyResult() *InspectionResult {
+ return &InspectionResult{
+ Groups: make([]*fodcv1.GroupLifecycleInfo, 0),
+ LifecycleStatuses: make([]*PodLifecycleStatus, 0),
+ }
+}
+
+// agentLifecycleData carries pod_name alongside the lifecycle data through the channel.
+type agentLifecycleData struct {
+ Data *fodcv1.LifecycleData
+ PodName string
+}
+
+// RequestSender is an interface for sending lifecycle data requests to agents.
+type RequestSender interface {
+ RequestLifecycleData(agentID string) error
+}
+
+// Manager manages lifecycle data from multiple agents.
+type Manager struct {
+ log *logger.Logger
+ registry *registry.AgentRegistry
+ grpcService RequestSender
+ collecting map[string]chan *agentLifecycleData
+ mu sync.RWMutex
+ collectingMu sync.RWMutex
+ collectingOp sync.Mutex
+}
+
+// NewManager creates a new lifecycle manager.
+func NewManager(registry *registry.AgentRegistry, grpcService RequestSender, log *logger.Logger) *Manager {
+ return &Manager{
+ registry: registry,
+ grpcService: grpcService,
+ log: log,
+ collecting: make(map[string]chan *agentLifecycleData),
+ }
+}
+
+// SetGRPCService sets the gRPC service for sending lifecycle data requests.
+func (m *Manager) SetGRPCService(grpcService RequestSender) {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ m.grpcService = grpcService
+}
+
+// UpdateLifecycle updates lifecycle data for a specific agent.
+func (m *Manager) UpdateLifecycle(agentID, podName string, data *fodcv1.LifecycleData) {
+ if data == nil {
+ data = &fodcv1.LifecycleData{}
+ }
+ m.collectingMu.RLock()
+ defer m.collectingMu.RUnlock()
+ collectCh, exists := m.collecting[agentID]
+ if !exists {
+ return
+ }
+ select {
+ case collectCh <- &agentLifecycleData{PodName: podName, Data: data}:
+ default:
+ m.log.Warn().Str("agent_id", agentID).Msg("Lifecycle collection channel full, dropping data")
+ }
+}
+
+// CollectLifecycle requests and collects lifecycle data from all agents.
+func (m *Manager) CollectLifecycle(ctx context.Context) *InspectionResult {
+ m.collectingOp.Lock()
+ defer m.collectingOp.Unlock()
+
+ if m.registry == nil {
+ m.log.Info().Msg("CollectLifecycle: registry is nil, returning empty")
+ return emptyResult()
+ }
+
+ agents := m.registry.ListAgents()
+ if len(agents) == 0 {
+ m.log.Info().Msg("CollectLifecycle: no agents registered, returning empty")
+ return emptyResult()
+ }
+
+ m.log.Info().Int("agent_count", len(agents)).Msg("CollectLifecycle: starting collection")
+
+ collectChs := make(map[string]chan *agentLifecycleData)
+
+ defer func() {
+ m.collectingMu.Lock()
+ for agentID, collectCh := range collectChs {
+ close(collectCh)
+ delete(m.collecting, agentID)
+ }
+ m.collectingMu.Unlock()
+ }()
+
+ requestedCount := 0
+ for _, agentInfo := range agents {
+ select {
+ case <-ctx.Done():
+ m.log.Info().Msg("CollectLifecycle: context canceled during request phase")
+ return emptyResult()
+ default:
+ }
+ if m.grpcService == nil {
+ m.log.Info().Str("agent_id", agentInfo.AgentID).Msg("CollectLifecycle: grpcService is nil, skipping request")
+ continue
+ }
+ if err := m.grpcService.RequestLifecycleData(agentInfo.AgentID); err != nil {
+ m.log.Info().Err(err).
+ Str("agent_id", agentInfo.AgentID).
+ Msg("Agent does not support lifecycle stream, skipping")
+ continue
+ }
+ collectCh := make(chan *agentLifecycleData, 1)
+ collectChs[agentInfo.AgentID] = collectCh
+ m.collectingMu.Lock()
+ m.collecting[agentInfo.AgentID] = collectCh
+ m.collectingMu.Unlock()
+ requestedCount++
+ }
+
+ m.log.Info().Int("requested", requestedCount).Int("waiting_for", len(collectChs)).Msg("CollectLifecycle: requests sent, waiting for responses")
+
+ allData := make([]*agentLifecycleData, 0, len(collectChs))
+ var dataMu sync.Mutex
+ var wg sync.WaitGroup
+
+ for agentID, collectCh := range collectChs {
+ wg.Add(1)
+ go func(id string, ch chan *agentLifecycleData) {
+ defer wg.Done()
+ agentCtx, agentCancel := context.WithTimeout(ctx, defaultCollectionTimeout)
+ defer agentCancel()
+ select {
+ case <-agentCtx.Done():
+ m.log.Warn().
+ Str("agent_id", id).
+ Msg("Timeout waiting for lifecycle data from agent")
+ case data := <-ch:
+ if data != nil {
+ m.log.Info().
+ Str("agent_id", id).
+ Str("pod_name", data.PodName).
+ Int("reports", len(data.Data.Reports)).
+ Int("groups", len(data.Data.Groups)).
+ Msg("CollectLifecycle: received data from agent")
+ dataMu.Lock()
+ allData = append(allData, data)
+ dataMu.Unlock()
+ } else {
+ m.log.Info().Str("agent_id", id).Msg("CollectLifecycle: received nil data from agent")
+ }
+ }
+ }(agentID, collectCh)
+ }
+
+ wg.Wait()
+ m.log.Info().Int("responses_with_data", len(allData)).Msg("CollectLifecycle: all responses collected, aggregating")
+
+ return m.buildInspectionResult(allData)
+}
+
+func (m *Manager) buildInspectionResult(allData []*agentLifecycleData) *InspectionResult {
+ return &InspectionResult{
+ Groups: m.mergeGroups(allData),
+ LifecycleStatuses: m.aggregateLifecycle(allData),
+ }
+}
+
+func (m *Manager) mergeGroups(allData []*agentLifecycleData) []*fodcv1.GroupLifecycleInfo {
+ groupMap := make(map[string]*fodcv1.GroupLifecycleInfo)
+ for _, ad := range allData {
+ if ad == nil || ad.Data == nil {
+ continue
+ }
+ for _, g := range ad.Data.Groups {
+ groupMap[g.Name] = g
+ }
+ }
+ groups := make([]*fodcv1.GroupLifecycleInfo, 0, len(groupMap))
+ for _, g := range groupMap {
+ groups = append(groups, g)
+ }
+ return groups
+}
+
+// aggregateLifecycle aggregates lifecycle statuses from multiple agents.
+func (m *Manager) aggregateLifecycle(allData []*agentLifecycleData) []*PodLifecycleStatus {
+ allStatuses := make([]*PodLifecycleStatus, 0)
+ for _, ad := range allData {
+ if ad == nil || ad.Data == nil {
+ continue
+ }
+ if len(ad.Data.Reports) == 0 {
+ continue
+ }
+ allStatuses = append(allStatuses, &PodLifecycleStatus{
+ PodName: ad.PodName,
+ Reports: ad.Data.Reports,
+ })
+ }
+ return allStatuses
+}
diff --git a/fodc/proxy/internal/lifecycle/manager_test.go b/fodc/proxy/internal/lifecycle/manager_test.go
new file mode 100644
index 000000000..7df3a2811
--- /dev/null
+++ b/fodc/proxy/internal/lifecycle/manager_test.go
@@ -0,0 +1,379 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package lifecycle
+
+import (
+ "context"
+ "fmt"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ fodcv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/fodc/v1"
+ "github.com/apache/skywalking-banyandb/fodc/proxy/internal/registry"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+)
+
+type mockRequestSender struct {
+ requestErrors map[string]error
+ requestedAgents []string
+ mu sync.Mutex
+}
+
+func newMockRequestSender() *mockRequestSender {
+ return &mockRequestSender{
+ requestErrors: make(map[string]error),
+ requestedAgents: make([]string, 0),
+ }
+}
+
+func (m *mockRequestSender) RequestLifecycleData(agentID string) error {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ m.requestedAgents = append(m.requestedAgents, agentID)
+ if err, exists := m.requestErrors[agentID]; exists {
+ return err
+ }
+ return nil
+}
+
+func (m *mockRequestSender) GetRequestedAgents() []string {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ result := make([]string, len(m.requestedAgents))
+ copy(result, m.requestedAgents)
+ return result
+}
+
+func (m *mockRequestSender) SetRequestError(agentID string, err error) {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ m.requestErrors[agentID] = err
+}
+
+func initTestLogger(t *testing.T) *logger.Logger {
+ t.Helper()
+ err := logger.Init(logger.Logging{Env: "dev", Level: "debug"})
+ require.NoError(t, err)
+ return logger.GetLogger("test", "lifecycle")
+}
+
+func TestNewManager(t *testing.T) {
+ log := initTestLogger(t)
+ testRegistry := registry.NewAgentRegistry(log, 5*time.Second, 10*time.Second, 100)
+ mgr := NewManager(testRegistry, nil, log)
+
+ require.NotNil(t, mgr)
+ assert.Equal(t, log, mgr.log)
+ assert.NotNil(t, mgr.collecting)
+ assert.Empty(t, mgr.collecting)
+}
+
+func TestManager_SetGRPCService(t *testing.T) {
+ log := initTestLogger(t)
+ testRegistry := registry.NewAgentRegistry(log, 5*time.Second, 10*time.Second, 100)
+ mgr := NewManager(testRegistry, nil, log)
+
+ mockSender := newMockRequestSender()
+ mgr.SetGRPCService(mockSender)
+ assert.Equal(t, mockSender, mgr.grpcService)
+}
+
+func TestManager_UpdateLifecycle_NoActiveCollection(t *testing.T) {
+ log := initTestLogger(t)
+ testRegistry := registry.NewAgentRegistry(log, 5*time.Second, 10*time.Second, 100)
+ mgr := NewManager(testRegistry, nil, log)
+
+ data := &fodcv1.LifecycleData{
+ Reports: []*fodcv1.LifecycleReport{
+ {Filename: "2026-03-26.json", ReportJson: "{}"},
+ },
+ }
+
+ // Should not panic when no collection is active
+ mgr.UpdateLifecycle("agent1", "pod-1", data)
+}
+
+func TestManager_CollectLifecycle_NoAgents(t *testing.T) {
+ log := initTestLogger(t)
+ testRegistry := registry.NewAgentRegistry(log, 5*time.Second, 10*time.Second, 100)
+ mgr := NewManager(testRegistry, nil, log)
+
+ result := mgr.CollectLifecycle(context.Background())
+ require.NotNil(t, result)
+ assert.Empty(t, result.Groups)
+ assert.Empty(t, result.LifecycleStatuses)
+}
+
+func TestManager_CollectLifecycle_NilRegistry(t *testing.T) {
+ log := initTestLogger(t)
+ mgr := NewManager(nil, nil, log)
+
+ result := mgr.CollectLifecycle(context.Background())
+ require.NotNil(t, result)
+ assert.Empty(t, result.Groups)
+}
+
+func TestManager_CollectLifecycle_MultipleAgents(t *testing.T) {
+ log := initTestLogger(t)
+ testRegistry := registry.NewAgentRegistry(log, 5*time.Second, 10*time.Second, 100)
+ defer testRegistry.Stop()
+ mockSender := newMockRequestSender()
+ mgr := NewManager(testRegistry, mockSender, log)
+
+ ctx := context.Background()
+ identity1 := registry.AgentIdentity{
+ Role: "ROLE_DATA",
+ PodName: "pod-1",
+ ContainerNames: []string{"banyandb"},
+ Labels: map[string]string{"env": "test"},
+ }
+ identity2 := registry.AgentIdentity{
+ Role: "ROLE_DATA",
+ PodName: "pod-2",
+ ContainerNames: []string{"banyandb"},
+ Labels: map[string]string{"env": "test"},
+ }
+ agentID1, err := testRegistry.RegisterAgent(ctx, identity1)
+ require.NoError(t, err)
+ agentID2, err := testRegistry.RegisterAgent(ctx, identity2)
+ require.NoError(t, err)
+
+ data1 := &fodcv1.LifecycleData{
+ Reports: []*fodcv1.LifecycleReport{
+ {Filename: "2026-03-25.json", ReportJson: "{}"},
+ },
+ }
+
+ data2 := &fodcv1.LifecycleData{
+ Reports: []*fodcv1.LifecycleReport{
+ {Filename: "2026-03-24.json", ReportJson: "{}"},
+ },
+ }
+
+ done := make(chan *InspectionResult)
+ go func() {
+ done <- mgr.CollectLifecycle(ctx)
+ }()
+
+ time.Sleep(50 * time.Millisecond)
+
+ requestedAgents := mockSender.GetRequestedAgents()
+ assert.Contains(t, requestedAgents, agentID1)
+ assert.Contains(t, requestedAgents, agentID2)
+
+ mgr.UpdateLifecycle(agentID1, "pod-1", data1)
+ mgr.UpdateLifecycle(agentID2, "pod-2", data2)
+
+ select {
+ case result := <-done:
+ require.NotNil(t, result)
+ // Groups are collected from liaison (nil in this test), so should be empty
+ assert.Empty(t, result.Groups)
+ // Should have 2 lifecycle statuses (one per pod)
+ assert.Equal(t, 2, len(result.LifecycleStatuses))
+ // Verify pod names
+ podNames := []string{result.LifecycleStatuses[0].PodName, result.LifecycleStatuses[1].PodName}
+ assert.Contains(t, podNames, "pod-1")
+ assert.Contains(t, podNames, "pod-2")
+
+ case <-time.After(5 * time.Second):
+ t.Fatal("Collection timed out")
+ }
+}
+
+func TestManager_CollectLifecycle_RequestError(t *testing.T) {
+ log := initTestLogger(t)
+ testRegistry := registry.NewAgentRegistry(log, 5*time.Second, 10*time.Second, 100)
+ defer testRegistry.Stop()
+ mockSender := newMockRequestSender()
+ mgr := NewManager(testRegistry, mockSender, log)
+
+ ctx := context.Background()
+ identity := registry.AgentIdentity{
+ Role: "ROLE_DATA",
+ PodName: "pod-1",
+ ContainerNames: []string{"banyandb"},
+ Labels: map[string]string{"env": "test"},
+ }
+ agentID, err := testRegistry.RegisterAgent(ctx, identity)
+ require.NoError(t, err)
+
+ mockSender.SetRequestError(agentID, fmt.Errorf("connection refused"))
+
+ result := mgr.CollectLifecycle(ctx)
+ require.NotNil(t, result)
+ // Should return empty data since request failed
+ assert.Empty(t, result.Groups)
+ assert.Empty(t, result.LifecycleStatuses)
+}
+
+func TestManager_AggregateLifecycle_Empty(t *testing.T) {
+ log := initTestLogger(t)
+ mgr := NewManager(nil, nil, log)
+
+ result := mgr.aggregateLifecycle(nil)
+ assert.Empty(t, result)
+
+ result = mgr.aggregateLifecycle([]*agentLifecycleData{})
+ assert.Empty(t, result)
+}
+
+func TestManager_AggregateLifecycle_StatusesConcatenated(t *testing.T) {
+ log := initTestLogger(t)
+ mgr := NewManager(nil, nil, log)
+
+ ad1 := &agentLifecycleData{
+ PodName: "pod-1",
+ Data: &fodcv1.LifecycleData{
+ Reports: []*fodcv1.LifecycleReport{{Filename: "2026-03-23.json"}},
+ },
+ }
+ ad2 := &agentLifecycleData{
+ PodName: "pod-2",
+ Data: &fodcv1.LifecycleData{
+ Reports: []*fodcv1.LifecycleReport{{Filename: "2026-03-22.json"}},
+ },
+ }
+
+ result := mgr.aggregateLifecycle([]*agentLifecycleData{ad1, ad2})
+ assert.Equal(t, 2, len(result))
+ assert.Equal(t, "pod-1", result[0].PodName)
+ assert.Equal(t, "pod-2", result[1].PodName)
+}
+
+func TestManager_AggregateLifecycle_NilData(t *testing.T) {
+ log := initTestLogger(t)
+ mgr := NewManager(nil, nil, log)
+
+ result := mgr.aggregateLifecycle([]*agentLifecycleData{nil, nil})
+ assert.Empty(t, result)
+}
+
+func TestManager_AggregateLifecycle_OnlyReports(t *testing.T) {
+ log := initTestLogger(t)
+ mgr := NewManager(nil, nil, log)
+
+ ad := &agentLifecycleData{
+ PodName: "pod-1",
+ Data: &fodcv1.LifecycleData{
+ Reports: []*fodcv1.LifecycleReport{{Filename: "2026-03-23.json"}},
+ },
+ }
+
+ result := mgr.aggregateLifecycle([]*agentLifecycleData{ad})
+ assert.Equal(t, 1, len(result))
+ assert.Equal(t, "pod-1", result[0].PodName)
+}
+
+func TestManager_CollectLifecycle_WithGroups(t *testing.T) {
+ log := initTestLogger(t)
+ testRegistry := registry.NewAgentRegistry(log, 5*time.Second, 10*time.Second, 100)
+ defer testRegistry.Stop()
+ mockSender := newMockRequestSender()
+ mgr := NewManager(testRegistry, mockSender, log)
+
+ ctx := context.Background()
+ agentID, err := testRegistry.RegisterAgent(ctx, registry.AgentIdentity{
+ Role: "ROLE_LIAISON", PodName: "liaison-pod", ContainerNames: []string{"banyandb"},
+ })
+ require.NoError(t, err)
+
+ done := make(chan *InspectionResult)
+ go func() { done <- mgr.CollectLifecycle(ctx) }()
+ time.Sleep(50 * time.Millisecond)
+
+ mgr.UpdateLifecycle(agentID, "liaison-pod", &fodcv1.LifecycleData{
+ Groups: []*fodcv1.GroupLifecycleInfo{
+ {Name: "sw_metric", Catalog: "CATALOG_MEASURE"},
+ {Name: "sw_record", Catalog: "CATALOG_STREAM"},
+ },
+ })
+
+ select {
+ case result := <-done:
+ require.NotNil(t, result)
+ assert.Equal(t, 2, len(result.Groups))
+ groupNames := []string{result.Groups[0].Name, result.Groups[1].Name}
+ assert.Contains(t, groupNames, "sw_metric")
+ assert.Contains(t, groupNames, "sw_record")
+ assert.Empty(t, result.LifecycleStatuses)
+ case <-time.After(5 * time.Second):
+ t.Fatal("Collection timed out")
+ }
+}
+
+func TestManager_CollectLifecycle_GroupsFromFirstAgent(t *testing.T) {
+ log := initTestLogger(t)
+ testRegistry := registry.NewAgentRegistry(log, 5*time.Second, 10*time.Second, 100)
+ defer testRegistry.Stop()
+ mockSender := newMockRequestSender()
+ mgr := NewManager(testRegistry, mockSender, log)
+
+ ctx := context.Background()
+ agentID1, err := testRegistry.RegisterAgent(ctx, registry.AgentIdentity{
+ Role: "ROLE_LIAISON", PodName: "liaison-1", ContainerNames: []string{"banyandb"},
+ })
+ require.NoError(t, err)
+ agentID2, err := testRegistry.RegisterAgent(ctx, registry.AgentIdentity{
+ Role: "ROLE_DATA", PodName: "data-1", ContainerNames: []string{"banyandb"},
+ })
+ require.NoError(t, err)
+
+ done := make(chan *InspectionResult)
+ go func() { done <- mgr.CollectLifecycle(ctx) }()
+ time.Sleep(50 * time.Millisecond)
+
+ mgr.UpdateLifecycle(agentID1, "liaison-1", &fodcv1.LifecycleData{
+ Groups: []*fodcv1.GroupLifecycleInfo{
+ {Name: "sw_metric", Catalog: "CATALOG_MEASURE"},
+ },
+ })
+ mgr.UpdateLifecycle(agentID2, "data-1", &fodcv1.LifecycleData{
+ Reports: []*fodcv1.LifecycleReport{
+ {Filename: "2026-03-26.json", ReportJson: "{}"},
+ },
+ })
+
+ select {
+ case result := <-done:
+ require.NotNil(t, result)
+ assert.Equal(t, 1, len(result.Groups))
+ assert.Equal(t, "sw_metric", result.Groups[0].Name)
+ assert.Equal(t, 1, len(result.LifecycleStatuses))
+ assert.Equal(t, "data-1", result.LifecycleStatuses[0].PodName)
+ case <-time.After(5 * time.Second):
+ t.Fatal("Collection timed out")
+ }
+}
+
+func TestManager_CollectLifecycle_ContextCanceled(t *testing.T) {
+ log := initTestLogger(t)
+ testRegistry := registry.NewAgentRegistry(log, 5*time.Second, 10*time.Second, 100)
+ defer testRegistry.Stop()
+ mgr := NewManager(testRegistry, nil, log)
+
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel() // Cancel immediately
+
+ result := mgr.CollectLifecycle(ctx)
+ require.NotNil(t, result)
+}
diff --git a/pkg/cmdsetup/liaison.go b/pkg/cmdsetup/liaison.go
index 69756ef83..806d82012 100644
--- a/pkg/cmdsetup/liaison.go
+++ b/pkg/cmdsetup/liaison.go
@@ -104,6 +104,7 @@ func newLiaisonCmd(runners ...run.Unit) *cobra.Command {
PropertyNodeRegistry: grpc.NewClusterNodeRegistry(data.TopicPropertyUpdate, tire2Client, propertyNodeSel),
TraceLiaisonNodeRegistry: grpc.NewClusterNodeRegistry(data.TopicTraceWrite, tire1Client, traceLiaisonNodeSel),
}, metricSvc, pm, routeProviders)
+ internalPipeline.SetMetadataRepo(metaSvc)
profSvc := observability.NewProfService()
httpServer := http.NewServer(grpcServer.GetAuthReloader())
var units []run.Unit