You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ha...@apache.org on 2025/07/19 22:42:52 UTC
(skywalking-banyandb) branch trace-module created (now c0bf4b27)
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a change to branch trace-module
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
at c0bf4b27 Add trace support with schema, metadata, and validation
This branch includes the following new commits:
new 113011c1 Add trace plan
new c0bf4b27 Add trace support with schema, metadata, and validation
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(skywalking-banyandb) 02/02: Add trace support with schema, metadata, and validation
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch trace-module
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit c0bf4b27772f4c21e6b7d670a462865ef22a0290
Author: Cursor Agent <cu...@cursor.com>
AuthorDate: Fri Jul 18 14:39:02 2025 +0000
Add trace support with schema, metadata, and validation
---
api/validate/validate.go | 35 +++++++
banyand/metadata/client.go | 4 +
banyand/metadata/metadata.go | 1 +
banyand/metadata/schema/checker.go | 7 ++
banyand/metadata/schema/etcd.go | 1 +
banyand/metadata/schema/kind.go | 15 ++-
banyand/metadata/schema/schema.go | 15 +++
banyand/metadata/schema/trace.go | 83 +++++++++++++++
banyand/trace/metadata.go | 184 ++++++++++++++++++++++++++++++++++
banyand/trace/svc_liaison.go | 190 +++++++++++++++++++++++++++++++++++
banyand/trace/svc_standalone.go | 200 +++++++++++++++++++++++++++++++++++++
banyand/trace/trace.go | 137 +++++++++++++++++++++++++
12 files changed, 870 insertions(+), 2 deletions(-)
diff --git a/api/validate/validate.go b/api/validate/validate.go
index c8e74bed..d9389a29 100644
--- a/api/validate/validate.go
+++ b/api/validate/validate.go
@@ -167,6 +167,41 @@ func Measure(measure *databasev1.Measure) error {
return tagFamily(measure.TagFamilies)
}
+// Trace validates the provided Trace object.
+// It checks for nil values, empty strings, and unspecified enum values.
+func Trace(trace *databasev1.Trace) error {
+ if trace == nil {
+ return errors.New("trace is nil")
+ }
+ if trace.Metadata == nil {
+ return errors.New("trace metadata is nil")
+ }
+ if trace.Metadata.Name == "" {
+ return errors.New("trace name is empty")
+ }
+ if trace.Metadata.Group == "" {
+ return errors.New("trace group is empty")
+ }
+ if len(trace.Tags) == 0 {
+ return errors.New("trace tags is empty")
+ }
+ if trace.TraceIdTagName == "" {
+ return errors.New("trace_id_tag_name is empty")
+ }
+ if trace.TimestampTagName == "" {
+ return errors.New("timestamp_tag_name is empty")
+ }
+ for i := range trace.Tags {
+ if trace.Tags[i].Name == "" {
+ return errors.New("trace tag name is empty")
+ }
+ if trace.Tags[i].Type == databasev1.TagType_TAG_TYPE_UNSPECIFIED {
+ return errors.New("trace tag type is unspecified")
+ }
+ }
+ return nil
+}
+
func tagFamily(tagFamilies []*databasev1.TagFamilySpec) error {
for i := range tagFamilies {
if tagFamilies[i].Name == "" {
diff --git a/banyand/metadata/client.go b/banyand/metadata/client.go
index f935f432..f192603c 100644
--- a/banyand/metadata/client.go
+++ b/banyand/metadata/client.go
@@ -243,6 +243,10 @@ func (s *clientService) MeasureRegistry() schema.Measure {
return s.schemaRegistry
}
+func (s *clientService) TraceRegistry() schema.Trace {
+ return s.schemaRegistry
+}
+
func (s *clientService) GroupRegistry() schema.Group {
return s.schemaRegistry
}
diff --git a/banyand/metadata/metadata.go b/banyand/metadata/metadata.go
index 14467c50..9a0024bc 100644
--- a/banyand/metadata/metadata.go
+++ b/banyand/metadata/metadata.go
@@ -45,6 +45,7 @@ type Repo interface {
IndexRuleRegistry() schema.IndexRule
IndexRuleBindingRegistry() schema.IndexRuleBinding
MeasureRegistry() schema.Measure
+ TraceRegistry() schema.Trace
GroupRegistry() schema.Group
TopNAggregationRegistry() schema.TopNAggregation
RegisterHandler(string, schema.Kind, schema.EventHandler)
diff --git a/banyand/metadata/schema/checker.go b/banyand/metadata/schema/checker.go
index 9dc140e2..7503360d 100644
--- a/banyand/metadata/schema/checker.go
+++ b/banyand/metadata/schema/checker.go
@@ -62,6 +62,13 @@ var CheckerMap = map[Kind]EqualityChecker{
protocmp.IgnoreFields(&commonv1.Metadata{}, "id", "create_revision", "mod_revision"),
protocmp.Transform())
},
+ KindTrace: func(a, b proto.Message) bool {
+ return cmp.Equal(a, b,
+ protocmp.IgnoreUnknown(),
+ protocmp.IgnoreFields(&databasev1.Trace{}, "updated_at"),
+ protocmp.IgnoreFields(&commonv1.Metadata{}, "id", "create_revision", "mod_revision"),
+ protocmp.Transform())
+ },
KindGroup: func(a, b proto.Message) bool {
return cmp.Equal(a, b,
protocmp.IgnoreUnknown(),
diff --git a/banyand/metadata/schema/etcd.go b/banyand/metadata/schema/etcd.go
index 73f8305a..a6372560 100644
--- a/banyand/metadata/schema/etcd.go
+++ b/banyand/metadata/schema/etcd.go
@@ -52,6 +52,7 @@ var (
_ IndexRuleBinding = (*etcdSchemaRegistry)(nil)
_ IndexRule = (*etcdSchemaRegistry)(nil)
_ Measure = (*etcdSchemaRegistry)(nil)
+ _ Trace = (*etcdSchemaRegistry)(nil)
_ Group = (*etcdSchemaRegistry)(nil)
errUnexpectedNumberOfEntities = errors.New("unexpected number of entities")
diff --git a/banyand/metadata/schema/kind.go b/banyand/metadata/schema/kind.go
index 3a955015..f164518e 100644
--- a/banyand/metadata/schema/kind.go
+++ b/banyand/metadata/schema/kind.go
@@ -36,15 +36,16 @@ const (
KindGroup Kind = 1 << iota
KindStream
KindMeasure
+ KindTrace
KindIndexRuleBinding
KindIndexRule
KindTopNAggregation
KindNode
KindProperty
- KindMask = KindGroup | KindStream | KindMeasure |
+ KindMask = KindGroup | KindStream | KindMeasure | KindTrace |
KindIndexRuleBinding | KindIndexRule |
KindTopNAggregation | KindNode | KindProperty
- KindSize = 8
+ KindSize = 9
)
func (k Kind) key() string {
@@ -55,6 +56,8 @@ func (k Kind) key() string {
return streamKeyPrefix
case KindMeasure:
return measureKeyPrefix
+ case KindTrace:
+ return traceKeyPrefix
case KindIndexRuleBinding:
return indexRuleBindingKeyPrefix
case KindIndexRule:
@@ -63,6 +66,8 @@ func (k Kind) key() string {
return topNAggregationKeyPrefix
case KindNode:
return nodeKeyPrefix
+ case KindProperty:
+ return propertyKeyPrefix
default:
return "unknown"
}
@@ -81,6 +86,8 @@ func (k Kind) Unmarshal(kv *mvccpb.KeyValue) (Metadata, error) {
m = &databasev1.Stream{}
case KindMeasure:
m = &databasev1.Measure{}
+ case KindTrace:
+ m = &databasev1.Trace{}
case KindIndexRuleBinding:
m = &databasev1.IndexRuleBinding{}
case KindIndexRule:
@@ -125,6 +132,8 @@ func (k Kind) String() string {
return "stream"
case KindMeasure:
return "measure"
+ case KindTrace:
+ return "trace"
case KindIndexRuleBinding:
return "indexRuleBinding"
case KindIndexRule:
@@ -133,6 +142,8 @@ func (k Kind) String() string {
return "topNAggregation"
case KindNode:
return "node"
+ case KindProperty:
+ return "property"
default:
return "unknown"
}
diff --git a/banyand/metadata/schema/schema.go b/banyand/metadata/schema/schema.go
index 6b98c05a..05e26f69 100644
--- a/banyand/metadata/schema/schema.go
+++ b/banyand/metadata/schema/schema.go
@@ -61,6 +61,7 @@ type Registry interface {
IndexRule
IndexRuleBinding
Measure
+ Trace
Group
TopNAggregation
Node
@@ -103,6 +104,11 @@ func (m Metadata) key() (string, error) {
Group: m.Group,
Name: m.Name,
}), nil
+ case KindTrace:
+ return formatTraceKey(&commonv1.Metadata{
+ Group: m.Group,
+ Name: m.Name,
+ }), nil
case KindIndexRule:
return formatIndexRuleKey(&commonv1.Metadata{
Group: m.Group,
@@ -180,6 +186,15 @@ type Measure interface {
TopNAggregations(ctx context.Context, metadata *commonv1.Metadata) ([]*databasev1.TopNAggregation, error)
}
+// Trace allows CRUD trace schemas in a group.
+type Trace interface {
+ GetTrace(ctx context.Context, metadata *commonv1.Metadata) (*databasev1.Trace, error)
+ ListTrace(ctx context.Context, opt ListOpt) ([]*databasev1.Trace, error)
+ CreateTrace(ctx context.Context, trace *databasev1.Trace) (int64, error)
+ UpdateTrace(ctx context.Context, trace *databasev1.Trace) (int64, error)
+ DeleteTrace(ctx context.Context, metadata *commonv1.Metadata) (bool, error)
+}
+
// Group allows CRUD groups which is namespaces of resources.
//
//go:generate mockgen -destination=group_mock.go -package=schema github.com/apache/skywalking-banyandb/banyand/metadata/schema Group
diff --git a/banyand/metadata/schema/trace.go b/banyand/metadata/schema/trace.go
new file mode 100644
index 00000000..74279407
--- /dev/null
+++ b/banyand/metadata/schema/trace.go
@@ -0,0 +1,83 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package schema
+
+import (
+ "context"
+
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+)
+
+var traceKeyPrefix = "/traces/"
+
+func (e *etcdSchemaRegistry) GetTrace(ctx context.Context, metadata *commonv1.Metadata) (*databasev1.Trace, error) {
+ var trace databasev1.Trace
+ if err := e.get(ctx, formatTraceKey(metadata), &trace); err != nil {
+ return nil, err
+ }
+ return &trace, nil
+}
+
+func (e *etcdSchemaRegistry) ListTrace(ctx context.Context, opt ListOpt) ([]*databasev1.Trace, error) {
+ messages, err := e.listWithPrefix(ctx, listPrefixesForEntity(opt.Group, traceKeyPrefix), KindTrace)
+ if err != nil {
+ return nil, err
+ }
+ traces := make([]*databasev1.Trace, len(messages))
+ for i, message := range messages {
+ traces[i] = message.(*databasev1.Trace)
+ }
+ return traces, nil
+}
+
+func (e *etcdSchemaRegistry) CreateTrace(ctx context.Context, trace *databasev1.Trace) (int64, error) {
+ return e.create(ctx, Metadata{
+ TypeMeta: TypeMeta{
+ Kind: KindTrace,
+ Group: trace.GetMetadata().GetGroup(),
+ Name: trace.GetMetadata().GetName(),
+ },
+ Spec: trace,
+ })
+}
+
+func (e *etcdSchemaRegistry) UpdateTrace(ctx context.Context, trace *databasev1.Trace) (int64, error) {
+ return e.update(ctx, Metadata{
+ TypeMeta: TypeMeta{
+ Kind: KindTrace,
+ Group: trace.GetMetadata().GetGroup(),
+ Name: trace.GetMetadata().GetName(),
+ },
+ Spec: trace,
+ })
+}
+
+func (e *etcdSchemaRegistry) DeleteTrace(ctx context.Context, metadata *commonv1.Metadata) (bool, error) {
+ return e.delete(ctx, Metadata{
+ TypeMeta: TypeMeta{
+ Kind: KindTrace,
+ Group: metadata.GetGroup(),
+ Name: metadata.GetName(),
+ },
+ })
+}
+
+func formatTraceKey(metadata *commonv1.Metadata) string {
+ return formatKey(traceKeyPrefix, metadata)
+}
diff --git a/banyand/trace/metadata.go b/banyand/trace/metadata.go
new file mode 100644
index 00000000..1286decb
--- /dev/null
+++ b/banyand/trace/metadata.go
@@ -0,0 +1,184 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package trace
+
+import (
+ "context"
+ "time"
+
+ "github.com/pkg/errors"
+
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ "github.com/apache/skywalking-banyandb/banyand/liaison/grpc"
+ "github.com/apache/skywalking-banyandb/banyand/metadata"
+ "github.com/apache/skywalking-banyandb/banyand/observability"
+ "github.com/apache/skywalking-banyandb/banyand/protector"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+ resourceSchema "github.com/apache/skywalking-banyandb/pkg/schema"
+ "github.com/apache/skywalking-banyandb/pkg/timestamp"
+)
+
+var (
+ metadataScope = traceScope.SubScope("metadata")
+ ErrTraceNotFound = errors.New("trace not found")
+)
+
+// SchemaService allows querying schema information.
+type SchemaService interface {
+ Query
+ Close()
+}
+
+type schemaRepo struct {
+ resourceSchema.Repository
+ l *logger.Logger
+ metadata metadata.Repo
+ path string
+}
+
+func newSchemaRepo(path string, svc *standalone, nodeLabels map[string]string) schemaRepo {
+ sr := schemaRepo{
+ l: svc.l,
+ path: path,
+ metadata: svc.metadata,
+ Repository: resourceSchema.NewRepository(
+ svc.metadata,
+ svc.l,
+ newSupplier(path, svc, nodeLabels),
+ resourceSchema.NewMetrics(svc.omr.With(metadataScope)),
+ ),
+ }
+ sr.start()
+ return sr
+}
+
+func newLiaisonSchemaRepo(path string, svc *liaison, traceDataNodeRegistry grpc.NodeRegistry) schemaRepo {
+ sr := schemaRepo{
+ l: svc.l,
+ path: path,
+ metadata: svc.metadata,
+ Repository: resourceSchema.NewRepository(
+ svc.metadata,
+ svc.l,
+ newQueueSupplier(path, svc, traceDataNodeRegistry),
+ resourceSchema.NewMetrics(svc.omr.With(metadataScope)),
+ ),
+ }
+ sr.start()
+ return sr
+}
+
+func (sr *schemaRepo) start() {
+ sr.l.Info().Str("path", sr.path).Msg("starting trace metadata repository")
+}
+
+func (sr *schemaRepo) LoadGroup(name string) (resourceSchema.Group, bool) {
+ return sr.Repository.LoadGroup(name)
+}
+
+func (sr *schemaRepo) Trace(metadata *commonv1.Metadata) (*trace, bool) {
+ sm, ok := sr.Repository.LoadResource(metadata)
+ if !ok {
+ return nil, false
+ }
+ t, ok := sm.Delegated().(*trace)
+ return t, ok
+}
+
+func (sr *schemaRepo) GetRemovalSegmentsTimeRange(group string) *timestamp.TimeRange {
+ panic("not implemented")
+}
+
+// supplier is the supplier for standalone service
+type supplier struct {
+ metadata metadata.Repo
+ omr observability.MetricsRegistry
+ pm protector.Memory
+ l *logger.Logger
+ schemaRepo *schemaRepo
+ nodeLabels map[string]string
+ path string
+ option option
+}
+
+func newSupplier(path string, svc *standalone, nodeLabels map[string]string) *supplier {
+ return &supplier{
+ metadata: svc.metadata,
+ omr: svc.omr,
+ pm: svc.pm,
+ l: svc.l,
+ nodeLabels: nodeLabels,
+ path: path,
+ option: svc.option,
+ }
+}
+
+func (s *supplier) OpenResource(spec resourceSchema.Resource) (resourceSchema.IndexListener, error) {
+ traceSchema := spec.Schema().(*databasev1.Trace)
+ return openTrace(traceSchema, s.l, s.pm, s.schemaRepo), nil
+}
+
+func (s *supplier) ResourceSchema(md *commonv1.Metadata) (resourceSchema.ResourceSchema, error) {
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ return s.metadata.TraceRegistry().GetTrace(ctx, md)
+}
+
+func (s *supplier) OpenDB(groupSchema *commonv1.Group) (resourceSchema.DB, error) {
+ panic("not implemented")
+}
+
+// queueSupplier is the supplier for liaison service
+type queueSupplier struct {
+ metadata metadata.Repo
+ omr observability.MetricsRegistry
+ pm protector.Memory
+ traceDataNodeRegistry grpc.NodeRegistry
+ l *logger.Logger
+ schemaRepo *schemaRepo
+ path string
+ option option
+}
+
+func newQueueSupplier(path string, svc *liaison, traceDataNodeRegistry grpc.NodeRegistry) *queueSupplier {
+ return &queueSupplier{
+ metadata: svc.metadata,
+ omr: svc.omr,
+ pm: svc.pm,
+ traceDataNodeRegistry: traceDataNodeRegistry,
+ l: svc.l,
+ path: path,
+ option: svc.option,
+ }
+}
+
+func (qs *queueSupplier) OpenResource(spec resourceSchema.Resource) (resourceSchema.IndexListener, error) {
+ traceSchema := spec.Schema().(*databasev1.Trace)
+ return openTrace(traceSchema, qs.l, qs.pm, qs.schemaRepo), nil
+}
+
+func (qs *queueSupplier) ResourceSchema(md *commonv1.Metadata) (resourceSchema.ResourceSchema, error) {
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ return qs.metadata.TraceRegistry().GetTrace(ctx, md)
+}
+
+func (qs *queueSupplier) OpenDB(groupSchema *commonv1.Group) (resourceSchema.DB, error) {
+ panic("not implemented")
+}
diff --git a/banyand/trace/svc_liaison.go b/banyand/trace/svc_liaison.go
new file mode 100644
index 00000000..49912e67
--- /dev/null
+++ b/banyand/trace/svc_liaison.go
@@ -0,0 +1,190 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package trace
+
+import (
+ "context"
+ "path"
+
+ "github.com/pkg/errors"
+
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ "github.com/apache/skywalking-banyandb/banyand/liaison/grpc"
+ "github.com/apache/skywalking-banyandb/banyand/metadata"
+ "github.com/apache/skywalking-banyandb/banyand/observability"
+ "github.com/apache/skywalking-banyandb/banyand/protector"
+ "github.com/apache/skywalking-banyandb/banyand/queue"
+ "github.com/apache/skywalking-banyandb/pkg/fs"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+ "github.com/apache/skywalking-banyandb/pkg/node"
+ "github.com/apache/skywalking-banyandb/pkg/run"
+ resourceSchema "github.com/apache/skywalking-banyandb/pkg/schema"
+ "github.com/apache/skywalking-banyandb/pkg/timestamp"
+)
+
+type liaison struct {
+ pm protector.Memory
+ metadata metadata.Repo
+ pipeline queue.Server
+ omr observability.MetricsRegistry
+ lfs fs.FileSystem
+ dataNodeSelector node.Selector
+ l *logger.Logger
+ schemaRepo schemaRepo
+ dataPath string
+ root string
+ option option
+ maxDiskUsagePercent int
+}
+
+var _ Service = (*liaison)(nil)
+
+// LiaisonService returns a new liaison service.
+func LiaisonService(ctx context.Context) (Service, error) {
+ return &liaison{}, nil
+}
+
+func (l *liaison) FlagSet() *run.FlagSet {
+ fs := run.NewFlagSet("trace")
+ fs.StringVar(&l.root, "trace-root-path", "/tmp", "the root path for trace data")
+ fs.StringVar(&l.dataPath, "trace-data-path", "", "the path for trace data (optional)")
+ fs.DurationVar(&l.option.flushTimeout, "trace-flush-timeout", defaultFlushTimeout, "the timeout for trace data flush")
+ fs.IntVar(&l.maxDiskUsagePercent, "trace-max-disk-usage-percent", 95, "the maximum disk usage percentage")
+ // Additional liaison-specific flags can be added here
+ return fs
+}
+
+func (l *liaison) Validate() error {
+ if l.root == "" {
+ return errEmptyRootPath
+ }
+ return nil
+}
+
+func (l *liaison) Name() string {
+ return "trace"
+}
+
+func (l *liaison) Role() databasev1.Role {
+ return databasev1.Role_ROLE_LIAISON
+}
+
+func (l *liaison) PreRun(ctx context.Context) error {
+ l.l = logger.GetLogger("trace")
+
+ // Initialize metadata
+ if l.metadata == nil {
+ return errors.New("metadata repo is required")
+ }
+
+ // Initialize filesystem
+ if l.lfs == nil {
+ l.lfs = fs.NewLocalFileSystem()
+ }
+
+ // Initialize protector
+ if l.pm == nil {
+ return errors.New("memory protector is required")
+ }
+
+ // Initialize pipeline
+ if l.pipeline == nil {
+ return errors.New("pipeline is required")
+ }
+
+ // Set up data path
+ if l.dataPath == "" {
+ l.dataPath = path.Join(l.root, "trace-data")
+ }
+
+ // Initialize data node registry - this would be provided by the system
+ var traceDataNodeRegistry grpc.NodeRegistry
+
+ // Initialize schema repository
+ l.schemaRepo = newLiaisonSchemaRepo(l.dataPath, l, traceDataNodeRegistry)
+
+ l.l.Info().
+ Str("root", l.root).
+ Str("dataPath", l.dataPath).
+ Msg("trace liaison service initialized")
+
+ return nil
+}
+
+func (l *liaison) Serve() run.StopNotify {
+ // As specified in the plan, no pipeline listeners should be implemented
+ l.l.Info().Msg("trace liaison service started")
+
+ // Return a channel that never closes since this service runs indefinitely
+ stopCh := make(chan struct{})
+ return stopCh
+}
+
+func (l *liaison) GracefulStop() {
+ if l.schemaRepo.Repository != nil {
+ l.schemaRepo.Repository.Close()
+ }
+ l.l.Info().Msg("trace liaison service stopped")
+}
+
+func (l *liaison) LoadGroup(name string) (resourceSchema.Group, bool) {
+ return l.schemaRepo.LoadGroup(name)
+}
+
+func (l *liaison) Trace(metadata *commonv1.Metadata) (Trace, error) {
+ sm, ok := l.schemaRepo.Trace(metadata)
+ if !ok {
+ return nil, ErrTraceNotFound
+ }
+ return sm, nil
+}
+
+func (l *liaison) GetRemovalSegmentsTimeRange(group string) *timestamp.TimeRange {
+ return l.schemaRepo.GetRemovalSegmentsTimeRange(group)
+}
+
+// SetMetadata sets the metadata repository
+func (l *liaison) SetMetadata(metadata metadata.Repo) {
+ l.metadata = metadata
+}
+
+// SetObservabilityRegistry sets the observability metrics registry
+func (l *liaison) SetObservabilityRegistry(omr observability.MetricsRegistry) {
+ l.omr = omr
+}
+
+// SetProtector sets the memory protector
+func (l *liaison) SetProtector(pm protector.Memory) {
+ l.pm = pm
+}
+
+// SetPipeline sets the pipeline server
+func (l *liaison) SetPipeline(pipeline queue.Server) {
+ l.pipeline = pipeline
+}
+
+// SetFileSystem sets the file system
+func (l *liaison) SetFileSystem(lfs fs.FileSystem) {
+ l.lfs = lfs
+}
+
+// SetDataNodeSelector sets the data node selector
+func (l *liaison) SetDataNodeSelector(selector node.Selector) {
+ l.dataNodeSelector = selector
+}
diff --git a/banyand/trace/svc_standalone.go b/banyand/trace/svc_standalone.go
new file mode 100644
index 00000000..6be6451e
--- /dev/null
+++ b/banyand/trace/svc_standalone.go
@@ -0,0 +1,200 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package trace
+
+import (
+ "context"
+ "path"
+ "path/filepath"
+
+ "github.com/pkg/errors"
+
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ "github.com/apache/skywalking-banyandb/banyand/metadata"
+ "github.com/apache/skywalking-banyandb/banyand/observability"
+ "github.com/apache/skywalking-banyandb/banyand/protector"
+ "github.com/apache/skywalking-banyandb/banyand/queue"
+ "github.com/apache/skywalking-banyandb/pkg/fs"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+ "github.com/apache/skywalking-banyandb/pkg/run"
+ resourceSchema "github.com/apache/skywalking-banyandb/pkg/schema"
+ "github.com/apache/skywalking-banyandb/pkg/timestamp"
+)
+
+var (
+ errEmptyRootPath = errors.New("root path is empty")
+ // ErrTraceNotExist denotes a trace doesn't exist in the metadata repo.
+ ErrTraceNotExist = errors.New("trace doesn't exist")
+)
+
+var _ Service = (*standalone)(nil)
+
+type standalone struct {
+ pm protector.Memory
+ pipeline queue.Server
+ localPipeline queue.Queue
+ omr observability.MetricsRegistry
+ lfs fs.FileSystem
+ metadata metadata.Repo
+ l *logger.Logger
+ schemaRepo schemaRepo
+ snapshotDir string
+ root string
+ dataPath string
+ option option
+ maxDiskUsagePercent int
+ maxFileSnapshotNum int
+}
+
+// StandaloneService returns a new standalone service.
+func StandaloneService(ctx context.Context) (Service, error) {
+ return &standalone{}, nil
+}
+
+func (s *standalone) FlagSet() *run.FlagSet {
+ fs := run.NewFlagSet("trace")
+ fs.StringVar(&s.root, "trace-root-path", "/tmp", "the root path for trace data")
+ fs.StringVar(&s.dataPath, "trace-data-path", "", "the path for trace data (optional)")
+ fs.DurationVar(&s.option.flushTimeout, "trace-flush-timeout", defaultFlushTimeout, "the timeout for trace data flush")
+ fs.IntVar(&s.maxDiskUsagePercent, "trace-max-disk-usage-percent", 95, "the maximum disk usage percentage")
+ fs.IntVar(&s.maxFileSnapshotNum, "trace-max-file-snapshot-num", 2, "the maximum number of file snapshots")
+ // Additional flags can be added here
+ return fs
+}
+
+func (s *standalone) Validate() error {
+ if s.root == "" {
+ return errEmptyRootPath
+ }
+ return nil
+}
+
+func (s *standalone) Name() string {
+ return "trace"
+}
+
+func (s *standalone) Role() databasev1.Role {
+ return databasev1.Role_ROLE_DATA
+}
+
+func (s *standalone) PreRun(ctx context.Context) error {
+ s.l = logger.GetLogger("trace")
+
+ // Initialize metadata
+ if s.metadata == nil {
+ return errors.New("metadata repo is required")
+ }
+
+ // Initialize filesystem
+ if s.lfs == nil {
+ s.lfs = fs.NewLocalFileSystem()
+ }
+
+ // Initialize protector
+ if s.pm == nil {
+ return errors.New("memory protector is required")
+ }
+
+ // Initialize pipeline
+ if s.pipeline == nil {
+ return errors.New("pipeline is required")
+ }
+
+ // Set up data path
+ if s.dataPath == "" {
+ s.dataPath = path.Join(s.root, "trace-data")
+ }
+
+ // Initialize schema repository
+ var nodeLabels map[string]string
+ s.schemaRepo = newSchemaRepo(s.dataPath, s, nodeLabels)
+
+ // Initialize snapshot directory
+ s.snapshotDir = filepath.Join(s.dataPath, "snapshot")
+
+ s.l.Info().
+ Str("root", s.root).
+ Str("dataPath", s.dataPath).
+ Str("snapshotDir", s.snapshotDir).
+ Msg("trace standalone service initialized")
+
+ return nil
+}
+
+func (s *standalone) Serve() run.StopNotify {
+ // As specified in the plan, no pipeline listeners should be implemented
+ s.l.Info().Msg("trace standalone service started")
+
+ // Return a channel that never closes since this service runs indefinitely
+ stopCh := make(chan struct{})
+ return stopCh
+}
+
+func (s *standalone) GracefulStop() {
+ if s.schemaRepo.Repository != nil {
+ s.schemaRepo.Repository.Close()
+ }
+ s.l.Info().Msg("trace standalone service stopped")
+}
+
+func (s *standalone) LoadGroup(name string) (resourceSchema.Group, bool) {
+ return s.schemaRepo.LoadGroup(name)
+}
+
+func (s *standalone) Trace(metadata *commonv1.Metadata) (Trace, error) {
+ sm, ok := s.schemaRepo.Trace(metadata)
+ if !ok {
+ return nil, ErrTraceNotFound
+ }
+ return sm, nil
+}
+
+func (s *standalone) GetRemovalSegmentsTimeRange(group string) *timestamp.TimeRange {
+ return s.schemaRepo.GetRemovalSegmentsTimeRange(group)
+}
+
+// SetMetadata sets the metadata repository
+func (s *standalone) SetMetadata(metadata metadata.Repo) {
+ s.metadata = metadata
+}
+
+// SetObservabilityRegistry sets the observability metrics registry
+func (s *standalone) SetObservabilityRegistry(omr observability.MetricsRegistry) {
+ s.omr = omr
+}
+
+// SetProtector sets the memory protector
+func (s *standalone) SetProtector(pm protector.Memory) {
+ s.pm = pm
+}
+
+// SetPipeline sets the pipeline server
+func (s *standalone) SetPipeline(pipeline queue.Server) {
+ s.pipeline = pipeline
+}
+
+// SetLocalPipeline sets the local pipeline queue
+func (s *standalone) SetLocalPipeline(localPipeline queue.Queue) {
+ s.localPipeline = localPipeline
+}
+
+// SetFileSystem sets the file system
+func (s *standalone) SetFileSystem(lfs fs.FileSystem) {
+ s.lfs = lfs
+}
diff --git a/banyand/trace/trace.go b/banyand/trace/trace.go
new file mode 100644
index 00000000..1f6d3e13
--- /dev/null
+++ b/banyand/trace/trace.go
@@ -0,0 +1,137 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Package trace implements a trace-based storage which consists of trace data.
+// Traces are composed of spans and support querying by trace ID and various tags.
+package trace
+
+import (
+ "sync/atomic"
+ "time"
+
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ "github.com/apache/skywalking-banyandb/banyand/observability"
+ "github.com/apache/skywalking-banyandb/banyand/protector"
+ "github.com/apache/skywalking-banyandb/banyand/queue"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+ "github.com/apache/skywalking-banyandb/pkg/run"
+ "github.com/apache/skywalking-banyandb/pkg/schema"
+ "github.com/apache/skywalking-banyandb/pkg/timestamp"
+)
+
+const (
+ defaultFlushTimeout = time.Second
+ defaultElementIndexFlushTimeout = time.Second
+)
+
+var traceScope = observability.RootScope.SubScope("trace")
+
+type option struct {
+ protector protector.Memory
+ tire2Client queue.Client
+ seriesCacheMaxSize run.Bytes
+ flushTimeout time.Duration
+ elementIndexFlushTimeout time.Duration
+}
+
+// Service allows inspecting the trace data.
+type Service interface {
+ run.PreRunner
+ run.Config
+ run.Service
+ Query
+}
+
+// Query allows retrieving traces.
+type Query interface {
+ LoadGroup(name string) (schema.Group, bool)
+ Trace(metadata *commonv1.Metadata) (Trace, error)
+ GetRemovalSegmentsTimeRange(group string) *timestamp.TimeRange
+}
+
+// Trace allows inspecting trace details.
+type Trace interface {
+ GetSchema() *databasev1.Trace
+ GetIndexRules() []*databasev1.IndexRule
+}
+
+type indexSchema struct {
+ tagMap map[string]*databasev1.TraceTagSpec
+ indexRuleLocators interface{} // Will be properly typed when trace partition support is added
+ indexRules []*databasev1.IndexRule
+}
+
+func (i *indexSchema) parse(schema *databasev1.Trace) {
+ // Note: This will need proper implementation when trace partition support is added
+ // For now, we'll just parse the tag map
+ i.tagMap = make(map[string]*databasev1.TraceTagSpec)
+ for _, tag := range schema.GetTags() {
+ i.tagMap[tag.GetName()] = tag
+ }
+}
+
+type trace struct {
+ pm protector.Memory
+ indexSchema atomic.Value
+ tsdb atomic.Value
+ l *logger.Logger
+ schema *databasev1.Trace
+ schemaRepo *schemaRepo
+ name string
+ group string
+}
+
+type traceSpec struct {
+ schema *databasev1.Trace
+}
+
+func (t *trace) GetSchema() *databasev1.Trace {
+ return t.schema
+}
+
+func (t *trace) GetIndexRules() []*databasev1.IndexRule {
+ if is := t.indexSchema.Load(); is != nil {
+ return is.(*indexSchema).indexRules
+ }
+ return nil
+}
+
+func (t *trace) OnIndexUpdate(index []*databasev1.IndexRule) {
+ var is indexSchema
+ is.indexRules = index
+ is.parse(t.schema)
+ t.indexSchema.Store(is)
+}
+
+func (t *trace) parseSpec() {
+ t.name, t.group = t.schema.GetMetadata().GetName(), t.schema.GetMetadata().GetGroup()
+ var is indexSchema
+ is.parse(t.schema)
+ t.indexSchema.Store(is)
+}
+
+func openTrace(schema *databasev1.Trace, l *logger.Logger, pm protector.Memory, schemaRepo *schemaRepo) *trace {
+ t := &trace{
+ schema: schema,
+ l: l,
+ pm: pm,
+ schemaRepo: schemaRepo,
+ }
+ t.parseSpec()
+ return t
+}
(skywalking-banyandb) 01/02: Add trace plan
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch trace-module
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit 113011c14dafe73f7e5e574031698d56ff9216d0
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Fri Jul 18 22:07:56 2025 +0800
Add trace plan
Signed-off-by: Gao Hongtao <ha...@gmail.com>
---
banyand/trace-plan.md | 264 ++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 264 insertions(+)
diff --git a/banyand/trace-plan.md b/banyand/trace-plan.md
new file mode 100644
index 00000000..5904076f
--- /dev/null
+++ b/banyand/trace-plan.md
@@ -0,0 +1,264 @@
+
+
+# Plan for Creating a New "trace" Package under `/banyand`
+
+Based on the analysis of the existing `stream` and `measure` packages, here's a comprehensive plan to create a new `trace` package following the established patterns:
+
+## 1. Package Structure
+
+```
+banyand/trace/
+├── svc_liaison.go # Liaison service implementation
+├── svc_standalone.go # Standalone service implementation
+├── metadata.go # Schema repository and suppliers
+├── trace.go # Core trace interface and implementation
+└── [other supporting files]
+```
+
+## 2. Core Interfaces and Types
+
+### 2.1 Service Interface
+```go
+// Service allows inspecting the trace data.
+type Service interface {
+ run.PreRunner
+ run.Config
+ run.Service
+ Query
+}
+
+// Query allows retrieving traces.
+type Query interface {
+ LoadGroup(name string) (schema.Group, bool)
+ Trace(metadata *commonv1.Metadata) (Trace, error)
+ GetRemovalSegmentsTimeRange(group string) *timestamp.TimeRange
+}
+
+// Trace allows inspecting trace details.
+type Trace interface {
+ GetSchema() *databasev1.Trace
+ GetIndexRules() []*databasev1.IndexRule
+ Query(ctx context.Context, opts model.TraceQueryOptions) (model.TraceQueryResult, error)
+}
+```
+
+### 2.2 Configuration Options
+```go
+type option struct {
+ mergePolicy *mergePolicy
+ protector protector.Memory
+ tire2Client queue.Client
+ seriesCacheMaxSize run.Bytes
+ flushTimeout time.Duration
+ elementIndexFlushTimeout time.Duration
+}
+```
+
+## 3. Service Implementations
+
+### 3.1 Liaison Service (`svc_liaison.go`)
+
+**Key Features:**
+- Implements `Service` interface
+- Role: `databasev1.Role_ROLE_LIAISON`
+- Uses `queueSupplier` for resource management
+- No pipeline listeners (as requested)
+- Handles distributed trace operations
+
+**Structure:**
+```go
+type liaison struct {
+ pm protector.Memory
+ metadata metadata.Repo
+ pipeline queue.Server
+ omr observability.MetricsRegistry
+ lfs fs.FileSystem
+ dataNodeSelector node.Selector
+ l *logger.Logger
+ schemaRepo schemaRepo
+ dataPath string
+ root string
+ option option
+ maxDiskUsagePercent int
+}
+```
+
+**Key Methods:**
+- `Trace(metadata *commonv1.Metadata) (Trace, error)`
+- `LoadGroup(name string) (resourceSchema.Group, bool)`
+- `GetRemovalSegmentsTimeRange(group string) *timestamp.TimeRange`
+- `FlagSet() *run.FlagSet`
+- `Validate() error`
+- `Name() string` (returns "trace")
+- `Role() databasev1.Role` (returns `ROLE_LIAISON`)
+- `PreRun(ctx context.Context) error`
+- `Serve() run.StopNotify`
+- `GracefulStop()`
+
+### 3.2 Standalone Service (`svc_standalone.go`)
+
+**Key Features:**
+- Implements `Service` interface
+- Role: `databasev1.Role_ROLE_DATA`
+- Uses `supplier` for resource management
+- No pipeline listeners (as requested)
+- Handles local trace operations
+
+**Structure:**
+```go
+type standalone struct {
+ pm protector.Memory
+ pipeline queue.Server
+ localPipeline queue.Queue
+ omr observability.MetricsRegistry
+ lfs fs.FileSystem
+ metadata metadata.Repo
+ l *logger.Logger
+ schemaRepo schemaRepo
+ snapshotDir string
+ root string
+ dataPath string
+ option option
+ maxDiskUsagePercent int
+ maxFileSnapshotNum int
+}
+```
+
+**Key Methods:**
+- `Trace(metadata *commonv1.Metadata) (Trace, error)`
+- `LoadGroup(name string) (resourceSchema.Group, bool)`
+- `GetRemovalSegmentsTimeRange(group string) *timestamp.TimeRange`
+- `FlagSet() *run.FlagSet`
+- `Validate() error`
+- `Name() string` (returns "trace")
+- `Role() databasev1.Role` (returns `ROLE_DATA`)
+- `PreRun(ctx context.Context) error`
+- `Serve() run.StopNotify`
+- `GracefulStop()`
+
+## 4. Schema Repository and Suppliers (`metadata.go`)
+
+### 4.1 Schema Repository
+```go
+type schemaRepo struct {
+ resourceSchema.Repository
+ l *logger.Logger
+ metadata metadata.Repo
+ path string
+}
+```
+
+### 4.2 Supplier (for Standalone)
+```go
+type supplier struct {
+ metadata metadata.Repo
+ omr observability.MetricsRegistry
+ pm protector.Memory
+ l *logger.Logger
+ schemaRepo *schemaRepo
+ nodeLabels map[string]string
+ path string
+ option option
+}
+```
+
+**Required Methods:**
+- `OpenResource(spec resourceSchema.Resource) (resourceSchema.IndexListener, error)`
+- `ResourceSchema(md *commonv1.Metadata) (resourceSchema.ResourceSchema, error)`
+- **Note: `OpenDB` method should NOT be implemented (as requested)**
+
+### 4.3 Queue Supplier (for Liaison)
+```go
+type queueSupplier struct {
+ metadata metadata.Repo
+ omr observability.MetricsRegistry
+ pm protector.Memory
+ traceDataNodeRegistry grpc.NodeRegistry
+ l *logger.Logger
+ schemaRepo *schemaRepo
+ path string
+ option option
+}
+```
+
+**Required Methods:**
+- `OpenResource(spec resourceSchema.Resource) (resourceSchema.IndexListener, error)`
+- `ResourceSchema(md *commonv1.Metadata) (resourceSchema.ResourceSchema, error)`
+- **Note: `OpenDB` method should NOT be implemented (as requested)**
+
+## 5. Core Trace Implementation (`trace.go`)
+
+```go
+type trace struct {
+ pm protector.Memory
+ indexSchema atomic.Value
+ tsdb atomic.Value
+ l *logger.Logger
+ schema *databasev1.Trace
+ schemaRepo *schemaRepo
+ name string
+ group string
+}
+
+type traceSpec struct {
+ schema *databasev1.Trace
+}
+```
+
+## 6. Configuration and Flags
+
+### 6.1 Liaison Service Flags
+- `--trace-root-path` (default: "/tmp")
+- `--trace-data-path` (optional)
+- `--trace-flush-timeout` (default: 1 second)
+- `--trace-max-disk-usage-percent` (default: 95)
+
+### 6.2 Standalone Service Flags
+- `--trace-root-path` (default: "/tmp")
+- `--trace-data-path` (optional)
+- `--trace-flush-timeout` (default: 1 second)
+- `--trace-max-fan-out-size` (merge policy)
+- `--trace-series-cache-max-size` (default: 32MB)
+- `--trace-max-disk-usage-percent` (default: 95)
+- `--trace-max-file-snapshot-num` (default: 2)
+
+## 7. Error Handling
+
+Define common errors:
+```go
+var (
+ errEmptyRootPath = errors.New("root path is empty")
+ ErrTraceNotExist = errors.New("trace doesn't exist")
+)
+```
+
+## 8. Integration Points
+
+### 8.1 Metadata Integration
+- Use `metadata.TraceRegistry()` for trace schema operations
+- Implement proper schema watching and updates
+
+### 8.2 Storage Integration
+- Use `storage.TSDB` for time-series data storage
+- Implement proper segment management and TTL handling
+
+### 8.3 Metrics Integration
+- Implement trace-specific metrics
+- Use observability framework for monitoring
+
+## 9. Implementation Order
+
+1. **Core Interfaces** (`trace.go`)
+2. **Schema Repository** (`metadata.go`)
+3. **Standalone Service** (`svc_standalone.go`)
+4. **Liaison Service** (`svc_liaison.go`)
+5. **Supporting Files** (metrics, query, write handlers, etc.)
+
+## 10. Key Differences from Stream/Measure
+
+1. **No Pipeline Listeners**: Both services should not implement any pipeline listeners as requested
+2. **No OpenDB Implementation**: Both suppliers should implement all methods except `OpenDB`
+3. **Trace-Specific Schema**: Use `databasev1.Trace` instead of `databasev1.Stream` or `databasev1.Measure`
+4. **Trace-Specific Queries**: Implement trace-specific query patterns and result types
+
+This plan follows the established patterns from the stream and measure packages while adapting them specifically for trace data handling, ensuring consistency with the existing codebase architecture.
\ No newline at end of file