You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by lu...@apache.org on 2022/04/04 11:11:04 UTC

[skywalking-banyandb] 01/01: add tests

This is an automated email from the ASF dual-hosted git repository.

lujiajing pushed a commit to branch add-liaison-measure-test
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git

commit ee6e81479ca024e93f2db0bf571bb52f1c39af7d
Author: Megrez Lu <lu...@gmail.com>
AuthorDate: Mon Apr 4 19:10:53 2022 +0800

    add tests
---
 .../grpc/{stream_test.go => measure_test.go}       | 122 +++++++--------------
 banyand/liaison/grpc/stream_test.go                |  25 +++--
 pkg/pb/v1/write.go                                 | 118 ++++++++++++++++++++
 3 files changed, 171 insertions(+), 94 deletions(-)

diff --git a/banyand/liaison/grpc/stream_test.go b/banyand/liaison/grpc/measure_test.go
similarity index 54%
copy from banyand/liaison/grpc/stream_test.go
copy to banyand/liaison/grpc/measure_test.go
index a116426..a7ae577 100644
--- a/banyand/liaison/grpc/stream_test.go
+++ b/banyand/liaison/grpc/measure_test.go
@@ -19,7 +19,6 @@ package grpc_test
 
 import (
 	"context"
-	"encoding/base64"
 	"io"
 	"path/filepath"
 	"runtime"
@@ -30,19 +29,14 @@ import (
 	grpclib "google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
 
-	streamv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/stream/v1"
-	"github.com/apache/skywalking-banyandb/banyand/discovery"
-	"github.com/apache/skywalking-banyandb/banyand/liaison/grpc"
+	measurev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/measure/v1"
 	"github.com/apache/skywalking-banyandb/banyand/metadata"
-	"github.com/apache/skywalking-banyandb/banyand/query"
-	"github.com/apache/skywalking-banyandb/banyand/queue"
-	"github.com/apache/skywalking-banyandb/banyand/stream"
 	pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
 	"github.com/apache/skywalking-banyandb/pkg/test"
-	teststream "github.com/apache/skywalking-banyandb/pkg/test/stream"
+	testmeasure "github.com/apache/skywalking-banyandb/pkg/test/measure"
 )
 
-var _ = Describe("Stream", func() {
+var _ = Describe("Measure", func() {
 	var rootPath, metadataPath string
 	var gracefulStop, deferRootFunc, deferMetadataFunc func()
 	var conn *grpclib.ClientConn
@@ -55,14 +49,14 @@ var _ = Describe("Stream", func() {
 	})
 	It("is a plain server", func() {
 		By("Verifying an empty server")
-		flags := []string{"--stream-root-path=" + rootPath, "--metadata-root-path=" + metadataPath}
+		flags := []string{"--measure-root-path=" + rootPath, "--metadata-root-path=" + metadataPath}
 		gracefulStop = setup(flags)
 		var err error
 		conn, err = grpclib.Dial("localhost:17912", grpclib.WithInsecure())
 		Expect(err).NotTo(HaveOccurred())
-		streamWrite(conn)
+		measureWrite(conn)
 		Eventually(func() (int, error) {
-			return streamQuery(conn)
+			return measureQuery(conn)
 		}).Should(Equal(1))
 		_ = conn.Close()
 		gracefulStop()
@@ -71,16 +65,16 @@ var _ = Describe("Stream", func() {
 		conn, err = grpclib.Dial("localhost:17912", grpclib.WithInsecure())
 		Expect(err).NotTo(HaveOccurred())
 		Eventually(func() int {
-			num, err := streamQuery(conn)
+			num, err := measureQuery(conn)
 			if err != nil {
-				GinkgoWriter.Printf("stream query err: %v \n", err)
+				GinkgoWriter.Printf("measure query err: %v \n", err)
 				return 0
 			}
 			return num
 		}).Should(Equal(1))
 	})
 	It("is a TLS server", func() {
-		flags := []string{"--tls=true", "--stream-root-path=" + rootPath, "--metadata-root-path=" + metadataPath}
+		flags := []string{"--tls=true", "--measure-root-path=" + rootPath, "--metadata-root-path=" + metadataPath}
 		_, currentFile, _, _ := runtime.Caller(0)
 		basePath := filepath.Dir(currentFile)
 		certFile := filepath.Join(basePath, "testdata/server_cert.pem")
@@ -94,9 +88,9 @@ var _ = Describe("Stream", func() {
 		Expect(err).NotTo(HaveOccurred())
 		conn, err = grpclib.Dial(addr, grpclib.WithTransportCredentials(creds))
 		Expect(err).NotTo(HaveOccurred())
-		streamWrite(conn)
+		measureWrite(conn)
 		Eventually(func() (int, error) {
-			return streamQuery(conn)
+			return measureQuery(conn)
 		}).Should(Equal(1))
 	})
 	AfterEach(func() {
@@ -107,90 +101,48 @@ var _ = Describe("Stream", func() {
 	})
 })
 
-func setup(flags []string) func() {
-	// Init `Discovery` module
-	repo, err := discovery.NewServiceRepo(context.Background())
-	Expect(err).NotTo(HaveOccurred())
-	// Init `Queue` module
-	pipeline, err := queue.NewQueue(context.TODO(), repo)
-	Expect(err).NotTo(HaveOccurred())
-	// Init `Metadata` module
-	metaSvc, err := metadata.NewService(context.TODO())
-	Expect(err).NotTo(HaveOccurred())
-	// Init `Stream` module
-	streamSvc, err := stream.NewService(context.TODO(), metaSvc, repo, pipeline)
-	Expect(err).NotTo(HaveOccurred())
-	// Init `Query` module
-	q, err := query.NewExecutor(context.TODO(), streamSvc, nil, metaSvc, repo, pipeline)
-	Expect(err).NotTo(HaveOccurred())
-
-	tcp := grpc.NewServer(context.TODO(), pipeline, repo, metaSvc)
-	preloadStreamSvc := &preloadStreamService{metaSvc: metaSvc}
-
-	return test.SetUpModules(
-		flags,
-		repo,
-		pipeline,
-		metaSvc,
-		preloadStreamSvc,
-		streamSvc,
-		q,
-		tcp,
-	)
+func writeMeasureData() *measurev1.WriteRequest {
+	return pbv1.NewMeasureWriteRequestBuilder().
+		Metadata("sw_metric", "service_cpm_minute").
+		Timestamp(time.Now()).
+		TagFamily(
+			pbv1.ID("1"),
+			"entity_1",
+		).Fields(100, 1).
+		Build()
 }
 
-type preloadStreamService struct {
+type preloadMeasureService struct {
 	metaSvc metadata.Service
 }
 
-func (p *preloadStreamService) Name() string {
+func (p *preloadMeasureService) Name() string {
 	return "preload-measure"
 }
 
-func (p *preloadStreamService) PreRun() error {
-	return teststream.PreloadSchema(p.metaSvc.SchemaRegistry())
-}
-
-func writeData() *streamv1.WriteRequest {
-	bb, _ := base64.StdEncoding.DecodeString("YWJjMTIzIT8kKiYoKSctPUB+")
-	return pbv1.NewStreamWriteRequestBuilder().
-		ID("1").
-		Metadata("default", "sw").
-		Timestamp(time.Now()).
-		TagFamily(bb).
-		TagFamily(
-			"trace_id-xxfff.111",
-			0,
-			"webapp_id",
-			"10.0.0.1_id",
-			"/home_id",
-			300,
-			1622933202000000000,
-		).
-		Build()
+func (p *preloadMeasureService) PreRun() error {
+	return testmeasure.PreloadSchema(p.metaSvc.SchemaRegistry())
 }
 
-func queryCriteria(baseTs time.Time) *streamv1.QueryRequest {
-	return pbv1.NewStreamQueryRequestBuilder().
-		Limit(10).
-		Offset(0).
-		Metadata("default", "sw").
+func queryMeasureCriteria(baseTs time.Time) *measurev1.QueryRequest {
+	return pbv1.NewMeasureQueryRequestBuilder().
+		Metadata("sw_metric", "service_cpm_minute").
 		TimeRange(baseTs.Add(-1*time.Minute), baseTs.Add(1*time.Minute)).
-		Projection("searchable", "trace_id").
+		TagProjection("searchable", "trace_id").
 		Build()
 }
 
-func streamWrite(conn *grpclib.ClientConn) {
-	client := streamv1.NewStreamServiceClient(conn)
+func measureWrite(conn *grpclib.ClientConn) {
+	c := measurev1.NewMeasureServiceClient(conn)
 	ctx := context.Background()
-	var writeClient streamv1.StreamService_WriteClient
+	var writeClient measurev1.MeasureService_WriteClient
 	Eventually(func(g Gomega) {
 		var err error
-		writeClient, err = client.Write(ctx)
+		writeClient, err = c.Write(ctx)
 		g.Expect(err).NotTo(HaveOccurred())
 	}).Should(Succeed())
 	Eventually(func() error {
-		return writeClient.Send(writeData())
+		return writeClient.Send(writeMeasureData())
 	}).ShouldNot(HaveOccurred())
 	Expect(writeClient.CloseSend()).Should(Succeed())
 	Eventually(func() error {
@@ -199,10 +151,10 @@ func streamWrite(conn *grpclib.ClientConn) {
 	}).Should(Equal(io.EOF))
 }
 
-func streamQuery(conn *grpclib.ClientConn) (int, error) {
-	client := streamv1.NewStreamServiceClient(conn)
+func measureQuery(conn *grpclib.ClientConn) (int, error) {
+	c := measurev1.NewMeasureServiceClient(conn)
 	ctx := context.Background()
-	resp, err := client.Query(ctx, queryCriteria(time.Now()))
+	resp, err := c.Query(ctx, queryMeasureCriteria(time.Now()))
 
-	return len(resp.GetElements()), err
+	return len(resp.GetDataPoints()), err
 }
diff --git a/banyand/liaison/grpc/stream_test.go b/banyand/liaison/grpc/stream_test.go
index a116426..dbc271d 100644
--- a/banyand/liaison/grpc/stream_test.go
+++ b/banyand/liaison/grpc/stream_test.go
@@ -33,6 +33,7 @@ import (
 	streamv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/stream/v1"
 	"github.com/apache/skywalking-banyandb/banyand/discovery"
 	"github.com/apache/skywalking-banyandb/banyand/liaison/grpc"
+	"github.com/apache/skywalking-banyandb/banyand/measure"
 	"github.com/apache/skywalking-banyandb/banyand/metadata"
 	"github.com/apache/skywalking-banyandb/banyand/query"
 	"github.com/apache/skywalking-banyandb/banyand/queue"
@@ -120,12 +121,16 @@ func setup(flags []string) func() {
 	// Init `Stream` module
 	streamSvc, err := stream.NewService(context.TODO(), metaSvc, repo, pipeline)
 	Expect(err).NotTo(HaveOccurred())
+	// Init `Measure` module
+	measureSvc, err := measure.NewService(context.TODO(), metaSvc, repo, pipeline)
+	Expect(err).NotTo(HaveOccurred())
 	// Init `Query` module
-	q, err := query.NewExecutor(context.TODO(), streamSvc, nil, metaSvc, repo, pipeline)
+	q, err := query.NewExecutor(context.TODO(), streamSvc, measureSvc, metaSvc, repo, pipeline)
 	Expect(err).NotTo(HaveOccurred())
 
 	tcp := grpc.NewServer(context.TODO(), pipeline, repo, metaSvc)
 	preloadStreamSvc := &preloadStreamService{metaSvc: metaSvc}
+	preloadMeasureSvc := &preloadMeasureService{metaSvc: metaSvc}
 
 	return test.SetUpModules(
 		flags,
@@ -133,7 +138,9 @@ func setup(flags []string) func() {
 		pipeline,
 		metaSvc,
 		preloadStreamSvc,
+		preloadMeasureSvc,
 		streamSvc,
+		measureSvc,
 		q,
 		tcp,
 	)
@@ -144,14 +151,14 @@ type preloadStreamService struct {
 }
 
 func (p *preloadStreamService) Name() string {
-	return "preload-measure"
+	return "preload-stream"
 }
 
 func (p *preloadStreamService) PreRun() error {
 	return teststream.PreloadSchema(p.metaSvc.SchemaRegistry())
 }
 
-func writeData() *streamv1.WriteRequest {
+func writeStreamData() *streamv1.WriteRequest {
 	bb, _ := base64.StdEncoding.DecodeString("YWJjMTIzIT8kKiYoKSctPUB+")
 	return pbv1.NewStreamWriteRequestBuilder().
 		ID("1").
@@ -170,7 +177,7 @@ func writeData() *streamv1.WriteRequest {
 		Build()
 }
 
-func queryCriteria(baseTs time.Time) *streamv1.QueryRequest {
+func queryStreamCriteria(baseTs time.Time) *streamv1.QueryRequest {
 	return pbv1.NewStreamQueryRequestBuilder().
 		Limit(10).
 		Offset(0).
@@ -181,16 +188,16 @@ func queryCriteria(baseTs time.Time) *streamv1.QueryRequest {
 }
 
 func streamWrite(conn *grpclib.ClientConn) {
-	client := streamv1.NewStreamServiceClient(conn)
+	c := streamv1.NewStreamServiceClient(conn)
 	ctx := context.Background()
 	var writeClient streamv1.StreamService_WriteClient
 	Eventually(func(g Gomega) {
 		var err error
-		writeClient, err = client.Write(ctx)
+		writeClient, err = c.Write(ctx)
 		g.Expect(err).NotTo(HaveOccurred())
 	}).Should(Succeed())
 	Eventually(func() error {
-		return writeClient.Send(writeData())
+		return writeClient.Send(writeStreamData())
 	}).ShouldNot(HaveOccurred())
 	Expect(writeClient.CloseSend()).Should(Succeed())
 	Eventually(func() error {
@@ -200,9 +207,9 @@ func streamWrite(conn *grpclib.ClientConn) {
 }
 
 func streamQuery(conn *grpclib.ClientConn) (int, error) {
-	client := streamv1.NewStreamServiceClient(conn)
+	c := streamv1.NewStreamServiceClient(conn)
 	ctx := context.Background()
-	resp, err := client.Query(ctx, queryCriteria(time.Now()))
+	resp, err := c.Query(ctx, queryStreamCriteria(time.Now()))
 
 	return len(resp.GetElements()), err
 }
diff --git a/pkg/pb/v1/write.go b/pkg/pb/v1/write.go
index e39a3a0..7eb8cbc 100644
--- a/pkg/pb/v1/write.go
+++ b/pkg/pb/v1/write.go
@@ -26,11 +26,14 @@ import (
 	"google.golang.org/protobuf/types/known/timestamppb"
 
 	commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+	measurev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/measure/v1"
 	modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
 	streamv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/stream/v1"
 	"github.com/apache/skywalking-banyandb/pkg/convert"
 )
 
+type ID string
+
 const strDelimiter = "\n"
 
 var ErrUnsupportedTagForIndexField = errors.New("the tag type(for example, null) can not be as the index field value")
@@ -131,6 +134,121 @@ func getTag(tag interface{}) *modelv1.TagValue {
 				BinaryData: t,
 			},
 		}
+	case ID:
+		return &modelv1.TagValue{
+			Value: &modelv1.TagValue_Id{
+				Id: &modelv1.ID{
+					Value: string(t),
+				},
+			},
+		}
+	}
+	return nil
+}
+
+type MeasureWriteRequestBuilder struct {
+	ec *measurev1.WriteRequest
+}
+
+func NewMeasureWriteRequestBuilder() *MeasureWriteRequestBuilder {
+	return &MeasureWriteRequestBuilder{
+		ec: &measurev1.WriteRequest{
+			DataPoint: &measurev1.DataPointValue{
+				TagFamilies: make([]*modelv1.TagFamilyForWrite, 0),
+				Fields:      make([]*modelv1.FieldValue, 0),
+			},
+		},
+	}
+}
+
+func (b *MeasureWriteRequestBuilder) Metadata(group, name string) *MeasureWriteRequestBuilder {
+	b.ec.Metadata = &commonv1.Metadata{
+		Group: group,
+		Name:  name,
+	}
+	return b
+}
+
+func (b *MeasureWriteRequestBuilder) TagFamily(tags ...interface{}) *MeasureWriteRequestBuilder {
+	tagFamily := &modelv1.TagFamilyForWrite{}
+	for _, tag := range tags {
+		tagFamily.Tags = append(tagFamily.Tags, getTag(tag))
+	}
+	b.ec.DataPoint.TagFamilies = append(b.ec.DataPoint.TagFamilies, tagFamily)
+	return b
+}
+
+func (b *MeasureWriteRequestBuilder) Fields(fields ...interface{}) *MeasureWriteRequestBuilder {
+	var fieldValues []*modelv1.FieldValue
+	for _, field := range fields {
+		fieldValues = append(fieldValues, getField(field))
+	}
+	b.ec.DataPoint.Fields = append(b.ec.DataPoint.Fields, fieldValues...)
+	return b
+}
+
+func (b *MeasureWriteRequestBuilder) Timestamp(t time.Time) *MeasureWriteRequestBuilder {
+	b.ec.DataPoint.Timestamp = timestamppb.New(t)
+	return b
+}
+
+func (b *MeasureWriteRequestBuilder) Build() *measurev1.WriteRequest {
+	return b.ec
+}
+
+func getField(field interface{}) *modelv1.FieldValue {
+	if field == nil {
+		return &modelv1.FieldValue{
+			Value: &modelv1.FieldValue_Null{},
+		}
+	}
+	switch t := field.(type) {
+	case int8:
+		return &modelv1.FieldValue{
+			Value: &modelv1.FieldValue_Int{
+				Int: &modelv1.Int{
+					Value: int64(t),
+				},
+			},
+		}
+	case int32:
+		return &modelv1.FieldValue{
+			Value: &modelv1.FieldValue_Int{
+				Int: &modelv1.Int{
+					Value: int64(t),
+				},
+			},
+		}
+	case int64:
+		return &modelv1.FieldValue{
+			Value: &modelv1.FieldValue_Int{
+				Int: &modelv1.Int{
+					Value: t,
+				},
+			},
+		}
+	case int:
+		return &modelv1.FieldValue{
+			Value: &modelv1.FieldValue_Int{
+				Int: &modelv1.Int{
+					Value: int64(t),
+				},
+			},
+		}
+	case string:
+		return &modelv1.FieldValue{
+			Value: &modelv1.FieldValue_Str{
+				Str: &modelv1.Str{
+					Value: t,
+				},
+			},
+		}
+	case []byte:
+		return &modelv1.FieldValue{
+			Value: &modelv1.FieldValue_BinaryData{
+				BinaryData: t,
+			},
+		}
 	}
 	return nil
 }