You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ha...@apache.org on 2022/07/27 01:13:55 UTC
[skywalking-banyandb] 01/01: Add stress test components
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch test-stress
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit c007fc40cd8734c34b273a4de02770ac4fe07334
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Tue Jul 26 02:11:06 2022 +0000
Add stress test components
Signed-off-by: Gao Hongtao <ha...@gmail.com>
---
banyand/Dockerfile | 2 ++
banyand/kv/kv.go | 1 +
banyand/observability/pprof.go | 2 +-
banyand/tsdb/index/writer.go | 9 +++++++++
pkg/index/inverted/inverted.go | 13 +++++++++++--
pkg/index/lsm/lsm.go | 7 ++++++-
pkg/index/metadata/metadata.go | 6 ++++++
pkg/partition/entity.go | 3 +++
pkg/pb/v1/write.go | 7 ++++++-
test/docker/base-compose.yml | 2 ++
test/stress/Makefile | 12 +++++++++---
test/stress/docker-compose.yaml | 28 ++++++++++++++++++++++++++--
test/stress/env | 1 +
test/stress/env.dev | 1 +
test/stress/prom/prometheus.yml | 23 +++++++++++++++++++++++
15 files changed, 107 insertions(+), 10 deletions(-)
diff --git a/banyand/Dockerfile b/banyand/Dockerfile
index a1baea8..e6da33b 100644
--- a/banyand/Dockerfile
+++ b/banyand/Dockerfile
@@ -54,6 +54,7 @@ COPY --from=certs /etc/ssl/certs /etc/ssl/certs
EXPOSE 17912
EXPOSE 17913
+EXPOSE 6060
ENTRYPOINT ["/banyand-server"]
@@ -64,5 +65,6 @@ COPY --from=base /bin/grpc-health-probe /grpc-health-probe
EXPOSE 17912
EXPOSE 17913
+EXPOSE 6060
ENTRYPOINT ["/banyand-server"]
\ No newline at end of file
diff --git a/banyand/kv/kv.go b/banyand/kv/kv.go
index 4f0b795..9ae630f 100644
--- a/banyand/kv/kv.go
+++ b/banyand/kv/kv.go
@@ -143,6 +143,7 @@ type Iterable interface {
type HandoverCallback func()
type IndexStore interface {
+ observability.Observable
Iterable
Reader
Handover(iterator Iterator) error
diff --git a/banyand/observability/pprof.go b/banyand/observability/pprof.go
index 6e9bfe8..9a07a2d 100644
--- a/banyand/observability/pprof.go
+++ b/banyand/observability/pprof.go
@@ -45,7 +45,7 @@ type pprofService struct {
func (p *pprofService) FlagSet() *run.FlagSet {
flagSet := run.NewFlagSet("prof")
- flagSet.StringVar(&p.listenAddr, "pprof-listener-addr", "127.0.0.1:6060", "listen addr for pprof")
+ flagSet.StringVar(&p.listenAddr, "pprof-listener-addr", ":6060", "listen addr for pprof")
return flagSet
}
diff --git a/banyand/tsdb/index/writer.go b/banyand/tsdb/index/writer.go
index 993a482..ae9fde2 100644
--- a/banyand/tsdb/index/writer.go
+++ b/banyand/tsdb/index/writer.go
@@ -131,6 +131,9 @@ func (s *Writer) writeGlobalIndex(scope tsdb.Entry, ruleIndex *partition.IndexRu
if err != nil {
return err
}
+ if values == nil {
+ return nil
+ }
var errWriting error
for _, val := range values {
indexShardID, err := partition.ShardID(val, s.shardNum)
@@ -176,6 +179,9 @@ func writeLocalIndex(writer tsdb.Writer, ruleIndex *partition.IndexRuleLocator,
if err != nil {
return err
}
+ if values == nil {
+ return nil
+ }
var errWriting error
for _, val := range values {
rule := ruleIndex.Rule
@@ -221,6 +227,9 @@ func getIndexValue(ruleIndex *partition.IndexRuleLocator, value Value) (val [][]
existInt = true
}
fv, err := pbv1.ParseIndexFieldValue(tag)
+ if errors.Is(err, pbv1.ErrNullValue) {
+ return nil, existInt, nil
+ }
if err != nil {
return nil, false, err
}
diff --git a/pkg/index/inverted/inverted.go b/pkg/index/inverted/inverted.go
index da8b8fc..8073d19 100644
--- a/pkg/index/inverted/inverted.go
+++ b/pkg/index/inverted/inverted.go
@@ -100,10 +100,19 @@ func (s *store) Flush() error {
return nil
}
-func (s *store) Stats() (stat observability.Statistics) {
+func (s *store) Stats() observability.Statistics {
+ stat := s.mainStats()
+ disk := s.diskTable.Stats()
+ stat.MaxMemBytes = disk.MaxMemBytes
+ term := s.termMetadata.Stats()
+ stat.MemBytes += term.MemBytes
+ stat.MaxMemBytes += term.MaxMemBytes
+ return stat
+}
+
+func (s *store) mainStats() (stat observability.Statistics) {
s.rwMutex.RLock()
defer s.rwMutex.RUnlock()
- // TODO: add MaxMem
main := s.memTable.Stats()
stat.MemBytes += main.MemBytes
if s.immutableMemTable != nil {
diff --git a/pkg/index/lsm/lsm.go b/pkg/index/lsm/lsm.go
index a6a3557..b2ef18c 100644
--- a/pkg/index/lsm/lsm.go
+++ b/pkg/index/lsm/lsm.go
@@ -42,7 +42,12 @@ func (*store) Flush() error {
}
func (s *store) Stats() observability.Statistics {
- return s.lsm.Stats()
+ store := s.lsm.Stats()
+ term := s.termMetadata.Stats()
+ return observability.Statistics{
+ MemBytes: store.MemBytes + term.MemBytes,
+ MaxMemBytes: store.MaxMemBytes + term.MaxMemBytes,
+ }
}
func (s *store) Close() error {
diff --git a/pkg/index/metadata/metadata.go b/pkg/index/metadata/metadata.go
index fd7d5d0..3e150e0 100644
--- a/pkg/index/metadata/metadata.go
+++ b/pkg/index/metadata/metadata.go
@@ -23,11 +23,13 @@ import (
"github.com/pkg/errors"
"github.com/apache/skywalking-banyandb/banyand/kv"
+ "github.com/apache/skywalking-banyandb/banyand/observability"
"github.com/apache/skywalking-banyandb/pkg/convert"
"github.com/apache/skywalking-banyandb/pkg/logger"
)
type Term interface {
+ observability.Observable
ID(term []byte) (id []byte, err error)
Literal(id []byte) (term []byte, err error)
io.Closer
@@ -71,3 +73,7 @@ func (t *term) Literal(id []byte) (term []byte, err error) {
func (t *term) Close() error {
return t.store.Close()
}
+
+func (t *term) Stats() observability.Statistics {
+ return t.store.Stats()
+}
diff --git a/pkg/partition/entity.go b/pkg/partition/entity.go
index 207d18e..4a8c698 100644
--- a/pkg/partition/entity.go
+++ b/pkg/partition/entity.go
@@ -56,6 +56,9 @@ func (e EntityLocator) Find(subject string, value []*modelv1.TagFamilyForWrite)
return nil, err
}
entry, errMarshal := pbv1.MarshalIndexFieldValue(tag)
+ if errors.Is(errMarshal, pbv1.ErrNullValue) {
+ continue
+ }
if errMarshal != nil {
return nil, errMarshal
}
diff --git a/pkg/pb/v1/write.go b/pkg/pb/v1/write.go
index ba9602d..fa5dd3c 100644
--- a/pkg/pb/v1/write.go
+++ b/pkg/pb/v1/write.go
@@ -35,7 +35,10 @@ type ID string
var strDelimiter = []byte("\n")
-var ErrUnsupportedTagForIndexField = errors.New("the tag type(for example, null) can not be as the index field value")
+var (
+ ErrUnsupportedTagForIndexField = errors.New("the tag type(for example, null) can not be as the index field value")
+ ErrNullValue = errors.New("the tag value is null")
+)
func MarshalIndexFieldValue(tagValue *modelv1.TagValue) ([]byte, error) {
fv, err := ParseIndexFieldValue(tagValue)
@@ -114,6 +117,8 @@ func (fv *FieldValue) marshalArr() []byte {
func ParseIndexFieldValue(tagValue *modelv1.TagValue) (FieldValue, error) {
switch x := tagValue.GetValue().(type) {
+ case *modelv1.TagValue_Null:
+ return FieldValue{}, ErrNullValue
case *modelv1.TagValue_Str:
return newValue([]byte(x.Str.GetValue())), nil
case *modelv1.TagValue_Int:
diff --git a/test/docker/base-compose.yml b/test/docker/base-compose.yml
index 1db2405..e6893a2 100644
--- a/test/docker/base-compose.yml
+++ b/test/docker/base-compose.yml
@@ -17,6 +17,8 @@ services:
banyandb:
expose:
- 17912
+ - 2121
+ - 6060
command: standalone
healthcheck:
test: ["CMD", "/grpc-health-probe", "-addr=localhost:17912"]
diff --git a/test/stress/Makefile b/test/stress/Makefile
index 1a12f92..e020015 100644
--- a/test/stress/Makefile
+++ b/test/stress/Makefile
@@ -16,15 +16,21 @@
# under the License.
#
-NAME := ui
+NAME := stress
+
+CLI_ARGS :=
+
+ifdef PROFILE
+ CLI_ARGS := $(CLI_ARGS) --profile $(PROFILE)
+endif
.PHONY: dev-up
dev-up:
- DOCKER_BUILDKIT=1 docker compose --env-file ./env.dev up --build
+ DOCKER_BUILDKIT=1 docker compose $(CLI_ARGS) --env-file ./env.dev up --build
.PHONY: up
up:
- DOCKER_BUILDKIT=1 docker compose --env-file ./env up --build
+ DOCKER_BUILDKIT=1 docker compose $(CLI_ARGS) --env-file ./env up --build
.PHONY: down
down:
diff --git a/test/stress/docker-compose.yaml b/test/stress/docker-compose.yaml
index ad92e1a..bb0ce4c 100644
--- a/test/stress/docker-compose.yaml
+++ b/test/stress/docker-compose.yaml
@@ -26,8 +26,10 @@ services:
- ../..:/app:rw,delegated
ports:
- 17913:17913
+ - 6060:6060
networks:
- test
+ - monitoring
oap:
extends:
@@ -86,7 +88,7 @@ services:
extends:
file: ../docker/base-compose.yml
service: traffic_loader
- command: run --duration 1h /scripts/consumer.js
+ command: run --vus ${VUS} --duration 1h /scripts/consumer.js
volumes:
- ./scripts:/scripts
networks:
@@ -98,8 +100,30 @@ services:
condition: service_healthy
consumer:
condition: service_healthy
+
+ prometheus:
+ image: prom/prometheus:latest
+ container_name: prometheus
+ restart: unless-stopped
+ profiles:
+ - "monitoring"
+ volumes:
+ - ./prom/prometheus.yml:/etc/prometheus/prometheus.yml
+ - prometheus_data:/prometheus
+ command:
+ - '--config.file=/etc/prometheus/prometheus.yml'
+ - '--storage.tsdb.path=/prometheus'
+ - '--web.console.libraries=/etc/prometheus/console_libraries'
+ - '--web.console.templates=/etc/prometheus/consoles'
+ - '--web.enable-lifecycle'
+ ports:
+ - 9090:9090
+ networks:
+ - monitoring
networks:
test:
+ monitoring:
volumes:
- sw_agent:
+ sw_agent: {}
+ prometheus_data: {}
diff --git a/test/stress/env b/test/stress/env
index 3571db7..78577df 100644
--- a/test/stress/env
+++ b/test/stress/env
@@ -29,3 +29,4 @@ SW_CTL_COMMIT=219876daf985fd474955834ef0b65013f0890e96
SW_OAP_COMMIT=dc39ce9bb44ed33d9c2bb0d5a054b1dfd5bbd657
TARGET=test
+VUS=10
diff --git a/test/stress/env.dev b/test/stress/env.dev
index 496b269..5234b7a 100644
--- a/test/stress/env.dev
+++ b/test/stress/env.dev
@@ -29,3 +29,4 @@ SW_CTL_COMMIT=219876daf985fd474955834ef0b65013f0890e96
SW_OAP_COMMIT=dc39ce9bb44ed33d9c2bb0d5a054b1dfd5bbd657
TARGET=dev
+VUS=1
diff --git a/test/stress/prom/prometheus.yml b/test/stress/prom/prometheus.yml
new file mode 100644
index 0000000..afa97c2
--- /dev/null
+++ b/test/stress/prom/prometheus.yml
@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+global:
+ scrape_interval: 1m
+
+scrape_configs:
+ - job_name: "banyandb"
+ static_configs:
+ - targets: ["banyandb:2121"]
+