You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ha...@apache.org on 2024/03/14 01:14:44 UTC
(skywalking-banyandb) 01/02: Refactor stress test
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch metadata
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit d78d3b003ee6bac892f1a944c125394375d90e7e
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Mon Mar 11 22:11:45 2024 +0000
Refactor stress test
Signed-off-by: Gao Hongtao <ha...@gmail.com>
---
banyand/metadata/schema/watcher.go | 8 +++
banyand/queue/pub/pub.go | 2 +-
test/stress/{ => classic}/Makefile | 0
test/stress/{ => classic}/docker-compose.yaml | 14 ++--
test/stress/{ => classic}/env | 0
test/stress/{ => classic}/env.dev | 0
test/stress/{ => classic}/log4j2.xml | 0
test/stress/{ => classic}/prom/prometheus.yml | 0
test/stress/{ => classic}/scripts/consumer.js | 0
test/stress/{cases => }/istio/Makefile | 0
test/stress/{cases => }/istio/istio_suite_test.go | 0
test/stress/{cases => }/istio/repo.go | 0
test/stress/{cases => }/istio/report.md | 0
.../{cases => }/istio/testdata/access.tar.bz2 | Bin
.../{cases => }/istio/testdata/groups/group.yaml | 0
.../measure-default-index-rule-binding.yaml | 0
.../measure-minute-index-rule-binding.yaml | 0
.../index-rules/measure-default-index-rule.yaml | 0
.../index-rules/measure-minute-index-rule.yaml | 0
.../istio/testdata/measures/measure-default.yaml | 0
.../istio/testdata/measures/measure-minute.yaml | 0
.../{cases => }/istio/testdata/metrics/data.csv | 0
.../istio/testdata/metrics/intermediate.csv | 0
.../{cases => }/istio/testdata/metrics/result.csv | 0
test/stress/{ => trace}/Makefile | 6 +-
test/stress/{ => trace}/docker-compose.yaml | 78 +++++----------------
test/stress/{env.dev => trace/env} | 3 +-
test/stress/{ => trace}/log4j2.xml | 0
test/stress/{prom => trace}/prometheus.yml | 0
29 files changed, 35 insertions(+), 76 deletions(-)
diff --git a/banyand/metadata/schema/watcher.go b/banyand/metadata/schema/watcher.go
index bd9b2b8f..c21e3dec 100644
--- a/banyand/metadata/schema/watcher.go
+++ b/banyand/metadata/schema/watcher.go
@@ -65,10 +65,16 @@ func (w *watcher) Close() error {
func (w *watcher) allEvents() int64 {
cli := w.cli
var resp *clientv3.GetResponse
+ start := time.Now()
+ var eventHandleTime time.Duration
+ var eventSize int
for {
var err error
if resp, err = cli.Get(w.closer.Ctx(), w.key, clientv3.WithPrefix()); err == nil {
+ startHandle := time.Now()
+ eventSize = len(resp.Kvs)
w.handleAllEvents(resp.Kvs)
+ eventHandleTime = time.Since(startHandle)
break
}
select {
@@ -77,6 +83,8 @@ func (w *watcher) allEvents() int64 {
case <-time.After(1 * time.Second):
}
}
+ w.l.Info().Dur("event_handle_time", eventHandleTime).Dur("total_time", time.Since(start)).
+ Int("event_size", eventSize).Str("key", w.key).Msg("watcher all events")
return resp.Header.Revision
}
diff --git a/banyand/queue/pub/pub.go b/banyand/queue/pub/pub.go
index 73cf9927..8e6de582 100644
--- a/banyand/queue/pub/pub.go
+++ b/banyand/queue/pub/pub.go
@@ -107,7 +107,7 @@ func (p *pub) Publish(topic bus.Topic, messages ...bus.Message) (bus.Future, err
return multierr.Append(err, fmt.Errorf("failed to get client for node %s", node))
}
stream, errCreateStream := client.client.Send(context.Background())
- if err != nil {
+ if errCreateStream != nil {
return multierr.Append(err, fmt.Errorf("failed to get stream for node %s: %w", node, errCreateStream))
}
errSend = stream.Send(r)
diff --git a/test/stress/Makefile b/test/stress/classic/Makefile
similarity index 100%
copy from test/stress/Makefile
copy to test/stress/classic/Makefile
diff --git a/test/stress/docker-compose.yaml b/test/stress/classic/docker-compose.yaml
similarity index 92%
copy from test/stress/docker-compose.yaml
copy to test/stress/classic/docker-compose.yaml
index 9d28ff79..af8a24f8 100644
--- a/test/stress/docker-compose.yaml
+++ b/test/stress/classic/docker-compose.yaml
@@ -25,11 +25,11 @@ services:
banyandb:
user: "${USER_ID}:${GROUP_ID}"
extends:
- file: ../docker/base-compose.yml
+ file: ../../docker/base-compose.yml
service: banyandb
build:
dockerfile: ./docker/Dockerfile
- context: ../..
+ context: ../../..
volumes:
- /tmp/banyandb-stress-agent:/tmp:rw,delgated
ports:
@@ -45,7 +45,7 @@ services:
oap:
extends:
- file: ../docker/base-compose.yml
+ file: ../../docker/base-compose.yml
service: oap
environment:
SW_STORAGE: banyandb
@@ -61,14 +61,14 @@ services:
agent:
extends:
- file: ../docker/base-compose.yml
+ file: ../../docker/base-compose.yml
service: agent
networks:
- test
provider:
extends:
- file: ../docker/base-compose.yml
+ file: ../../docker/base-compose.yml
service: provider
ports:
- 9090
@@ -82,7 +82,7 @@ services:
consumer:
extends:
- file: ../docker/base-compose.yml
+ file: ../../docker/base-compose.yml
service: consumer
ports:
- 9092
@@ -98,7 +98,7 @@ services:
traffic_loader:
extends:
- file: ../docker/base-compose.yml
+ file: ../../docker/base-compose.yml
service: traffic_loader
command: run --vus ${VUS} --duration 1h /scripts/consumer.js
volumes:
diff --git a/test/stress/env b/test/stress/classic/env
similarity index 100%
rename from test/stress/env
rename to test/stress/classic/env
diff --git a/test/stress/env.dev b/test/stress/classic/env.dev
similarity index 100%
copy from test/stress/env.dev
copy to test/stress/classic/env.dev
diff --git a/test/stress/log4j2.xml b/test/stress/classic/log4j2.xml
similarity index 100%
copy from test/stress/log4j2.xml
copy to test/stress/classic/log4j2.xml
diff --git a/test/stress/prom/prometheus.yml b/test/stress/classic/prom/prometheus.yml
similarity index 100%
copy from test/stress/prom/prometheus.yml
copy to test/stress/classic/prom/prometheus.yml
diff --git a/test/stress/scripts/consumer.js b/test/stress/classic/scripts/consumer.js
similarity index 100%
rename from test/stress/scripts/consumer.js
rename to test/stress/classic/scripts/consumer.js
diff --git a/test/stress/cases/istio/Makefile b/test/stress/istio/Makefile
similarity index 100%
rename from test/stress/cases/istio/Makefile
rename to test/stress/istio/Makefile
diff --git a/test/stress/cases/istio/istio_suite_test.go b/test/stress/istio/istio_suite_test.go
similarity index 100%
rename from test/stress/cases/istio/istio_suite_test.go
rename to test/stress/istio/istio_suite_test.go
diff --git a/test/stress/cases/istio/repo.go b/test/stress/istio/repo.go
similarity index 100%
rename from test/stress/cases/istio/repo.go
rename to test/stress/istio/repo.go
diff --git a/test/stress/cases/istio/report.md b/test/stress/istio/report.md
similarity index 100%
rename from test/stress/cases/istio/report.md
rename to test/stress/istio/report.md
diff --git a/test/stress/cases/istio/testdata/access.tar.bz2 b/test/stress/istio/testdata/access.tar.bz2
similarity index 100%
rename from test/stress/cases/istio/testdata/access.tar.bz2
rename to test/stress/istio/testdata/access.tar.bz2
diff --git a/test/stress/cases/istio/testdata/groups/group.yaml b/test/stress/istio/testdata/groups/group.yaml
similarity index 100%
rename from test/stress/cases/istio/testdata/groups/group.yaml
rename to test/stress/istio/testdata/groups/group.yaml
diff --git a/test/stress/cases/istio/testdata/index-rule-bindings/measure-default-index-rule-binding.yaml b/test/stress/istio/testdata/index-rule-bindings/measure-default-index-rule-binding.yaml
similarity index 100%
rename from test/stress/cases/istio/testdata/index-rule-bindings/measure-default-index-rule-binding.yaml
rename to test/stress/istio/testdata/index-rule-bindings/measure-default-index-rule-binding.yaml
diff --git a/test/stress/cases/istio/testdata/index-rule-bindings/measure-minute-index-rule-binding.yaml b/test/stress/istio/testdata/index-rule-bindings/measure-minute-index-rule-binding.yaml
similarity index 100%
rename from test/stress/cases/istio/testdata/index-rule-bindings/measure-minute-index-rule-binding.yaml
rename to test/stress/istio/testdata/index-rule-bindings/measure-minute-index-rule-binding.yaml
diff --git a/test/stress/cases/istio/testdata/index-rules/measure-default-index-rule.yaml b/test/stress/istio/testdata/index-rules/measure-default-index-rule.yaml
similarity index 100%
rename from test/stress/cases/istio/testdata/index-rules/measure-default-index-rule.yaml
rename to test/stress/istio/testdata/index-rules/measure-default-index-rule.yaml
diff --git a/test/stress/cases/istio/testdata/index-rules/measure-minute-index-rule.yaml b/test/stress/istio/testdata/index-rules/measure-minute-index-rule.yaml
similarity index 100%
rename from test/stress/cases/istio/testdata/index-rules/measure-minute-index-rule.yaml
rename to test/stress/istio/testdata/index-rules/measure-minute-index-rule.yaml
diff --git a/test/stress/cases/istio/testdata/measures/measure-default.yaml b/test/stress/istio/testdata/measures/measure-default.yaml
similarity index 100%
rename from test/stress/cases/istio/testdata/measures/measure-default.yaml
rename to test/stress/istio/testdata/measures/measure-default.yaml
diff --git a/test/stress/cases/istio/testdata/measures/measure-minute.yaml b/test/stress/istio/testdata/measures/measure-minute.yaml
similarity index 100%
rename from test/stress/cases/istio/testdata/measures/measure-minute.yaml
rename to test/stress/istio/testdata/measures/measure-minute.yaml
diff --git a/test/stress/cases/istio/testdata/metrics/data.csv b/test/stress/istio/testdata/metrics/data.csv
similarity index 100%
rename from test/stress/cases/istio/testdata/metrics/data.csv
rename to test/stress/istio/testdata/metrics/data.csv
diff --git a/test/stress/cases/istio/testdata/metrics/intermediate.csv b/test/stress/istio/testdata/metrics/intermediate.csv
similarity index 100%
rename from test/stress/cases/istio/testdata/metrics/intermediate.csv
rename to test/stress/istio/testdata/metrics/intermediate.csv
diff --git a/test/stress/cases/istio/testdata/metrics/result.csv b/test/stress/istio/testdata/metrics/result.csv
similarity index 100%
rename from test/stress/cases/istio/testdata/metrics/result.csv
rename to test/stress/istio/testdata/metrics/result.csv
diff --git a/test/stress/Makefile b/test/stress/trace/Makefile
similarity index 89%
rename from test/stress/Makefile
rename to test/stress/trace/Makefile
index 789c4934..ca57b2db 100644
--- a/test/stress/Makefile
+++ b/test/stress/trace/Makefile
@@ -28,11 +28,7 @@ cli_env := DOCKER_BUILDKIT=1 USER_ID=$(shell id -u) GROUP_ID=$(shell id -g)
.PHONY: clean
clean:
- rm -rf /tmp/banyandb-stress-agent
-
-.PHONY: dev-up
-dev-up: clean
- $(cli_env) docker compose $(CLI_ARGS) --env-file ./env.dev up --build
+ rm -rf /tmp/banyandb-stress-trace
.PHONY: up
up: clean
diff --git a/test/stress/docker-compose.yaml b/test/stress/trace/docker-compose.yaml
similarity index 61%
rename from test/stress/docker-compose.yaml
rename to test/stress/trace/docker-compose.yaml
index 9d28ff79..9554ddf3 100644
--- a/test/stress/docker-compose.yaml
+++ b/test/stress/trace/docker-compose.yaml
@@ -20,22 +20,28 @@ services:
group_add:
- '${GROUP_ID}'
volumes:
- - /tmp/banyandb-stress-agent:/tmp/change-ownership
+ - /tmp/banyandb-stress-trace:/tmp/change-ownership
command: chown -R ${USER_ID}:${GROUP_ID} /tmp/change-ownership
+
banyandb:
user: "${USER_ID}:${GROUP_ID}"
extends:
- file: ../docker/base-compose.yml
+ file: ../../docker/base-compose.yml
service: banyandb
build:
dockerfile: ./docker/Dockerfile
- context: ../..
+ context: ../../..
volumes:
- - /tmp/banyandb-stress-agent:/tmp:rw,delgated
+ - /tmp/banyandb-stress-trace:/tmp:rw,delgated
ports:
- 17913:17913
- 6060:6060
- 2121:2121
+ deploy:
+ resources:
+ limits:
+ cpus: "4"
+ memory: 4G
networks:
- test
- monitoring
@@ -45,12 +51,17 @@ services:
oap:
extends:
- file: ../docker/base-compose.yml
+ file: ../../docker/base-compose.yml
service: oap
environment:
SW_STORAGE: banyandb
ports:
- 12800
+ deploy:
+ resources:
+ limits:
+ cpus: "4"
+ memory: 4G
volumes:
- ./log4j2.xml:/skywalking/config/log4j2.xml
networks:
@@ -59,60 +70,6 @@ services:
banyandb:
condition: service_healthy
- agent:
- extends:
- file: ../docker/base-compose.yml
- service: agent
- networks:
- - test
-
- provider:
- extends:
- file: ../docker/base-compose.yml
- service: provider
- ports:
- - 9090
- networks:
- - test
- depends_on:
- oap:
- condition: service_healthy
- agent:
- condition: service_completed_successfully
-
- consumer:
- extends:
- file: ../docker/base-compose.yml
- service: consumer
- ports:
- - 9092
- networks:
- - test
- depends_on:
- oap:
- condition: service_healthy
- provider:
- condition: service_healthy
- agent:
- condition: service_completed_successfully
-
- traffic_loader:
- extends:
- file: ../docker/base-compose.yml
- service: traffic_loader
- command: run --vus ${VUS} --duration 1h /scripts/consumer.js
- volumes:
- - ./scripts:/scripts
- networks:
- - test
- depends_on:
- oap:
- condition: service_healthy
- provider:
- condition: service_healthy
- consumer:
- condition: service_healthy
-
prometheus:
image: prom/prometheus:latest
container_name: prometheus
@@ -120,7 +77,7 @@ services:
profiles:
- "monitoring"
volumes:
- - ./prom/prometheus.yml:/etc/prometheus/prometheus.yml
+ - ./prometheus.yml:/etc/prometheus/prometheus.yml
- prometheus_data:/prometheus
command:
- '--config.file=/etc/prometheus/prometheus.yml'
@@ -137,5 +94,4 @@ networks:
monitoring:
volumes:
- sw_agent: {}
prometheus_data: {}
diff --git a/test/stress/env.dev b/test/stress/trace/env
similarity index 99%
rename from test/stress/env.dev
rename to test/stress/trace/env
index 6e0d3224..94bce219 100644
--- a/test/stress/env.dev
+++ b/test/stress/trace/env
@@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+
SW_AGENT_JAVA_COMMIT=3f88d735ba2bfd1196aff946502447d4b14450c8
SW_AGENT_SATELLITE_COMMIT=ea27a3f4e126a24775fe12e2aa2695bcb23d99c3
SW_AGENT_NGINX_LUA_COMMIT=c3cee4841798a147d83b96a10914d4ac0e11d0aa
@@ -27,5 +28,3 @@ SW_CTL_COMMIT=6b2eb0011e38b630db6af7203db215806bd141ed
SW_OAP_COMMIT=e5c308d4358d5b02a658b8ad7f153aec99b8e63a
SW_AGENT_E2E_SERVICE_PROVIDER_COMMIT=cc7a2c9e97fd2c421adbe3e9c471688459a446d9
-
-VUS=1
diff --git a/test/stress/log4j2.xml b/test/stress/trace/log4j2.xml
similarity index 100%
rename from test/stress/log4j2.xml
rename to test/stress/trace/log4j2.xml
diff --git a/test/stress/prom/prometheus.yml b/test/stress/trace/prometheus.yml
similarity index 100%
rename from test/stress/prom/prometheus.yml
rename to test/stress/trace/prometheus.yml