You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ha...@apache.org on 2024/02/28 00:35:24 UTC
(skywalking-banyandb) branch ci created (now bab97782)
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a change to branch ci
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
at bab97782 Fix CI tasks
This branch includes the following new commits:
new bab97782 Fix CI tasks
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(skywalking-banyandb) 01/01: Fix CI tasks
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch ci
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit bab977823e5665feb3db44fca20dcd3e1ad99008
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Wed Feb 28 00:33:06 2024 +0000
Fix CI tasks
Signed-off-by: Gao Hongtao <ha...@gmail.com>
---
.github/workflows/flaky-test.yml | 2 +-
banyand/measure/merger_policy.go | 2 +-
bydbctl/internal/cmd/root.go | 3 +
test/docker/base-compose.yml | 2 +-
test/integration/load/load_suite_test.go | 141 ++++++++++++++++++-------------
test/stress/Makefile | 16 ++--
test/stress/docker-compose.yaml | 16 +++-
test/stress/env | 8 +-
test/stress/env.dev | 8 +-
9 files changed, 122 insertions(+), 76 deletions(-)
diff --git a/.github/workflows/flaky-test.yml b/.github/workflows/flaky-test.yml
index 0ca35b3a..3420da1d 100644
--- a/.github/workflows/flaky-test.yml
+++ b/.github/workflows/flaky-test.yml
@@ -28,5 +28,5 @@ jobs:
if: github.repository == 'apache/skywalking-banyandb'
uses: ./.github/workflows/test.yml
with:
- options: --repeat 49 --label-filter \\(integration\\&\\&\\!slow\\)\\|\\|banyand
+ options: --repeat 49 --label-filter '(integration&&!slow)||banyand'
timeout-minutes: 40
diff --git a/banyand/measure/merger_policy.go b/banyand/measure/merger_policy.go
index cdb7e9ad..b04e26cc 100644
--- a/banyand/measure/merger_policy.go
+++ b/banyand/measure/merger_policy.go
@@ -32,7 +32,7 @@ type mergePolicy struct {
// NewDefaultMergePolicy create a MergePolicy with default parameters.
func newDefaultMergePolicy() *mergePolicy {
- return newMergePolicy(15, 1.7, math.MaxUint64)
+ return newMergePolicy(8, 1.7, math.MaxUint64)
}
func newDefaultMergePolicyForTesting() *mergePolicy {
diff --git a/bydbctl/internal/cmd/root.go b/bydbctl/internal/cmd/root.go
index 9aff040d..f9645fca 100644
--- a/bydbctl/internal/cmd/root.go
+++ b/bydbctl/internal/cmd/root.go
@@ -79,6 +79,9 @@ func init() {
func initConfig() {
if cfgFile != "" {
+ if cfgFile == "-" {
+ return
+ }
// Use config file from the flag.
viper.SetConfigFile(cfgFile)
} else {
diff --git a/test/docker/base-compose.yml b/test/docker/base-compose.yml
index 49b4e3d5..7379151c 100644
--- a/test/docker/base-compose.yml
+++ b/test/docker/base-compose.yml
@@ -21,7 +21,7 @@ services:
- 6060
command: standalone
healthcheck:
- test: ["CMD", "./bydbctl", "health", "--addr=http://banyandb:17913"]
+ test: ["CMD", "./bydbctl", "health", "--config=-", "--addr=http://banyandb:17913"]
interval: 5s
timeout: 10s
retries: 120
diff --git a/test/integration/load/load_suite_test.go b/test/integration/load/load_suite_test.go
index 7071bc5e..65011cd4 100644
--- a/test/integration/load/load_suite_test.go
+++ b/test/integration/load/load_suite_test.go
@@ -18,6 +18,7 @@
package integration_load_test
import (
+ "context"
"testing"
"time"
@@ -27,12 +28,14 @@ import (
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+ streamv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/stream/v1"
"github.com/apache/skywalking-banyandb/pkg/grpchelper"
"github.com/apache/skywalking-banyandb/pkg/logger"
"github.com/apache/skywalking-banyandb/pkg/test/flags"
"github.com/apache/skywalking-banyandb/pkg/test/helpers"
"github.com/apache/skywalking-banyandb/pkg/test/setup"
- cases_stream "github.com/apache/skywalking-banyandb/test/cases/stream"
cases_stream_data "github.com/apache/skywalking-banyandb/test/cases/stream/data"
)
@@ -41,64 +44,88 @@ func TestIntegrationLoad(t *testing.T) {
RunSpecs(t, "Integration Load Suite", Label("integration", "slow"))
}
-var (
- connection *grpc.ClientConn
- now time.Time
- deferFunc func()
- goods []gleak.Goroutine
-)
+var _ = Describe("Test Suite", func() {
+ var (
+ connection *grpc.ClientConn
+ now time.Time
+ deferFunc func()
+ goods []gleak.Goroutine
+ addr string
+ )
-var _ = SynchronizedBeforeSuite(func() []byte {
- goods = gleak.Goroutines()
- Expect(logger.Init(logger.Logging{
- Env: "dev",
- Level: flags.LogLevel,
- })).To(Succeed())
- var addr string
- addr, _, deferFunc = setup.Standalone()
- Eventually(
- helpers.HealthCheck(addr, 10*time.Second, 10*time.Second, grpc.WithTransportCredentials(insecure.NewCredentials())),
- flags.EventuallyTimeout).Should(Succeed())
- conn, err := grpchelper.Conn(addr, 10*time.Second, grpc.WithTransportCredentials(insecure.NewCredentials()))
- Expect(err).NotTo(HaveOccurred())
- days := 7
- hours := 24
- minutes := 60
- interval := 10 * time.Second
- c := time.Now()
- for i := 0; i < days; i++ {
- date := c.Add(-time.Hour * time.Duration((days-i)*24))
- for h := 0; h < hours; h++ {
- hour := date.Add(time.Hour * time.Duration(h))
- start := time.Now()
- for j := 0; j < minutes; j++ {
- n := hour.Add(time.Minute * time.Duration(j))
- ns := n.UnixNano()
- now = time.Unix(0, ns-ns%int64(time.Minute))
- // stream
- cases_stream_data.Write(conn, "data.json", now, interval)
+ BeforeEach(func() {
+ Expect(logger.Init(logger.Logging{
+ Env: "dev",
+ Level: flags.LogLevel,
+ })).Should(Succeed())
+ goods = gleak.Goroutines()
+ addr, _, deferFunc = setup.Standalone()
+ Eventually(
+ helpers.HealthCheck(addr, 10*time.Second, 10*time.Second, grpc.WithTransportCredentials(insecure.NewCredentials())),
+ flags.EventuallyTimeout).Should(Succeed())
+ var err error
+ connection, err = grpchelper.Conn(addr, 10*time.Second, grpc.WithTransportCredentials(insecure.NewCredentials()))
+ Expect(err).NotTo(HaveOccurred())
+ days := 7
+ hours := 24
+ minutes := 60
+ interval := 10 * time.Second
+ c := time.Now()
+ for i := 0; i < days; i++ {
+ date := c.Add(-time.Hour * time.Duration((days-i)*24))
+ for h := 0; h < hours; h++ {
+ hour := date.Add(time.Hour * time.Duration(h))
+ start := time.Now()
+ for j := 0; j < minutes; j++ {
+ n := hour.Add(time.Minute * time.Duration(j))
+ ns := n.UnixNano()
+ now = time.Unix(0, ns-ns%int64(time.Minute))
+ // stream
+ cases_stream_data.Write(connection, "data.json", now, interval)
+ }
+ logger.Infof("written stream in %s took %s \n", hour, time.Since(start))
}
- GinkgoWriter.Printf("written stream in %s took %s \n", hour, time.Since(start))
}
- }
- Expect(conn.Close()).To(Succeed())
- return []byte(addr)
-}, func(address []byte) {
- var err error
- connection, err = grpchelper.Conn(string(address), 10*time.Second,
- grpc.WithTransportCredentials(insecure.NewCredentials()))
- cases_stream.SharedContext = helpers.SharedContext{
- Connection: connection,
- BaseTime: now,
- }
- Expect(err).NotTo(HaveOccurred())
-})
-
-var _ = SynchronizedAfterSuite(func() {
- if connection != nil {
Expect(connection.Close()).To(Succeed())
- }
-}, func() {
- deferFunc()
- Eventually(gleak.Goroutines, flags.EventuallyTimeout).ShouldNot(gleak.HaveLeaked(goods))
+ })
+
+ It("should read data", func() {
+ var err error
+ connection, err = grpchelper.Conn(addr, 10*time.Second,
+ grpc.WithTransportCredentials(insecure.NewCredentials()))
+ Expect(err).NotTo(HaveOccurred())
+ sharedContext := helpers.SharedContext{
+ Connection: connection,
+ BaseTime: now,
+ }
+ query := &streamv1.QueryRequest{
+ Metadata: &commonv1.Metadata{
+ Name: "sw",
+ Group: "default",
+ },
+ Projection: &modelv1.TagProjection{
+ TagFamilies: []*modelv1.TagProjection_TagFamily{
+ {
+ Name: "searchable",
+ Tags: []string{"trace_id"},
+ },
+ },
+ },
+ }
+ query.TimeRange = helpers.TimeRange(helpers.Args{Input: "all", Duration: 1 * time.Hour}, sharedContext)
+ c := streamv1.NewStreamServiceClient(sharedContext.Connection)
+ ctx := context.Background()
+ resp, err := c.Query(ctx, query)
+ Expect(err).NotTo(HaveOccurred())
+ GinkgoWriter.Printf("query result: %s elements\n", resp.GetElements())
+ Expect(len(resp.GetElements())).To(BeNumerically(">", 0))
+ })
+
+ AfterEach(func() {
+ if connection != nil {
+ Expect(connection.Close()).To(Succeed())
+ }
+ deferFunc()
+ Eventually(gleak.Goroutines, flags.EventuallyTimeout).ShouldNot(gleak.HaveLeaked(goods))
+ })
})
diff --git a/test/stress/Makefile b/test/stress/Makefile
index e0200152..789c4934 100644
--- a/test/stress/Makefile
+++ b/test/stress/Makefile
@@ -24,14 +24,20 @@ ifdef PROFILE
CLI_ARGS := $(CLI_ARGS) --profile $(PROFILE)
endif
+cli_env := DOCKER_BUILDKIT=1 USER_ID=$(shell id -u) GROUP_ID=$(shell id -g)
+
+.PHONY: clean
+clean:
+ rm -rf /tmp/banyandb-stress-agent
+
.PHONY: dev-up
-dev-up:
- DOCKER_BUILDKIT=1 docker compose $(CLI_ARGS) --env-file ./env.dev up --build
+dev-up: clean
+ $(cli_env) docker compose $(CLI_ARGS) --env-file ./env.dev up --build
.PHONY: up
-up:
- DOCKER_BUILDKIT=1 docker compose $(CLI_ARGS) --env-file ./env up --build
+up: clean
+ $(cli_env) docker compose $(CLI_ARGS) --env-file ./env up --build
.PHONY: down
down:
- DOCKER_BUILDKIT=1 docker compose down
+ docker compose down
diff --git a/test/stress/docker-compose.yaml b/test/stress/docker-compose.yaml
index 1278a19e..9d28ff79 100644
--- a/test/stress/docker-compose.yaml
+++ b/test/stress/docker-compose.yaml
@@ -14,7 +14,16 @@
# limitations under the License.
services:
+ change-vol-ownership:
+ image: ubuntu
+ user: "root"
+ group_add:
+ - '${GROUP_ID}'
+ volumes:
+ - /tmp/banyandb-stress-agent:/tmp/change-ownership
+ command: chown -R ${USER_ID}:${GROUP_ID} /tmp/change-ownership
banyandb:
+ user: "${USER_ID}:${GROUP_ID}"
extends:
file: ../docker/base-compose.yml
service: banyandb
@@ -22,9 +31,7 @@ services:
dockerfile: ./docker/Dockerfile
context: ../..
volumes:
- - ../..:/app:rw,delegated
- # Uncomment below line to mount an external volume
- # - /tmp:/tmp:rw,delgated
+ - /tmp/banyandb-stress-agent:/tmp:rw,delgated
ports:
- 17913:17913
- 6060:6060
@@ -32,6 +39,9 @@ services:
networks:
- test
- monitoring
+ depends_on:
+ change-vol-ownership:
+ condition: service_completed_successfully
oap:
extends:
diff --git a/test/stress/env b/test/stress/env
index 1560b70d..25350fb1 100644
--- a/test/stress/env
+++ b/test/stress/env
@@ -22,11 +22,11 @@ SW_AGENT_GO_COMMIT=4af380c2db6243106b0fc650b6003ce3b3eb82a0
SW_AGENT_PYTHON_COMMIT=c76a6ec51a478ac91abb20ec8f22a99b8d4d6a58
SW_AGENT_CLIENT_JS_COMMIT=af0565a67d382b683c1dbd94c379b7080db61449
SW_AGENT_CLIENT_JS_TEST_COMMIT=4f1eb1dcdbde3ec4a38534bf01dded4ab5d2f016
-SW_KUBERNETES_COMMIT_SHA=b670c41d94a82ddefcf466d54bab5c492d88d772
-SW_ROVER_COMMIT=d956eaede57b62108b78bca48045bd09ba88e653
-SW_CTL_COMMIT=e684fae0107045fc23799146d62f04cb68bd5a3b
+SW_KUBERNETES_COMMIT_SHA=e2c61c6774cf377b23516fca6f8a1e119d3191c5
+SW_ROVER_COMMIT=fc8d074c6d34ecfee585a7097cbd5aef1ca680a5
+SW_CTL_COMMIT=6b2eb0011e38b630db6af7203db215806bd141ed
SW_OAP_COMMIT=e5c308d4358d5b02a658b8ad7f153aec99b8e63a
-SW_AGENT_E2E_SERVICE_PROVIDER_COMMIT=828e6e2f2b57a0f06bb0d507e3296d2377943d9a
+SW_AGENT_E2E_SERVICE_PROVIDER_COMMIT=cc7a2c9e97fd2c421adbe3e9c471688459a446d9
VUS=10
diff --git a/test/stress/env.dev b/test/stress/env.dev
index 69aa1ce1..6e0d3224 100644
--- a/test/stress/env.dev
+++ b/test/stress/env.dev
@@ -21,11 +21,11 @@ SW_AGENT_GO_COMMIT=4af380c2db6243106b0fc650b6003ce3b3eb82a0
SW_AGENT_PYTHON_COMMIT=c76a6ec51a478ac91abb20ec8f22a99b8d4d6a58
SW_AGENT_CLIENT_JS_COMMIT=af0565a67d382b683c1dbd94c379b7080db61449
SW_AGENT_CLIENT_JS_TEST_COMMIT=4f1eb1dcdbde3ec4a38534bf01dded4ab5d2f016
-SW_KUBERNETES_COMMIT_SHA=b670c41d94a82ddefcf466d54bab5c492d88d772
-SW_ROVER_COMMIT=d956eaede57b62108b78bca48045bd09ba88e653
-SW_CTL_COMMIT=e684fae0107045fc23799146d62f04cb68bd5a3b
+SW_KUBERNETES_COMMIT_SHA=e2c61c6774cf377b23516fca6f8a1e119d3191c5
+SW_ROVER_COMMIT=fc8d074c6d34ecfee585a7097cbd5aef1ca680a5
+SW_CTL_COMMIT=6b2eb0011e38b630db6af7203db215806bd141ed
SW_OAP_COMMIT=e5c308d4358d5b02a658b8ad7f153aec99b8e63a
-SW_AGENT_E2E_SERVICE_PROVIDER_COMMIT=828e6e2f2b57a0f06bb0d507e3296d2377943d9a
+SW_AGENT_E2E_SERVICE_PROVIDER_COMMIT=cc7a2c9e97fd2c421adbe3e9c471688459a446d9
VUS=1