You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@apisix.apache.org by sp...@apache.org on 2021/08/05 02:13:31 UTC
[apisix] branch master updated: test: add chaos test on network
delay between apisix and etcd (#4315)
This is an automated email from the ASF dual-hosted git repository.
spacewander pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new a99a547 test: add chaos test on network delay between apisix and etcd (#4315)
a99a547 is described below
commit a99a54794848de60b7c419d3ba0e2fefd60e7e63
Author: Shuyang Wu <wo...@gmail.com>
AuthorDate: Wed Aug 4 22:13:24 2021 -0400
test: add chaos test on network delay between apisix and etcd (#4315)
---
t/chaos/delayetcd/delayetcd.go | 194 +++++++++++++++++++++++++++++++++++++++++
t/chaos/e2e.go | 1 +
t/chaos/killetcd/killetcd.go | 63 +++----------
t/chaos/utils/utils.go | 84 ++++++++++++++----
4 files changed, 271 insertions(+), 71 deletions(-)
diff --git a/t/chaos/delayetcd/delayetcd.go b/t/chaos/delayetcd/delayetcd.go
new file mode 100644
index 0000000..69f1af4
--- /dev/null
+++ b/t/chaos/delayetcd/delayetcd.go
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package delayetcd
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "strconv"
+ "time"
+
+ "github.com/chaos-mesh/chaos-mesh/api/v1alpha1"
+ "github.com/gavv/httpexpect"
+ "github.com/onsi/ginkgo"
+ "github.com/onsi/gomega"
+ v1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "sigs.k8s.io/controller-runtime/pkg/client"
+
+ "github.com/apache/apisix/t/chaos/utils"
+)
+
+func getEtcdDelayChaos(delay int) *v1alpha1.NetworkChaos {
+ return &v1alpha1.NetworkChaos{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "etcd-delay",
+ Namespace: metav1.NamespaceDefault,
+ },
+ Spec: v1alpha1.NetworkChaosSpec{
+ Selector: v1alpha1.SelectorSpec{
+ LabelSelectors: map[string]string{"app.kubernetes.io/instance": "etcd"},
+ },
+ Action: v1alpha1.DelayAction,
+ Mode: v1alpha1.AllPodMode,
+ TcParameter: v1alpha1.TcParameter{
+ Delay: &v1alpha1.DelaySpec{
+ Latency: strconv.Itoa(delay) + "ms",
+ },
+ },
+ },
+ }
+}
+
+func setRouteMultipleTimes(e *httpexpect.Expect, times int, status httpexpect.StatusRange) time.Duration {
+ now := time.Now()
+ timeLast := now
+ var timeList []string
+ for i := 0; i < times; i++ {
+ utils.SetRoute(e, status)
+ timeList = append(timeList, time.Since(timeLast).String())
+ timeLast = time.Now()
+ }
+ fmt.Fprintf(ginkgo.GinkgoWriter, "takes %v separately\n", timeList)
+ return time.Since(now) / time.Duration(times)
+}
+
+func setRouteMultipleTimesIgnoreError(e *httpexpect.Expect, times int) (time.Duration, int) {
+ now := time.Now()
+ var resp *httpexpect.Response
+ for i := 0; i < times; i++ {
+ resp = utils.SetRouteIgnoreError(e)
+ }
+ // use status code of the last time is enough to show the accessibility of apisix
+ return time.Since(now) / time.Duration(times), resp.Raw().StatusCode
+}
+
+func deleteChaosAndCheck(eSilent *httpexpect.Expect, cliSet *utils.ClientSet, chaos *v1alpha1.NetworkChaos) {
+ err := cliSet.CtrlCli.Delete(context.Background(), chaos)
+ gomega.Expect(err).To(gomega.BeNil())
+ time.Sleep(1 * time.Second)
+
+ var setDuration time.Duration
+ var statusCode int
+ for range [10]int{} {
+ setDuration, statusCode = setRouteMultipleTimesIgnoreError(eSilent, 5)
+ if setDuration < 15*time.Millisecond && statusCode == http.StatusOK {
+ break
+ }
+ time.Sleep(5 * time.Second)
+ }
+ gomega.Ω(setDuration).Should(gomega.BeNumerically("<", 15*time.Millisecond))
+ gomega.Ω(statusCode).Should(gomega.BeNumerically("==", http.StatusOK))
+}
+
+var _ = ginkgo.Describe("Test APISIX Delay When Add ETCD Delay", func() {
+ ctx := context.Background()
+ e := httpexpect.New(ginkgo.GinkgoT(), utils.Host)
+ eSilent := utils.GetSilentHttpexpectClient()
+
+ var cliSet *utils.ClientSet
+ var apisixPod *v1.Pod
+ var err error
+ ginkgo.It("init client set", func() {
+ cliSet, err = utils.InitClientSet()
+ gomega.Expect(err).To(gomega.BeNil())
+ listOption := client.MatchingLabels{"app": "apisix-gw"}
+ apisixPods, err := utils.GetPods(cliSet.CtrlCli, metav1.NamespaceDefault, listOption)
+ gomega.Expect(err).To(gomega.BeNil())
+ gomega.Ω(len(apisixPods)).Should(gomega.BeNumerically(">", 0))
+ apisixPod = &apisixPods[0]
+ })
+
+ ginkgo.It("check if everything works", func() {
+ utils.SetRoute(e, httpexpect.Status2xx)
+ utils.GetRouteList(e, http.StatusOK)
+
+ utils.WaitUntilMethodSucceed(e, http.MethodGet, 1)
+ utils.TestPrometheusEtcdMetric(e, 1)
+ })
+
+ // get default
+ ginkgo.It("get default apisix delay", func() {
+ timeStart := time.Now()
+ setDuration := setRouteMultipleTimes(eSilent, 5, httpexpect.Status2xx)
+ gomega.Ω(setDuration).Should(gomega.BeNumerically("<", 15*time.Millisecond))
+
+ errorLog, err := utils.Log(apisixPod, cliSet.KubeCli, timeStart)
+ gomega.Expect(err).To(gomega.BeNil())
+ gomega.Ω(errorLog).ShouldNot(gomega.ContainSubstring("error"))
+ })
+
+ // 30ms delay
+ ginkgo.It("generate a 30ms delay between etcd and apisix", func() {
+ timeStart := time.Now()
+ chaos := getEtcdDelayChaos(30)
+ err := cliSet.CtrlCli.Create(ctx, chaos)
+ gomega.Expect(err).To(gomega.BeNil())
+ time.Sleep(1 * time.Second)
+
+ defer deleteChaosAndCheck(eSilent, cliSet, chaos)
+
+ setDuration := setRouteMultipleTimes(eSilent, 5, httpexpect.Status2xx)
+ gomega.Ω(setDuration).Should(gomega.BeNumerically("<", 400*time.Millisecond))
+
+ errorLog, err := utils.Log(apisixPod, cliSet.KubeCli, timeStart)
+ gomega.Expect(err).To(gomega.BeNil())
+ gomega.Ω(errorLog).ShouldNot(gomega.ContainSubstring("error"))
+ })
+
+ // 300ms delay
+ ginkgo.It("generate a 300ms delay between etcd and apisix", func() {
+ timeStart := time.Now()
+ chaos := getEtcdDelayChaos(300)
+ err := cliSet.CtrlCli.Create(ctx, chaos)
+ gomega.Expect(err).To(gomega.BeNil())
+ time.Sleep(1 * time.Second)
+
+ defer deleteChaosAndCheck(eSilent, cliSet, chaos)
+
+ setDuration := setRouteMultipleTimes(eSilent, 5, httpexpect.Status2xx)
+ gomega.Ω(setDuration).Should(gomega.BeNumerically("<", 4*time.Second))
+
+ errorLog, err := utils.Log(apisixPod, cliSet.KubeCli, timeStart)
+ gomega.Expect(err).To(gomega.BeNil())
+ gomega.Ω(errorLog).ShouldNot(gomega.ContainSubstring("error"))
+ })
+
+ // 3s delay and cause error
+ ginkgo.It("generate a 3s delay between etcd and apisix", func() {
+ timeStart := time.Now()
+ chaos := getEtcdDelayChaos(3000)
+ err := cliSet.CtrlCli.Create(ctx, chaos)
+ gomega.Expect(err).To(gomega.BeNil())
+ time.Sleep(1 * time.Second)
+
+ defer deleteChaosAndCheck(eSilent, cliSet, chaos)
+
+ _ = setRouteMultipleTimes(e, 2, httpexpect.Status5xx)
+
+ errorLog, err := utils.Log(apisixPod, cliSet.KubeCli, timeStart)
+ gomega.Expect(err).To(gomega.BeNil())
+ gomega.Ω(errorLog).Should(gomega.ContainSubstring("error"))
+ })
+
+ ginkgo.It("restore test environment", func() {
+ utils.WaitUntilMethodSucceed(e, http.MethodPut, 5)
+ utils.DeleteRoute(e)
+ })
+})
diff --git a/t/chaos/e2e.go b/t/chaos/e2e.go
index d0cd543..51094df 100644
--- a/t/chaos/e2e.go
+++ b/t/chaos/e2e.go
@@ -18,6 +18,7 @@
package e2e
import (
+ _ "github.com/apache/apisix/t/chaos/delayetcd"
_ "github.com/apache/apisix/t/chaos/killetcd"
)
diff --git a/t/chaos/killetcd/killetcd.go b/t/chaos/killetcd/killetcd.go
index 7967a52..81612be 100644
--- a/t/chaos/killetcd/killetcd.go
+++ b/t/chaos/killetcd/killetcd.go
@@ -29,7 +29,6 @@ import (
"github.com/onsi/gomega"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- "k8s.io/kubernetes/pkg/api/v1/pod"
"sigs.k8s.io/controller-runtime/pkg/client"
"github.com/apache/apisix/t/chaos/utils"
@@ -65,13 +64,7 @@ func getEtcdKillChaos() *v1alpha1.PodChaos {
var _ = ginkgo.Describe("Test Get Success When Etcd Got Killed", func() {
e := httpexpect.New(ginkgo.GinkgoT(), utils.Host)
- eSilent := httpexpect.WithConfig(httpexpect.Config{
- BaseURL: utils.Host,
- Reporter: httpexpect.NewAssertReporter(ginkgo.GinkgoT()),
- Printers: []httpexpect.Printer{
- utils.NewSilentPrinter(ginkgo.GinkgoT()),
- },
- })
+ eSilent := utils.GetSilentHttpexpectClient()
var cliSet *utils.ClientSet
var apisixPod *v1.Pod
@@ -87,32 +80,12 @@ var _ = ginkgo.Describe("Test Get Success When Etcd Got Killed", func() {
})
stopChan := make(chan bool)
- defer ginkgo.It("restore test environment", func() {
- stopChan <- true
- cliSet.CtrlCli.Delete(context.Background(), getEtcdKillChaos())
- utils.DeleteRoute(e)
- })
ginkgo.It("check if everything works", func() {
utils.SetRoute(e, httpexpect.Status2xx)
utils.GetRouteList(e, http.StatusOK)
- var resp *httpexpect.Response
-
- resp = utils.GetRouteIgnoreError(e)
- // wait 1s seems not enough, wait some more time to make sure nothing goes wrong
- if resp.Raw().StatusCode != http.StatusOK {
- for i := range [60]int{} {
- timeWait := fmt.Sprintf("wait for %ds\n", i)
- fmt.Fprint(ginkgo.GinkgoWriter, timeWait)
- resp = utils.GetRouteIgnoreError(e)
- if resp.Raw().StatusCode != http.StatusOK {
- time.Sleep(time.Second)
- } else {
- break
- }
- }
- }
- gomega.Ω(resp.Raw().StatusCode).Should(gomega.BeNumerically("==", http.StatusOK))
+
+ utils.WaitUntilMethodSucceed(e, http.MethodGet, 1)
utils.TestPrometheusEtcdMetric(e, 1)
})
@@ -176,30 +149,16 @@ var _ = ginkgo.Describe("Test Get Success When Etcd Got Killed", func() {
})
ginkgo.It("ingress bandwidth per second not change much", func() {
- fmt.Fprintf(ginkgo.GinkgoWriter, "bandwidth before: %f, after: %f", bandwidthBefore, bandwidthAfter)
- fmt.Fprintf(ginkgo.GinkgoWriter, "duration before: %f, after: %f", durationBefore, durationAfter)
- fmt.Fprintf(ginkgo.GinkgoWriter, "bps before: %f, after: %f", bpsBefore, bpsAfter)
+ fmt.Fprintf(ginkgo.GinkgoWriter, "bandwidth before: %f, after: %f\n", bandwidthBefore, bandwidthAfter)
+ fmt.Fprintf(ginkgo.GinkgoWriter, "duration before: %f, after: %f\n", durationBefore, durationAfter)
+ fmt.Fprintf(ginkgo.GinkgoWriter, "bps before: %f, after: %f\n", bpsBefore, bpsAfter)
gomega.Expect(utils.RoughCompare(bpsBefore, bpsAfter)).To(gomega.BeTrue())
})
- ginkgo.It("wait till etcd return to normal", func() {
- listOption := client.MatchingLabels{"app.kubernetes.io/instance": "etcd"}
- var etcdPod []v1.Pod
- successCount := 0
- for i := 0; i < 6; i++ {
- etcdPods, err := utils.GetPods(cliSet.CtrlCli, metav1.NamespaceDefault, listOption)
- gomega.Expect(err).To(gomega.BeNil())
- for _, p := range etcdPods {
- if pod.IsPodReady(&p) {
- successCount++
- }
- }
- if successCount == len(etcdPod) {
- break
- }
- time.Sleep(5 * time.Second)
- successCount = 0
- }
- gomega.Ω(successCount).Should(gomega.BeNumerically("==", len(etcdPod)))
+ ginkgo.It("restore test environment", func() {
+ stopChan <- true
+ cliSet.CtrlCli.Delete(context.Background(), getEtcdKillChaos())
+ utils.WaitUntilMethodSucceed(e, http.MethodPut, 5)
+ utils.DeleteRoute(e)
})
})
diff --git a/t/chaos/utils/utils.go b/t/chaos/utils/utils.go
index 0e4ba9d..4e64c9c 100644
--- a/t/chaos/utils/utils.go
+++ b/t/chaos/utils/utils.go
@@ -25,12 +25,29 @@ import (
"time"
"github.com/gavv/httpexpect"
+ "github.com/onsi/ginkgo"
"github.com/onsi/gomega"
)
var (
- token = "edd1c9f034335f136f87ad84b625c8f1"
- Host = "http://127.0.0.1:9080"
+ token = "edd1c9f034335f136f87ad84b625c8f1"
+ Host = "http://127.0.0.1:9080"
+ setRouteBody = `{
+ "uri": "/get",
+ "plugins": {
+ "prometheus": {}
+ },
+ "upstream": {
+ "nodes": {
+ "httpbin.default.svc.cluster.local:8000": 1
+ },
+ "type": "roundrobin"
+ }
+ }`
+ ignoreErrorFuncMap = map[string]func(e *httpexpect.Expect) *httpexpect.Response{
+ http.MethodGet: GetRouteIgnoreError,
+ http.MethodPut: SetRouteIgnoreError,
+ }
)
type httpTestCase struct {
@@ -91,26 +108,26 @@ func caseCheck(tc httpTestCase) *httpexpect.Response {
func SetRoute(e *httpexpect.Expect, expectStatusRange httpexpect.StatusRange) *httpexpect.Response {
return caseCheck(httpTestCase{
- E: e,
- Method: http.MethodPut,
- Path: "/apisix/admin/routes/1",
- Headers: map[string]string{"X-API-KEY": token},
- Body: `{
- "uri": "/get",
- "plugins": {
- "prometheus": {}
- },
- "upstream": {
- "nodes": {
- "httpbin.default.svc.cluster.local:8000": 1
- },
- "type": "roundrobin"
- }
- }`,
+ E: e,
+ Method: http.MethodPut,
+ Path: "/apisix/admin/routes/1",
+ Headers: map[string]string{"X-API-KEY": token},
+ Body: setRouteBody,
ExpectStatusRange: expectStatusRange,
})
}
+func SetRouteIgnoreError(e *httpexpect.Expect) *httpexpect.Response {
+ return caseCheck(httpTestCase{
+ E: e,
+ Method: http.MethodPut,
+ Path: "/apisix/admin/routes/1",
+ Headers: map[string]string{"X-API-KEY": token},
+ Body: setRouteBody,
+ IgnoreError: true,
+ })
+}
+
func GetRoute(e *httpexpect.Expect, expectStatus int) *httpexpect.Response {
return caseCheck(httpTestCase{
E: e,
@@ -196,6 +213,35 @@ func GetEgressBandwidthPerSecond(e *httpexpect.Expect) (float64, float64) {
return bandWidthEnd - bandWidthStart, duration.Seconds()
}
+func GetSilentHttpexpectClient() *httpexpect.Expect {
+ return httpexpect.WithConfig(httpexpect.Config{
+ BaseURL: Host,
+ Reporter: httpexpect.NewAssertReporter(ginkgo.GinkgoT()),
+ Printers: []httpexpect.Printer{
+ newSilentPrinter(ginkgo.GinkgoT()),
+ },
+ })
+}
+
+func WaitUntilMethodSucceed(e *httpexpect.Expect, method string, interval int) {
+ f, ok := ignoreErrorFuncMap[method]
+ gomega.Expect(ok).To(gomega.BeTrue())
+ resp := f(e)
+ if resp.Raw().StatusCode != http.StatusOK {
+ for i := range [60]int{} {
+ timeWait := fmt.Sprintf("wait for %ds\n", i*interval)
+ fmt.Fprint(ginkgo.GinkgoWriter, timeWait)
+ resp = f(e)
+ if resp.Raw().StatusCode != http.StatusOK {
+ time.Sleep(5 * time.Second)
+ } else {
+ break
+ }
+ }
+ }
+ gomega.Ω(resp.Raw().StatusCode).Should(gomega.BeNumerically("==", http.StatusOK))
+}
+
func RoughCompare(a float64, b float64) bool {
ratio := a / b
if ratio < 1.3 && ratio > 0.7 {
@@ -208,7 +254,7 @@ type silentPrinter struct {
logger httpexpect.Logger
}
-func NewSilentPrinter(logger httpexpect.Logger) silentPrinter {
+func newSilentPrinter(logger httpexpect.Logger) silentPrinter {
return silentPrinter{logger}
}