You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@apisix.apache.org by GitBox <gi...@apache.org> on 2021/05/27 00:08:48 UTC
[GitHub] [apisix] Yiyiyimu opened a new pull request #4315: ci: add chaos test on network delay between apisix and etcd
Yiyiyimu opened a new pull request #4315:
URL: https://github.com/apache/apisix/pull/4315
Signed-off-by: yiyiyimu <wo...@gmail.com>
### What this PR does / why we need it:
1. add chaos test on network delay between apisix and etcd
2. refactor
### Pre-submission checklist:
* [ ] Did you explain what problem does this PR solve? Or what new features have been added?
* [ ] Have you added corresponding test cases?
* [ ] Have you modified the corresponding document?
* [ ] Is this PR backward compatible? **If it is not backward compatible, please discuss on the [mailing list](https://github.com/apache/apisix/tree/master#community) first**
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [apisix] Yiyiyimu commented on a change in pull request #4315: test: add chaos test on network delay between apisix and etcd
Posted by GitBox <gi...@apache.org>.
Yiyiyimu commented on a change in pull request #4315:
URL: https://github.com/apache/apisix/pull/4315#discussion_r682905582
##########
File path: t/chaos/delayetcd/delayetcd.go
##########
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package delayetcd
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "strconv"
+ "time"
+
+ "github.com/chaos-mesh/chaos-mesh/api/v1alpha1"
+ "github.com/gavv/httpexpect"
+ "github.com/onsi/ginkgo"
+ "github.com/onsi/gomega"
+ v1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "sigs.k8s.io/controller-runtime/pkg/client"
+
+ "github.com/apache/apisix/t/chaos/utils"
+)
+
+func getEtcdDelayChaos(delay int) *v1alpha1.NetworkChaos {
+ return &v1alpha1.NetworkChaos{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "etcd-delay",
+ Namespace: metav1.NamespaceDefault,
+ },
+ Spec: v1alpha1.NetworkChaosSpec{
+ Selector: v1alpha1.SelectorSpec{
+ LabelSelectors: map[string]string{"app.kubernetes.io/instance": "etcd"},
+ },
+ Action: v1alpha1.DelayAction,
+ Mode: v1alpha1.AllPodMode,
+ TcParameter: v1alpha1.TcParameter{
+ Delay: &v1alpha1.DelaySpec{
+ Latency: strconv.Itoa(delay) + "ms",
+ },
+ },
+ },
+ }
+}
+
+func setRouteMultipleTimes(e *httpexpect.Expect, times int, status httpexpect.StatusRange) time.Duration {
+ now := time.Now()
+ timeLast := now
+ var timeList []string
+ for i := 0; i < times; i++ {
+ utils.SetRoute(e, status)
+ timeList = append(timeList, time.Since(timeLast).String())
+ timeLast = time.Now()
+ }
+ fmt.Fprintf(ginkgo.GinkgoWriter, "takes %v separately", timeList)
+ return time.Since(now) / time.Duration(times)
+}
+
+var _ = ginkgo.Describe("Test APISIX Delay When Add ETCD Delay", func() {
+ ctx := context.Background()
+ e := httpexpect.New(ginkgo.GinkgoT(), utils.Host)
+ eSilent := utils.GetSilentHttpexpectClient()
+
+ var cliSet *utils.ClientSet
+ var apisixPod *v1.Pod
+ var err error
+ ginkgo.It("init client set", func() {
+ cliSet, err = utils.InitClientSet()
+ gomega.Expect(err).To(gomega.BeNil())
+ listOption := client.MatchingLabels{"app": "apisix-gw"}
+ apisixPods, err := utils.GetPods(cliSet.CtrlCli, metav1.NamespaceDefault, listOption)
+ gomega.Expect(err).To(gomega.BeNil())
+ gomega.Ω(len(apisixPods)).Should(gomega.BeNumerically(">", 0))
+ apisixPod = &apisixPods[0]
+ })
+
+ defer ginkgo.It("restore test environment", func() {
+ utils.DeleteRoute(e)
+ })
+
+ ginkgo.It("check if everything works", func() {
+ utils.SetRoute(e, httpexpect.Status2xx)
+ utils.GetRouteList(e, http.StatusOK)
+
+ utils.CheckMethodSucceed(e, http.MethodGet, 1)
+ utils.TestPrometheusEtcdMetric(e, 1)
+ })
+
+ // get default
+ ginkgo.It("get default apisix delay", func() {
+ timeStart := time.Now()
+ setDuration := setRouteMultipleTimes(eSilent, 5, httpexpect.Status2xx)
+ gomega.Ω(setDuration).Should(gomega.BeNumerically("<", 15*time.Millisecond))
+
+ errorLog, err := utils.Log(apisixPod, cliSet.KubeCli, timeStart)
+ gomega.Expect(err).To(gomega.BeNil())
+ gomega.Ω(errorLog).ShouldNot(gomega.ContainSubstring("error"))
+ })
+
+ // 30ms delay
+ ginkgo.It("generate a 30ms delay between etcd and apisix", func() {
+ timeStart := time.Now()
+ chaos := getEtcdDelayChaos(30)
+ err := cliSet.CtrlCli.Create(ctx, chaos)
+ gomega.Expect(err).To(gomega.BeNil())
+ time.Sleep(1 * time.Second)
+
+ defer func() {
+ err = cliSet.CtrlCli.Delete(ctx, chaos)
+ gomega.Expect(err).To(gomega.BeNil())
+ time.Sleep(1 * time.Second)
Review comment:
Thanks fixed. PTAL
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [apisix] Yiyiyimu commented on a change in pull request #4315: test: add chaos test on network delay between apisix and etcd
Posted by GitBox <gi...@apache.org>.
Yiyiyimu commented on a change in pull request #4315:
URL: https://github.com/apache/apisix/pull/4315#discussion_r682244507
##########
File path: t/chaos/killetcd/killetcd.go
##########
@@ -90,15 +84,30 @@ var _ = ginkgo.Describe("Test Get Success When Etcd Got Killed", func() {
defer ginkgo.It("restore test environment", func() {
stopChan <- true
cliSet.CtrlCli.Delete(context.Background(), getEtcdKillChaos())
+
+ resp := utils.SetRouteIgnoreError(e)
Review comment:
Thanks! fixed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [apisix] spacewander commented on a change in pull request #4315: test: add chaos test on network delay between apisix and etcd
Posted by GitBox <gi...@apache.org>.
spacewander commented on a change in pull request #4315:
URL: https://github.com/apache/apisix/pull/4315#discussion_r682212888
##########
File path: t/chaos/killetcd/killetcd.go
##########
@@ -90,15 +84,30 @@ var _ = ginkgo.Describe("Test Get Success When Etcd Got Killed", func() {
defer ginkgo.It("restore test environment", func() {
stopChan <- true
cliSet.CtrlCli.Delete(context.Background(), getEtcdKillChaos())
+
+ resp := utils.SetRouteIgnoreError(e)
Review comment:
Need to avoid repeating the waiting code
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [apisix] Yiyiyimu commented on a change in pull request #4315: test: add chaos test on network delay between apisix and etcd
Posted by GitBox <gi...@apache.org>.
Yiyiyimu commented on a change in pull request #4315:
URL: https://github.com/apache/apisix/pull/4315#discussion_r682905661
##########
File path: t/chaos/utils/utils.go
##########
@@ -219,6 +223,25 @@ func GetSilentHttpexpectClient() *httpexpect.Expect {
})
}
+func CheckMethodSucceed(e *httpexpect.Expect, method string, interval int) {
Review comment:
Thanks fixed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [apisix] spacewander commented on a change in pull request #4315: test: add chaos test on network delay between apisix and etcd
Posted by GitBox <gi...@apache.org>.
spacewander commented on a change in pull request #4315:
URL: https://github.com/apache/apisix/pull/4315#discussion_r682262234
##########
File path: t/chaos/utils/utils.go
##########
@@ -219,6 +223,25 @@ func GetSilentHttpexpectClient() *httpexpect.Expect {
})
}
+func CheckMethodSucceed(e *httpexpect.Expect, method string, interval int) {
Review comment:
Should be `WaitUntilMethodSucceed`?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [apisix] spacewander merged pull request #4315: test: add chaos test on network delay between apisix and etcd
Posted by GitBox <gi...@apache.org>.
spacewander merged pull request #4315:
URL: https://github.com/apache/apisix/pull/4315
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [apisix] tokers commented on a change in pull request #4315: test: add chaos test on network delay between apisix and etcd
Posted by GitBox <gi...@apache.org>.
tokers commented on a change in pull request #4315:
URL: https://github.com/apache/apisix/pull/4315#discussion_r682282750
##########
File path: t/chaos/delayetcd/delayetcd.go
##########
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package delayetcd
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "strconv"
+ "time"
+
+ "github.com/chaos-mesh/chaos-mesh/api/v1alpha1"
+ "github.com/gavv/httpexpect"
+ "github.com/onsi/ginkgo"
+ "github.com/onsi/gomega"
+ v1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "sigs.k8s.io/controller-runtime/pkg/client"
+
+ "github.com/apache/apisix/t/chaos/utils"
+)
+
+func getEtcdDelayChaos(delay int) *v1alpha1.NetworkChaos {
+ return &v1alpha1.NetworkChaos{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "etcd-delay",
+ Namespace: metav1.NamespaceDefault,
+ },
+ Spec: v1alpha1.NetworkChaosSpec{
+ Selector: v1alpha1.SelectorSpec{
+ LabelSelectors: map[string]string{"app.kubernetes.io/instance": "etcd"},
+ },
+ Action: v1alpha1.DelayAction,
+ Mode: v1alpha1.AllPodMode,
+ TcParameter: v1alpha1.TcParameter{
+ Delay: &v1alpha1.DelaySpec{
+ Latency: strconv.Itoa(delay) + "ms",
+ },
+ },
+ },
+ }
+}
+
+func setRouteMultipleTimes(e *httpexpect.Expect, times int, status httpexpect.StatusRange) time.Duration {
+ now := time.Now()
+ timeLast := now
+ var timeList []string
+ for i := 0; i < times; i++ {
+ utils.SetRoute(e, status)
+ timeList = append(timeList, time.Since(timeLast).String())
+ timeLast = time.Now()
+ }
+ fmt.Fprintf(ginkgo.GinkgoWriter, "takes %v separately", timeList)
+ return time.Since(now) / time.Duration(times)
+}
+
+var _ = ginkgo.Describe("Test APISIX Delay When Add ETCD Delay", func() {
+ ctx := context.Background()
+ e := httpexpect.New(ginkgo.GinkgoT(), utils.Host)
+ eSilent := utils.GetSilentHttpexpectClient()
+
+ var cliSet *utils.ClientSet
+ var apisixPod *v1.Pod
+ var err error
+ ginkgo.It("init client set", func() {
+ cliSet, err = utils.InitClientSet()
+ gomega.Expect(err).To(gomega.BeNil())
+ listOption := client.MatchingLabels{"app": "apisix-gw"}
+ apisixPods, err := utils.GetPods(cliSet.CtrlCli, metav1.NamespaceDefault, listOption)
+ gomega.Expect(err).To(gomega.BeNil())
+ gomega.Ω(len(apisixPods)).Should(gomega.BeNumerically(">", 0))
+ apisixPod = &apisixPods[0]
+ })
+
+ defer ginkgo.It("restore test environment", func() {
+ utils.DeleteRoute(e)
+ })
+
+ ginkgo.It("check if everything works", func() {
+ utils.SetRoute(e, httpexpect.Status2xx)
+ utils.GetRouteList(e, http.StatusOK)
+
+ utils.CheckMethodSucceed(e, http.MethodGet, 1)
+ utils.TestPrometheusEtcdMetric(e, 1)
+ })
+
+ // get default
+ ginkgo.It("get default apisix delay", func() {
+ timeStart := time.Now()
+ setDuration := setRouteMultipleTimes(eSilent, 5, httpexpect.Status2xx)
+ gomega.Ω(setDuration).Should(gomega.BeNumerically("<", 15*time.Millisecond))
+
+ errorLog, err := utils.Log(apisixPod, cliSet.KubeCli, timeStart)
+ gomega.Expect(err).To(gomega.BeNil())
+ gomega.Ω(errorLog).ShouldNot(gomega.ContainSubstring("error"))
+ })
+
+ // 30ms delay
+ ginkgo.It("generate a 30ms delay between etcd and apisix", func() {
+ timeStart := time.Now()
+ chaos := getEtcdDelayChaos(30)
+ err := cliSet.CtrlCli.Create(ctx, chaos)
+ gomega.Expect(err).To(gomega.BeNil())
+ time.Sleep(1 * time.Second)
+
+ defer func() {
+ err = cliSet.CtrlCli.Delete(ctx, chaos)
+ gomega.Expect(err).To(gomega.BeNil())
+ time.Sleep(1 * time.Second)
Review comment:
The sleep mechanism might not be so reliable, I'm afraid it's unstable in the Github Action.
We may try to make sure the environment is clean before each case runs.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [apisix] spacewander merged pull request #4315: test: add chaos test on network delay between apisix and etcd
Posted by GitBox <gi...@apache.org>.
spacewander merged pull request #4315:
URL: https://github.com/apache/apisix/pull/4315
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [apisix] Yiyiyimu commented on a change in pull request #4315: test: add chaos test on network delay between apisix and etcd
Posted by GitBox <gi...@apache.org>.
Yiyiyimu commented on a change in pull request #4315:
URL: https://github.com/apache/apisix/pull/4315#discussion_r682905582
##########
File path: t/chaos/delayetcd/delayetcd.go
##########
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package delayetcd
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "strconv"
+ "time"
+
+ "github.com/chaos-mesh/chaos-mesh/api/v1alpha1"
+ "github.com/gavv/httpexpect"
+ "github.com/onsi/ginkgo"
+ "github.com/onsi/gomega"
+ v1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "sigs.k8s.io/controller-runtime/pkg/client"
+
+ "github.com/apache/apisix/t/chaos/utils"
+)
+
+func getEtcdDelayChaos(delay int) *v1alpha1.NetworkChaos {
+ return &v1alpha1.NetworkChaos{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "etcd-delay",
+ Namespace: metav1.NamespaceDefault,
+ },
+ Spec: v1alpha1.NetworkChaosSpec{
+ Selector: v1alpha1.SelectorSpec{
+ LabelSelectors: map[string]string{"app.kubernetes.io/instance": "etcd"},
+ },
+ Action: v1alpha1.DelayAction,
+ Mode: v1alpha1.AllPodMode,
+ TcParameter: v1alpha1.TcParameter{
+ Delay: &v1alpha1.DelaySpec{
+ Latency: strconv.Itoa(delay) + "ms",
+ },
+ },
+ },
+ }
+}
+
+func setRouteMultipleTimes(e *httpexpect.Expect, times int, status httpexpect.StatusRange) time.Duration {
+ now := time.Now()
+ timeLast := now
+ var timeList []string
+ for i := 0; i < times; i++ {
+ utils.SetRoute(e, status)
+ timeList = append(timeList, time.Since(timeLast).String())
+ timeLast = time.Now()
+ }
+ fmt.Fprintf(ginkgo.GinkgoWriter, "takes %v separately", timeList)
+ return time.Since(now) / time.Duration(times)
+}
+
+var _ = ginkgo.Describe("Test APISIX Delay When Add ETCD Delay", func() {
+ ctx := context.Background()
+ e := httpexpect.New(ginkgo.GinkgoT(), utils.Host)
+ eSilent := utils.GetSilentHttpexpectClient()
+
+ var cliSet *utils.ClientSet
+ var apisixPod *v1.Pod
+ var err error
+ ginkgo.It("init client set", func() {
+ cliSet, err = utils.InitClientSet()
+ gomega.Expect(err).To(gomega.BeNil())
+ listOption := client.MatchingLabels{"app": "apisix-gw"}
+ apisixPods, err := utils.GetPods(cliSet.CtrlCli, metav1.NamespaceDefault, listOption)
+ gomega.Expect(err).To(gomega.BeNil())
+ gomega.Ω(len(apisixPods)).Should(gomega.BeNumerically(">", 0))
+ apisixPod = &apisixPods[0]
+ })
+
+ defer ginkgo.It("restore test environment", func() {
+ utils.DeleteRoute(e)
+ })
+
+ ginkgo.It("check if everything works", func() {
+ utils.SetRoute(e, httpexpect.Status2xx)
+ utils.GetRouteList(e, http.StatusOK)
+
+ utils.CheckMethodSucceed(e, http.MethodGet, 1)
+ utils.TestPrometheusEtcdMetric(e, 1)
+ })
+
+ // get default
+ ginkgo.It("get default apisix delay", func() {
+ timeStart := time.Now()
+ setDuration := setRouteMultipleTimes(eSilent, 5, httpexpect.Status2xx)
+ gomega.Ω(setDuration).Should(gomega.BeNumerically("<", 15*time.Millisecond))
+
+ errorLog, err := utils.Log(apisixPod, cliSet.KubeCli, timeStart)
+ gomega.Expect(err).To(gomega.BeNil())
+ gomega.Ω(errorLog).ShouldNot(gomega.ContainSubstring("error"))
+ })
+
+ // 30ms delay
+ ginkgo.It("generate a 30ms delay between etcd and apisix", func() {
+ timeStart := time.Now()
+ chaos := getEtcdDelayChaos(30)
+ err := cliSet.CtrlCli.Create(ctx, chaos)
+ gomega.Expect(err).To(gomega.BeNil())
+ time.Sleep(1 * time.Second)
+
+ defer func() {
+ err = cliSet.CtrlCli.Delete(ctx, chaos)
+ gomega.Expect(err).To(gomega.BeNil())
+ time.Sleep(1 * time.Second)
Review comment:
Thanks fixed. PTAL
##########
File path: t/chaos/utils/utils.go
##########
@@ -219,6 +223,25 @@ func GetSilentHttpexpectClient() *httpexpect.Expect {
})
}
+func CheckMethodSucceed(e *httpexpect.Expect, method string, interval int) {
Review comment:
Thanks fixed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org