You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@apisix.apache.org by zh...@apache.org on 2023/03/22 02:51:37 UTC

[apisix-ingress-controller] branch master updated: e2e: more stability (#1739)

This is an automated email from the ASF dual-hosted git repository.

zhangjintao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix-ingress-controller.git


The following commit(s) were added to refs/heads/master by this push:
     new 277669e7 e2e: more stability (#1739)
277669e7 is described below

commit 277669e7d970c3fe147d509479ff02aa75a23fd9
Author: Xin Rong <al...@gmail.com>
AuthorDate: Wed Mar 22 10:51:30 2023 +0800

    e2e: more stability (#1739)
---
 .github/workflows/e2e-test-ci.yml                  |   1 +
 test/e2e/e2e.go                                    |   1 +
 test/e2e/scaffold/k8s.go                           |   2 -
 test/e2e/scaffold/scaffold.go                      |  23 +-
 test/e2e/suite-cluster/apisix_cluster_config.go    | 259 +++++++++++++++++++++
 test/e2e/suite-features/external-sd.go             |  10 +-
 test/e2e/suite-features/external-service.go        |  23 +-
 test/e2e/suite-features/global_rule.go             |  75 ------
 .../suite-ingress-features/ingress-class.go        | 149 ------------
 9 files changed, 302 insertions(+), 241 deletions(-)

diff --git a/.github/workflows/e2e-test-ci.yml b/.github/workflows/e2e-test-ci.yml
index b68e03ce..94f81f68 100644
--- a/.github/workflows/e2e-test-ci.yml
+++ b/.github/workflows/e2e-test-ci.yml
@@ -196,4 +196,5 @@ jobs:
           E2E_FLAKE_ATTEMPTS: "2"
           E2E_ENV: "ci"
         run: |
+          echo ${E2E_FOCUS} | grep "suite-cluster" && export E2E_NODES=1
           make e2e-test
diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go
index 7a58fce0..0393cacb 100644
--- a/test/e2e/e2e.go
+++ b/test/e2e/e2e.go
@@ -17,6 +17,7 @@ package e2e
 import (
 	_ "github.com/apache/apisix-ingress-controller/test/e2e/suite-annotations"
 	_ "github.com/apache/apisix-ingress-controller/test/e2e/suite-chore"
+	_ "github.com/apache/apisix-ingress-controller/test/e2e/suite-cluster"
 	_ "github.com/apache/apisix-ingress-controller/test/e2e/suite-features"
 	_ "github.com/apache/apisix-ingress-controller/test/e2e/suite-gateway"
 	_ "github.com/apache/apisix-ingress-controller/test/e2e/suite-ingress/suite-ingress-features"
diff --git a/test/e2e/scaffold/k8s.go b/test/e2e/scaffold/k8s.go
index 3f208b8c..db3d0dfb 100644
--- a/test/e2e/scaffold/k8s.go
+++ b/test/e2e/scaffold/k8s.go
@@ -126,8 +126,6 @@ func (s *Scaffold) CreateApisixRoute(name string, rules []ApisixRouteRule) {
 // CreateResourceFromString creates resource from a loaded yaml string.
 func (s *Scaffold) CreateResourceFromString(yaml string) error {
 	err := k8s.KubectlApplyFromStringE(s.t, s.kubectlOptions, yaml)
-	time.Sleep(5 * time.Second)
-
 	// if the error raised, it may be a &shell.ErrWithCmdOutput, which is useless in debug
 	if err != nil {
 		err = fmt.Errorf(err.Error())
diff --git a/test/e2e/scaffold/scaffold.go b/test/e2e/scaffold/scaffold.go
index bd35ecf4..2e05f941 100644
--- a/test/e2e/scaffold/scaffold.go
+++ b/test/e2e/scaffold/scaffold.go
@@ -78,6 +78,7 @@ type Scaffold struct {
 	httpbinService     *corev1.Service
 	testBackendService *corev1.Service
 	finalizers         []func()
+	label              map[string]string
 
 	apisixAdminTunnel      *k8s.Tunnel
 	apisixHttpTunnel       *k8s.Tunnel
@@ -438,17 +439,17 @@ func (s *Scaffold) beforeEach() {
 	}
 	s.finalizers = nil
 
-	label := map[string]string{}
-	if !s.opts.DisableNamespaceLabel {
-		if s.opts.NamespaceSelectorLabel == nil {
-			label["apisix.ingress.watch"] = s.namespace
-			s.opts.NamespaceSelectorLabel = label
-		} else {
-			label = s.opts.NamespaceSelectorLabel
-		}
+	if s.opts.NamespaceSelectorLabel != nil {
+		s.label = s.opts.NamespaceSelectorLabel
+	} else {
+		s.label = map[string]string{"apisix.ingress.watch": s.namespace}
 	}
 
-	k8s.CreateNamespaceWithMetadata(s.t, s.kubectlOptions, metav1.ObjectMeta{Name: s.namespace, Labels: label})
+	var nsLabel map[string]string
+	if !s.opts.DisableNamespaceLabel {
+		nsLabel = s.label
+	}
+	k8s.CreateNamespaceWithMetadata(s.t, s.kubectlOptions, metav1.ObjectMeta{Name: s.namespace, Labels: nsLabel})
 
 	s.nodes, err = k8s.GetReadyNodesE(s.t, s.kubectlOptions)
 	assert.Nil(s.t, err, "querying ready nodes")
@@ -690,12 +691,12 @@ func (s *Scaffold) DeleteResource(resourceType, name string) error {
 
 func (s *Scaffold) NamespaceSelectorLabelStrings() []string {
 	var labels []string
-	for k, v := range s.opts.NamespaceSelectorLabel {
+	for k, v := range s.label {
 		labels = append(labels, fmt.Sprintf("%s=%s", k, v))
 	}
 	return labels
 }
 
 func (s *Scaffold) NamespaceSelectorLabel() map[string]string {
-	return s.opts.NamespaceSelectorLabel
+	return s.label
 }
diff --git a/test/e2e/suite-cluster/apisix_cluster_config.go b/test/e2e/suite-cluster/apisix_cluster_config.go
new file mode 100644
index 00000000..3a561003
--- /dev/null
+++ b/test/e2e/suite-cluster/apisix_cluster_config.go
@@ -0,0 +1,259 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package cluster
+
+import (
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/apache/apisix-ingress-controller/pkg/config"
+	"github.com/apache/apisix-ingress-controller/pkg/id"
+	ginkgo "github.com/onsi/ginkgo/v2"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/apache/apisix-ingress-controller/test/e2e/scaffold"
+)
+
+var _ = ginkgo.Describe("suite-cluster: ApisixClusterConfig with v2 and v2beta3 ", func() {
+	suites := func(scaffoldFunc func() *scaffold.Scaffold) {
+		s := scaffoldFunc()
+
+		ginkgo.It("enable prometheus", func() {
+			adminSvc, adminPort := s.ApisixAdminServiceAndPort()
+			assert.Nil(ginkgo.GinkgoT(), s.NewApisixClusterConfig("default", true, true), "creating ApisixClusterConfig")
+
+			defer func() {
+				assert.Nil(ginkgo.GinkgoT(), s.DeleteApisixClusterConfig("default", true, true))
+			}()
+
+			// Wait until the ApisixClusterConfig create event was delivered.
+			time.Sleep(3 * time.Second)
+
+			ar := fmt.Sprintf(`
+apiVersion: apisix.apache.org/v2beta3
+kind: ApisixRoute
+metadata:
+  name: default
+spec:
+  http:
+  - name: public-api
+    match:
+      paths:
+      - /apisix/prometheus/metrics
+    backends:
+    - serviceName: %s
+      servicePort: %d
+    plugins:
+    - name: public-api
+      enable: true
+`, adminSvc, adminPort)
+
+			err := s.CreateVersionedApisixResource(ar)
+			assert.Nil(ginkgo.GinkgoT(), err, "creating ApisixRouteConfig")
+
+			time.Sleep(3 * time.Second)
+
+			grs, err := s.ListApisixGlobalRules()
+			assert.Nil(ginkgo.GinkgoT(), err, "listing global_rules")
+			assert.Len(ginkgo.GinkgoT(), grs, 1)
+			assert.Equal(ginkgo.GinkgoT(), grs[0].ID, id.GenID("default"))
+			assert.Len(ginkgo.GinkgoT(), grs[0].Plugins, 1)
+			_, ok := grs[0].Plugins["prometheus"]
+			assert.Equal(ginkgo.GinkgoT(), ok, true)
+
+			resp := s.NewAPISIXClient().GET("/apisix/prometheus/metrics").Expect()
+			resp.Status(http.StatusOK)
+			resp.Body().Contains("# HELP apisix_etcd_modify_indexes Etcd modify index for APISIX keys")
+			resp.Body().Contains("# HELP apisix_etcd_reachable Config server etcd reachable from APISIX, 0 is unreachable")
+			resp.Body().Contains("# HELP apisix_node_info Info of APISIX node")
+
+			time.Sleep(3 * time.Second)
+
+			if s.ApisixResourceVersion() != config.ApisixV2beta3 {
+				resp1 := s.NewAPISIXClient().GET("/apisix/prometheus/metrics").Expect()
+				resp1.Status(http.StatusOK)
+				resp1.Body().Contains("public-api")
+			}
+		})
+	}
+
+	ginkgo.Describe("suite-cluster: scaffold v2beta3", func() {
+		suites(scaffold.NewDefaultV2beta3Scaffold)
+	})
+	ginkgo.Describe("suite-cluster: scaffold v2", func() {
+		suites(scaffold.NewDefaultV2Scaffold)
+	})
+})
+
+var _ = ginkgo.Describe("suite-cluster: Testin ApisixClusterConfig with IngressClass apisix", func() {
+	s := scaffold.NewScaffold(&scaffold.Options{
+		Name:                  "ingress-class",
+		IngressAPISIXReplicas: 1,
+		IngressClass:          "apisix",
+	})
+
+	ginkgo.It("ApisiClusterConfig should be ignored", func() {
+		// create ApisixConsumer resource with ingressClassName: ignore
+		acc := `
+apiVersion: apisix.apache.org/v2
+kind: ApisixClusterConfig
+metadata:
+  name: default
+spec:
+  ingressClassName: ignore
+  monitoring:
+    prometheus:
+      enable: true
+      prefer_name: true
+`
+		assert.Nil(ginkgo.GinkgoT(), s.CreateResourceFromStringWithNamespace(acc, ""))
+		time.Sleep(6 * time.Second)
+
+		agrs, err := s.ListApisixGlobalRules()
+		assert.Nil(ginkgo.GinkgoT(), err)
+		assert.Len(ginkgo.GinkgoT(), agrs, 0)
+	})
+
+	ginkgo.It("ApisiClusterConfig should be handled", func() {
+		// create ApisixConsumer resource without ingressClassName
+		acc := `
+apiVersion: apisix.apache.org/v2
+kind: ApisixClusterConfig
+metadata:
+  name: default
+spec:
+  monitoring:
+    prometheus:
+      enable: true
+      prefer_name: true
+`
+		assert.Nil(ginkgo.GinkgoT(), s.CreateResourceFromStringWithNamespace(acc, ""))
+		time.Sleep(6 * time.Second)
+
+		agrs, err := s.ListApisixGlobalRules()
+		assert.Nil(ginkgo.GinkgoT(), err)
+		assert.Len(ginkgo.GinkgoT(), agrs, 1)
+		assert.Equal(ginkgo.GinkgoT(), agrs[0].ID, id.GenID("default"))
+		assert.Len(ginkgo.GinkgoT(), agrs[0].Plugins, 1)
+		_, ok := agrs[0].Plugins["prometheus"]
+		assert.Equal(ginkgo.GinkgoT(), ok, true)
+
+		// update ApisixConsumer resource with ingressClassName: apisix
+		acc = `
+apiVersion: apisix.apache.org/v2
+kind: ApisixClusterConfig
+metadata:
+  name: default
+spec:
+  ingressClassName: apisix
+  monitoring:
+    prometheus:
+      enable: true
+      prefer_name: true
+`
+		assert.Nil(ginkgo.GinkgoT(), s.CreateResourceFromStringWithNamespace(acc, ""))
+		time.Sleep(6 * time.Second)
+
+		agrs, err = s.ListApisixGlobalRules()
+		assert.Nil(ginkgo.GinkgoT(), err)
+		assert.Len(ginkgo.GinkgoT(), agrs, 1)
+		assert.Equal(ginkgo.GinkgoT(), agrs[0].ID, id.GenID("default"))
+		assert.Len(ginkgo.GinkgoT(), agrs[0].Plugins, 1)
+		_, ok = agrs[0].Plugins["prometheus"]
+		assert.Equal(ginkgo.GinkgoT(), ok, true)
+	})
+})
+
+var _ = ginkgo.Describe("suite-cluster: Testing ApisixClusterConfig with IngressClass apisix-and-all", func() {
+	s := scaffold.NewScaffold(&scaffold.Options{
+		Name:                  "ingress-class",
+		IngressAPISIXReplicas: 1,
+		IngressClass:          "apisix-and-all",
+	})
+
+	ginkgo.It("ApisiClusterConfig should be handled", func() {
+		// create ApisixConsumer resource without ingressClassName
+		acc := `
+apiVersion: apisix.apache.org/v2
+kind: ApisixClusterConfig
+metadata:
+  name: default
+spec:
+  monitoring:
+    prometheus:
+      enable: true
+      prefer_name: true
+`
+		assert.Nil(ginkgo.GinkgoT(), s.CreateResourceFromStringWithNamespace(acc, ""))
+		time.Sleep(6 * time.Second)
+
+		agrs, err := s.ListApisixGlobalRules()
+		assert.Nil(ginkgo.GinkgoT(), err)
+		assert.Len(ginkgo.GinkgoT(), agrs, 1)
+		assert.Equal(ginkgo.GinkgoT(), agrs[0].ID, id.GenID("default"))
+		assert.Len(ginkgo.GinkgoT(), agrs[0].Plugins, 1)
+		_, ok := agrs[0].Plugins["prometheus"]
+		assert.Equal(ginkgo.GinkgoT(), ok, true)
+
+		// update ApisixConsumer resource with ingressClassName: apisix
+		acc = `
+apiVersion: apisix.apache.org/v2
+kind: ApisixClusterConfig
+metadata:
+  name: default
+spec:
+  ingressClassName: apisix
+  monitoring:
+    prometheus:
+      enable: true
+      prefer_name: true
+`
+		assert.Nil(ginkgo.GinkgoT(), s.CreateResourceFromStringWithNamespace(acc, ""))
+		time.Sleep(6 * time.Second)
+
+		agrs, err = s.ListApisixGlobalRules()
+		assert.Nil(ginkgo.GinkgoT(), err)
+		assert.Len(ginkgo.GinkgoT(), agrs, 1)
+		assert.Equal(ginkgo.GinkgoT(), agrs[0].ID, id.GenID("default"))
+		assert.Len(ginkgo.GinkgoT(), agrs[0].Plugins, 1)
+		_, ok = agrs[0].Plugins["prometheus"]
+		assert.Equal(ginkgo.GinkgoT(), ok, true)
+
+		// update ApisixConsumer resource with ingressClassName: watch
+		acc = `
+apiVersion: apisix.apache.org/v2
+kind: ApisixClusterConfig
+metadata:
+  name: default
+spec:
+  ingressClassName: watch
+  monitoring:
+    prometheus:
+      enable: true
+      prefer_name: true
+`
+		assert.Nil(ginkgo.GinkgoT(), s.CreateResourceFromStringWithNamespace(acc, ""))
+		time.Sleep(6 * time.Second)
+
+		agrs, err = s.ListApisixGlobalRules()
+		assert.Nil(ginkgo.GinkgoT(), err)
+		assert.Len(ginkgo.GinkgoT(), agrs, 1)
+		assert.Equal(ginkgo.GinkgoT(), agrs[0].ID, id.GenID("default"))
+		assert.Len(ginkgo.GinkgoT(), agrs[0].Plugins, 1)
+		_, ok = agrs[0].Plugins["prometheus"]
+		assert.Equal(ginkgo.GinkgoT(), ok, true)
+	})
+})
diff --git a/test/e2e/suite-features/external-sd.go b/test/e2e/suite-features/external-sd.go
index 613a547b..9d33ac0f 100644
--- a/test/e2e/suite-features/external-sd.go
+++ b/test/e2e/suite-features/external-sd.go
@@ -224,6 +224,7 @@ spec:
 			// We use it for service discovery
 			PhaseCreateApisixUpstream(s, "httpbin-upstream", "dns", "httpbin-temp")
 			PhaseCreateApisixRoute(s, "httpbin-route", "httpbin-upstream")
+			time.Sleep(time.Second * 6)
 
 			// -- validation --
 			upstreamId := PhaseValidateFirstUpstream(s, 1, fqdn, "dns")
@@ -236,10 +237,12 @@ spec:
 			// -- Data preparation --
 			fqdn := PhaseCreateHttpbin(s, "httpbin-temp")
 			PhaseCreateApisixRoute(s, "httpbin-route", "httpbin-upstream")
-			PhaseValidateNoUpstreams(s)
+			time.Sleep(time.Second * 6)
 
+			PhaseValidateNoUpstreams(s)
 			// -- Data Update --
 			PhaseCreateApisixUpstream(s, "httpbin-upstream", "dns", "httpbin-temp")
+			time.Sleep(time.Second * 6)
 
 			// -- validation --
 			upstreamId := PhaseValidateFirstUpstream(s, 1, fqdn, "dns")
@@ -249,11 +252,13 @@ spec:
 		ginkgo.It("should be able to create the target service later", func() {
 			// -- Data preparation --
 			PhaseCreateApisixRoute(s, "httpbin-route", "httpbin-upstream")
+			time.Sleep(time.Second * 6)
 			PhaseValidateNoUpstreams(s)
-			PhaseCreateApisixUpstream(s, "httpbin-upstream", "dns", "httpbin-temp")
 
+			PhaseCreateApisixUpstream(s, "httpbin-upstream", "dns", "httpbin-temp")
 			// -- Data Update --
 			fqdn := PhaseCreateHttpbin(s, "httpbin-temp")
+			time.Sleep(time.Second * 6)
 
 			// -- validation --
 			upstreamId := PhaseValidateFirstUpstream(s, 1, fqdn, "dns")
@@ -267,6 +272,7 @@ spec:
 			fqdn := PhaseCreateHttpbin(s, "httpbin-temp")
 			PhaseCreateApisixUpstream(s, "httpbin-upstream", "dns", "httpbin-temp")
 			PhaseCreateApisixRoute(s, "httpbin-route", "httpbin-upstream")
+			time.Sleep(time.Second * 6)
 
 			// -- validation --
 			upstreamId := PhaseValidateFirstUpstream(s, 1, fqdn, "dns")
diff --git a/test/e2e/suite-features/external-service.go b/test/e2e/suite-features/external-service.go
index a15c1641..6eefa318 100644
--- a/test/e2e/suite-features/external-service.go
+++ b/test/e2e/suite-features/external-service.go
@@ -262,6 +262,7 @@ spec:
 			// -- Data preparation --
 			PhaseCreateApisixUpstream(s, "httpbin-upstream", v2.ExternalTypeDomain, "httpbin.org")
 			PhaseCreateApisixRoute(s, "httpbin-route", "httpbin-upstream")
+			time.Sleep(time.Second * 6)
 
 			// -- validation --
 			upstreamId := PhaseValidateFirstUpstream(s, 1, "httpbin.org", 80, translation.DefaultWeight)
@@ -271,6 +272,7 @@ spec:
 			// -- Data preparation --
 			PhaseCreateApisixUpstream(s, "httpbin-upstream", v2.ExternalTypeDomain, "httpbun.org")
 			PhaseCreateApisixRoute(s, "httpbin-route", "httpbin-upstream")
+			time.Sleep(time.Second * 6)
 
 			// -- Expect failed --
 			upstreamId := PhaseValidateFirstUpstream(s, 1, "httpbun.org", 80, translation.DefaultWeight)
@@ -278,6 +280,7 @@ spec:
 
 			// -- update --
 			PhaseCreateApisixRouteWithHostRewrite(s, "httpbin-route", "httpbin-upstream", "httpbun.org")
+			time.Sleep(time.Second * 6)
 
 			// -- validation --
 			upstreamId = PhaseValidateFirstUpstream(s, 1, "httpbun.org", 80, translation.DefaultWeight)
@@ -288,6 +291,7 @@ spec:
 			PhaseCreateExternalService(s, "ext-httpbin", "httpbin.org")
 			PhaseCreateApisixUpstream(s, "httpbin-upstream", v2.ExternalTypeService, "ext-httpbin")
 			PhaseCreateApisixRoute(s, "httpbin-route", "httpbin-upstream")
+			time.Sleep(time.Second * 6)
 
 			// -- validation --
 			upstreamId := PhaseValidateFirstUpstream(s, 1, "httpbin.org", 80, translation.DefaultWeight)
@@ -296,12 +300,14 @@ spec:
 		ginkgo.It("should be able to access in-cluster ExternalName service", func() {
 			// -- Data preparation --
 			fqdn := PhaseCreateHttpbin(s, "httpbin-temp")
+			time.Sleep(time.Second * 10)
 
 			// We are only testing the functionality of the external service and do not care which namespace the service is in.
 			// The namespace of the external service should be watched.
 			PhaseCreateExternalService(s, "ext-httpbin", fqdn)
 			PhaseCreateApisixUpstream(s, "httpbin-upstream", v2.ExternalTypeService, "ext-httpbin")
 			PhaseCreateApisixRoute(s, "httpbin-route", "httpbin-upstream")
+			time.Sleep(time.Second * 6)
 
 			// -- validation --
 			upstreamId := PhaseValidateFirstUpstream(s, 1, fqdn, 80, translation.DefaultWeight)
@@ -463,6 +469,7 @@ spec:
 			PhaseCreateApisixUpstreamWithMultipleExternalNodes(s, "httpbin-upstream",
 				v2.ExternalTypeDomain, "httpbin.org", v2.ExternalTypeDomain, "httpbun.org")
 			PhaseCreateApisixRouteWithHostRewrite(s, "httpbin-route", "httpbin-upstream", "httpbun.org")
+			time.Sleep(time.Second * 6)
 
 			// -- validation --
 			PhaseValidateMultipleNodes(s, 2, map[string]*validateFactor{
@@ -479,8 +486,10 @@ spec:
 		ginkgo.It("should be able to use backends and upstreams together", func() {
 			// -- Data preparation --
 			PhaseCreateHttpbin(s, "httpbin-temp")
+			time.Sleep(time.Second * 10)
 			PhaseCreateApisixUpstream(s, "httpbin-upstream", v2.ExternalTypeDomain, "httpbun.org")
 			PhaseCreateApisixRouteWithHostRewriteAndBackend(s, "httpbin-route", "httpbin-upstream", "httpbun.org", "httpbin-temp", 80)
+			time.Sleep(time.Second * 6)
 
 			svc, err := s.GetServiceByName("httpbin-temp")
 			assert.Nil(ginkgo.GinkgoT(), err, "get httpbin service")
@@ -506,10 +515,12 @@ spec:
 		ginkgo.It("should be able to create the ApisixUpstream later", func() {
 			// -- Data preparation --
 			PhaseCreateApisixRoute(s, "httpbin-route", "httpbin-upstream")
+			time.Sleep(time.Second * 6)
 			PhaseValidateNoUpstreams(s)
 
 			// -- Data Update --
 			PhaseCreateApisixUpstream(s, "httpbin-upstream", v2.ExternalTypeDomain, "httpbin.org")
+			time.Sleep(time.Second * 6)
 
 			// -- validation --
 			upstreamId := PhaseValidateFirstUpstream(s, 1, "httpbin.org", 80, translation.DefaultWeight)
@@ -518,13 +529,15 @@ spec:
 		ginkgo.It("should be able to create the ExternalName service later", func() {
 			// -- Data preparation --
 			fqdn := PhaseCreateHttpbin(s, "httpbin-temp")
+			time.Sleep(time.Second * 10)
 			PhaseCreateApisixUpstream(s, "httpbin-upstream", v2.ExternalTypeService, "ext-httpbin")
 			PhaseCreateApisixRoute(s, "httpbin-route", "httpbin-upstream")
+			time.Sleep(time.Second * 6)
 			PhaseValidateNoUpstreams(s)
 
 			// -- Data update --
 			PhaseCreateExternalService(s, "ext-httpbin", fqdn)
-
+			time.Sleep(time.Second * 6)
 			// -- validation --
 			upstreamId := PhaseValidateFirstUpstream(s, 1, fqdn, 80, translation.DefaultWeight)
 			PhaseValidateRouteAccess(s, upstreamId)
@@ -532,13 +545,16 @@ spec:
 		ginkgo.It("should be able to update the ApisixUpstream later", func() {
 			// -- Data preparation --
 			fqdn := PhaseCreateHttpbin(s, "httpbin-temp")
+			time.Sleep(time.Second * 10)
 			PhaseCreateExternalService(s, "ext-httpbin", fqdn)
 			PhaseCreateApisixUpstream(s, "httpbin-upstream", v2.ExternalTypeService, "doesnt-exist")
 			PhaseCreateApisixRoute(s, "httpbin-route", "httpbin-upstream")
+			time.Sleep(time.Second * 6)
 			PhaseValidateNoUpstreams(s)
 
 			// -- Data update --
 			PhaseCreateApisixUpstream(s, "httpbin-upstream", v2.ExternalTypeService, "ext-httpbin")
+			time.Sleep(time.Second * 6)
 
 			// -- validation --
 			upstreamId := PhaseValidateFirstUpstream(s, 1, fqdn, 80, translation.DefaultWeight)
@@ -549,10 +565,12 @@ spec:
 			PhaseCreateExternalService(s, "ext-httpbin", "unknown.org")
 			PhaseCreateApisixUpstream(s, "httpbin-upstream", v2.ExternalTypeService, "ext-httpbin")
 			PhaseCreateApisixRoute(s, "httpbin-route", "httpbin-upstream")
+			time.Sleep(time.Second * 6)
 			PhaseValidateFirstUpstream(s, 1, "unknown.org", 80, translation.DefaultWeight)
 
 			// -- Data update --
 			PhaseCreateExternalService(s, "ext-httpbin", "httpbin.org")
+			time.Sleep(time.Second * 6)
 
 			// -- validation --
 			upstreamId := PhaseValidateFirstUpstream(s, 1, "httpbin.org", 80, translation.DefaultWeight)
@@ -564,6 +582,7 @@ spec:
 			// -- Data preparation --
 			PhaseCreateApisixUpstream(s, "httpbin-upstream", v2.ExternalTypeDomain, "httpbin.org")
 			PhaseCreateApisixRoute(s, "httpbin-route", "httpbin-upstream")
+			time.Sleep(time.Second * 6)
 
 			// -- validation --
 			upstreamId := PhaseValidateFirstUpstream(s, 1, "httpbin.org", 80, translation.DefaultWeight)
@@ -572,7 +591,7 @@ spec:
 			// -- delete --
 			assert.Nil(ginkgo.GinkgoT(), s.DeleteResource("ar", "httpbin-route"), "delete route")
 			assert.Nil(ginkgo.GinkgoT(), s.DeleteResource("au", "httpbin-upstream"), "delete upstream")
-			time.Sleep(time.Second * 15)
+			time.Sleep(time.Second * 6)
 
 			// -- validate --
 			PhaseValidateNoRoutes(s)
diff --git a/test/e2e/suite-features/global_rule.go b/test/e2e/suite-features/global_rule.go
index b457ddd9..3f490261 100644
--- a/test/e2e/suite-features/global_rule.go
+++ b/test/e2e/suite-features/global_rule.go
@@ -15,89 +15,14 @@
 package features
 
 import (
-	"fmt"
-	"net/http"
 	"time"
 
-	"github.com/apache/apisix-ingress-controller/pkg/config"
-	"github.com/apache/apisix-ingress-controller/pkg/id"
 	ginkgo "github.com/onsi/ginkgo/v2"
 	"github.com/stretchr/testify/assert"
 
 	"github.com/apache/apisix-ingress-controller/test/e2e/scaffold"
 )
 
-var _ = ginkgo.Describe("suite-features: ApisixClusterConfig", func() {
-	suites := func(scaffoldFunc func() *scaffold.Scaffold) {
-		s := scaffoldFunc()
-
-		ginkgo.It("enable prometheus", func() {
-			adminSvc, adminPort := s.ApisixAdminServiceAndPort()
-			assert.Nil(ginkgo.GinkgoT(), s.NewApisixClusterConfig("default", true, true), "creating ApisixClusterConfig")
-
-			defer func() {
-				assert.Nil(ginkgo.GinkgoT(), s.DeleteApisixClusterConfig("default", true, true))
-			}()
-
-			// Wait until the ApisixClusterConfig create event was delivered.
-			time.Sleep(3 * time.Second)
-
-			ar := fmt.Sprintf(`
-apiVersion: apisix.apache.org/v2beta3
-kind: ApisixRoute
-metadata:
-  name: default
-spec:
-  http:
-  - name: public-api
-    match:
-      paths:
-      - /apisix/prometheus/metrics
-    backends:
-    - serviceName: %s
-      servicePort: %d
-    plugins:
-    - name: public-api
-      enable: true
-`, adminSvc, adminPort)
-
-			err := s.CreateVersionedApisixResource(ar)
-			assert.Nil(ginkgo.GinkgoT(), err, "creating ApisixRouteConfig")
-
-			time.Sleep(3 * time.Second)
-
-			grs, err := s.ListApisixGlobalRules()
-			assert.Nil(ginkgo.GinkgoT(), err, "listing global_rules")
-			assert.Len(ginkgo.GinkgoT(), grs, 1)
-			assert.Equal(ginkgo.GinkgoT(), grs[0].ID, id.GenID("default"))
-			assert.Len(ginkgo.GinkgoT(), grs[0].Plugins, 1)
-			_, ok := grs[0].Plugins["prometheus"]
-			assert.Equal(ginkgo.GinkgoT(), ok, true)
-
-			resp := s.NewAPISIXClient().GET("/apisix/prometheus/metrics").Expect()
-			resp.Status(http.StatusOK)
-			resp.Body().Contains("# HELP apisix_etcd_modify_indexes Etcd modify index for APISIX keys")
-			resp.Body().Contains("# HELP apisix_etcd_reachable Config server etcd reachable from APISIX, 0 is unreachable")
-			resp.Body().Contains("# HELP apisix_node_info Info of APISIX node")
-
-			time.Sleep(3 * time.Second)
-
-			if s.ApisixResourceVersion() != config.ApisixV2beta3 {
-				resp1 := s.NewAPISIXClient().GET("/apisix/prometheus/metrics").Expect()
-				resp1.Status(http.StatusOK)
-				resp1.Body().Contains("public-api")
-			}
-		})
-	}
-
-	ginkgo.Describe("suite-features: scaffold v2beta3", func() {
-		suites(scaffold.NewDefaultV2beta3Scaffold)
-	})
-	ginkgo.Describe("suite-features: scaffold v2", func() {
-		suites(scaffold.NewDefaultV2Scaffold)
-	})
-})
-
 var _ = ginkgo.Describe("suite-features: ApisiGlobalRule", func() {
 	s := scaffold.NewDefaultScaffold()
 
diff --git a/test/e2e/suite-ingress/suite-ingress-features/ingress-class.go b/test/e2e/suite-ingress/suite-ingress-features/ingress-class.go
index ea4222ca..dc5c0672 100644
--- a/test/e2e/suite-ingress/suite-ingress-features/ingress-class.go
+++ b/test/e2e/suite-ingress/suite-ingress-features/ingress-class.go
@@ -19,7 +19,6 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/apache/apisix-ingress-controller/pkg/id"
 	ginkgo "github.com/onsi/ginkgo/v2"
 	"github.com/stretchr/testify/assert"
 
@@ -473,81 +472,6 @@ spec:
 		assert.Contains(ginkgo.GinkgoT(), acs[0].Username, "james")
 		assert.Equal(ginkgo.GinkgoT(), map[string]interface{}{"key": "james-key"}, acs[0].Plugins["key-auth"])
 	})
-
-	ginkgo.It("ApisiClusterConfig should be ignored", func() {
-		// create ApisixConsumer resource with ingressClassName: ignore
-		acc := `
-apiVersion: apisix.apache.org/v2
-kind: ApisixClusterConfig
-metadata:
-  name: default
-spec:
-  ingressClassName: ignore
-  monitoring:
-    prometheus:
-      enable: true
-      prefer_name: true
-`
-		assert.Nil(ginkgo.GinkgoT(), s.CreateResourceFromString(acc))
-		time.Sleep(6 * time.Second)
-
-		agrs, err := s.ListApisixGlobalRules()
-		assert.Nil(ginkgo.GinkgoT(), err)
-		assert.Len(ginkgo.GinkgoT(), agrs, 1)
-		assert.Equal(ginkgo.GinkgoT(), agrs[0].ID, id.GenID("default"))
-		assert.Len(ginkgo.GinkgoT(), agrs[0].Plugins, 1)
-		_, ok := agrs[0].Plugins["prometheus"]
-		assert.Equal(ginkgo.GinkgoT(), ok, true)
-	})
-
-	ginkgo.It("ApisiClusterConfig should be handled", func() {
-		// create ApisixConsumer resource without ingressClassName
-		acc := `
-apiVersion: apisix.apache.org/v2
-kind: ApisixClusterConfig
-metadata:
-  name: default
-spec:
-  monitoring:
-    prometheus:
-      enable: true
-      prefer_name: true
-`
-		assert.Nil(ginkgo.GinkgoT(), s.CreateResourceFromString(acc))
-		time.Sleep(6 * time.Second)
-
-		agrs, err := s.ListApisixGlobalRules()
-		assert.Nil(ginkgo.GinkgoT(), err)
-		assert.Len(ginkgo.GinkgoT(), agrs, 1)
-		assert.Equal(ginkgo.GinkgoT(), agrs[0].ID, id.GenID("default"))
-		assert.Len(ginkgo.GinkgoT(), agrs[0].Plugins, 1)
-		_, ok := agrs[0].Plugins["prometheus"]
-		assert.Equal(ginkgo.GinkgoT(), ok, true)
-
-		// update ApisixConsumer resource with ingressClassName: apisix
-		acc = `
-apiVersion: apisix.apache.org/v2
-kind: ApisixClusterConfig
-metadata:
-  name: default
-spec:
-  ingressClassName: apisix
-  monitoring:
-    prometheus:
-      enable: true
-      prefer_name: true
-`
-		assert.Nil(ginkgo.GinkgoT(), s.CreateResourceFromString(acc))
-		time.Sleep(6 * time.Second)
-
-		agrs, err = s.ListApisixGlobalRules()
-		assert.Nil(ginkgo.GinkgoT(), err)
-		assert.Len(ginkgo.GinkgoT(), agrs, 1)
-		assert.Equal(ginkgo.GinkgoT(), agrs[0].ID, id.GenID("default"))
-		assert.Len(ginkgo.GinkgoT(), agrs[0].Plugins, 1)
-		_, ok = agrs[0].Plugins["prometheus"]
-		assert.Equal(ginkgo.GinkgoT(), ok, true)
-	})
 })
 
 var _ = ginkgo.Describe("suite-ingress-features: Testing CRDs with IngressClass apisix-and-all", func() {
@@ -846,77 +770,4 @@ spec:
 		assert.Contains(ginkgo.GinkgoT(), acs[0].Username, "james")
 		assert.Equal(ginkgo.GinkgoT(), map[string]interface{}{"key": "james-password"}, acs[0].Plugins["key-auth"])
 	})
-
-	ginkgo.It("ApisiClusterConfig should be handled", func() {
-		// create ApisixConsumer resource without ingressClassName
-		acc := `
-apiVersion: apisix.apache.org/v2
-kind: ApisixClusterConfig
-metadata:
-  name: default
-spec:
-  monitoring:
-    prometheus:
-      enable: true
-      prefer_name: true
-`
-		assert.Nil(ginkgo.GinkgoT(), s.CreateResourceFromString(acc))
-		time.Sleep(6 * time.Second)
-
-		agrs, err := s.ListApisixGlobalRules()
-		assert.Nil(ginkgo.GinkgoT(), err)
-		assert.Len(ginkgo.GinkgoT(), agrs, 1)
-		assert.Equal(ginkgo.GinkgoT(), agrs[0].ID, id.GenID("default"))
-		assert.Len(ginkgo.GinkgoT(), agrs[0].Plugins, 1)
-		_, ok := agrs[0].Plugins["prometheus"]
-		assert.Equal(ginkgo.GinkgoT(), ok, true)
-
-		// update ApisixConsumer resource with ingressClassName: apisix
-		acc = `
-apiVersion: apisix.apache.org/v2
-kind: ApisixClusterConfig
-metadata:
-  name: default
-spec:
-  ingressClassName: apisix
-  monitoring:
-    prometheus:
-      enable: true
-      prefer_name: true
-`
-		assert.Nil(ginkgo.GinkgoT(), s.CreateResourceFromString(acc))
-		time.Sleep(6 * time.Second)
-
-		agrs, err = s.ListApisixGlobalRules()
-		assert.Nil(ginkgo.GinkgoT(), err)
-		assert.Len(ginkgo.GinkgoT(), agrs, 1)
-		assert.Equal(ginkgo.GinkgoT(), agrs[0].ID, id.GenID("default"))
-		assert.Len(ginkgo.GinkgoT(), agrs[0].Plugins, 1)
-		_, ok = agrs[0].Plugins["prometheus"]
-		assert.Equal(ginkgo.GinkgoT(), ok, true)
-
-		// update ApisixConsumer resource with ingressClassName: watch
-		acc = `
-apiVersion: apisix.apache.org/v2
-kind: ApisixClusterConfig
-metadata:
-  name: default
-spec:
-  ingressClassName: watch
-  monitoring:
-    prometheus:
-      enable: true
-      prefer_name: true
-`
-		assert.Nil(ginkgo.GinkgoT(), s.CreateResourceFromString(acc))
-		time.Sleep(6 * time.Second)
-
-		agrs, err = s.ListApisixGlobalRules()
-		assert.Nil(ginkgo.GinkgoT(), err)
-		assert.Len(ginkgo.GinkgoT(), agrs, 1)
-		assert.Equal(ginkgo.GinkgoT(), agrs[0].ID, id.GenID("default"))
-		assert.Len(ginkgo.GinkgoT(), agrs[0].Plugins, 1)
-		_, ok = agrs[0].Plugins["prometheus"]
-		assert.Equal(ginkgo.GinkgoT(), ok, true)
-	})
 })