[apisix-ingress-controller] branch master updated: feat: support endpointslice, and improve test/e2e/endpoints.go tests. (#1140)

[zh...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

zhangjintao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix-ingress-controller.git


The following commit(s) were added to refs/heads/master by this push:
     new a73b52d3 feat:  support endpointslice, and improve test/e2e/endpoints.go tests. (#1140)
a73b52d3 is described below

commit a73b52d3946dcc08473888445a0c66152e659120
Author: Xin Rong <13...@qq.com>
AuthorDate: Mon Jul 11 13:19:50 2022 +0800

    feat:  support endpointslice, and improve test/e2e/endpoints.go tests. (#1140)
---
 pkg/ingress/endpoint.go               | 18 +++++-----
 pkg/ingress/endpointslice.go          | 13 +++++++
 test/e2e/scaffold/k8s.go              | 14 ++++++++
 test/e2e/scaffold/scaffold.go         |  6 ++--
 test/e2e/suite-endpoints/endpoints.go | 65 ++++++++++++++++++++++++-----------
 5 files changed, 85 insertions(+), 31 deletions(-)

diff --git a/pkg/ingress/endpoint.go b/pkg/ingress/endpoint.go
index 76aed6cb..a3c62ff0 100644
--- a/pkg/ingress/endpoint.go
+++ b/pkg/ingress/endpoint.go
@@ -90,23 +90,23 @@ func (c *endpointsController) sync(ctx context.Context, ev *types.Event) error {
 	if err != nil {
 		return err
 	}
-	newestEp, err := c.controller.epLister.GetEndpoint(ns, ep.ServiceName())
-	if err != nil {
-		if !errors.IsNotFound(err) {
-			return err
-		}
-		newestEp = ep
-	}
-	if ev.Type == types.EventDelete && newestEp != nil {
+	if ev.Type == types.EventDelete {
 		clusterName := c.controller.cfg.APISIX.DefaultClusterName
 		err = c.controller.apisix.Cluster(clusterName).UpstreamServiceRelation().Delete(ctx,
 			&v1.UpstreamServiceRelation{
-				ServiceName: ns + "_" + newestEp.ServiceName(),
+				ServiceName: ns + "_" + ep.ServiceName(),
 			})
 		if err != nil {
 			return err
 		}
 	}
+	newestEp, err := c.controller.epLister.GetEndpoint(ns, ep.ServiceName())
+	if err != nil {
+		if !errors.IsNotFound(err) {
+			return err
+		}
+		newestEp = ep
+	}
 	return c.controller.syncEndpoint(ctx, newestEp)
 }
 
diff --git a/pkg/ingress/endpointslice.go b/pkg/ingress/endpointslice.go
index 1b8d83b9..7b658b33 100644
--- a/pkg/ingress/endpointslice.go
+++ b/pkg/ingress/endpointslice.go
@@ -26,6 +26,7 @@ import (
 
 	"github.com/apache/apisix-ingress-controller/pkg/log"
 	"github.com/apache/apisix-ingress-controller/pkg/types"
+	v1 "github.com/apache/apisix-ingress-controller/pkg/types/apisix/v1"
 )
 
 const (
@@ -98,6 +99,18 @@ func (c *endpointSliceController) sync(ctx context.Context, ev *types.Event) err
 		log.Errorf("found endpointSlice object with bad namespace/name: %s, ignore it", epEvent.Key)
 		return nil
 	}
+	if ev.Type == types.EventDelete {
+		log.Debugw("endpointsplice upstream serviece sync",
+			zap.String("service_name", epEvent.ServiceName))
+		clusterName := c.controller.cfg.APISIX.DefaultClusterName
+		err = c.controller.apisix.Cluster(clusterName).UpstreamServiceRelation().Delete(ctx,
+			&v1.UpstreamServiceRelation{
+				ServiceName: namespace + "_" + epEvent.ServiceName,
+			})
+		if err != nil {
+			return err
+		}
+	}
 	ep, err := c.controller.epLister.GetEndpointSlices(namespace, epEvent.ServiceName)
 	if err != nil {
 		log.Errorf("failed to get all endpointSlices for service %s: %s",
diff --git a/test/e2e/scaffold/k8s.go b/test/e2e/scaffold/k8s.go
index b422995c..84a7009d 100644
--- a/test/e2e/scaffold/k8s.go
+++ b/test/e2e/scaffold/k8s.go
@@ -263,6 +263,20 @@ func (s *Scaffold) EnsureNumApisixTlsCreated(desired int) error {
 	return s.ensureNumApisixCRDsCreated(u.String(), desired)
 }
 
+// EnsureNumListUpstreamNodesNth waits until desired number of upstreams[n-1].Nodes created in
+// APISIX cluster.
+// The upstreams[n-1].Nodes number is equal to desired.
+func (s *Scaffold) EnsureNumListUpstreamNodesNth(n, desired int) error {
+	condFunc := func() (bool, error) {
+		ups, err := s.ListApisixUpstreams()
+		if err != nil || len(ups) < n || len(ups[n-1].Nodes) != desired {
+			return false, fmt.Errorf("EnsureNumListUpstreamNodes failed")
+		}
+		return true, nil
+	}
+	return wait.Poll(3*time.Second, 35*time.Second, condFunc)
+}
+
 // CreateApisixRouteByApisixAdmin create or update a route
 func (s *Scaffold) CreateApisixRouteByApisixAdmin(routeID string, body []byte) error {
 	u := url.URL{
diff --git a/test/e2e/scaffold/scaffold.go b/test/e2e/scaffold/scaffold.go
index 8d901a41..7703792b 100644
--- a/test/e2e/scaffold/scaffold.go
+++ b/test/e2e/scaffold/scaffold.go
@@ -433,8 +433,10 @@ func (s *Scaffold) beforeEach() {
 	err = s.newIngressAPISIXController()
 	assert.Nil(s.t, err, "initializing ingress apisix controller")
 
-	err = s.WaitAllIngressControllerPodsAvailable()
-	assert.Nil(s.t, err, "waiting for ingress apisix controller ready")
+	if s.opts.IngressAPISIXReplicas != 0 {
+		err = s.WaitAllIngressControllerPodsAvailable()
+		assert.Nil(s.t, err, "waiting for ingress apisix controller ready")
+	}
 }
 
 func (s *Scaffold) afterEach() {
diff --git a/test/e2e/suite-endpoints/endpoints.go b/test/e2e/suite-endpoints/endpoints.go
index 4c5a411b..e4aa36df 100644
--- a/test/e2e/suite-endpoints/endpoints.go
+++ b/test/e2e/suite-endpoints/endpoints.go
@@ -17,7 +17,6 @@ package endpoints
 import (
 	"fmt"
 	"net/http"
-	"time"
 
 	ginkgo "github.com/onsi/ginkgo/v2"
 	"github.com/stretchr/testify/assert"
@@ -28,10 +27,8 @@ import (
 var _ = ginkgo.Describe("suite-endpoints: endpoints", func() {
 	suites := func(s *scaffold.Scaffold) {
 		ginkgo.It("ignore applied only if there is an ApisixRoute referenced", func() {
-			time.Sleep(5 * time.Second)
-			assert.Nil(ginkgo.GinkgoT(), s.EnsureNumApisixUpstreamsCreated(0), "checking number of upstreams")
 			backendSvc, backendSvcPort := s.DefaultHTTPBackend()
-			ups := fmt.Sprintf(`
+			ar := fmt.Sprintf(`
 apiVersion: apisix.apache.org/v2beta3
 kind: ApisixRoute
 metadata:
@@ -48,12 +45,14 @@ spec:
     - serviceName: %s
       servicePort: %d
 `, backendSvc, backendSvcPort[0])
-			assert.Nil(ginkgo.GinkgoT(), s.CreateResourceFromString(ups))
+			assert.Nil(ginkgo.GinkgoT(), s.CreateVersionedApisixResource(ar))
+			assert.Nil(ginkgo.GinkgoT(), s.EnsureNumApisixRoutesCreated(1), "checking number of routes")
 			assert.Nil(ginkgo.GinkgoT(), s.EnsureNumApisixUpstreamsCreated(1), "checking number of upstreams")
+
+			s.NewAPISIXClient().GET("/ip").WithHeader("Host", "httpbin.org").Expect().Status(http.StatusOK)
 		})
 
 		ginkgo.It("upstream nodes should be reset to empty when Service/Endpoints was deleted", func() {
-			ginkgo.Skip("now we don't handle endpoints delete event")
 			backendSvc, backendSvcPort := s.DefaultHTTPBackend()
 			apisixRoute := fmt.Sprintf(`
 apiVersion: apisix.apache.org/v2beta3
@@ -66,18 +65,48 @@ spec:
     match:
       hosts:
       - httpbin.com
-      paths: /ip
+      paths:
+      - /ip
     backends:
     - serviceName: %s
       servicePort: %d
 `, backendSvc, backendSvcPort[0])
-			assert.Nil(ginkgo.GinkgoT(), s.CreateResourceFromString(apisixRoute))
+			assert.Nil(ginkgo.GinkgoT(), s.CreateVersionedApisixResource(apisixRoute))
 			assert.Nil(ginkgo.GinkgoT(), s.EnsureNumApisixUpstreamsCreated(1), "checking number of upstreams")
 			s.NewAPISIXClient().GET("/ip").WithHeader("Host", "httpbin.com").Expect().Status(http.StatusOK)
 
 			// Now delete the backend httpbin service resource.
 			assert.Nil(ginkgo.GinkgoT(), s.DeleteHTTPBINService())
-			time.Sleep(3 * time.Second)
+			assert.Nil(ginkgo.GinkgoT(), s.EnsureNumListUpstreamNodesNth(1, 0))
+			s.NewAPISIXClient().GET("/ip").WithHeader("Host", "httpbin.com").Expect().Status(http.StatusServiceUnavailable)
+		})
+
+		ginkgo.It("when endpoint is 0, upstream nodes is also 0", func() {
+			backendSvc, backendSvcPort := s.DefaultHTTPBackend()
+			apisixRoute := fmt.Sprintf(`
+apiVersion: apisix.apache.org/v2beta3
+kind: ApisixRoute
+metadata:
+ name: httpbin-route
+spec:
+  http:
+  - name: rule1
+    match:
+      hosts:
+      - httpbin.com
+      paths: 
+      - /ip
+    backends:
+    - serviceName: %s
+      servicePort: %d
+`, backendSvc, backendSvcPort[0])
+			assert.Nil(ginkgo.GinkgoT(), s.CreateVersionedApisixResource(apisixRoute))
+			assert.Nil(ginkgo.GinkgoT(), s.EnsureNumApisixRoutesCreated(1), "checking number of upstreams")
+			assert.Nil(ginkgo.GinkgoT(), s.EnsureNumListUpstreamNodesNth(1, 1))
+
+			// scale HTTPBIN, so the endpoints controller has the opportunity to update upstream.
+			assert.Nil(ginkgo.GinkgoT(), s.ScaleHTTPBIN(0))
+			assert.Nil(ginkgo.GinkgoT(), s.EnsureNumListUpstreamNodesNth(1, 0))
 			s.NewAPISIXClient().GET("/ip").WithHeader("Host", "httpbin.com").Expect().Status(http.StatusServiceUnavailable)
 		})
 	}
@@ -110,25 +139,21 @@ spec:
     - serviceName: %s
       servicePort: %d
 `, backendSvc, backendSvcPort[0])
-			assert.Nil(ginkgo.GinkgoT(), s.CreateResourceFromString(apisixRoute))
-			time.Sleep(12 * time.Second)
-			ups, err := s.ListApisixUpstreams()
-			assert.Nil(ginkgo.GinkgoT(), err, "listing APISIX upstreams")
-			assert.Len(ginkgo.GinkgoT(), ups, 1)
-			assert.Len(ginkgo.GinkgoT(), ups[0].Nodes, 1)
+			assert.Nil(ginkgo.GinkgoT(), s.CreateVersionedApisixResource(apisixRoute))
+			assert.Nil(ginkgo.GinkgoT(), s.EnsureNumListUpstreamNodesNth(1, 1))
 
 			// port in nodes is still the targetPort, not the service port
+			ups, err := s.ListApisixUpstreams()
+			assert.Nil(ginkgo.GinkgoT(), err, "listing APISIX upstreams")
 			assert.Equal(ginkgo.GinkgoT(), ups[0].Nodes[0].Port, 80)
 
 			// scale HTTPBIN, so the endpoints controller has the opportunity to update upstream.
 			assert.Nil(ginkgo.GinkgoT(), s.ScaleHTTPBIN(3))
-			time.Sleep(30 * time.Second)
-			ups, err = s.ListApisixUpstreams()
-			assert.Nil(ginkgo.GinkgoT(), err, "listing APISIX upstreams")
-			assert.Len(ginkgo.GinkgoT(), ups, 1)
-			assert.Len(ginkgo.GinkgoT(), ups[0].Nodes, 3)
+			assert.Nil(ginkgo.GinkgoT(), s.EnsureNumListUpstreamNodesNth(1, 3))
 
 			// port in nodes is still the targetPort, not the service port
+			ups, err = s.ListApisixUpstreams()
+			assert.Nil(ginkgo.GinkgoT(), err, "listing APISIX upstreams")
 			assert.Equal(ginkgo.GinkgoT(), ups[0].Nodes[0].Port, 80)
 			assert.Equal(ginkgo.GinkgoT(), ups[0].Nodes[1].Port, 80)
 			assert.Equal(ginkgo.GinkgoT(), ups[0].Nodes[2].Port, 80)