You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@apisix.apache.org by li...@apache.org on 2022/05/12 10:06:38 UTC

[apisix-ingress-controller] branch master updated: e2e-test: add e2e tests and CRDs for ApisixTls v2 (#1014)

This is an automated email from the ASF dual-hosted git repository.

lingsamuel pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix-ingress-controller.git


The following commit(s) were added to refs/heads/master by this push:
     new cd5063f0 e2e-test: add e2e tests and CRDs for ApisixTls v2 (#1014)
cd5063f0 is described below

commit cd5063f04881248338595cb85100ef1009e75a80
Author: Sarasa Kisaragi <li...@gmail.com>
AuthorDate: Thu May 12 18:06:34 2022 +0800

    e2e-test: add e2e tests and CRDs for ApisixTls v2 (#1014)
    
    Signed-off-by: Ling Samuel <li...@gmail.com>
---
 pkg/kube/apisix/apis/config/v2/types.go |   2 +-
 samples/deploy/crd/v1/ApisixRoute.yaml  | 248 +++++++++++++++++++++++++++++++-
 samples/deploy/crd/v1/ApisixTls.yaml    | 155 ++++++++++++++++++++
 test/e2e/scaffold/scaffold.go           |   3 +-
 test/e2e/scaffold/ssl.go                |  10 +-
 test/e2e/suite-features/consumer.go     |   2 +-
 test/e2e/suite-features/scheme.go       |  14 +-
 test/e2e/suite-ingress/ingress.go       |   2 +-
 test/e2e/suite-ingress/ssl.go           | 228 +++++++++++++++--------------
 test/e2e/suite-ingress/upstream_tls.go  |  16 +--
 10 files changed, 538 insertions(+), 142 deletions(-)

diff --git a/pkg/kube/apisix/apis/config/v2/types.go b/pkg/kube/apisix/apis/config/v2/types.go
index 95fa439b..a25f42d3 100644
--- a/pkg/kube/apisix/apis/config/v2/types.go
+++ b/pkg/kube/apisix/apis/config/v2/types.go
@@ -341,7 +341,7 @@ type ApisixConsumerBasicAuth struct {
 // ApisixConsumerBasicAuthValue defines the in-place username and password configuration for basic auth.
 type ApisixConsumerBasicAuthValue struct {
 	Username string `json:"username" yaml:"username"`
-	Password string `json:"password" yaml:"username"`
+	Password string `json:"password" yaml:"password"`
 }
 
 // ApisixConsumerKeyAuth defines the configuration for the key auth.
diff --git a/samples/deploy/crd/v1/ApisixRoute.yaml b/samples/deploy/crd/v1/ApisixRoute.yaml
index 07576d02..d0e8ee92 100644
--- a/samples/deploy/crd/v1/ApisixRoute.yaml
+++ b/samples/deploy/crd/v1/ApisixRoute.yaml
@@ -274,7 +274,7 @@ spec:
                         type: integer
     - name: v2beta3
       served: true
-      storage: true
+      storage: false
       subresources:
         status: { }
       additionalPrinterColumns:
@@ -518,3 +518,249 @@ spec:
                         type: string
                       observedGeneration:
                         type: integer
+    - name: v2
+      served: true
+      storage: true
+      subresources:
+        status: { }
+      additionalPrinterColumns:
+        - jsonPath: .spec.http[].match.hosts
+          name: Hosts
+          type: string
+          priority: 0
+        - jsonPath: .spec.http[].match.paths
+          name: URIs
+          type: string
+          priority: 0
+        - jsonPath: .spec.http[].backends[].serviceName
+          name: Target Service(HTTP)
+          type: string
+          priority: 1
+        - jsonPath: .spec.tcp[].match.ingressPort
+          name: Ingress Server Port(TCP)
+          type: integer
+          priority: 1
+        - jsonPath: .spec.tcp[].match.backend.serviceName
+          name: Target Service(TCP)
+          type: string
+          priority: 1
+        - jsonPath: .metadata.creationTimestamp
+          name: Age
+          type: date
+          priority: 0
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            spec:
+              type: object
+              anyOf:
+                - required: [ "http" ]
+                - required: [ "stream" ]
+              properties:
+                http:
+                  type: array
+                  minItems: 1
+                  items:
+                    type: object
+                    required: [ "name", "match", "backends" ]
+                    properties:
+                      name:
+                        type: string
+                        minLength: 1
+                      priority:
+                        type: integer
+                      timeout:
+                        type: object
+                        properties:
+                          connect:
+                            type: string
+                          send:
+                            type: string
+                          read:
+                            type: string
+                      match:
+                        type: object
+                        required:
+                          - paths
+                        properties:
+                          paths:
+                            type: array
+                            minItems: 1
+                            items:
+                              type: string
+                              pattern: "^/[a-zA-Z0-9\\-._~%!$&'()+,;=:@/]*\\*?$"
+                          hosts:
+                            type: array
+                            minItems: 1
+                            items:
+                              type: string
+                              pattern: "^\\*?[0-9a-zA-Z-._]+$"
+                          methods:
+                            type: array
+                            minItems: 1
+                            items:
+                              type: string
+                              enum: [ "CONNECT", "DELETE", "GET", "HEAD", "OPTIONS", "PATCH", "POST", "PUT", "TRACE" ]
+                          remoteAddrs:
+                            type: array
+                            minItems: 1
+                            items:
+                              type: string
+                          exprs:
+                            type: array
+                            minItems: 1
+                            items:
+                              type: object
+                              properties:
+                                subject:
+                                  type: object
+                                  properties:
+                                    scope:
+                                      type: string
+                                      enum: [ "Cookie", "Header", "Path", "Query" ]
+                                    name:
+                                      type: string
+                                      minLength: 1
+                                  required:
+                                    - scope
+                                op:
+                                  type: string
+                                  enum:
+                                    - Equal
+                                    - NotEqual
+                                    - GreaterThan
+                                    - LessThan
+                                    - In
+                                    - NotIn
+                                    - RegexMatch
+                                    - RegexNotMatch
+                                    - RegexMatchCaseInsensitive
+                                    - RegexNotMatchCaseInsensitive
+                                value:
+                                  type: string
+                                set:
+                                  type: array
+                                  items:
+                                    type: string
+                              oneOf:
+                                - required: [ "subject", "op", "value" ]
+                                - required: [ "subject", "op", "set" ]
+                      websocket:
+                        type: boolean
+                      plugin_config_name:
+                        type: string
+                        minLength: 1
+                      backends:
+                        type: array
+                        minItems: 1
+                        items:
+                          type: object
+                          properties:
+                            serviceName:
+                              type: string
+                              minLength: 1
+                            servicePort:
+                              type: integer
+                              minimum: 1
+                              maximum: 65535
+                            resolveGranularity:
+                              type: string
+                              enum: [ "endpoint", "service" ]
+                            weight:
+                              type: integer
+                              minimum: 0
+                            subset:
+                              type: string
+                        required:
+                          - serviceName
+                          - servicePort
+                      plugins:
+                        type: array
+                        items:
+                          type: object
+                          properties:
+                            name:
+                              type: string
+                              minLength: 1
+                            enable:
+                              type: boolean
+                            config:
+                              type: object
+                              x-kubernetes-preserve-unknown-fields: true # we have to enable it since plugin config
+                        required:
+                          - name
+                          - enable
+                      authentication:
+                        type: object
+                        properties:
+                          enable:
+                            type: boolean
+                          type:
+                            type: string
+                            enum: [ "basicAuth", "keyAuth" ]
+                          keyAuth:
+                            type: object
+                            properties:
+                              header:
+                                type: string
+                        required:
+                          - enable
+                stream:
+                  type: array
+                  minItems: 1
+                  items:
+                    type: object
+                    required: [ "name", "match", "backend", "protocol" ]
+                    properties:
+                      "protocol":
+                        type: string
+                        enum: [ "TCP", "UDP" ]
+                      name:
+                        type: string
+                        minLength: 1
+                      match:
+                        type: object
+                        properties:
+                          ingressPort:
+                            type: integer
+                            minimum: 1
+                            maximum: 65535
+                        required:
+                          - ingressPort
+                      backend:
+                        type: object
+                        properties:
+                          serviceName:
+                            type: string
+                            minLength: 1
+                          servicePort:
+                            type: integer
+                            minimum: 1
+                            maximum: 65535
+                          resolveGranularity:
+                            type: string
+                            enum: [ "endpoint", "service" ]
+                          subset:
+                            type: string
+                        required:
+                          - serviceName
+                          - servicePort
+            status:
+              type: object
+              properties:
+                conditions:
+                  type: array
+                  items:
+                    type: object
+                    properties:
+                      "type":
+                        type: string
+                      reason:
+                        type: string
+                      status:
+                        type: string
+                      message:
+                        type: string
+                      observedGeneration:
+                        type: integer
\ No newline at end of file
diff --git a/samples/deploy/crd/v1/ApisixTls.yaml b/samples/deploy/crd/v1/ApisixTls.yaml
index da50833f..d740905c 100644
--- a/samples/deploy/crd/v1/ApisixTls.yaml
+++ b/samples/deploy/crd/v1/ApisixTls.yaml
@@ -31,6 +31,161 @@ spec:
   preserveUnknownFields: false
   versions:
   - name: v2beta3
+    served: true
+    storage: false
+    subresources:
+      status: {}
+    additionalPrinterColumns:
+    - jsonPath: .spec.hosts
+      name: SNIs
+      type: string
+    - jsonPath: .spec.secret.name
+      name: Secret Name
+      type: string
+    - jsonPath: .spec.secret.namespace
+      name: Secret Namespace
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    schema:
+      openAPIV3Schema:
+        description: ApisixTls defines SSL resource in APISIX.
+        type: object
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: ApisixTlsSpec is the specification of ApisixSSL.
+            type: object
+            required:
+            - hosts
+            - secret
+            properties:
+              client:
+                description: ApisixMutualTlsClientConfig describes the mutual TLS CA
+                  and verify depth
+                type: object
+                properties:
+                  caSecret:
+                    description: ApisixSecret describes the Kubernetes Secret name and
+                      namespace.
+                    type: object
+                    required:
+                    - name
+                    - namespace
+                    properties:
+                      name:
+                        type: string
+                        minLength: 1
+                      namespace:
+                        type: string
+                        minLength: 1
+                  depth:
+                    type: integer
+              hosts:
+                type: array
+                minItems: 1
+                items:
+                  type: string
+                  pattern: ^\*?[0-9a-zA-Z-.]+$
+              secret:
+                description: ApisixSecret describes the Kubernetes Secret name and namespace.
+                type: object
+                required:
+                - name
+                - namespace
+                properties:
+                  name:
+                    type: string
+                    minLength: 1
+                  namespace:
+                    type: string
+                    minLength: 1
+          status:
+            description: ApisixStatus is the status report for Apisix ingress Resources
+            type: object
+            properties:
+              conditions:
+                type: array
+                items:
+                  description: "Condition contains details for one aspect of the current
+                    state of this API Resource. --- This struct is intended for direct
+                    use as an array at the field path .status.conditions.  For example,
+                    type FooStatus struct{     // Represents the observations of a foo's
+                    current state.     // Known .status.conditions.type are: \"Available\",
+                    \"Progressing\", and \"Degraded\"     // +patchMergeKey=type     //
+                    +patchStrategy=merge     // +listType=map     // +listMapKey=type
+                    \    Conditions []metav1.Condition `json:\"conditions,omitempty\"
+                    patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`
+                    \n     // other fields }"
+                  type: object
+                  required:
+                  - lastTransitionTime
+                  - message
+                  - reason
+                  - status
+                  - type
+                  properties:
+                    lastTransitionTime:
+                      description: lastTransitionTime is the last time the condition
+                        transitioned from one status to another. This should be when
+                        the underlying condition changed.  If that is not known, then
+                        using the time when the API field changed is acceptable.
+                      type: string
+                      format: date-time
+                    message:
+                      description: message is a human readable message indicating details
+                        about the transition. This may be an empty string.
+                      type: string
+                      maxLength: 32768
+                    observedGeneration:
+                      description: observedGeneration represents the .metadata.generation
+                        that the condition was set based upon. For instance, if .metadata.generation
+                        is currently 12, but the .status.conditions[x].observedGeneration
+                        is 9, the condition is out of date with respect to the current
+                        state of the instance.
+                      type: integer
+                      format: int64
+                      minimum: 0
+                    reason:
+                      description: reason contains a programmatic identifier indicating
+                        the reason for the condition's last transition. Producers of
+                        specific condition types may define expected values and meanings
+                        for this field, and whether the values are considered a guaranteed
+                        API. The value should be a CamelCase string. This field may
+                        not be empty.
+                      type: string
+                      maxLength: 1024
+                      minLength: 1
+                      pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
+                    status:
+                      description: status of the condition, one of True, False, Unknown.
+                      type: string
+                      enum:
+                      - "True"
+                      - "False"
+                      - Unknown
+                    type:
+                      description: type of condition in CamelCase or in foo.example.com/CamelCase.
+                        --- Many .condition.type values are consistent across resources
+                        like Available, but because arbitrary conditions can be useful
+                        (see .node.status.conditions), the ability to deconflict is
+                        important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)
+                      type: string
+                      maxLength: 316
+                      pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+  - name: v2
     served: true
     storage: true
     subresources:
diff --git a/test/e2e/scaffold/scaffold.go b/test/e2e/scaffold/scaffold.go
index 3dfa86bf..a6fff937 100644
--- a/test/e2e/scaffold/scaffold.go
+++ b/test/e2e/scaffold/scaffold.go
@@ -149,7 +149,8 @@ func NewDefaultV2Scaffold() *Scaffold {
 		APISIXConfigPath:      "testdata/apisix-gw-config.yaml",
 		IngressAPISIXReplicas: 1,
 		HTTPBinServicePort:    80,
-		APISIXRouteVersion:    kube.ApisixRouteV2beta3,
+		APISIXRouteVersion:    kube.ApisixRouteV2,
+		APISIXTlsVersion:      config.ApisixV2,
 		EnableWebhooks:        false,
 		APISIXPublishAddress:  "",
 	}
diff --git a/test/e2e/scaffold/ssl.go b/test/e2e/scaffold/ssl.go
index 3e762e6d..4beb5352 100644
--- a/test/e2e/scaffold/ssl.go
+++ b/test/e2e/scaffold/ssl.go
@@ -50,7 +50,7 @@ data:
   cert: %s
 `
 	_api6tlsTemplate = `
-apiVersion: apisix.apache.org/v2beta3
+apiVersion: %s
 kind: ApisixTls
 metadata:
   name: %s
@@ -62,7 +62,7 @@ spec:
     namespace: %s
 `
 	_api6tlsWithClientCATemplate = `
-apiVersion: apisix.apache.org/v2beta3
+apiVersion: %s
 kind: ApisixTls
 metadata:
   name: %s
@@ -114,7 +114,7 @@ func (s *Scaffold) NewClientCASecret(name, cert, key string) error {
 
 // NewApisixTls new a ApisixTls CRD
 func (s *Scaffold) NewApisixTls(name, host, secretName string) error {
-	tls := fmt.Sprintf(_api6tlsTemplate, name, host, secretName, s.kubectlOptions.Namespace)
+	tls := fmt.Sprintf(_api6tlsTemplate, s.opts.APISIXTlsVersion, name, host, secretName, s.kubectlOptions.Namespace)
 	if err := k8s.KubectlApplyFromStringE(s.t, s.kubectlOptions, tls); err != nil {
 		return err
 	}
@@ -123,7 +123,7 @@ func (s *Scaffold) NewApisixTls(name, host, secretName string) error {
 
 // NewApisixTlsWithClientCA new a ApisixTls CRD
 func (s *Scaffold) NewApisixTlsWithClientCA(name, host, secretName, clientCASecret string) error {
-	tls := fmt.Sprintf(_api6tlsWithClientCATemplate, name, host, secretName, s.kubectlOptions.Namespace, clientCASecret, s.kubectlOptions.Namespace)
+	tls := fmt.Sprintf(_api6tlsWithClientCATemplate, s.opts.APISIXTlsVersion, name, host, secretName, s.kubectlOptions.Namespace, clientCASecret, s.kubectlOptions.Namespace)
 	if err := k8s.KubectlApplyFromStringE(s.t, s.kubectlOptions, tls); err != nil {
 		return err
 	}
@@ -132,7 +132,7 @@ func (s *Scaffold) NewApisixTlsWithClientCA(name, host, secretName, clientCASecr
 
 // DeleteApisixTls remove ApisixTls CRD
 func (s *Scaffold) DeleteApisixTls(name string, host, secretName string) error {
-	tls := fmt.Sprintf(_api6tlsTemplate, name, host, secretName, s.kubectlOptions.Namespace)
+	tls := fmt.Sprintf(_api6tlsTemplate, s.opts.APISIXTlsVersion, name, host, secretName, s.kubectlOptions.Namespace)
 	if err := k8s.KubectlDeleteFromStringE(s.t, s.kubectlOptions, tls); err != nil {
 		return err
 	}
diff --git a/test/e2e/suite-features/consumer.go b/test/e2e/suite-features/consumer.go
index 785981ab..b0cd8fd0 100644
--- a/test/e2e/suite-features/consumer.go
+++ b/test/e2e/suite-features/consumer.go
@@ -26,7 +26,7 @@ import (
 )
 
 var _ = ginkgo.Describe("suite-features: ApisixConsumer", func() {
-	s := scaffold.NewDefaultV2Scaffold()
+	s := scaffold.NewDefaultScaffold()
 
 	ginkgo.It("ApisixRoute with basicAuth consumer", func() {
 		ac := `
diff --git a/test/e2e/suite-features/scheme.go b/test/e2e/suite-features/scheme.go
index d580ab14..2e7391bc 100644
--- a/test/e2e/suite-features/scheme.go
+++ b/test/e2e/suite-features/scheme.go
@@ -16,7 +16,6 @@
 package features
 
 import (
-	"fmt"
 	"io/ioutil"
 	"time"
 
@@ -163,18 +162,7 @@ spec:
        servicePort: 50052
 `))
 
-		assert.NoError(ginkgo.GinkgoT(), s.CreateResourceFromString(fmt.Sprintf(`
-apiVersion: apisix.apache.org/v2beta3
-kind: ApisixTls
-metadata:
-  name: grpc-secret
-spec:
-  hosts:
-    - "e2e.apisix.local"
-  secret:
-    name: grpc-secret
-    namespace: %s
-`, s.Namespace())))
+		assert.NoError(ginkgo.GinkgoT(), s.NewApisixTls("grpc-secret", "e2e.apisix.local", "grpc-secret"))
 
 		time.Sleep(2 * time.Second)
 		ups, err := s.ListApisixUpstreams()
diff --git a/test/e2e/suite-ingress/ingress.go b/test/e2e/suite-ingress/ingress.go
index aa8ad33a..cd59ebcc 100644
--- a/test/e2e/suite-ingress/ingress.go
+++ b/test/e2e/suite-ingress/ingress.go
@@ -30,7 +30,7 @@ import (
 )
 
 var _ = ginkgo.Describe("suite-ingress: support ingress https", func() {
-	s := scaffold.NewDefaultV2Scaffold()
+	s := scaffold.NewDefaultScaffold()
 
 	rootCA := `-----BEGIN CERTIFICATE-----
 MIIF9zCCA9+gAwIBAgIUFKuzAJZgm/fsFS6JDrd+lcpVZr8wDQYJKoZIhvcNAQEL
diff --git a/test/e2e/suite-ingress/ssl.go b/test/e2e/suite-ingress/ssl.go
index e0448869..3b9c4f07 100644
--- a/test/e2e/suite-ingress/ssl.go
+++ b/test/e2e/suite-ingress/ssl.go
@@ -29,10 +29,11 @@ import (
 )
 
 var _ = ginkgo.Describe("suite-ingress: SSL Testing", func() {
-	s := scaffold.NewDefaultScaffold()
-	ginkgo.It("create a SSL from ApisixTls ", func() {
-		secretName := "test-apisix-tls"
-		cert := `-----BEGIN CERTIFICATE-----
+	suites := func(scaffoldFunc func() *scaffold.Scaffold) {
+		s := scaffoldFunc()
+		ginkgo.It("create a SSL from ApisixTls ", func() {
+			secretName := "test-apisix-tls"
+			cert := `-----BEGIN CERTIFICATE-----
 MIIDSjCCAjICCQC/34ZwGz7ZXjANBgkqhkiG9w0BAQsFADBmMQswCQYDVQQGEwJD
 TjEQMA4GA1UECAwHSmlhbmdzdTEPMA0GA1UEBwwGU3V6aG91MQ8wDQYDVQQKDAZ6
 aGlsaXUxEDAOBgNVBAsMB3NlY3Rpb24xETAPBgNVBAMMCHRlc3QuY29tMCAXDTIx
@@ -52,7 +53,7 @@ iYpt+TDAuySnLhAcd3GfE5ml6am2dOsOKpxHU/8clUSaz+21fckRopWo+xL6rSVC
 PpET/mPDrcb4bGsZkW/cu0LrPSUVp12br5TAYaXqYS0Ex+jAVTXML9SeEQuvU3dH
 5Uw2wVHxQXHglsdCYUXXFd3HZffb4rSQH+Mk0CBI
 -----END CERTIFICATE-----`
-		key := `-----BEGIN RSA PRIVATE KEY-----
+			key := `-----BEGIN RSA PRIVATE KEY-----
 MIIEpQIBAAKCAQEA3DEQ5K9PVYicINTHt3arqrsrftrhotyBuGqMxxqGMVO/E2SA
 a/81fC1UCcjYV4Wila0kl8i5fa8HjtVm5UWlrqxeFLOS3E0Wv2QYw46BGZJY4InE
 9zKwYyC2DkBxE6p14JRjmtW/MQPNaOFjJ4bmCuRHsEzmQIGRM0b7oKHjfFwv6l7B
@@ -79,23 +80,23 @@ YOpduE8CgYEAu9k9WOQuRX4f6i5LBIxyaYn6Hw6oJn8e/w+p2+HNBXdyVQNqUHBG
 V5rgnBwhc5LeIFbehKvQOvYSWbwbA1VunMpdYgV6+EBLayumJNqV6jGei4okx2of
 wrw7im4TNSAdwVX4Y1F4svJ2as5SJn5QYGAzXDixNuwzXYrpP9rzA2s=
 -----END RSA PRIVATE KEY-----`
-		// create secret
-		err := s.NewSecret(secretName, cert, key)
-		assert.Nil(ginkgo.GinkgoT(), err, "create secret error")
-		// create ApisixTls resource
-		tlsName := "tls-name"
-		host := "api6.com"
-		err = s.NewApisixTls(tlsName, host, secretName)
-		assert.Nil(ginkgo.GinkgoT(), err, "create tls error")
-		// check ssl in APISIX
-		time.Sleep(10 * time.Second)
-		tls, err := s.ListApisixSsl()
-		assert.Nil(ginkgo.GinkgoT(), err, "list tls error")
-		assert.Len(ginkgo.GinkgoT(), tls, 1, "tls number not expect")
-	})
-	ginkgo.It("update a SSL from ApisixTls ", func() {
-		secretName := "test-apisix-tls"
-		cert := `-----BEGIN CERTIFICATE-----
+			// create secret
+			err := s.NewSecret(secretName, cert, key)
+			assert.Nil(ginkgo.GinkgoT(), err, "create secret error")
+			// create ApisixTls resource
+			tlsName := "tls-name"
+			host := "api6.com"
+			err = s.NewApisixTls(tlsName, host, secretName)
+			assert.Nil(ginkgo.GinkgoT(), err, "create tls error")
+			// check ssl in APISIX
+			time.Sleep(10 * time.Second)
+			tls, err := s.ListApisixSsl()
+			assert.Nil(ginkgo.GinkgoT(), err, "list tls error")
+			assert.Len(ginkgo.GinkgoT(), tls, 1, "tls number not expect")
+		})
+		ginkgo.It("update a SSL from ApisixTls ", func() {
+			secretName := "test-apisix-tls"
+			cert := `-----BEGIN CERTIFICATE-----
 MIIDSDCCAjACCQDf02nwtW2VrzANBgkqhkiG9w0BAQsFADBmMQswCQYDVQQGEwJj
 bjEQMA4GA1UECAwHamlhbmdzdTEPMA0GA1UEBwwGc3V6aG91MQ8wDQYDVQQKDAZ6
 aGlsaXUxEDAOBgNVBAsMB3NlY3Rpb24xETAPBgNVBAMMCGFwaTYuY29tMB4XDTIx
@@ -115,7 +116,7 @@ x4cosJI+W55Kzejiqgm/wzBbr4OpjW4DDz1YBJFXCc1TN9pf2ALkWZ8j3HfMrn2y
 HvOefA8g628WpNtPZodWe/zC8hanCzRMp37JPbh85+RwlGhi7gIkhvjf78EiAZBy
 eHg1iDgdVUzlXn+LNPCAbjxCaTqn6zmIb+GkhA==
 -----END CERTIFICATE-----`
-		key := `-----BEGIN RSA PRIVATE KEY-----
+			key := `-----BEGIN RSA PRIVATE KEY-----
 MIIEpAIBAAKCAQEAxAXOahtVW6LE5rRwQi1kvO3eAqJ9RLVv6w3l3TLtIYp4CmOG
 BKXRuXXEVcQbIW20I2UVTT5p1sodckuw0Vs7+XzGL7bXyKFhnWYzvTzxPwgkzh3i
 OnHQ4oVlJT23Cj5XE0tPAWxq3YaIHf0oyI2ks5KGEh0r0EpPz6/pSqoxuRigIJE5
@@ -142,32 +143,32 @@ RU+QPRECgYB6XW24EI5+w3STbpnc6VoTS+sy9I9abTJPYo9LpCJwfMYc9Tg9Cx2K
 29PnmSrLFpU2fvE0ijpyHRr7gGmINTxbrmTmfMBI01m+GpPuvDcBQ2tsFJ+A3DzN
 9xJulR2NZUZdDIIIqx983ANE6S4Zb8rAbsoHQdqpjUrcVxI2OJBp3Q==
 -----END RSA PRIVATE KEY-----`
-		// create secret
-		err := s.NewSecret(secretName, cert, key)
-		assert.Nil(ginkgo.GinkgoT(), err, "create secret error")
-		// create ApisixTls resource
-		tlsName := "tls-name"
-		host := "api6.com"
-		err = s.NewApisixTls(tlsName, host, secretName)
-		assert.Nil(ginkgo.GinkgoT(), err, "create tls error")
-		// update ApisixTls resource
-		host = "api7.com"
-		err = s.NewApisixTls(tlsName, host, secretName)
-		assert.Nil(ginkgo.GinkgoT(), err, "update tls error")
+			// create secret
+			err := s.NewSecret(secretName, cert, key)
+			assert.Nil(ginkgo.GinkgoT(), err, "create secret error")
+			// create ApisixTls resource
+			tlsName := "tls-name"
+			host := "api6.com"
+			err = s.NewApisixTls(tlsName, host, secretName)
+			assert.Nil(ginkgo.GinkgoT(), err, "create tls error")
+			// update ApisixTls resource
+			host = "api7.com"
+			err = s.NewApisixTls(tlsName, host, secretName)
+			assert.Nil(ginkgo.GinkgoT(), err, "update tls error")
 
-		// check ssl in APISIX
-		time.Sleep(10 * time.Second)
-		tls, err := s.ListApisixSsl()
-		assert.Nil(ginkgo.GinkgoT(), err, "list tls error")
-		assert.Len(ginkgo.GinkgoT(), tls, 1, "tls number not expect")
-		assert.Equal(ginkgo.GinkgoT(), tls[0].Snis[0], host, "tls host is error")
-		assert.Equal(ginkgo.GinkgoT(), tls[0].Labels, map[string]string{
-			"managed-by": "apisix-ingress-controller",
+			// check ssl in APISIX
+			time.Sleep(10 * time.Second)
+			tls, err := s.ListApisixSsl()
+			assert.Nil(ginkgo.GinkgoT(), err, "list tls error")
+			assert.Len(ginkgo.GinkgoT(), tls, 1, "tls number not expect")
+			assert.Equal(ginkgo.GinkgoT(), tls[0].Snis[0], host, "tls host is error")
+			assert.Equal(ginkgo.GinkgoT(), tls[0].Labels, map[string]string{
+				"managed-by": "apisix-ingress-controller",
+			})
 		})
-	})
-	ginkgo.It("delete a SSL from ApisixTls ", func() {
-		secretName := "test-apisix-tls"
-		cert := `-----BEGIN CERTIFICATE-----
+		ginkgo.It("delete a SSL from ApisixTls ", func() {
+			secretName := "test-apisix-tls"
+			cert := `-----BEGIN CERTIFICATE-----
 MIIDSDCCAjACCQDf02nwtW2VrzANBgkqhkiG9w0BAQsFADBmMQswCQYDVQQGEwJj
 bjEQMA4GA1UECAwHamlhbmdzdTEPMA0GA1UEBwwGc3V6aG91MQ8wDQYDVQQKDAZ6
 aGlsaXUxEDAOBgNVBAsMB3NlY3Rpb24xETAPBgNVBAMMCGFwaTYuY29tMB4XDTIx
@@ -187,7 +188,7 @@ x4cosJI+W55Kzejiqgm/wzBbr4OpjW4DDz1YBJFXCc1TN9pf2ALkWZ8j3HfMrn2y
 HvOefA8g628WpNtPZodWe/zC8hanCzRMp37JPbh85+RwlGhi7gIkhvjf78EiAZBy
 eHg1iDgdVUzlXn+LNPCAbjxCaTqn6zmIb+GkhA==
 -----END CERTIFICATE-----`
-		key := `-----BEGIN RSA PRIVATE KEY-----
+			key := `-----BEGIN RSA PRIVATE KEY-----
 MIIEpAIBAAKCAQEAxAXOahtVW6LE5rRwQi1kvO3eAqJ9RLVv6w3l3TLtIYp4CmOG
 BKXRuXXEVcQbIW20I2UVTT5p1sodckuw0Vs7+XzGL7bXyKFhnWYzvTzxPwgkzh3i
 OnHQ4oVlJT23Cj5XE0tPAWxq3YaIHf0oyI2ks5KGEh0r0EpPz6/pSqoxuRigIJE5
@@ -214,30 +215,38 @@ RU+QPRECgYB6XW24EI5+w3STbpnc6VoTS+sy9I9abTJPYo9LpCJwfMYc9Tg9Cx2K
 29PnmSrLFpU2fvE0ijpyHRr7gGmINTxbrmTmfMBI01m+GpPuvDcBQ2tsFJ+A3DzN
 9xJulR2NZUZdDIIIqx983ANE6S4Zb8rAbsoHQdqpjUrcVxI2OJBp3Q==
 -----END RSA PRIVATE KEY-----`
-		// create secret
-		err := s.NewSecret(secretName, cert, key)
-		assert.Nil(ginkgo.GinkgoT(), err, "create secret error")
-		// create ApisixTls resource
-		tlsName := "tls-name"
-		host := "api6.com"
-		err = s.NewApisixTls(tlsName, host, secretName)
-		assert.Nil(ginkgo.GinkgoT(), err, "create tls error")
+			// create secret
+			err := s.NewSecret(secretName, cert, key)
+			assert.Nil(ginkgo.GinkgoT(), err, "create secret error")
+			// create ApisixTls resource
+			tlsName := "tls-name"
+			host := "api6.com"
+			err = s.NewApisixTls(tlsName, host, secretName)
+			assert.Nil(ginkgo.GinkgoT(), err, "create tls error")
 
-		// check ssl in APISIX
-		time.Sleep(10 * time.Second)
-		tls, err := s.ListApisixSsl()
-		assert.Nil(ginkgo.GinkgoT(), err, "list tls error")
-		assert.Len(ginkgo.GinkgoT(), tls, 1, "tls number not expect")
-		assert.Equal(ginkgo.GinkgoT(), tls[0].Snis[0], host, "tls host is error")
+			// check ssl in APISIX
+			time.Sleep(10 * time.Second)
+			tls, err := s.ListApisixSsl()
+			assert.Nil(ginkgo.GinkgoT(), err, "list tls error")
+			assert.Len(ginkgo.GinkgoT(), tls, 1, "tls number not expect")
+			assert.Equal(ginkgo.GinkgoT(), tls[0].Snis[0], host, "tls host is error")
+
+			// delete ApisixTls
+			err = s.DeleteApisixTls(tlsName, host, secretName)
+			assert.Nil(ginkgo.GinkgoT(), err, "delete tls error")
+			// check ssl in APISIX
+			time.Sleep(10 * time.Second)
+			tls, err = s.ListApisixSsl()
+			assert.Nil(ginkgo.GinkgoT(), err, "list tls error")
+			assert.Len(ginkgo.GinkgoT(), tls, 0, "tls number not expect")
+		})
+	}
 
-		// delete ApisixTls
-		err = s.DeleteApisixTls(tlsName, host, secretName)
-		assert.Nil(ginkgo.GinkgoT(), err, "delete tls error")
-		// check ssl in APISIX
-		time.Sleep(10 * time.Second)
-		tls, err = s.ListApisixSsl()
-		assert.Nil(ginkgo.GinkgoT(), err, "list tls error")
-		assert.Len(ginkgo.GinkgoT(), tls, 0, "tls number not expect")
+	ginkgo.Describe("suite-ingress: scaffold v2beta3", func() {
+		suites(scaffold.NewDefaultScaffold)
+	})
+	ginkgo.Describe("suite-ingress: scaffold v2", func() {
+		suites(scaffold.NewDefaultV2Scaffold)
 	})
 })
 
@@ -462,28 +471,29 @@ RAmKucXUNJSR8wYGSg5ymvsnChTaYHLL1gmIdQli2y8XxqUaYC1tXrEt4g5z4a/O
 -----END RSA PRIVATE KEY-----
 `
 
-	s := scaffold.NewDefaultV2Scaffold()
-	ginkgo.It("create a SSL with client CA", func() {
-		// create secrets
-		err := s.NewSecret(serverCertSecret, serverCert, serverKey)
-		assert.Nil(ginkgo.GinkgoT(), err, "create server cert secret error")
-		err = s.NewClientCASecret(clientCASecret, rootCA, "")
-		assert.Nil(ginkgo.GinkgoT(), err, "create client CA cert secret error")
+	suites := func(scaffoldFunc func() *scaffold.Scaffold) {
+		s := scaffoldFunc()
+		ginkgo.It("create a SSL with client CA", func() {
+			// create secrets
+			err := s.NewSecret(serverCertSecret, serverCert, serverKey)
+			assert.Nil(ginkgo.GinkgoT(), err, "create server cert secret error")
+			err = s.NewClientCASecret(clientCASecret, rootCA, "")
+			assert.Nil(ginkgo.GinkgoT(), err, "create client CA cert secret error")
 
-		// create ApisixTls resource
-		tlsName := "tls-with-client-ca"
-		host := "mtls.httpbin.local"
-		err = s.NewApisixTlsWithClientCA(tlsName, host, serverCertSecret, clientCASecret)
-		assert.Nil(ginkgo.GinkgoT(), err, "create ApisixTls with client CA error")
-		// check ssl in APISIX
-		time.Sleep(10 * time.Second)
-		apisixSsls, err := s.ListApisixSsl()
-		assert.Nil(ginkgo.GinkgoT(), err, "list ssl error")
-		assert.Len(ginkgo.GinkgoT(), apisixSsls, 1, "ssl number not expect")
+			// create ApisixTls resource
+			tlsName := "tls-with-client-ca"
+			host := "mtls.httpbin.local"
+			err = s.NewApisixTlsWithClientCA(tlsName, host, serverCertSecret, clientCASecret)
+			assert.Nil(ginkgo.GinkgoT(), err, "create ApisixTls with client CA error")
+			// check ssl in APISIX
+			time.Sleep(10 * time.Second)
+			apisixSsls, err := s.ListApisixSsl()
+			assert.Nil(ginkgo.GinkgoT(), err, "list ssl error")
+			assert.Len(ginkgo.GinkgoT(), apisixSsls, 1, "ssl number not expect")
 
-		// create route
-		backendSvc, backendSvcPort := s.DefaultHTTPBackend()
-		apisixRoute := fmt.Sprintf(`
+			// create route
+			backendSvc, backendSvcPort := s.DefaultHTTPBackend()
+			apisixRoute := fmt.Sprintf(`
 apiVersion: apisix.apache.org/v2beta3
 kind: ApisixRoute
 metadata:
@@ -500,25 +510,33 @@ spec:
     - serviceName: %s
       servicePort: %d
 `, backendSvc, backendSvcPort[0])
-		assert.Nil(ginkgo.GinkgoT(), s.CreateResourceFromString(apisixRoute))
-		time.Sleep(10 * time.Second)
+			assert.Nil(ginkgo.GinkgoT(), s.CreateResourceFromString(apisixRoute))
+			time.Sleep(10 * time.Second)
+
+			apisixRoutes, err := s.ListApisixRoutes()
+			assert.Nil(ginkgo.GinkgoT(), err, "list routes error")
+			assert.Len(ginkgo.GinkgoT(), apisixRoutes, 1, "route number not expect")
 
-		apisixRoutes, err := s.ListApisixRoutes()
-		assert.Nil(ginkgo.GinkgoT(), err, "list routes error")
-		assert.Len(ginkgo.GinkgoT(), apisixRoutes, 1, "route number not expect")
+			// Without Client Cert
+			s.NewAPISIXHttpsClient(host).GET("/ip").WithHeader("Host", host).Expect().Status(http.StatusBadRequest).Body().Raw()
 
-		// Without Client Cert
-		s.NewAPISIXHttpsClient(host).GET("/ip").WithHeader("Host", host).Expect().Status(http.StatusBadRequest).Body().Raw()
+			// With client cert
+			caCertPool := x509.NewCertPool()
+			ok := caCertPool.AppendCertsFromPEM([]byte(rootCA))
+			assert.True(ginkgo.GinkgoT(), ok, "Append cert to CA pool")
 
-		// With client cert
-		caCertPool := x509.NewCertPool()
-		ok := caCertPool.AppendCertsFromPEM([]byte(rootCA))
-		assert.True(ginkgo.GinkgoT(), ok, "Append cert to CA pool")
+			cert, err := tls.X509KeyPair([]byte(clientCert), []byte(clientKey))
+			assert.Nil(ginkgo.GinkgoT(), err, "generate cert")
 
-		cert, err := tls.X509KeyPair([]byte(clientCert), []byte(clientKey))
-		assert.Nil(ginkgo.GinkgoT(), err, "generate cert")
+			s.NewAPISIXHttpsClientWithCertificates(host, true, caCertPool, []tls.Certificate{cert}).
+				GET("/ip").WithHeader("Host", host).Expect().Status(http.StatusOK)
+		})
+	}
 
-		s.NewAPISIXHttpsClientWithCertificates(host, true, caCertPool, []tls.Certificate{cert}).
-			GET("/ip").WithHeader("Host", host).Expect().Status(http.StatusOK)
+	ginkgo.Describe("suite-ingress: scaffold v2beta3", func() {
+		suites(scaffold.NewDefaultScaffold)
+	})
+	ginkgo.Describe("suite-ingress: scaffold v2", func() {
+		suites(scaffold.NewDefaultV2Scaffold)
 	})
 })
diff --git a/test/e2e/suite-ingress/upstream_tls.go b/test/e2e/suite-ingress/upstream_tls.go
index b0fa97b2..5c57c78d 100644
--- a/test/e2e/suite-ingress/upstream_tls.go
+++ b/test/e2e/suite-ingress/upstream_tls.go
@@ -16,7 +16,6 @@
 package ingress
 
 import (
-	"fmt"
 	"io/ioutil"
 	"net/http"
 	"time"
@@ -39,7 +38,7 @@ var _ = ginkgo.Describe("suite-ingress: ApisixUpstreams mTLS test", func() {
 	assert.NoError(ginkgo.GinkgoT(), err, "read client key")
 	clientKey := string(f)
 
-	s := scaffold.NewDefaultV2Scaffold()
+	s := scaffold.NewDefaultScaffold()
 	ginkgo.It("create ApisixUpstreams with http mTLS", func() {
 		// create client secret
 		err := s.NewSecret(clientSecret, clientCert, clientKey)
@@ -117,18 +116,7 @@ spec:
        servicePort: 50053
 `))
 
-		assert.NoError(ginkgo.GinkgoT(), s.CreateResourceFromString(fmt.Sprintf(`
-apiVersion: apisix.apache.org/v2beta3
-kind: ApisixTls
-metadata:
-  name: grpc-secret
-spec:
-  hosts:
-    - "e2e.apisix.local"
-  secret:
-    name: grpc-secret
-    namespace: %s
-`, s.Namespace())))
+		assert.NoError(ginkgo.GinkgoT(), s.NewApisixTls("grpc-secret", "e2e.apisix.local", "grpc-secret"))
 
 		assert.NoError(ginkgo.GinkgoT(), err, "create ApisixRoute for backend that require mTLS")
 		time.Sleep(10 * time.Second)