You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@apisix.apache.org by we...@apache.org on 2020/12/25 07:14:40 UTC
[apisix-ingress-controller] branch master updated: chore: hone e2e
test scaffold and run cases in CI (#122)
This is an automated email from the ASF dual-hosted git repository.
wenming pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix-ingress-controller.git
The following commit(s) were added to refs/heads/master by this push:
new d339fe5 chore: hone e2e test scaffold and run cases in CI (#122)
d339fe5 is described below
commit d339fe5fc02f6457e64de7fa0fd289b431e4fa5a
Author: Alex Zhang <zc...@gmail.com>
AuthorDate: Fri Dec 25 15:14:32 2020 +0800
chore: hone e2e test scaffold and run cases in CI (#122)
---
.github/workflows/e2e-test-ci.yml | 38 ++++++++
.github/workflows/lint-checker.yml | 1 -
.github/workflows/unit-test-ci.yml | 1 -
.gitignore | 1 +
Makefile | 14 +++
test/e2e/e2e.go | 69 +++++++++++++++
test/e2e/{main_test.go => e2e_test.go} | 3 +-
test/e2e/go.mod | 2 +
test/e2e/ingress/sanity.go | 62 +++++++++++++
test/e2e/main.go | 21 -----
test/e2e/proxy/sanity.go | 28 ------
test/e2e/scaffold/apisix.go | 69 +++++++++++++--
test/e2e/scaffold/crd.go | 150 ++++++++++++++++++++++++++++++++
test/e2e/scaffold/etcd.go | 38 +++++++-
test/e2e/scaffold/ingress.go | 39 ++++++++-
test/e2e/scaffold/scaffold.go | 69 ++++++++++++---
test/e2e/testdata/apisix-gw-config.yaml | 3 +-
17 files changed, 530 insertions(+), 78 deletions(-)
diff --git a/.github/workflows/e2e-test-ci.yml b/.github/workflows/e2e-test-ci.yml
new file mode 100644
index 0000000..ff4f400
--- /dev/null
+++ b/.github/workflows/e2e-test-ci.yml
@@ -0,0 +1,38 @@
+name: e2e-test-ci
+
+on:
+ push:
+ branches:
+ - master
+ pull_request:
+ branches:
+ - master
+ - test/e2e-sanity-case
+jobs:
+ e2e-test:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2
+ - name: Install minikube
+ uses: abstractj/setup-minikube@issue-12
+ - name: Output cluster info
+ run: kubectl cluster-info
+ - name: Add images
+ run: |
+ IMAGE_TAG=dev make build-image
+ minikube cache add apache/apisix:latest -v 7 --alsologtostderr
+ minikube cache add bitnami/etcd:3.4.14-debian-10-r0 -v 7 --alsologtostderr
+ minikube cache add kennethreitz/httpbin -v 7 --alsologtostderr
+ minikube cache add apisix-ingress-controller:dev -v 7 --alsologtostderr
+ - name: Setup Go Env
+ uses: actions/setup-go@v1
+ with:
+ go-version: '1.13'
+ - name: Run e2e test cases
+ working-directory: ./
+ run: |
+ make e2e-test
+ - name: upload coverage profile
+ working-directory: ./test/e2e
+ run: |
+ bash <(curl -s https://codecov.io/bash)
diff --git a/.github/workflows/lint-checker.yml b/.github/workflows/lint-checker.yml
index 31d65ab..c1f59d8 100644
--- a/.github/workflows/lint-checker.yml
+++ b/.github/workflows/lint-checker.yml
@@ -4,7 +4,6 @@ on:
push:
branches:
- master
- - chore/gofmt
pull_request:
branches:
- master
diff --git a/.github/workflows/unit-test-ci.yml b/.github/workflows/unit-test-ci.yml
index 527ed96..62d03f5 100644
--- a/.github/workflows/unit-test-ci.yml
+++ b/.github/workflows/unit-test-ci.yml
@@ -4,7 +4,6 @@ on:
push:
branches:
- master
- - ci/go-unit-test-coverage-report
pull_request:
branches:
- master
diff --git a/.gitignore b/.gitignore
index 0fdeedf..384d362 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,5 +14,6 @@
.idea
.DS_Store
coverage.txt
+test/e2e/coverage.txt
apisix-ingress-controller
.actions/openwhisk-utilities
diff --git a/Makefile b/Makefile
index b1d54e1..2a7b3fc 100644
--- a/Makefile
+++ b/Makefile
@@ -17,9 +17,11 @@
default: help
VERSION ?= 0.0.0
+IMAGE_TAG ?= "latest"
GITSHA ?= $(shell git rev-parse --short=7 HEAD)
OSNAME ?= $(shell uname -s | tr A-Z a-z)
OSARCH ?= $(shell uname -m | tr A-Z a-z)
+PWD ?= $(shell pwd)
ifeq ($(OSARCH), x86_64)
OSARCH = amd64
endif
@@ -36,6 +38,10 @@ build:
-ldflags $(GO_LDFLAGS) \
main.go
+### build-image: Build apisix-ingress-controller image
+build-image:
+ docker build -t apisix-ingress-controller:$(IMAGE_TAG) .
+
### lint: Do static lint check
lint:
golangci-lint run
@@ -48,6 +54,14 @@ gofmt:
unit-test:
go test -cover -coverprofile=coverage.txt ./...
+### e2e-test: Run e2e test cases
+e2e-test:
+ export APISIX_ROUTE_DEF=$(PWD)/samples/deploy/crd/v1beta1/ApisixRoute.yaml && \
+ export APISIX_UPSTREAM_DEF=$(PWD)/samples/deploy/crd/v1beta1/ApisixUpstream.yaml && \
+ export APISIX_SERVICE_DEF=$(PWD)/samples/deploy/crd/v1beta1/ApisixService.yaml && \
+ export APISIX_TLS_DEF=$(PWD)/samples/deploy/crd/v1beta1/ApisixTls.yaml && \
+ cd test/e2e && go test -cover -coverprofile=coverage.txt github.com/api7/ingress-controller/test/e2e
+
### license-check: Do Apache License Header check
license-check:
ifeq ("$(wildcard .actions/openwhisk-utilities/scancode/scanCode.py)", "")
diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go
new file mode 100644
index 0000000..54a408e
--- /dev/null
+++ b/test/e2e/e2e.go
@@ -0,0 +1,69 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package e2e
+
+import (
+ "os"
+
+ "github.com/gruntwork-io/terratest/modules/k8s"
+ "github.com/onsi/ginkgo"
+
+ _ "github.com/api7/ingress-controller/test/e2e/ingress"
+ "github.com/api7/ingress-controller/test/e2e/scaffold"
+)
+
+var (
+ _apisixRouteDef string
+ _apisixUpstreamDef string
+ _apisixServiceDef string
+ _apisixTLSDef string
+)
+
+func runE2E() {
+ if v := os.Getenv("APISIX_ROUTE_DEF"); v != "" {
+ _apisixRouteDef = v
+ } else {
+ panic("no specified ApisixRoute definition file")
+ }
+ if v := os.Getenv("APISIX_UPSTREAM_DEF"); v != "" {
+ _apisixUpstreamDef = v
+ } else {
+ panic("no specified ApisixUpstream resource definition file")
+ }
+ if v := os.Getenv("APISIX_UPSTREAM_DEF"); v != "" {
+ _apisixUpstreamDef = v
+ } else {
+ panic("no specified ApisixUpstream resource definition file")
+ }
+ if v := os.Getenv("APISIX_SERVICE_DEF"); v != "" {
+ _apisixServiceDef = v
+ } else {
+ panic("no specified ApisixService resource definition file")
+ }
+ if v := os.Getenv("APISIX_TLS_DEF"); v != "" {
+ _apisixTLSDef = v
+ } else {
+ panic("no specified ApisixTls resource definition file")
+ }
+
+ kubeconfig := scaffold.GetKubeconfig()
+ opts := &k8s.KubectlOptions{
+ ConfigPath: kubeconfig,
+ }
+ k8s.KubectlApply(ginkgo.GinkgoT(), opts, _apisixRouteDef)
+ k8s.KubectlApply(ginkgo.GinkgoT(), opts, _apisixUpstreamDef)
+ k8s.KubectlApply(ginkgo.GinkgoT(), opts, _apisixServiceDef)
+ k8s.KubectlApply(ginkgo.GinkgoT(), opts, _apisixTLSDef)
+}
diff --git a/test/e2e/main_test.go b/test/e2e/e2e_test.go
similarity index 97%
rename from test/e2e/main_test.go
rename to test/e2e/e2e_test.go
index 01d4be0..989adf7 100644
--- a/test/e2e/main_test.go
+++ b/test/e2e/e2e_test.go
@@ -13,7 +13,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package main
+package e2e
import (
"testing"
@@ -22,5 +22,6 @@ import (
)
func TestRunE2E(t *testing.T) {
+ runE2E()
ginkgo.RunSpecs(t, "ingress-apisix e2e test suites")
}
diff --git a/test/e2e/go.mod b/test/e2e/go.mod
index 01479bc..fb68a99 100644
--- a/test/e2e/go.mod
+++ b/test/e2e/go.mod
@@ -10,6 +10,8 @@ require (
github.com/onsi/ginkgo v1.14.2
github.com/sergi/go-diff v1.1.0 // indirect
github.com/stretchr/testify v1.6.1
+ gopkg.in/yaml.v2 v2.3.0
k8s.io/api v0.19.3
+ k8s.io/apimachinery v0.19.3
k8s.io/client-go v0.19.3
)
diff --git a/test/e2e/ingress/sanity.go b/test/e2e/ingress/sanity.go
new file mode 100644
index 0000000..3b107f9
--- /dev/null
+++ b/test/e2e/ingress/sanity.go
@@ -0,0 +1,62 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package ingress
+
+import (
+ "encoding/json"
+ "net/http"
+
+ "github.com/onsi/ginkgo"
+ "github.com/stretchr/testify/assert"
+
+ "github.com/api7/ingress-controller/test/e2e/scaffold"
+)
+
+type ip struct {
+ IP string `json:"ip"`
+}
+
+var _ = ginkgo.Describe("single-route", func() {
+ s := scaffold.NewDefaultScaffold()
+ ginkgo.It("/ip should return your ip", func() {
+ backendSvc, backendSvcPort := s.DefaultHTTPBackend()
+ s.CreateApisixRoute("httpbin-route", []scaffold.ApisixRouteRule{
+ {
+ Host: "httpbin.com",
+ HTTP: scaffold.ApisixRouteRuleHTTP{
+ Paths: []scaffold.ApisixRouteRuleHTTPPath{
+ {
+ Path: "/ip",
+ Backend: scaffold.ApisixRouteRuleHTTPBackend{
+ ServiceName: backendSvc,
+ ServicePort: backendSvcPort[0],
+ },
+ },
+ },
+ },
+ },
+ })
+ err := s.EnsureNumApisixRoutesCreated(1)
+ assert.Nil(ginkgo.GinkgoT(), err, "checking number of routes")
+ err = s.EnsureNumApisixUpstreamsCreated(1)
+ assert.Nil(ginkgo.GinkgoT(), err, "checking number of upstreams")
+ body := s.NewAPISIXClient().GET("/ip").WithHeader("Host", "httpbin.com").Expect().Status(http.StatusOK).Body().Raw()
+ var placeholder ip
+ err = json.Unmarshal([]byte(body), &placeholder)
+ assert.Nil(ginkgo.GinkgoT(), err, "unmarshalling IP")
+ // It's not our focus point to check the IP address returned by httpbin,
+ // so here skip the IP address validation.
+ })
+})
diff --git a/test/e2e/main.go b/test/e2e/main.go
deleted file mode 100644
index 3e5e488..0000000
--- a/test/e2e/main.go
+++ /dev/null
@@ -1,21 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-package main
-
-import (
- _ "github.com/api7/ingress-controller/test/e2e/proxy"
-)
-
-func main() {}
diff --git a/test/e2e/proxy/sanity.go b/test/e2e/proxy/sanity.go
deleted file mode 100644
index eab3b28..0000000
--- a/test/e2e/proxy/sanity.go
+++ /dev/null
@@ -1,28 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-package proxy
-
-import (
- "github.com/onsi/ginkgo"
-
- "github.com/api7/ingress-controller/test/e2e/scaffold"
-)
-
-var _ = ginkgo.Describe("proxy-sanity", func() {
- _ = scaffold.NewDefaultScaffold()
-
- ginkgo.It("make sure the environment is OK", func() {
- })
-})
diff --git a/test/e2e/scaffold/apisix.go b/test/e2e/scaffold/apisix.go
index 8f0f0e6..194a49e 100644
--- a/test/e2e/scaffold/apisix.go
+++ b/test/e2e/scaffold/apisix.go
@@ -17,10 +17,14 @@ package scaffold
import (
"errors"
"fmt"
+ "net"
+ "strconv"
"strings"
"github.com/gruntwork-io/terratest/modules/k8s"
+ "github.com/onsi/ginkgo"
corev1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
var (
@@ -59,16 +63,16 @@ spec:
containers:
- livenessProbe:
failureThreshold: 3
- initialDelaySeconds: 2
- periodSeconds: 5
+ initialDelaySeconds: 1
+ periodSeconds: 2
successThreshold: 1
tcpSocket:
port: 9080
timeoutSeconds: 2
readinessProbe:
failureThreshold: 3
- initialDelaySeconds: 2
- periodSeconds: 5
+ initialDelaySeconds: 1
+ periodSeconds: 2
successThreshold: 1
tcpSocket:
port: 9080
@@ -120,15 +124,40 @@ func (s *Scaffold) apisixServiceURL() (string, error) {
if len(s.nodes) == 0 {
return "", errors.New("no available node")
}
+ var addr string
+ for _, node := range s.nodes {
+ if len(node.Status.Addresses) > 0 {
+ addr = node.Status.Addresses[0].Address
+ break
+ }
+ }
for _, port := range s.apisixService.Spec.Ports {
if port.Name == "http" {
- // Basically we use minikube, so just use the first node.
- return fmt.Sprintf("http://%s:%d", s.nodes[0], port.NodePort), nil
+ return net.JoinHostPort(addr, strconv.Itoa(int(port.NodePort))), nil
}
}
return "", errors.New("no http port in apisix service")
}
+func (s *Scaffold) apisixAdminServiceURL() (string, error) {
+ if len(s.nodes) == 0 {
+ return "", errors.New("no available node")
+ }
+ var addr string
+ for _, node := range s.nodes {
+ if len(node.Status.Addresses) > 0 {
+ addr = node.Status.Addresses[0].Address
+ break
+ }
+ }
+ for _, port := range s.apisixService.Spec.Ports {
+ if port.Name == "http-admin" {
+ return net.JoinHostPort(addr, strconv.Itoa(int(port.NodePort))), nil
+ }
+ }
+ return "", errors.New("no http-admin port in apisix admin service")
+}
+
func (s *Scaffold) newAPISIX() (*corev1.Service, error) {
defaultData, err := s.renderConfig(s.opts.APISIXDefaultConfigPath)
if err != nil {
@@ -165,3 +194,31 @@ func indent(data string) string {
}
return strings.Join(list, "\n")
}
+
+func (s *Scaffold) waitAllAPISIXPodsAvailable() error {
+ opts := metav1.ListOptions{
+ LabelSelector: "app=apisix-deployment-e2e-test",
+ }
+ condFunc := func() (bool, error) {
+ items, err := k8s.ListPodsE(s.t, s.kubectlOptions, opts)
+ if err != nil {
+ return false, err
+ }
+ if len(items) == 0 {
+ ginkgo.GinkgoT().Log("no apisix pods created")
+ return false, nil
+ }
+ for _, item := range items {
+ for _, cond := range item.Status.Conditions {
+ if cond.Type != corev1.PodReady {
+ continue
+ }
+ if cond.Status != "True" {
+ return false, nil
+ }
+ }
+ }
+ return true, nil
+ }
+ return waitExponentialBackoff(condFunc)
+}
diff --git a/test/e2e/scaffold/crd.go b/test/e2e/scaffold/crd.go
new file mode 100644
index 0000000..39b5014
--- /dev/null
+++ b/test/e2e/scaffold/crd.go
@@ -0,0 +1,150 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package scaffold
+
+import (
+ "encoding/json"
+ "net/http"
+ "net/url"
+ "strconv"
+ "time"
+
+ "github.com/gruntwork-io/terratest/modules/k8s"
+ "github.com/onsi/ginkgo"
+ "github.com/stretchr/testify/assert"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/util/wait"
+)
+
+type counter struct {
+ Count string `json:"count"`
+}
+
+// ApisixRoute is the ApisixRoute CRD definition.
+// We don't use the definition in apisix-ingress-controller,
+// since the k8s dependencies in terratest and
+// apisix-ingress-controller are conflicted.
+type apisixRoute struct {
+ metav1.TypeMeta `json:",inline"`
+ metav1.ObjectMeta `json:"metadata"`
+ Spec apisixRouteSpec `json:"spec"`
+}
+
+type apisixRouteSpec struct {
+ Rules []ApisixRouteRule `json:"rules"`
+}
+
+// ApisixRouteRule defines the route policies of ApisixRoute.
+type ApisixRouteRule struct {
+ Host string `json:"host"`
+ HTTP ApisixRouteRuleHTTP `json:"http"`
+}
+
+// ApisixRouteRuleHTTP defines the HTTP part of route policies.
+type ApisixRouteRuleHTTP struct {
+ Paths []ApisixRouteRuleHTTPPath `json:"paths"`
+}
+
+// ApisixRouteRuleHTTP defines a route in the HTTP part of ApisixRoute.
+type ApisixRouteRuleHTTPPath struct {
+ Path string `json:"path"`
+ Backend ApisixRouteRuleHTTPBackend `json:"backend"`
+}
+
+// ApisixRouteRuleHTTPBackend defines a HTTP backend.
+type ApisixRouteRuleHTTPBackend struct {
+ ServiceName string `json:"serviceName"`
+ ServicePort int32 `json:"servicePort"`
+}
+
+// CreateApisixRoute creates an ApisixRoute object.
+func (s *Scaffold) CreateApisixRoute(name string, rules []ApisixRouteRule) {
+ route := &apisixRoute{
+ TypeMeta: metav1.TypeMeta{
+ Kind: "ApisixRoute",
+ APIVersion: "apisix.apache.org/v1",
+ },
+ ObjectMeta: metav1.ObjectMeta{
+ Name: name,
+ },
+ Spec: apisixRouteSpec{
+ Rules: rules,
+ },
+ }
+ data, err := json.Marshal(route)
+ assert.Nil(s.t, err)
+ k8s.KubectlApplyFromString(s.t, s.kubectlOptions, string(data))
+}
+
+func ensureNumApisixCRDsCreated(url string, desired int) error {
+ condFunc := func() (bool, error) {
+ resp, err := http.Get(url)
+ if err != nil {
+ ginkgo.GinkgoT().Logf("failed to get resources from APISIX: %s", err.Error())
+ return false, nil
+ }
+ if resp.StatusCode != http.StatusOK {
+ ginkgo.GinkgoT().Logf("got status code %d from APISIX", resp.StatusCode)
+ return false, nil
+ }
+ var c counter
+ dec := json.NewDecoder(resp.Body)
+ if err := dec.Decode(&c); err != nil {
+ return false, err
+ }
+ // NOTE count field is a string.
+ count, err := strconv.Atoi(c.Count)
+ if err != nil {
+ return false, err
+ }
+ // 1 for dir.
+ if count != desired+1 {
+ ginkgo.GinkgoT().Logf("mismatched number of items, expected %d but found %d", desired, count-1)
+ return false, nil
+ }
+ return true, nil
+ }
+ return wait.Poll(3*time.Second, 15*time.Second, condFunc)
+}
+
+// EnsureNumApisixRoutesCreated waits until desired number of Routes are created in
+// APISIX cluster.
+func (s *Scaffold) EnsureNumApisixRoutesCreated(desired int) error {
+ host, err := s.apisixAdminServiceURL()
+ if err != nil {
+ return err
+ }
+ u := url.URL{
+ Scheme: "http",
+ Host: host,
+ Path: "/apisix/admin/routes",
+ }
+ return ensureNumApisixCRDsCreated(u.String(), desired)
+}
+
+// EnsureNumApisixUpstreamsCreated waits until desired number of Upstreams are created in
+// APISIX cluster.
+func (s *Scaffold) EnsureNumApisixUpstreamsCreated(desired int) error {
+ host, err := s.apisixAdminServiceURL()
+ if err != nil {
+ return err
+ }
+ u := url.URL{
+ Scheme: "http",
+ Host: host,
+ Path: "/apisix/admin/upstreams",
+ }
+ return ensureNumApisixCRDsCreated(u.String(), desired)
+}
diff --git a/test/e2e/scaffold/etcd.go b/test/e2e/scaffold/etcd.go
index c5272d6..2ad9ff7 100644
--- a/test/e2e/scaffold/etcd.go
+++ b/test/e2e/scaffold/etcd.go
@@ -18,7 +18,9 @@ import (
"fmt"
"github.com/gruntwork-io/terratest/modules/k8s"
+ "github.com/onsi/ginkgo"
corev1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
var (
@@ -49,16 +51,16 @@ spec:
value: "yes"
livenessProbe:
failureThreshold: 3
- initialDelaySeconds: 2
- periodSeconds: 5
+ initialDelaySeconds: 1
+ periodSeconds: 2
successThreshold: 1
tcpSocket:
port: 2379
timeoutSeconds: 2
readinessProbe:
failureThreshold: 3
- initialDelaySeconds: 2
- periodSeconds: 5
+ initialDelaySeconds: 1
+ periodSeconds: 2
successThreshold: 1
tcpSocket:
port: 2379
@@ -103,3 +105,31 @@ func (s *Scaffold) newEtcd() (*corev1.Service, error) {
s.EtcdServiceFQDN = fmt.Sprintf("etcd-service-e2e-test.%s.svc.cluster.local", svc.Namespace)
return svc, nil
}
+
+func (s *Scaffold) waitAllEtcdPodsAvailable() error {
+ opts := metav1.ListOptions{
+ LabelSelector: "app=etcd-deployment-e2e-test",
+ }
+ condFunc := func() (bool, error) {
+ items, err := k8s.ListPodsE(s.t, s.kubectlOptions, opts)
+ if err != nil {
+ return false, err
+ }
+ if len(items) == 0 {
+ ginkgo.GinkgoT().Log("no etcd pods created")
+ return false, nil
+ }
+ for _, item := range items {
+ for _, cond := range item.Status.Conditions {
+ if cond.Type != corev1.PodReady {
+ continue
+ }
+ if cond.Status != "True" {
+ return false, nil
+ }
+ }
+ }
+ return true, nil
+ }
+ return waitExponentialBackoff(condFunc)
+}
diff --git a/test/e2e/scaffold/ingress.go b/test/e2e/scaffold/ingress.go
index a53b523..f0ba3dd 100644
--- a/test/e2e/scaffold/ingress.go
+++ b/test/e2e/scaffold/ingress.go
@@ -18,6 +18,9 @@ import (
"fmt"
"github.com/gruntwork-io/terratest/modules/k8s"
+ "github.com/onsi/ginkgo"
+ corev1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
const (
@@ -60,16 +63,16 @@ spec:
containers:
- livenessProbe:
failureThreshold: 3
- initialDelaySeconds: 2
- periodSeconds: 5
+ initialDelaySeconds: 1
+ periodSeconds: 2
successThreshold: 1
tcpSocket:
port: 8080
timeoutSeconds: 2
readinessProbe:
failureThreshold: 3
- initialDelaySeconds: 2
- periodSeconds: 5
+ initialDelaySeconds: 1
+ periodSeconds: 2
successThreshold: 1
tcpSocket:
port: 8080
@@ -110,3 +113,31 @@ func (s *Scaffold) newIngressAPISIXController() error {
}
return nil
}
+
+func (s *Scaffold) waitAllIngressControllerPodsAvailable() error {
+ opts := metav1.ListOptions{
+ LabelSelector: "app=ingress-apisix-controller-deployment-e2e-test",
+ }
+ condFunc := func() (bool, error) {
+ items, err := k8s.ListPodsE(s.t, s.kubectlOptions, opts)
+ if err != nil {
+ return false, err
+ }
+ if len(items) == 0 {
+ ginkgo.GinkgoT().Log("no ingress-apisix-controller pods created")
+ return false, nil
+ }
+ for _, item := range items {
+ for _, cond := range item.Status.Conditions {
+ if cond.Type != corev1.PodReady {
+ continue
+ }
+ if cond.Status != "True" {
+ return false, nil
+ }
+ }
+ }
+ return true, nil
+ }
+ return waitExponentialBackoff(condFunc)
+}
diff --git a/test/e2e/scaffold/scaffold.go b/test/e2e/scaffold/scaffold.go
index 3363636..1383f79 100644
--- a/test/e2e/scaffold/scaffold.go
+++ b/test/e2e/scaffold/scaffold.go
@@ -18,6 +18,10 @@ import (
"fmt"
"io/ioutil"
"net/http"
+ "net/url"
+ "os"
+ "os/user"
+ "path/filepath"
"strings"
"text/template"
"time"
@@ -29,6 +33,7 @@ import (
"github.com/stretchr/testify/assert"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/util/wait"
)
type Options struct {
@@ -53,6 +58,26 @@ type Scaffold struct {
EtcdServiceFQDN string
}
+// Getkubeconfig returns the kubeconfig file path.
+// Order:
+// env KUBECONFIG;
+// ~/.kube/config;
+// "" (in case in-cluster configuration will be used).
+func GetKubeconfig() string {
+ kubeconfig := os.Getenv("KUBECONFIG")
+ if kubeconfig == "" {
+ u, err := user.Current()
+ if err != nil {
+ panic(err)
+ }
+ kubeconfig = filepath.Join(u.HomeDir, ".kube", "config")
+ if _, err := os.Stat(kubeconfig); err != nil && !os.IsNotExist(err) {
+ kubeconfig = ""
+ }
+ }
+ return kubeconfig
+}
+
// NewScaffold creates an e2e test scaffold.
func NewScaffold(o *Options) *Scaffold {
defer ginkgo.GinkgoRecover()
@@ -68,12 +93,13 @@ func NewScaffold(o *Options) *Scaffold {
return s
}
+// NewDefaultScaffold creates a scaffold with some default options.
func NewDefaultScaffold() *Scaffold {
opts := &Options{
- Name: "sample",
- Kubeconfig: "/Users/alex/.kube/config",
- APISIXConfigPath: "/Users/alex/Workstation/tokers/apisix-ingress-controller/test/e2e/testdata/apisix-gw-config.yaml",
- APISIXDefaultConfigPath: "/Users/alex/Workstation/tokers/apisix-ingress-controller/test/e2e/testdata/apisix-gw-config-default.yaml",
+ Name: "default",
+ Kubeconfig: GetKubeconfig(),
+ APISIXConfigPath: "testdata/apisix-gw-config.yaml",
+ APISIXDefaultConfigPath: "testdata/apisix-gw-config-default.yaml",
}
return NewScaffold(opts)
}
@@ -88,12 +114,16 @@ func (s *Scaffold) DefaultHTTPBackend() (string, []int32) {
return s.httpbinService.Name, ports
}
-// NewHTTPClient creates the default HTTP client.
-func (s *Scaffold) NewHTTPClient() *httpexpect.Expect {
- url, err := s.apisixServiceURL()
+// NewAPISIXClient creates the default HTTP client.
+func (s *Scaffold) NewAPISIXClient() *httpexpect.Expect {
+ host, err := s.apisixServiceURL()
assert.Nil(s.t, err, "getting apisix service url")
+ u := url.URL{
+ Scheme: "http",
+ Host: host,
+ }
return httpexpect.WithConfig(httpexpect.Config{
- BaseURL: url,
+ BaseURL: u.String(),
Client: &http.Client{
Transport: &http.Transport{},
},
@@ -126,18 +156,27 @@ func (s *Scaffold) beforeEach() {
s.etcdService, err = s.newEtcd()
assert.Nil(s.t, err, "initializing etcd")
- k8s.WaitUntilServiceAvailable(s.t, s.kubectlOptions, s.etcdService.Name, 3, 2*time.Second)
+ // We don't use k8s.WaitUntilServiceAvailable since it hacks for Minikube.
+ err = s.waitAllEtcdPodsAvailable()
+ assert.Nil(s.t, err, "waiting for etcd ready")
s.apisixService, err = s.newAPISIX()
assert.Nil(s.t, err, "initializing Apache APISIX")
- k8s.WaitUntilServiceAvailable(s.t, s.kubectlOptions, s.apisixService.Name, 3, 2*time.Second)
+ // We don't use k8s.WaitUntilServiceAvailable since it hacks for Minikube.
+ err = s.waitAllAPISIXPodsAvailable()
+ assert.Nil(s.t, err, "waiting for apisix ready")
s.httpbinService, err = s.newHTTPBIN()
assert.Nil(s.t, err, "initializing httpbin")
+ k8s.WaitUntilServiceAvailable(s.t, s.kubectlOptions, s.httpbinService.Name, 3, 2*time.Second)
+
err = s.newIngressAPISIXController()
assert.Nil(s.t, err, "initializing ingress apisix controller")
+
+ err = s.waitAllIngressControllerPodsAvailable()
+ assert.Nil(s.t, err, "waiting for ingress apisix controller ready")
}
func (s *Scaffold) afterEach() {
@@ -159,3 +198,13 @@ func (s *Scaffold) renderConfig(path string) (string, error) {
}
return buf.String(), nil
}
+
+func waitExponentialBackoff(condFunc func() (bool, error)) error {
+ backoff := wait.Backoff{
+ Duration: 100 * time.Millisecond,
+ Factor: 3,
+ Jitter: 0,
+ Steps: 6,
+ }
+ return wait.ExponentialBackoff(backoff, condFunc)
+}
diff --git a/test/e2e/testdata/apisix-gw-config.yaml b/test/e2e/testdata/apisix-gw-config.yaml
index ad68fcf..8124b17 100644
--- a/test/e2e/testdata/apisix-gw-config.yaml
+++ b/test/e2e/testdata/apisix-gw-config.yaml
@@ -45,9 +45,8 @@ apisix:
port_admin: 9180 # use a separate port
allow_admin:
- - 172.17.0.0/16
- - 10.0.0.0/24
- 127.0.0.0/24
+ - 0.0.0.0/0
delete_uri_tail_slash: false # delete the '/' at the end of the URI
router: