You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@apisix.apache.org by al...@apache.org on 2025/12/08 16:29:06 UTC
(apisix-ingress-controller) branch master updated: feat: support benchmark test (#2663)
This is an automated email from the ASF dual-hosted git repository.
alinsran pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix-ingress-controller.git
The following commit(s) were added to refs/heads/master by this push:
new 0c0c5f95 feat: support benchmark test (#2663)
0c0c5f95 is described below
commit 0c0c5f95f0274c673afe3cf4212253d760c59b15
Author: AlinsRan <al...@apache.org>
AuthorDate: Tue Dec 9 00:29:00 2025 +0800
feat: support benchmark test (#2663)
---
.github/workflows/benchmark-test.yml | 113 ++++++
Makefile | 11 +-
go.mod | 11 +
go.sum | 22 ++
internal/adc/cache/store.go | 2 +-
internal/adc/client/client.go | 1 +
internal/controller/status/updater.go | 7 +-
internal/controller/utils.go | 2 +-
internal/manager/readiness/manager.go | 11 +-
internal/provider/apisix/provider.go | 9 +-
test/benchmark/benchmark_test.go | 430 +++++++++++++++++++++
test/benchmark/suite_test.go | 41 ++
test/benchmark/utis.go | 76 ++++
.../e2e/framework/manifests/apisix-standalone.yaml | 153 --------
test/e2e/framework/manifests/apisix.yaml | 20 +
test/e2e/scaffold/scaffold.go | 106 +++++
16 files changed, 848 insertions(+), 167 deletions(-)
diff --git a/.github/workflows/benchmark-test.yml b/.github/workflows/benchmark-test.yml
new file mode 100644
index 00000000..565fb1d0
--- /dev/null
+++ b/.github/workflows/benchmark-test.yml
@@ -0,0 +1,113 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Benchmark Test
+
+on:
+ push:
+ branches:
+ - master
+ pull_request:
+ branches:
+ - master
+ workflow_dispatch:
+ inputs:
+ routes:
+ description: "Number of routes for benchmark test"
+ required: false
+ default: "2000"
+ consumers:
+ description: "Number of consumers for benchmark test"
+ required: false
+ default: "2000"
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+
+env:
+ ADC_VERSION: dev
+ BENCHMARK_ROUTES: ${{ github.event.inputs.routes }}
+ BENCHMARK_CONSUMERS: ${{ github.event.inputs.consumers }}
+
+jobs:
+ e2e-test:
+ strategy:
+ matrix:
+ provider_type:
+ - apisix-standalone
+ - apisix
+ fail-fast: false
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v4
+ with:
+ submodules: recursive
+
+ - name: Setup Go Env
+ uses: actions/setup-go@v4
+ with:
+ go-version: "1.24"
+
+ - name: Install kind
+ run: |
+ go install sigs.k8s.io/kind@v0.23.0
+
+ - name: Install ginkgo
+ run: |
+ make install-ginkgo
+
+ - name: Build images
+ env:
+ TAG: dev
+ ARCH: amd64
+ ENABLE_PROXY: "false"
+ BASE_IMAGE_TAG: "debug"
+ run: |
+ echo "building images..."
+ make build-image
+
+ - name: Launch Kind Cluster
+ run: |
+ make kind-up
+
+ - name: Loading Docker Image to Kind Cluster
+ run: |
+ make kind-load-images
+
+ - name: Install Gateway API And CRDs
+ run: |
+ make install
+
+ - name: Extract adc binary
+ if: ${{ env.ADC_VERSION == 'dev' }}
+ run: |
+ docker create --name adc-temp ghcr.io/api7/adc:dev
+ docker cp adc-temp:main.js adc.js
+ docker rm adc-temp
+ node $(pwd)/adc.js -v
+ echo "ADC_BIN=node $(pwd)/adc.js" >> $GITHUB_ENV
+
+ - name: Run Benchmark Test
+ shell: bash
+ env:
+ PROVIDER_TYPE: ${{ matrix.provider_type }}
+ TEST_LABEL: ${{ matrix.cases_subset }}
+ TEST_ENV: CI
+ run: |
+ make benchmark-test
diff --git a/Makefile b/Makefile
index 69b94c71..037621e5 100644
--- a/Makefile
+++ b/Makefile
@@ -27,7 +27,7 @@ IMG ?= apache/apisix-ingress-controller:$(IMAGE_TAG)
ENVTEST_K8S_VERSION = 1.30.0
KIND_NAME ?= apisix-ingress-cluster
-ADC_VERSION ?= 0.21.2
+ADC_VERSION ?= 0.23.1
DIR := $(shell pwd)
@@ -57,6 +57,9 @@ CONFORMANCE_TEST_REPORT_OUTPUT ?= $(DIR)/apisix-ingress-controller-conformance-r
## https://github.com/kubernetes-sigs/gateway-api/blob/v1.3.0/conformance/utils/suite/profiles.go
CONFORMANCE_PROFILES ?= GATEWAY-HTTP,GATEWAY-GRPC,GATEWAY-TLS
+TEST_EXCLUDES ?= /e2e /conformance /benchmark
+TEST_PACKAGES = $(shell go list ./... $(foreach p,$(TEST_EXCLUDES),| grep -v $(p)))
+
# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
ifeq (,$(shell go env GOBIN))
GOBIN=$(shell go env GOPATH)/bin
@@ -128,7 +131,7 @@ vet: ## Run go vet against code.
.PHONY: test
test: manifests generate fmt vet envtest ## Run tests.
- KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e | grep -v /conformance) -coverprofile cover.out
+ KUBEBUILDER_ASSETS="$$( $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path )" go test $(TEST_PACKAGES) -coverprofile cover.out
.PHONY: kind-e2e-test
kind-e2e-test: kind-up build-image kind-load-images e2e-test
@@ -153,6 +156,10 @@ conformance-test:
--conformance-profiles=$(CONFORMANCE_PROFILES) \
--report-output=$(CONFORMANCE_TEST_REPORT_OUTPUT)
+.PHONY: benchmark-test
+benchmark-test:
+ go test -v ./test/benchmark -test.timeout=$(TEST_TIMEOUT) -v -ginkgo.v
+
.PHONY: lint
lint: sort-import golangci-lint ## Run golangci-lint linter
$(GOLANGCI_LINT) run
diff --git a/go.mod b/go.mod
index 60da4110..757f8e28 100644
--- a/go.mod
+++ b/go.mod
@@ -18,8 +18,10 @@ require (
github.com/hashicorp/go-memdb v1.3.4
github.com/imdario/mergo v0.3.16
github.com/incubator4/go-resty-expr v0.1.1
+ github.com/olekukonko/tablewriter v1.1.1
github.com/onsi/ginkgo/v2 v2.22.0
github.com/onsi/gomega v1.36.1
+ github.com/panjf2000/ants/v2 v2.11.3
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.19.1
github.com/samber/lo v1.47.0
@@ -33,6 +35,7 @@ require (
k8s.io/apiextensions-apiserver v0.32.3
k8s.io/apimachinery v0.32.3
k8s.io/client-go v0.32.3
+ k8s.io/code-generator v0.32.3
k8s.io/kubectl v0.30.3
k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738
sigs.k8s.io/controller-runtime v0.20.4
@@ -91,6 +94,9 @@ require (
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
+ github.com/clipperhouse/displaywidth v0.3.1 // indirect
+ github.com/clipperhouse/stringish v0.1.1 // indirect
+ github.com/clipperhouse/uax29/v2 v2.2.0 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/emicklei/go-restful/v3 v3.12.0 // indirect
@@ -139,6 +145,7 @@ require (
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
+ github.com/mattn/go-runewidth v0.0.19 // indirect
github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326 // indirect
github.com/miekg/dns v1.1.65 // indirect
github.com/mitchellh/copystructure v1.2.0 // indirect
@@ -150,6 +157,9 @@ require (
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
+ github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 // indirect
+ github.com/olekukonko/errors v1.1.0 // indirect
+ github.com/olekukonko/ll v0.1.2 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/pquerna/otp v1.4.0 // indirect
github.com/prometheus/client_model v0.6.1 // indirect
@@ -203,6 +213,7 @@ require (
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
k8s.io/apiserver v0.32.3 // indirect
k8s.io/component-base v0.32.3 // indirect
+ k8s.io/gengo/v2 v2.0.0-20240911193312-2b36238f13e9 // indirect
k8s.io/klog/v2 v2.130.1 // indirect
k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect
moul.io/http2curl/v2 v2.3.0 // indirect
diff --git a/go.sum b/go.sum
index 8904243c..eddf72b5 100644
--- a/go.sum
+++ b/go.sum
@@ -111,6 +111,12 @@ github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK3
github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/clipperhouse/displaywidth v0.3.1 h1:k07iN9gD32177o1y4O1jQMzbLdCrsGJh+blirVYybsk=
+github.com/clipperhouse/displaywidth v0.3.1/go.mod h1:tgLJKKyaDOCadywag3agw4snxS5kYEuYR6Y9+qWDDYM=
+github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs=
+github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA=
+github.com/clipperhouse/uax29/v2 v2.2.0 h1:ChwIKnQN3kcZteTXMgb1wztSgaU+ZemkgWdohwgs8tY=
+github.com/clipperhouse/uax29/v2 v2.2.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0=
@@ -285,6 +291,8 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw=
+github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
github.com/mattn/go-zglob v0.0.1/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo=
github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326 h1:ofNAzWCcyTALn2Zv40+8XitdzCgXY6e9qvXwN9W0YXg=
github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo=
@@ -311,12 +319,22 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
+github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 h1:zrbMGy9YXpIeTnGj4EljqMiZsIcE09mmF8XsD5AYOJc=
+github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6/go.mod h1:rEKTHC9roVVicUIfZK7DYrdIoM0EOr8mK1Hj5s3JjH0=
+github.com/olekukonko/errors v1.1.0 h1:RNuGIh15QdDenh+hNvKrJkmxxjV4hcS50Db478Ou5sM=
+github.com/olekukonko/errors v1.1.0/go.mod h1:ppzxA5jBKcO1vIpCXQ9ZqgDh8iwODz6OXIGKU8r5m4Y=
+github.com/olekukonko/ll v0.1.2 h1:lkg/k/9mlsy0SxO5aC+WEpbdT5K83ddnNhAepz7TQc0=
+github.com/olekukonko/ll v0.1.2/go.mod h1:b52bVQRRPObe+yyBl0TxNfhesL0nedD4Cht0/zx55Ew=
+github.com/olekukonko/tablewriter v1.1.1 h1:b3reP6GCfrHwmKkYwNRFh2rxidGHcT6cgxj/sHiDDx0=
+github.com/olekukonko/tablewriter v1.1.1/go.mod h1:De/bIcTF+gpBDB3Alv3fEsZA+9unTsSzAg/ZGADCtn4=
github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo=
github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg=
github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo=
github.com/onsi/gomega v1.36.1 h1:bJDPBO7ibjxcbHMgSCoo4Yj18UWbKDlLwX1x9sybDcw=
github.com/onsi/gomega v1.36.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog=
+github.com/panjf2000/ants/v2 v2.11.3 h1:AfI0ngBoXJmYOpDh9m516vjqoUu2sLrIVgppI9TZVpg=
+github.com/panjf2000/ants/v2 v2.11.3/go.mod h1:8u92CYMUc6gyvTIw8Ru7Mt7+/ESnJahz5EVtqfrilek=
github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo=
github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7/go.mod h1:zO8QMzTeZd5cpnIkz/Gn6iK0jDfGicM1nynOkkPIl28=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
@@ -579,8 +597,12 @@ k8s.io/apiserver v0.32.3 h1:kOw2KBuHOA+wetX1MkmrxgBr648ksz653j26ESuWNY8=
k8s.io/apiserver v0.32.3/go.mod h1:q1x9B8E/WzShF49wh3ADOh6muSfpmFL0I2t+TG0Zdgc=
k8s.io/client-go v0.32.3 h1:RKPVltzopkSgHS7aS98QdscAgtgah/+zmpAogooIqVU=
k8s.io/client-go v0.32.3/go.mod h1:3v0+3k4IcT9bXTc4V2rt+d2ZPPG700Xy6Oi0Gdl2PaY=
+k8s.io/code-generator v0.32.3 h1:31p2TVzC9+hVdSkAFruAk3JY+iSfzrJ83Qij1yZutyw=
+k8s.io/code-generator v0.32.3/go.mod h1:+mbiYID5NLsBuqxjQTygKM/DAdKpAjvBzrJd64NU1G8=
k8s.io/component-base v0.32.3 h1:98WJvvMs3QZ2LYHBzvltFSeJjEx7t5+8s71P7M74u8k=
k8s.io/component-base v0.32.3/go.mod h1:LWi9cR+yPAv7cu2X9rZanTiFKB2kHA+JjmhkKjCZRpI=
+k8s.io/gengo/v2 v2.0.0-20240911193312-2b36238f13e9 h1:si3PfKm8dDYxgfbeA6orqrtLkvvIeH8UqffFJDl0bz4=
+k8s.io/gengo/v2 v2.0.0-20240911193312-2b36238f13e9/go.mod h1:EJykeLsmFC60UQbYJezXkEsG2FLrt0GPNkU5iK5GWxU=
k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f h1:GA7//TjRY9yWGy1poLzYYJJ4JRdzg3+O6e8I+e+8T5Y=
diff --git a/internal/adc/cache/store.go b/internal/adc/cache/store.go
index 6c5a1f23..d41188ed 100644
--- a/internal/adc/cache/store.go
+++ b/internal/adc/cache/store.go
@@ -40,7 +40,7 @@ func NewStore(log logr.Logger) *Store {
return &Store{
cacheMap: make(map[string]Cache),
pluginMetadataMap: make(map[string]adctypes.PluginMetadata),
- log: log,
+ log: log.WithName("store"),
}
}
diff --git a/internal/adc/client/client.go b/internal/adc/client/client.go
index 8a498b13..2419dde9 100644
--- a/internal/adc/client/client.go
+++ b/internal/adc/client/client.go
@@ -198,6 +198,7 @@ func (c *Client) Sync(ctx context.Context) (map[string]types.ADCExecutionErrors,
if resources == nil {
continue
}
+ c.log.Info("syncing resources for config", "service_number", len(resources.Services))
if err := c.sync(ctx, Task{
Name: name + "-sync",
diff --git a/internal/controller/status/updater.go b/internal/controller/status/updater.go
index d00aec76..e2ef06ed 100644
--- a/internal/controller/status/updater.go
+++ b/internal/controller/status/updater.go
@@ -37,7 +37,7 @@ import (
pkgmetrics "github.com/apache/apisix-ingress-controller/pkg/metrics"
)
-const UpdateChannelBufferSize = 1000
+const UpdateChannelBufferSize = 10000
type Update struct {
NamespacedName k8stypes.NamespacedName
@@ -119,7 +119,7 @@ func (u *UpdateHandler) updateStatus(ctx context.Context, update Update) error {
newObj.SetUID(obj.GetUID())
- u.log.Info("updating status", "name", update.NamespacedName.Name,
+ u.log.V(1).Info("updating status", "name", update.NamespacedName.Name,
"namespace", update.NamespacedName.Namespace,
"kind", types.KindOf(newObj),
)
@@ -140,11 +140,10 @@ func (u *UpdateHandler) Start(ctx context.Context) error {
case update := <-u.updateChannel:
// Decrement queue length after removing item from queue
pkgmetrics.DecStatusQueueLength()
- u.log.V(1).Info("received a status update", "namespace", update.NamespacedName.Namespace,
+ u.log.Info("received a status update", "namespace", update.NamespacedName.Namespace,
"name", update.NamespacedName.Name,
"kind", types.KindOf(update.Resource),
)
-
u.apply(ctx, update)
}
}
diff --git a/internal/controller/utils.go b/internal/controller/utils.go
index 869efefc..892d027c 100644
--- a/internal/controller/utils.go
+++ b/internal/controller/utils.go
@@ -1368,7 +1368,7 @@ func ProcessIngressClassParameters(tctx *provider.TranslateContext, c client.Cli
return err
}
- log.Info("found GatewayProxy for IngressClass", "ingressClass", ingressClass.Name, "gatewayproxy", gatewayProxy.Name)
+ log.V(1).Info("found GatewayProxy for IngressClass", "ingressClass", ingressClass.Name, "gatewayproxy", gatewayProxy.Name)
tctx.GatewayProxies[ingressClassKind] = *gatewayProxy
tctx.ResourceParentRefs[objKind] = append(tctx.ResourceParentRefs[objKind], ingressClassKind)
diff --git a/internal/manager/readiness/manager.go b/internal/manager/readiness/manager.go
index eca380eb..b140bd83 100644
--- a/internal/manager/readiness/manager.go
+++ b/internal/manager/readiness/manager.go
@@ -125,7 +125,8 @@ func (r *readinessManager) Start(ctx context.Context) error {
})
}
if len(expected) > 0 {
- r.log.V(1).Info("registering readiness state", "gvk", gvk, "expected", expected)
+ r.log.Info("registering readiness state", "gvk", gvk, "registered_count", len(expected))
+ r.log.V(1).Info("registered resources for readiness", "gvk", gvk, "resources", expected)
r.registerState(gvk, expected)
}
}
@@ -135,13 +136,12 @@ func (r *readinessManager) Start(ctx context.Context) error {
r.isReady.Store(true)
close(r.done)
}
+ r.log.Info("readiness manager started")
})
return err
}
func (r *readinessManager) registerState(gvk schema.GroupVersionKind, list []k8stypes.NamespacedName) {
- r.mu.Lock()
- defer r.mu.Unlock()
if _, ok := r.state[gvk]; !ok {
r.state[gvk] = make(map[k8stypes.NamespacedName]struct{})
}
@@ -155,9 +155,12 @@ func (r *readinessManager) Done(obj client.Object, nn k8stypes.NamespacedName) {
if r.IsReady() {
return
}
+ <-r.started
+
r.mu.Lock()
defer r.mu.Unlock()
gvk := types.GvkOf(obj)
+ r.log.Info("marking resource as done", "gvk", gvk, "name", nn, "state_count", len(r.state[gvk]))
if _, ok := r.state[gvk]; !ok {
return
}
@@ -191,7 +194,7 @@ func (r *readinessManager) WaitReady(ctx context.Context, timeout time.Duration)
case <-ctx.Done():
return false
case <-time.After(timeout):
- return true
+ return false
case <-r.done:
return true
}
diff --git a/internal/provider/apisix/provider.go b/internal/provider/apisix/provider.go
index d0d8e48a..029675e2 100644
--- a/internal/provider/apisix/provider.go
+++ b/internal/provider/apisix/provider.go
@@ -76,7 +76,9 @@ func New(log logr.Logger, updater status.Updater, readier readiness.ReadinessMan
o.DefaultBackendMode = ProviderTypeAPISIX
}
- cli, err := adcclient.New(log, o.DefaultBackendMode, o.SyncTimeout)
+ logger := log.WithName("provider")
+
+ cli, err := adcclient.New(logger, o.DefaultBackendMode, o.SyncTimeout)
if err != nil {
return nil, err
}
@@ -88,7 +90,7 @@ func New(log logr.Logger, updater status.Updater, readier readiness.ReadinessMan
updater: updater,
readier: readier,
syncCh: make(chan struct{}, 1),
- log: log.WithName("provider"),
+ log: logger,
}, nil
}
@@ -249,7 +251,10 @@ func (d *apisixProvider) buildConfig(tctx *provider.TranslateContext, nnk types.
}
func (d *apisixProvider) Start(ctx context.Context) error {
+ d.log.Info("starting provider, waiting for readiness")
d.readier.WaitReady(ctx, 5*time.Minute)
+ d.log.Info("Ready detected, starting sync loop")
+
initalSyncDelay := d.InitSyncDelay
if initalSyncDelay > 0 {
time.AfterFunc(initalSyncDelay, d.syncNotify)
diff --git a/test/benchmark/benchmark_test.go b/test/benchmark/benchmark_test.go
new file mode 100644
index 00000000..f32f8cf7
--- /dev/null
+++ b/test/benchmark/benchmark_test.go
@@ -0,0 +1,430 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package benchmark
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "net/http"
+ "os"
+ "time"
+
+ "github.com/api7/gopkg/pkg/log"
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+
+ adctypes "github.com/apache/apisix-ingress-controller/api/adc"
+ "github.com/apache/apisix-ingress-controller/test/e2e/framework"
+ "github.com/apache/apisix-ingress-controller/test/e2e/scaffold"
+)
+
+var report = &BenchmarkReport{}
+var totalRoutes = 2000
+var totalConsumers = 2000
+
+var _ = BeforeSuite(func() {
+ routes := os.Getenv("BENCHMARK_ROUTES")
+ if routes != "" {
+ _, err := fmt.Sscanf(routes, "%d", &totalRoutes)
+ Expect(err).NotTo(HaveOccurred(), "parsing BENCHMARK_ROUTES")
+ }
+ consumers := os.Getenv("BENCHMARK_CONSUMERS")
+ if consumers != "" {
+ _, err := fmt.Sscanf(consumers, "%d", &totalConsumers)
+ Expect(err).NotTo(HaveOccurred(), "parsing BENCHMARK_CONSUMERS")
+ }
+})
+var _ = AfterSuite(func() {
+ report.PrintTable()
+})
+
+const gatewayProxyYaml = `
+apiVersion: apisix.apache.org/v1alpha1
+kind: GatewayProxy
+metadata:
+ name: apisix-proxy-config
+spec:
+ provider:
+ type: ControlPlane
+ controlPlane:
+ service:
+ name: %s
+ port: 9180
+ auth:
+ type: AdminKey
+ adminKey:
+ value: "%s"
+`
+
+var _ = Describe("Benchmark Test", func() {
+ var (
+ s = scaffold.NewDefaultScaffold()
+ controlAPIClient scaffold.ControlAPIClient
+ )
+
+ BeforeEach(func() {
+ By("port-forward to control api service")
+ var err error
+ controlAPIClient, err = s.ControlAPIClient()
+ Expect(err).NotTo(HaveOccurred(), "create control api client")
+ })
+
+ Context("Benchmark ApisixRoute", func() {
+ const ingressClassYaml = `
+apiVersion: networking.k8s.io/v1
+kind: IngressClass
+metadata:
+ name: apisix
+spec:
+ controller: "%s"
+ parameters:
+ apiGroup: "apisix.apache.org"
+ kind: "GatewayProxy"
+ name: "apisix-proxy-config"
+ namespace: %s
+ scope: "Namespace"
+`
+
+ const apisixRouteSpec = `
+apiVersion: apisix.apache.org/v2
+kind: ApisixRoute
+metadata:
+ name: %s
+spec:
+ ingressClassName: apisix
+ http:
+ - name: rule0
+ match:
+ paths:
+ - /get
+ exprs:
+ - subject:
+ scope: Header
+ name: X-Route-Name
+ op: Equal
+ value: %s
+ backends:
+ - serviceName: httpbin-service-e2e-test
+ servicePort: 80
+`
+ var apisixRouteSpecHeaders = `
+apiVersion: apisix.apache.org/v2
+kind: ApisixRoute
+metadata:
+ name: %s
+spec:
+ ingressClassName: apisix
+ http:
+ - name: rule0
+ match:
+ paths:
+ - /headers
+ exprs:
+ - subject:
+ scope: Header
+ name: X-Route-Name
+ op: Equal
+ value: %s
+ backends:
+ - serviceName: httpbin-service-e2e-test
+ servicePort: 80
+`
+
+ var apisixUpstreamSpec = `
+apiVersion: apisix.apache.org/v2
+kind: ApisixUpstream
+metadata:
+ name: httpbin-service-e2e-test
+spec:
+ ingressClassName: apisix
+ scheme: https
+`
+ var apisixRouteSpecKeyAuth = `
+apiVersion: apisix.apache.org/v2
+kind: ApisixRoute
+metadata:
+ name: key-auth
+spec:
+ ingressClassName: apisix
+ http:
+ - name: rule0
+ match:
+ paths:
+ - /get
+ backends:
+ - serviceName: httpbin-service-e2e-test
+ servicePort: 80
+ authentication:
+ enable: true
+ type: keyAuth
+`
+ var keyAuth = `
+apiVersion: apisix.apache.org/v2
+kind: ApisixConsumer
+metadata:
+ name: %s
+spec:
+ ingressClassName: apisix
+ authParameter:
+ keyAuth:
+ value:
+ key: %s
+`
+
+ getRouteName := func(i int) string {
+ return fmt.Sprintf("test-route-%04d", i)
+ }
+
+ createBatchApisixRoutes := func(number int) string {
+ var buf bytes.Buffer
+ for i := 0; i < number; i++ {
+ name := getRouteName(i)
+ fmt.Fprintf(&buf, apisixRouteSpec, name, name)
+ buf.WriteString("\n---\n")
+ }
+ return buf.String()
+ }
+ getConsumerName := func(i int) string {
+ return fmt.Sprintf("consumer-%04d", i)
+ }
+ createBatchConsumers := func(number int) string {
+ var buf bytes.Buffer
+ for i := 0; i < number; i++ {
+ name := getConsumerName(i)
+ fmt.Fprintf(&buf, keyAuth, name, name)
+ buf.WriteString("\n---\n")
+ }
+ return buf.String()
+ }
+
+ benchmark := func(scenario string) {
+ s.Deployer.ScaleIngress(0)
+ By(fmt.Sprintf("prepare %d ApisixRoutes", totalRoutes))
+ err := s.CreateResourceFromString(createBatchApisixRoutes(totalRoutes))
+ Expect(err).NotTo(HaveOccurred(), "creating ApisixRoutes")
+ s.Deployer.ScaleIngress(1)
+
+ now := time.Now()
+ By(fmt.Sprintf("start cale time for applying %d ApisixRoutes to take effect", totalRoutes))
+ err = s.EnsureNumService(controlAPIClient, func(actual int) bool { return actual == totalRoutes })
+ Expect(err).ShouldNot(HaveOccurred())
+ costTime := time.Since(now)
+ report.Add(scenario, fmt.Sprintf("Apply %d ApisixRoutes", totalRoutes), costTime)
+
+ By("Test the time required for an ApisixRoute update to take effect")
+ name := getRouteName(int(time.Now().Unix()))
+ err = s.CreateResourceFromString(fmt.Sprintf(apisixRouteSpecHeaders, name, name))
+ Expect(err).NotTo(HaveOccurred())
+ now = time.Now()
+ Eventually(func() int {
+ return s.NewAPISIXClient().GET("/headers").WithHeader("X-Route-Name", name).Expect().Raw().StatusCode
+ }).WithTimeout(15 * time.Minute).ProbeEvery(100 * time.Millisecond).Should(Equal(http.StatusOK))
+ report.Add(scenario, fmt.Sprintf("Update a single ApisixRoute base on %d ApisixRoutes", totalRoutes), time.Since(now))
+
+ By("Test the time required for a service endpoint change to take effect")
+ err = s.ScaleHTTPBIN(2)
+ Expect(err).NotTo(HaveOccurred(), "scale httpbin deployment")
+ now = time.Now()
+ err = s.EnsureNumUpstreamNodes(controlAPIClient, "", 2)
+ Expect(err).ShouldNot(HaveOccurred())
+ costTime = time.Since(now)
+ report.Add(scenario, fmt.Sprintf("Service endpoint change base on %d ApisixRoutes", totalRoutes), costTime)
+
+ By("Test the time required for an ApisixUpstream update to take effect")
+ err = s.CreateResourceFromString(apisixUpstreamSpec)
+ Expect(err).NotTo(HaveOccurred(), "creating ApisixUpstream")
+ now = time.Now()
+ err = s.ExpectUpstream(controlAPIClient, "", func(upstream adctypes.Upstream) bool {
+ if upstream.Scheme != "https" {
+ log.Warnf("expect upstream: [%s] scheme to be https, but got [%s]", upstream.Name, upstream.Scheme)
+ return false
+ }
+ return true
+ })
+ Expect(err).ShouldNot(HaveOccurred())
+ costTime = time.Since(now)
+ report.Add(scenario, fmt.Sprintf("Update ApisixUpstream base on %d ApisixRoutes", totalRoutes), costTime)
+ }
+
+ BeforeEach(func() {
+ By("create GatewayProxy")
+ gatewayProxy := fmt.Sprintf(gatewayProxyYaml, framework.ProviderType, s.AdminKey())
+ err := s.CreateResourceFromStringWithNamespace(gatewayProxy, s.Namespace())
+ Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy")
+ time.Sleep(5 * time.Second)
+
+ By("create IngressClass")
+ err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(ingressClassYaml, s.GetControllerName(), s.Namespace()), "")
+ Expect(err).NotTo(HaveOccurred(), "creating IngressClass")
+ time.Sleep(5 * time.Second)
+ })
+ It("benchmark ApisixRoute", func() {
+ benchmark("ApisixRoute Benchmark")
+ })
+ It("10 apisix-standalone pod scale benchmark", func() {
+ if framework.ProviderType != framework.ProviderTypeAPISIXStandalone {
+ Skip("only apisix-standalone support scale benchmark")
+ }
+ s.Deployer.ScaleDataplane(10)
+ benchmark("ApisixRoute Benchmark with 10 apisix-standalone pods")
+ })
+ It("ApisixRoute With Consumers benchmark", func() {
+ s.Deployer.ScaleIngress(0)
+ By(fmt.Sprintf("prepare %d ApisixConsumers", totalRoutes))
+ err := s.CreateResourceFromString(createBatchConsumers(totalRoutes))
+ Expect(err).NotTo(HaveOccurred(), "creating ApisixConsumers")
+ err = s.CreateResourceFromString(apisixRouteSpecKeyAuth)
+ Expect(err).NotTo(HaveOccurred(), "creating ApisixRoute with KeyAuth")
+ s.Deployer.ScaleIngress(1)
+
+ now := time.Now()
+ Eventually(func() error {
+ consumer, err := s.DefaultDataplaneResource().Consumer().List(context.Background())
+ if err != nil {
+ return err
+ }
+ if len(consumer) != totalConsumers {
+ return fmt.Errorf("expect %d consumers, but got %d", totalConsumers, len(consumer))
+ }
+ return nil
+ }).WithTimeout(15*time.Minute).ProbeEvery(1*time.Second).ShouldNot(HaveOccurred(), "waiting for all consumers to be synced to APISIX")
+ costTime := time.Since(now)
+ report.AddResult(TestResult{
+ Scenario: "ApisixRoute With Consumers Benchmark",
+ CaseName: fmt.Sprintf("Apply %d ApisixConsumers and ApisixRoute with KeyAuth", totalConsumers),
+ CostTime: costTime,
+ IsRequestGateway: true,
+ })
+ })
+ })
+
+ Context("Benchmark HTTPRoute", func() {
+ const httpRouteSpec = `
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+ name: %s
+spec:
+ parentRefs:
+ - name: %s
+ rules:
+ - matches:
+ - path:
+ type: Exact
+ value: /get
+ headers:
+ - type: Exact
+ name: X-Route-Name
+ value: %s
+ # name: get
+ backendRefs:
+ - name: httpbin-service-e2e-test
+ port: 80
+`
+ const httpRouteSpecHeaders = `
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+ name: %s
+spec:
+ parentRefs:
+ - name: %s
+ rules:
+ - matches:
+ - path:
+ type: Exact
+ value: /headers
+ headers:
+ - type: Exact
+ name: X-Route-Name
+ value: %s
+ # name: get
+ backendRefs:
+ - name: httpbin-service-e2e-test
+ port: 80
+`
+
+ createBatchHTTPRoutes := func(number int, parentGateway string) string {
+ var buf bytes.Buffer
+ for i := 0; i < number; i++ {
+ name := getRouteName(i)
+ fmt.Fprintf(&buf, httpRouteSpec, name, parentGateway, name)
+ buf.WriteString("\n---\n")
+ }
+ return buf.String()
+ }
+
+ BeforeEach(func() {
+ By("create GatewayProxy")
+ gatewayProxy := fmt.Sprintf(gatewayProxyYaml, framework.ProviderType, s.AdminKey())
+ err := s.CreateResourceFromStringWithNamespace(gatewayProxy, s.Namespace())
+ Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy")
+ time.Sleep(5 * time.Second)
+
+ By("create GatewayClass")
+ Expect(s.CreateResourceFromString(s.GetGatewayClassYaml())).NotTo(HaveOccurred(), "creating GatewayClass")
+
+ By("create Gateway")
+ Expect(s.CreateResourceFromString(s.GetGatewayYaml())).NotTo(HaveOccurred(), "creating Gateway")
+ time.Sleep(5 * time.Second)
+ })
+
+ It("benchmark HTTPRoute", func() {
+ s.Deployer.ScaleIngress(0)
+ By(fmt.Sprintf("prepare %d HTTPRoute", totalRoutes))
+ err := s.CreateResourceFromString(createBatchHTTPRoutes(totalRoutes, s.Namespace()))
+ Expect(err).NotTo(HaveOccurred(), "creating HTTPRoute")
+ s.Deployer.ScaleIngress(1)
+
+ now := time.Now()
+ By(fmt.Sprintf("start cale time for applying %d HTTPRoute to take effect", totalRoutes))
+ err = s.EnsureNumService(controlAPIClient, func(actual int) bool { return actual == totalRoutes })
+ Expect(err).ShouldNot(HaveOccurred())
+ costTime := time.Since(now)
+ report.Add("HTTPRoute Benchmark", fmt.Sprintf("Apply %d HTTPRoute", totalRoutes), costTime)
+
+ By("Test the time required for an HTTPRoute update to take effect")
+ name := getRouteName(int(time.Now().Unix()))
+ err = s.CreateResourceFromString(fmt.Sprintf(httpRouteSpecHeaders, name, s.Namespace(), name))
+ Expect(err).NotTo(HaveOccurred())
+ now = time.Now()
+ Eventually(func() int {
+ return s.NewAPISIXClient().GET("/headers").WithHeader("X-Route-Name", name).Expect().Raw().StatusCode
+ }).WithTimeout(5 * time.Minute).ProbeEvery(100 * time.Millisecond).Should(Equal(http.StatusOK))
+ report.AddResult(TestResult{
+ Scenario: "HTTPRoute Benchmark",
+ CaseName: fmt.Sprintf("Update a single HTTPRoute base on %d HTTPRoute", totalRoutes),
+ CostTime: time.Since(now),
+ })
+
+ By("Test the time required for a service endpoint change to take effect")
+ err = s.ScaleHTTPBIN(2)
+ Expect(err).NotTo(HaveOccurred(), "scale httpbin deployment")
+ now = time.Now()
+ err = s.EnsureNumUpstreamNodes(controlAPIClient, "", 2)
+ Expect(err).ShouldNot(HaveOccurred())
+ costTime = time.Since(now)
+ report.Add("HTTPRoute Benchmark", fmt.Sprintf("Service endpoint change base on %d HTTPRoute", totalRoutes), costTime)
+ })
+ })
+})
+
+func getRouteName(i int) string {
+ return fmt.Sprintf("test-route-%04d", i)
+}
diff --git a/test/benchmark/suite_test.go b/test/benchmark/suite_test.go
new file mode 100644
index 00000000..32f60af8
--- /dev/null
+++ b/test/benchmark/suite_test.go
@@ -0,0 +1,41 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package benchmark
+
+import (
+ "fmt"
+ "testing"
+
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+
+ "github.com/apache/apisix-ingress-controller/test/e2e/framework"
+ "github.com/apache/apisix-ingress-controller/test/e2e/scaffold"
+)
+
+// Run long-term-stability tests using Ginkgo runner.
+func TestBenchmark(t *testing.T) {
+ RegisterFailHandler(Fail)
+ var f = framework.NewFramework()
+ _ = f
+
+ scaffold.NewDeployer = scaffold.NewAPISIXDeployer
+
+ _, _ = fmt.Fprintf(GinkgoWriter, "Starting Benchmark Tests\n")
+ RunSpecs(t, "Benchmark Tests Suite")
+}
diff --git a/test/benchmark/utis.go b/test/benchmark/utis.go
new file mode 100644
index 00000000..35de0b90
--- /dev/null
+++ b/test/benchmark/utis.go
@@ -0,0 +1,76 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package benchmark
+
+import (
+ "encoding/json"
+ "fmt"
+ "os"
+ "time"
+
+ "github.com/api7/gopkg/pkg/log"
+ "github.com/olekukonko/tablewriter"
+ "go.uber.org/zap"
+)
+
+type TestResult struct {
+ Scenario string `json:"scenario"`
+ CaseName string `json:"case_name"`
+ CostTime time.Duration `json:"cost_time"`
+ IsRequestGateway bool `json:"is_request_gateway,omitempty"`
+}
+
+type BenchmarkReport struct {
+ Results []TestResult
+}
+
+func (r *BenchmarkReport) PrintTable() {
+ table := tablewriter.NewWriter(os.Stdout)
+ table.Header([]string{"Scenario", "Case", "Cost", "IsRequestGateway"})
+
+ for _, res := range r.Results {
+ if err := table.Append([]any{
+ res.Scenario,
+ res.CaseName,
+ res.CostTime.String(),
+ res.IsRequestGateway,
+ }); err != nil {
+ log.Errorw("failed to append row to table", zap.Error(err))
+ }
+ }
+ if err := table.Render(); err != nil {
+ log.Errorw("failed to render table", zap.Error(err))
+ }
+}
+
+func (r *BenchmarkReport) PrintJSON() {
+ b, _ := json.MarshalIndent(r.Results, "", " ")
+ fmt.Println(string(b))
+}
+
+func (r *BenchmarkReport) AddResult(result TestResult) {
+ r.Results = append(r.Results, result)
+}
+
+func (r *BenchmarkReport) Add(scenario, caseName string, cost time.Duration) {
+ r.Results = append(r.Results, TestResult{
+ Scenario: scenario,
+ CaseName: caseName,
+ CostTime: cost,
+ })
+}
diff --git a/test/e2e/framework/manifests/apisix-standalone.yaml b/test/e2e/framework/manifests/apisix-standalone.yaml
deleted file mode 100644
index 0eda2bc8..00000000
--- a/test/e2e/framework/manifests/apisix-standalone.yaml
+++ /dev/null
@@ -1,153 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: apisix-conf
-data:
- config.yaml: |
- deployment:
- role: traditional
- role_traditional:
- config_provider: yaml
- admin:
- allow_admin:
- - 0.0.0.0/0
- admin_key:
- - key: {{ .AdminKey }}
- name: admin
- role: admin
- nginx_config:
- worker_processes: 2
- error_log_level: info
- apisix:
- proxy_mode: http&stream
- stream_proxy: # TCP/UDP proxy
- tcp: # TCP proxy port list
- - 9100
- - addr: 9110
- tls: true
- udp: # UDP proxy port list
- - 9200
- discovery:
- dns:
- servers:
- - "10.96.0.10:53" # use the real address of your dns server.
- # currently we use KIND as the standard test environment, so here we can hard-code the default DNS address first.
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: apisix
- labels:
- app.kubernetes.io/name: apisix
-spec:
- replicas: 1
- selector:
- matchLabels:
- app.kubernetes.io/name: apisix
- template:
- metadata:
- labels:
- app.kubernetes.io/name: apisix
- spec:
- initContainers:
- - name: config-setup
- image: apache/apisix:dev
- command:
- - sh
- - -c
- - |
- echo "Copying default config directory to writable volume"
- cp -r /usr/local/apisix/conf/* /tmp/apisix-conf/
- echo "Overwriting config.yaml with custom configuration"
- cp /tmp/config-source/config.yaml /tmp/apisix-conf/config.yaml
- echo "Config setup completed successfully"
- ls -la /tmp/apisix-conf/
- volumeMounts:
- - name: config-source
- mountPath: /tmp/config-source
- - name: config-writable
- mountPath: /tmp/apisix-conf
- containers:
- - name: apisix
- image: apache/apisix:dev
- ports:
- - name: http
- containerPort: 9080
- protocol: TCP
- - name: https
- containerPort: 9443
- protocol: TCP
- - name: admin
- containerPort: 9180
- protocol: TCP
- - name: tcp
- containerPort: 9100
- protocol: TCP
- - name: udp
- containerPort: 9200
- protocol: UDP
- - name: tls
- containerPort: 9110
- protocol: TCP
- volumeMounts:
- - name: config-writable
- mountPath: /usr/local/apisix/conf
- volumes:
- - name: config-source
- configMap:
- name: apisix-conf
- - name: config-writable
- emptyDir: {}
----
-apiVersion: v1
-kind: Service
-metadata:
- name: {{ .ServiceName }}
- labels:
- app.kubernetes.io/name: apisix
-spec:
- ports:
- - port: {{ .ServiceHTTPPort }}
- name: http
- protocol: TCP
- targetPort: 9080
- - port: {{ .ServiceHTTPSPort }}
- name: https
- protocol: TCP
- targetPort: 9443
- - port: 9180
- name: admin
- protocol: TCP
- targetPort: 9180
- - name: tcp
- port: 9100
- protocol: TCP
- targetPort: 9100
- - name: udp
- port: 9200
- protocol: UDP
- targetPort: 9200
- - name: tls
- port: 9110
- protocol: TCP
- targetPort: 9110
- selector:
- app.kubernetes.io/name: apisix
- type: {{ .ServiceType | default "NodePort" }}
diff --git a/test/e2e/framework/manifests/apisix.yaml b/test/e2e/framework/manifests/apisix.yaml
index 31581bcc..eada5041 100644
--- a/test/e2e/framework/manifests/apisix.yaml
+++ b/test/e2e/framework/manifests/apisix.yaml
@@ -42,6 +42,9 @@ data:
nginx_config:
worker_processes: 2
error_log_level: info
+ meta:
+ lua_shared_dict:
+ standalone-config: 50m
apisix:
proxy_mode: http&stream
stream_proxy: # TCP/UDP proxy
@@ -168,3 +171,20 @@ spec:
selector:
app.kubernetes.io/name: apisix
type: {{ .ServiceType | default "NodePort" }}
+
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+ name: apisix-control-api
+ labels:
+ app.kubernetes.io/name: apisix-control-api
+spec:
+ ports:
+ - port: 9090
+ name: control
+ protocol: TCP
+ targetPort: 9090
+ selector:
+ app.kubernetes.io/name: apisix
diff --git a/test/e2e/scaffold/scaffold.go b/test/e2e/scaffold/scaffold.go
index c23acd6a..799cc798 100644
--- a/test/e2e/scaffold/scaffold.go
+++ b/test/e2e/scaffold/scaffold.go
@@ -20,11 +20,13 @@ package scaffold
import (
"context"
"crypto/tls"
+ "encoding/json"
"fmt"
"net/http"
"net/url"
"os"
"strings"
+ "time"
"github.com/api7/gopkg/pkg/log"
"github.com/gavv/httpexpect/v2"
@@ -32,10 +34,13 @@ import (
"github.com/gruntwork-io/terratest/modules/testing"
. "github.com/onsi/ginkgo/v2" //nolint:staticcheck
. "github.com/onsi/gomega" //nolint:staticcheck
+ "go.uber.org/zap"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/util/wait"
"sigs.k8s.io/controller-runtime/pkg/client"
+ adctypes "github.com/apache/apisix-ingress-controller/api/adc"
apiv2 "github.com/apache/apisix-ingress-controller/api/v2"
"github.com/apache/apisix-ingress-controller/test/e2e/framework"
)
@@ -556,3 +561,104 @@ func (s *Scaffold) GetMetricsEndpoint() string {
s.addFinalizers(tunnel.Close)
return fmt.Sprintf("http://%s/metrics", tunnel.Endpoint())
}
+
+func (s *Scaffold) ControlAPIClient() (ControlAPIClient, error) {
+ tunnel := k8s.NewTunnel(s.kubectlOptions, k8s.ResourceTypeService, "apisix-control-api", 9090, 9090)
+ if err := tunnel.ForwardPortE(s.t); err != nil {
+ return nil, err
+ }
+ s.addFinalizers(tunnel.Close)
+
+ return &controlAPI{
+ client: NewClient("http", tunnel.Endpoint()),
+ }, nil
+}
+
+func (s *Scaffold) EnsureNumService(controlAPIClient ControlAPIClient, matcher func(result int) bool) error {
+ times := 0
+ return wait.PollUntilContextTimeout(context.Background(), 100*time.Millisecond, 10*time.Minute, true, func(ctx context.Context) (done bool, err error) {
+ times++
+ results, _, err := controlAPIClient.ListServices()
+ if err != nil {
+ log.Errorw("failed to ListServices", zap.Error(err))
+ return false, nil
+ }
+ if !matcher(len(results)) {
+ log.Debugw("number of effective services", zap.Int("number", len(results)), zap.Int("times", times))
+ return false, nil
+ }
+ return true, nil
+ })
+}
+
+func (s *Scaffold) ExpectUpstream(controlAPIClient ControlAPIClient, name string, matcher func(upstream adctypes.Upstream) bool) error {
+ times := 0
+ return wait.PollUntilContextTimeout(context.Background(), 1*time.Second, 10*time.Minute, true, func(ctx context.Context) (done bool, err error) {
+ times++
+ upstreams, _, err := controlAPIClient.ListUpstreams()
+ if err != nil {
+ log.Errorw("failed to ListServices", zap.Error(err))
+ return false, nil
+ }
+ for _, upstream := range upstreams {
+ upsValue := upstream.(map[string]any)
+ data, err := json.Marshal(upsValue["value"])
+ if err != nil {
+ return false, fmt.Errorf("failed to marshal upstream: %v", err)
+ }
+
+ var ups adctypes.Upstream
+ if err := json.Unmarshal(data, &ups); err != nil {
+ return false, fmt.Errorf("failed to unmarshal upstream: %v", err)
+ }
+ if name != "" && ups.Name != name {
+ continue
+ }
+ if ok := matcher(ups); !ok {
+ return false, nil
+ }
+ }
+ return true, nil
+ })
+}
+
+func (s *Scaffold) EnsureNumUpstreamNodes(controlAPIClient ControlAPIClient, name string, number int) error {
+ return s.ExpectUpstream(controlAPIClient, name, func(upstream adctypes.Upstream) bool {
+ if len(upstream.Nodes) != number {
+ log.Warnf("expect upstream: [%s] nodes num to be %d, but got %d", upstream.Name, number, len(upstream.Nodes))
+ return false
+ }
+ return true
+ })
+}
+
+type ControlAPIClient interface {
+ ListServices() ([]any, int64, error)
+ ListUpstreams() ([]any, int64, error)
+}
+
+type controlAPI struct {
+ client *httpexpect.Expect
+}
+
+func (c *controlAPI) ListUpstreams() (result []any, total int64, err error) {
+ resp := c.client.Request(http.MethodGet, "/v1/upstreams").Expect()
+ if resp.Raw().StatusCode != http.StatusOK {
+ return nil, 0, fmt.Errorf("unexpected status code: %v, message: %s", resp.Raw().StatusCode, resp.Body().Raw())
+ }
+ if err = json.Unmarshal([]byte(resp.Body().Raw()), &result); err != nil {
+ return nil, 0, fmt.Errorf("failed to unmarshal response body: %w", err)
+ }
+ return result, int64(len(result)), err
+}
+
+func (c *controlAPI) ListServices() (result []any, total int64, err error) {
+ resp := c.client.Request(http.MethodGet, "/v1/services").Expect()
+ if resp.Raw().StatusCode != http.StatusOK {
+ return nil, 0, fmt.Errorf("unexpected status code: %v, message: %s", resp.Raw().StatusCode, resp.Body().Raw())
+ }
+ if err = json.Unmarshal([]byte(resp.Body().Raw()), &result); err != nil {
+ return nil, 0, fmt.Errorf("failed to unmarshal response body: %w", err)
+ }
+ return result, int64(len(result)), err
+}