You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@apisix.apache.org by to...@apache.org on 2021/05/07 02:06:21 UTC
[apisix-ingress-controller] branch master updated: feat: add
ApisixClusterConfig controller loop (#416)
This is an automated email from the ASF dual-hosted git repository.
tokers pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix-ingress-controller.git
The following commit(s) were added to refs/heads/master by this push:
new f613f36 feat: add ApisixClusterConfig controller loop (#416)
f613f36 is described below
commit f613f36bab842089ab54f681ed1b4bf126e4c162
Author: Alex Zhang <zc...@gmail.com>
AuthorDate: Fri May 7 10:06:13 2021 +0800
feat: add ApisixClusterConfig controller loop (#416)
---
Makefile | 4 +-
docs/en/latest/concepts/apisix_cluster_config.md | 81 +++++++
docs/en/latest/references/apisix_cluster_config.md | 38 ++++
pkg/apisix/apisix.go | 2 +
pkg/apisix/cluster.go | 37 +--
pkg/apisix/nonexistentclient.go | 4 +
pkg/config/config.go | 2 +-
pkg/ingress/apisix_cluster_config.go | 250 +++++++++++++++++++++
pkg/ingress/controller.go | 75 ++++---
pkg/kube/apisix/apis/config/v2alpha1/register.go | 2 +
pkg/kube/apisix/apis/config/v2alpha1/types.go | 8 +-
pkg/kube/translation/global_rule.go | 47 ++++
pkg/kube/translation/global_rule_test.go | 53 +++++
pkg/kube/translation/translator.go | 3 +
pkg/types/apisix/v1/types.go | 2 +-
samples/deploy/rbac/apisix_view_clusterrole.yaml | 1 +
test/e2e/features/global_rule.go | 73 ++++++
test/e2e/scaffold/ingress.go | 5 +-
test/e2e/scaffold/k8s.go | 20 ++
19 files changed, 650 insertions(+), 57 deletions(-)
diff --git a/Makefile b/Makefile
index 2ec758f..f74ae3b 100644
--- a/Makefile
+++ b/Makefile
@@ -68,9 +68,7 @@ unit-test:
### e2e-test: Run e2e test cases (kind is required)
.PHONY: e2e-test
e2e-test: ginkgo-check push-images-to-kind
- kubectl apply -f $(PWD)/samples/deploy/crd/v1beta1/ApisixRoute.yaml
- kubectl apply -f $(PWD)/samples/deploy/crd/v1beta1/ApisixUpstream.yaml
- kubectl apply -f $(PWD)/samples/deploy/crd/v1beta1/ApisixTls.yaml
+ kubectl apply -k $(PWD)/samples/deploy/crd/v1beta1
cd test/e2e && ginkgo -cover -coverprofile=coverage.txt -r --randomizeSuites --randomizeAllSpecs --trace -p --nodes=$(E2E_CONCURRENCY)
.PHONY: ginkgo-check
diff --git a/docs/en/latest/concepts/apisix_cluster_config.md b/docs/en/latest/concepts/apisix_cluster_config.md
new file mode 100644
index 0000000..1c98edb
--- /dev/null
+++ b/docs/en/latest/concepts/apisix_cluster_config.md
@@ -0,0 +1,81 @@
+---
+title: ApisixClusterConfig
+---
+
+<!--
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+-->
+
+`ApisixClusterConfig` is a CRD resource which used to describe an APISIX cluster, currently it's not a required
+resource but its existence can enrich an APISIX cluster, for instance, enabling cluster-wide monitoring, rate limiting and so on.
+
+monitoring features like collecting [Prometheus](https://prometheus.io/) metrics
+and [skywalking](https://skywalking.apache.org/) spans
+
+Monitoring
+----------
+
+By default, monitoring are not enabled for the APISIX cluster, this is not favorable
+if you'd like to learn the real running status of your cluster. In such a case, you
+could create a `ApisixClusterConfig` to enable these features explicitly.
+
+```yaml
+apiVersion: apisix.apache.org/v2alpha1
+kind: ApisixClusterConfig
+metadata:
+ name: default
+spec:
+ monitoring:
+ prometheus:
+ enable: true
+ skywalking:
+ enable: true
+ sampleRatio: 0.5
+```
+
+The above example enables both the Prometheus and Skywalking for the APISIX cluster which name is "default".
+Please see [Prometheus in APISIX](http://apisix.apache.org/docs/apisix/plugins/prometheus) and [Skywalking in APISIX](http://apisix.apache.org/docs/apisix/plugins/skywalking) for the details.
+
+Admin Config
+------------
+
+The default APISIX cluster is configured through command line options like `--default-apisix-cluster-xxx`. They are constant in apisix-ingress-controller's lifecycle, you have to change the definition
+of Deployment or Pod template. Now with the help of `ApisixClusterConfig`, you can change some administrative fields on it.
+
+```yaml
+apiVersion: apisix.apache.org/v2alpha1
+kind: ApisixClusterConfig
+metadata:
+ name: default
+spec:
+ admin:
+ baseURL: http://apisix-gw.default.svc.cluster.local:9180/apisix/admin
+ adminKey: "123456"
+```
+
+The above `ApisixClusterConfig` sets the base url and admin key for the APISIX cluster `"default"`. Once this
+resource is processed, resources like Route, Upstream and others will be pushed to the new address with the new admin key (for authentication).
+
+Multiple Clusters Management
+----------------------------
+
+`ApisixClusterConfig` is also designed for supporting multiple clusters management, but currently this function IS NOT ENABLED YET.
+Only the `ApisixClusterConfig` with the same named configured in `--default-apisix-cluster-name` option will be handled by apisix-ingress-controller, other instances will be neglected.
+
+The current delete event for `ApisixClusterConfig` doesn't mean the apisix-ingress-controller will lose the view of the corresponding APISIX cluster but
+resetting all the features on it, so the running of APISIX cluster is not influenced by this event.
diff --git a/docs/en/latest/references/apisix_cluster_config.md b/docs/en/latest/references/apisix_cluster_config.md
new file mode 100644
index 0000000..d4e3850
--- /dev/null
+++ b/docs/en/latest/references/apisix_cluster_config.md
@@ -0,0 +1,38 @@
+---
+title: ApisixRoute/v2alpha1 Reference
+---
+
+<!--
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+-->
+
+## Spec
+
+Spec describes the desired state of an ApisixClusterConfig object.
+
+| Field | Type | Description |
+|---------------|----------|-----------------------|
+| monitoring | object | Monitoring settings. |
+| monitoring.prometheus | object | Prometheus settings. |
+| monitoring.prometheus.enable | boolean | Whether to enable Prometheus or not. |
+| monitoring.skywalking | object | Skywalking settings. |
+| monitoring.skywalking.enable | boolean | Whether to enable Skywalking or not. |
+| monitoring.skywalking.sampleRatio | number | The sample ratio for spans, value should be in `[0, 1]`.|
+| admin | object | Administrative settings. |
+| admin.baseURL | string | the base url for APISIX cluster. |
+| admin.AdminKey | string | admin key used for authentication with APISIX cluster. |
diff --git a/pkg/apisix/apisix.go b/pkg/apisix/apisix.go
index e6c346c..a33c716 100644
--- a/pkg/apisix/apisix.go
+++ b/pkg/apisix/apisix.go
@@ -44,6 +44,8 @@ type Cluster interface {
SSL() SSL
// StreamRoute returns a StreamRoute interface that can operate StreamRoute resources.
StreamRoute() StreamRoute
+ // GlobalRule returns a GlobalRule interface that can operate GlobalRule resources.
+ GlobalRule() GlobalRule
// String exposes the client information in human readable format.
String() string
// HasSynced checks whether all resources in APISIX cluster is synced to cache.
diff --git a/pkg/apisix/cluster.go b/pkg/apisix/cluster.go
index bc31ee8..8ca93c7 100644
--- a/pkg/apisix/cluster.go
+++ b/pkg/apisix/cluster.go
@@ -280,23 +280,28 @@ func (c *cluster) StreamRoute() StreamRoute {
return c.streamRoute
}
-func (s *cluster) applyAuth(req *http.Request) {
- if s.adminKey != "" {
- req.Header.Set("X-API-Key", s.adminKey)
+// GlobalRule implements Cluster.GlobalRule method.
+func (c *cluster) GlobalRule() GlobalRule {
+ return c.globalRules
+}
+
+func (c *cluster) applyAuth(req *http.Request) {
+ if c.adminKey != "" {
+ req.Header.Set("X-API-Key", c.adminKey)
}
}
-func (s *cluster) do(req *http.Request) (*http.Response, error) {
- s.applyAuth(req)
- return s.cli.Do(req)
+func (c *cluster) do(req *http.Request) (*http.Response, error) {
+ c.applyAuth(req)
+ return c.cli.Do(req)
}
-func (s *cluster) getResource(ctx context.Context, url string) (*getResponse, error) {
+func (c *cluster) getResource(ctx context.Context, url string) (*getResponse, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
- resp, err := s.do(req)
+ resp, err := c.do(req)
if err != nil {
return nil, err
}
@@ -320,12 +325,12 @@ func (s *cluster) getResource(ctx context.Context, url string) (*getResponse, er
return &res, nil
}
-func (s *cluster) listResource(ctx context.Context, url string) (*listResponse, error) {
+func (c *cluster) listResource(ctx context.Context, url string) (*listResponse, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
- resp, err := s.do(req)
+ resp, err := c.do(req)
if err != nil {
return nil, err
}
@@ -345,12 +350,12 @@ func (s *cluster) listResource(ctx context.Context, url string) (*listResponse,
return &list, nil
}
-func (s *cluster) createResource(ctx context.Context, url string, body io.Reader) (*createResponse, error) {
+func (c *cluster) createResource(ctx context.Context, url string, body io.Reader) (*createResponse, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodPut, url, body)
if err != nil {
return nil, err
}
- resp, err := s.do(req)
+ resp, err := c.do(req)
if err != nil {
return nil, err
}
@@ -371,12 +376,12 @@ func (s *cluster) createResource(ctx context.Context, url string, body io.Reader
return &cr, nil
}
-func (s *cluster) updateResource(ctx context.Context, url string, body io.Reader) (*updateResponse, error) {
+func (c *cluster) updateResource(ctx context.Context, url string, body io.Reader) (*updateResponse, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodPut, url, body)
if err != nil {
return nil, err
}
- resp, err := s.do(req)
+ resp, err := c.do(req)
if err != nil {
return nil, err
}
@@ -395,12 +400,12 @@ func (s *cluster) updateResource(ctx context.Context, url string, body io.Reader
return &ur, nil
}
-func (s *cluster) deleteResource(ctx context.Context, url string) error {
+func (c *cluster) deleteResource(ctx context.Context, url string) error {
req, err := http.NewRequestWithContext(ctx, http.MethodDelete, url, nil)
if err != nil {
return err
}
- resp, err := s.do(req)
+ resp, err := c.do(req)
if err != nil {
return err
}
diff --git a/pkg/apisix/nonexistentclient.go b/pkg/apisix/nonexistentclient.go
index d965cda..d1a2c45 100644
--- a/pkg/apisix/nonexistentclient.go
+++ b/pkg/apisix/nonexistentclient.go
@@ -172,6 +172,10 @@ func (nc *nonExistentCluster) StreamRoute() StreamRoute {
return nc.streamRoute
}
+func (nc *nonExistentCluster) GlobalRule() GlobalRule {
+ return nc.globalRule
+}
+
func (nc *nonExistentCluster) HasSynced(_ context.Context) error {
return nil
}
diff --git a/pkg/config/config.go b/pkg/config/config.go
index aa7f681..86ba8dc 100644
--- a/pkg/config/config.go
+++ b/pkg/config/config.go
@@ -91,7 +91,7 @@ type APISIXConfig struct {
BaseURL string `json:"base_url" yaml:"base_url"`
// AdminKey is same to DefaultClusterAdminKey.
// Deprecated: use DefaultClusterAdminKey instead. AdminKey will be removed
- // // once v1.0.0 is released.
+ // once v1.0.0 is released.
AdminKey string `json:"admin_key" yaml:"admin_key"`
}
diff --git a/pkg/ingress/apisix_cluster_config.go b/pkg/ingress/apisix_cluster_config.go
new file mode 100644
index 0000000..c275a3c
--- /dev/null
+++ b/pkg/ingress/apisix_cluster_config.go
@@ -0,0 +1,250 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package ingress
+
+import (
+ "context"
+ "time"
+
+ "go.uber.org/zap"
+ k8serrors "k8s.io/apimachinery/pkg/api/errors"
+ "k8s.io/client-go/tools/cache"
+ "k8s.io/client-go/util/workqueue"
+
+ "github.com/apache/apisix-ingress-controller/pkg/apisix"
+ configv2alpha1 "github.com/apache/apisix-ingress-controller/pkg/kube/apisix/apis/config/v2alpha1"
+ "github.com/apache/apisix-ingress-controller/pkg/log"
+ "github.com/apache/apisix-ingress-controller/pkg/types"
+)
+
+type apisixClusterConfigController struct {
+ controller *Controller
+ workqueue workqueue.RateLimitingInterface
+ workers int
+}
+
+func (c *Controller) newApisixClusterConfigController() *apisixClusterConfigController {
+ ctl := &apisixClusterConfigController{
+ controller: c,
+ workqueue: workqueue.NewNamedRateLimitingQueue(workqueue.NewItemFastSlowRateLimiter(time.Second, 60*time.Second, 5), "ApisixClusterConfig"),
+ workers: 1,
+ }
+ c.apisixClusterConfigInformer.AddEventHandler(
+ cache.ResourceEventHandlerFuncs{
+ AddFunc: ctl.onAdd,
+ UpdateFunc: ctl.onUpdate,
+ DeleteFunc: ctl.onDelete,
+ },
+ )
+ return ctl
+}
+
+func (c *apisixClusterConfigController) run(ctx context.Context) {
+ log.Info("ApisixClusterConfig controller started")
+ defer log.Info("ApisixClusterConfig controller exited")
+ if ok := cache.WaitForCacheSync(ctx.Done(), c.controller.apisixClusterConfigInformer.HasSynced); !ok {
+ log.Error("cache sync failed")
+ return
+ }
+ for i := 0; i < c.workers; i++ {
+ go c.runWorker(ctx)
+ }
+ <-ctx.Done()
+ c.workqueue.ShutDown()
+}
+
+func (c *apisixClusterConfigController) runWorker(ctx context.Context) {
+ for {
+ obj, quit := c.workqueue.Get()
+ if quit {
+ return
+ }
+ err := c.sync(ctx, obj.(*types.Event))
+ c.workqueue.Done(obj)
+ c.handleSyncErr(obj, err)
+ }
+}
+
+func (c *apisixClusterConfigController) sync(ctx context.Context, ev *types.Event) error {
+ key := ev.Object.(string)
+ _, name, err := cache.SplitMetaNamespaceKey(key)
+ if err != nil {
+ log.Errorf("found ApisixClusterConfig resource with invalid meta key %s: %s", key, err)
+ return err
+ }
+ acc, err := c.controller.apisixClusterConfigLister.Get(name)
+ if err != nil {
+ if !k8serrors.IsNotFound(err) {
+ log.Errorf("failed to get ApisixClusterConfig %s: %s", key, err)
+ return err
+ }
+ if ev.Type != types.EventDelete {
+ log.Warnf("ApisixClusterConfig %s was deleted before it can be delivered", key)
+ return nil
+ }
+ }
+ if ev.Type == types.EventDelete {
+ if acc != nil {
+ // We still find the resource while we are processing the DELETE event,
+ // that means object with same namespace and name was created, discarding
+ // this stale DELETE event.
+ log.Warnf("discard the stale ApisixClusterConfig delete event since the %s exists", key)
+ return nil
+ }
+ acc = ev.Tombstone.(*configv2alpha1.ApisixClusterConfig)
+ }
+
+ // Currently we don't handle multiple cluster, so only process
+ // the default apisix cluster.
+ if acc.Name != c.controller.cfg.APISIX.DefaultClusterName {
+ log.Infow("ignore non-default apisix cluster config",
+ zap.String("default_cluster_name", c.controller.cfg.APISIX.DefaultClusterName),
+ zap.Any("ApisixClusterConfig", acc),
+ )
+ return nil
+ }
+ // Cluster delete is dangerous.
+ // TODO handle delete?
+ if ev.Type == types.EventDelete {
+ log.Error("ApisixClusterConfig delete event for default apisix cluster will be ignored")
+ return nil
+ }
+
+ if acc.Spec.Admin != nil {
+ clusterOpts := &apisix.ClusterOptions{
+ Name: acc.Name,
+ BaseURL: acc.Spec.Admin.BaseURL,
+ AdminKey: acc.Spec.Admin.AdminKey,
+ }
+ log.Infow("updating cluster",
+ zap.Any("opts", clusterOpts),
+ )
+ // TODO we may first call AddCluster.
+ // Since now we already have the default cluster, we just call UpdateCluster.
+ if err := c.controller.apisix.UpdateCluster(clusterOpts); err != nil {
+ log.Errorw("failed to update cluster",
+ zap.String("cluster_name", acc.Name),
+ zap.Error(err),
+ zap.Any("opts", clusterOpts),
+ )
+ return err
+ }
+ }
+
+ globalRule, err := c.controller.translator.TranslateClusterConfig(acc)
+ if err != nil {
+ // TODO add status
+ log.Errorw("failed to translate ApisixClusterConfig",
+ zap.Error(err),
+ zap.String("key", key),
+ zap.Any("object", acc),
+ )
+ return err
+ }
+ log.Debugw("translated global_rule",
+ zap.Any("object", globalRule),
+ )
+
+ // TODO multiple cluster support
+ if ev.Type == types.EventAdd {
+ _, err = c.controller.apisix.Cluster(acc.Name).GlobalRule().Create(ctx, globalRule)
+ } else {
+ _, err = c.controller.apisix.Cluster(acc.Name).GlobalRule().Update(ctx, globalRule)
+ }
+ if err != nil {
+ log.Errorw("failed to reflect global_rule changes to apisix cluster",
+ zap.Any("global_rule", globalRule),
+ zap.Any("cluster", acc.Name),
+ )
+ return err
+ }
+ return nil
+}
+
+func (c *apisixClusterConfigController) handleSyncErr(obj interface{}, err error) {
+ if err == nil {
+ c.workqueue.Forget(obj)
+ return
+ }
+ log.Warnw("sync ApisixClusterConfig failed, will retry",
+ zap.Any("object", obj),
+ zap.Error(err),
+ )
+ c.workqueue.AddRateLimited(obj)
+}
+
+func (c *apisixClusterConfigController) onAdd(obj interface{}) {
+ key, err := cache.MetaNamespaceKeyFunc(obj)
+ if err != nil {
+ log.Errorf("found ApisixClusterConfig resource with bad meta key: %s", err.Error())
+ return
+ }
+ log.Debugw("ApisixClusterConfig add event arrived",
+ zap.String("key", key),
+ zap.Any("object", obj),
+ )
+
+ c.workqueue.AddRateLimited(&types.Event{
+ Type: types.EventAdd,
+ Object: key,
+ })
+}
+
+func (c *apisixClusterConfigController) onUpdate(oldObj, newObj interface{}) {
+ prev := oldObj.(*configv2alpha1.ApisixClusterConfig)
+ curr := newObj.(*configv2alpha1.ApisixClusterConfig)
+ if prev.ResourceVersion >= curr.ResourceVersion {
+ return
+ }
+ key, err := cache.MetaNamespaceKeyFunc(newObj)
+ if err != nil {
+ log.Errorf("found ApisixClusterConfig with bad meta key: %s", err)
+ return
+ }
+ log.Debugw("ApisixClusterConfig update event arrived",
+ zap.Any("new object", curr),
+ zap.Any("old object", prev),
+ )
+
+ c.workqueue.AddRateLimited(&types.Event{
+ Type: types.EventUpdate,
+ Object: key,
+ })
+}
+
+func (c *apisixClusterConfigController) onDelete(obj interface{}) {
+ acc, ok := obj.(*configv2alpha1.ApisixClusterConfig)
+ if !ok {
+ tombstone, ok := obj.(*cache.DeletedFinalStateUnknown)
+ if !ok {
+ return
+ }
+ acc = tombstone.Obj.(*configv2alpha1.ApisixClusterConfig)
+ }
+
+ key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj)
+ if err != nil {
+ log.Errorf("found ApisixClusterConfig resource with bad meta key: %s", err)
+ return
+ }
+ log.Debugw("ApisixClusterConfig delete event arrived",
+ zap.Any("final state", acc),
+ )
+ c.workqueue.AddRateLimited(&types.Event{
+ Type: types.EventDelete,
+ Object: key,
+ Tombstone: acc,
+ })
+}
diff --git a/pkg/ingress/controller.go b/pkg/ingress/controller.go
index 7b2c9f9..a388e11 100644
--- a/pkg/ingress/controller.go
+++ b/pkg/ingress/controller.go
@@ -41,6 +41,7 @@ import (
crdclientset "github.com/apache/apisix-ingress-controller/pkg/kube/apisix/client/clientset/versioned"
"github.com/apache/apisix-ingress-controller/pkg/kube/apisix/client/informers/externalversions"
listersv1 "github.com/apache/apisix-ingress-controller/pkg/kube/apisix/client/listers/config/v1"
+ listersv2alpha1 "github.com/apache/apisix-ingress-controller/pkg/kube/apisix/client/listers/config/v2alpha1"
"github.com/apache/apisix-ingress-controller/pkg/kube/translation"
"github.com/apache/apisix-ingress-controller/pkg/log"
"github.com/apache/apisix-ingress-controller/pkg/metrics"
@@ -82,29 +83,32 @@ type Controller struct {
secretSSLMap *sync.Map
// common informers and listers
- epInformer cache.SharedIndexInformer
- epLister listerscorev1.EndpointsLister
- svcInformer cache.SharedIndexInformer
- svcLister listerscorev1.ServiceLister
- ingressLister kube.IngressLister
- ingressInformer cache.SharedIndexInformer
- secretInformer cache.SharedIndexInformer
- secretLister listerscorev1.SecretLister
- apisixUpstreamInformer cache.SharedIndexInformer
- apisixUpstreamLister listersv1.ApisixUpstreamLister
- apisixRouteLister kube.ApisixRouteLister
- apisixRouteInformer cache.SharedIndexInformer
- apisixTlsLister listersv1.ApisixTlsLister
- apisixTlsInformer cache.SharedIndexInformer
+ epInformer cache.SharedIndexInformer
+ epLister listerscorev1.EndpointsLister
+ svcInformer cache.SharedIndexInformer
+ svcLister listerscorev1.ServiceLister
+ ingressLister kube.IngressLister
+ ingressInformer cache.SharedIndexInformer
+ secretInformer cache.SharedIndexInformer
+ secretLister listerscorev1.SecretLister
+ apisixUpstreamInformer cache.SharedIndexInformer
+ apisixUpstreamLister listersv1.ApisixUpstreamLister
+ apisixRouteLister kube.ApisixRouteLister
+ apisixRouteInformer cache.SharedIndexInformer
+ apisixTlsLister listersv1.ApisixTlsLister
+ apisixTlsInformer cache.SharedIndexInformer
+ apisixClusterConfigLister listersv2alpha1.ApisixClusterConfigLister
+ apisixClusterConfigInformer cache.SharedIndexInformer
// resource controllers
endpointsController *endpointsController
ingressController *ingressController
secretController *secretController
- apisixUpstreamController *apisixUpstreamController
- apisixRouteController *apisixRouteController
- apisixTlsController *apisixTlsController
+ apisixUpstreamController *apisixUpstreamController
+ apisixRouteController *apisixRouteController
+ apisixTlsController *apisixTlsController
+ apisixClusterConfigController *apisixClusterConfigController
}
// NewController creates an ingress apisix controller object.
@@ -183,20 +187,22 @@ func NewController(cfg *config.Config) (*Controller, error) {
secretSSLMap: new(sync.Map),
recorder: eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: _component}),
- epInformer: kube.CoreSharedInformerFactory.Core().V1().Endpoints().Informer(),
- epLister: kube.CoreSharedInformerFactory.Core().V1().Endpoints().Lister(),
- svcInformer: kube.CoreSharedInformerFactory.Core().V1().Services().Informer(),
- svcLister: kube.CoreSharedInformerFactory.Core().V1().Services().Lister(),
- ingressLister: ingressLister,
- ingressInformer: ingressInformer,
- secretInformer: kube.CoreSharedInformerFactory.Core().V1().Secrets().Informer(),
- secretLister: kube.CoreSharedInformerFactory.Core().V1().Secrets().Lister(),
- apisixRouteInformer: apisixRouteInformer,
- apisixRouteLister: apisixRouteLister,
- apisixUpstreamInformer: sharedInformerFactory.Apisix().V1().ApisixUpstreams().Informer(),
- apisixUpstreamLister: sharedInformerFactory.Apisix().V1().ApisixUpstreams().Lister(),
- apisixTlsInformer: sharedInformerFactory.Apisix().V1().ApisixTlses().Informer(),
- apisixTlsLister: sharedInformerFactory.Apisix().V1().ApisixTlses().Lister(),
+ epInformer: kube.CoreSharedInformerFactory.Core().V1().Endpoints().Informer(),
+ epLister: kube.CoreSharedInformerFactory.Core().V1().Endpoints().Lister(),
+ svcInformer: kube.CoreSharedInformerFactory.Core().V1().Services().Informer(),
+ svcLister: kube.CoreSharedInformerFactory.Core().V1().Services().Lister(),
+ ingressLister: ingressLister,
+ ingressInformer: ingressInformer,
+ secretInformer: kube.CoreSharedInformerFactory.Core().V1().Secrets().Informer(),
+ secretLister: kube.CoreSharedInformerFactory.Core().V1().Secrets().Lister(),
+ apisixRouteInformer: apisixRouteInformer,
+ apisixRouteLister: apisixRouteLister,
+ apisixUpstreamInformer: sharedInformerFactory.Apisix().V1().ApisixUpstreams().Informer(),
+ apisixUpstreamLister: sharedInformerFactory.Apisix().V1().ApisixUpstreams().Lister(),
+ apisixTlsInformer: sharedInformerFactory.Apisix().V1().ApisixTlses().Informer(),
+ apisixTlsLister: sharedInformerFactory.Apisix().V1().ApisixTlses().Lister(),
+ apisixClusterConfigInformer: sharedInformerFactory.Apisix().V2alpha1().ApisixClusterConfigs().Informer(),
+ apisixClusterConfigLister: sharedInformerFactory.Apisix().V2alpha1().ApisixClusterConfigs().Lister(),
}
c.translator = translation.NewTranslator(&translation.TranslatorOptions{
EndpointsLister: c.epLister,
@@ -208,6 +214,7 @@ func NewController(cfg *config.Config) (*Controller, error) {
c.endpointsController = c.newEndpointsController()
c.apisixUpstreamController = c.newApisixUpstreamController()
c.apisixRouteController = c.newApisixRouteController()
+ c.apisixClusterConfigController = c.newApisixClusterConfigController()
c.apisixTlsController = c.newApisixTlsController()
c.ingressController = c.newIngressController()
c.secretController = c.newSecretController()
@@ -350,6 +357,9 @@ func (c *Controller) run(ctx context.Context) {
c.apisixUpstreamInformer.Run(ctx.Done())
})
c.goAttach(func() {
+ c.apisixClusterConfigInformer.Run(ctx.Done())
+ })
+ c.goAttach(func() {
c.secretInformer.Run(ctx.Done())
})
c.goAttach(func() {
@@ -368,6 +378,9 @@ func (c *Controller) run(ctx context.Context) {
c.apisixRouteController.run(ctx)
})
c.goAttach(func() {
+ c.apisixClusterConfigController.run(ctx)
+ })
+ c.goAttach(func() {
c.apisixTlsController.run(ctx)
})
c.goAttach(func() {
diff --git a/pkg/kube/apisix/apis/config/v2alpha1/register.go b/pkg/kube/apisix/apis/config/v2alpha1/register.go
index 5af6a1b..779ef2f 100644
--- a/pkg/kube/apisix/apis/config/v2alpha1/register.go
+++ b/pkg/kube/apisix/apis/config/v2alpha1/register.go
@@ -42,6 +42,8 @@ func addKnownTypes(scheme *runtime.Scheme) error {
scheme.AddKnownTypes(SchemeGroupVersion,
&ApisixRoute{},
&ApisixRouteList{},
+ &ApisixClusterConfig{},
+ &ApisixClusterConfigList{},
)
// register the type in the scheme
diff --git a/pkg/kube/apisix/apis/config/v2alpha1/types.go b/pkg/kube/apisix/apis/config/v2alpha1/types.go
index 2c01b6f..c96448d 100644
--- a/pkg/kube/apisix/apis/config/v2alpha1/types.go
+++ b/pkg/kube/apisix/apis/config/v2alpha1/types.go
@@ -271,10 +271,10 @@ type ApisixClusterConfigSpec struct {
type ApisixClusterMonitoringConfig struct {
// Prometheus is the config for using Prometheus in APISIX Cluster.
// +optional
- Prometheus ApisixClusterPrometheusConfig
+ Prometheus ApisixClusterPrometheusConfig `json:"prometheus" yaml:"prometheus"`
// Skywalking is the config for using Skywalking in APISIX Cluster.
// +optional
- Skywalking ApisixClusterSkywalkingConfig
+ Skywalking ApisixClusterSkywalkingConfig `json:"skywalking" yaml:"skywalking"`
}
// ApisixClusterPrometheusConfig is the config for using Prometheus in APISIX Cluster.
@@ -295,9 +295,9 @@ type ApisixClusterSkywalkingConfig struct {
type ApisixClusterAdminConfig struct {
// BaseURL is the base URL for the APISIX Admin API.
// It looks like "http://apisix-admin.default.svc.cluster.local:9080/apisix/admin"
- BaseURL string
+ BaseURL string `json:"baseURL" yaml:"baseURL"`
// AdminKey is used to verify the admin API user.
- AdminKey string
+ AdminKey string `json:"adminKey" yaml:"adminKey"`
}
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
diff --git a/pkg/kube/translation/global_rule.go b/pkg/kube/translation/global_rule.go
new file mode 100644
index 0000000..d830e04
--- /dev/null
+++ b/pkg/kube/translation/global_rule.go
@@ -0,0 +1,47 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package translation
+
+import (
+ "github.com/apache/apisix-ingress-controller/pkg/id"
+ configv2alpha1 "github.com/apache/apisix-ingress-controller/pkg/kube/apisix/apis/config/v2alpha1"
+ apisixv1 "github.com/apache/apisix-ingress-controller/pkg/types/apisix/v1"
+)
+
+type prometheusPluginConfig struct{}
+
+type skywalkingPluginConfig struct {
+ SampleRatio float64 `json:"sample_ratio,omitempty"`
+}
+
+func (t *translator) TranslateClusterConfig(acc *configv2alpha1.ApisixClusterConfig) (*apisixv1.GlobalRule, error) {
+ globalRule := &apisixv1.GlobalRule{
+ ID: id.GenID(acc.Name),
+ Plugins: make(apisixv1.Plugins),
+ }
+
+ if acc.Spec.Monitoring != nil {
+ if acc.Spec.Monitoring.Prometheus.Enable {
+ globalRule.Plugins["prometheus"] = &prometheusPluginConfig{}
+ }
+ if acc.Spec.Monitoring.Skywalking.Enable {
+ globalRule.Plugins["skywalking"] = &skywalkingPluginConfig{
+ SampleRatio: acc.Spec.Monitoring.Skywalking.SampleRatio,
+ }
+ }
+ }
+
+ return globalRule, nil
+}
diff --git a/pkg/kube/translation/global_rule_test.go b/pkg/kube/translation/global_rule_test.go
new file mode 100644
index 0000000..313e649
--- /dev/null
+++ b/pkg/kube/translation/global_rule_test.go
@@ -0,0 +1,53 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package translation
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+ "github.com/apache/apisix-ingress-controller/pkg/id"
+ configv2alpha1 "github.com/apache/apisix-ingress-controller/pkg/kube/apisix/apis/config/v2alpha1"
+)
+
+func TestTranslateClusterConfig(t *testing.T) {
+ tr := &translator{}
+
+ acc := &configv2alpha1.ApisixClusterConfig{
+ TypeMeta: metav1.TypeMeta{},
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "qa-apisix",
+ },
+ Spec: configv2alpha1.ApisixClusterConfigSpec{
+ Monitoring: &configv2alpha1.ApisixClusterMonitoringConfig{
+ Prometheus: configv2alpha1.ApisixClusterPrometheusConfig{
+ Enable: true,
+ },
+ Skywalking: configv2alpha1.ApisixClusterSkywalkingConfig{
+ Enable: true,
+ SampleRatio: 0.5,
+ },
+ },
+ },
+ }
+ gr, err := tr.TranslateClusterConfig(acc)
+ assert.Nil(t, err, "translating ApisixClusterConfig")
+ assert.Equal(t, gr.ID, id.GenID("qa-apisix"), "checking global_rule id")
+ assert.Len(t, gr.Plugins, 2)
+ assert.Equal(t, gr.Plugins["prometheus"], &prometheusPluginConfig{})
+ assert.Equal(t, gr.Plugins["skywalking"], &skywalkingPluginConfig{SampleRatio: 0.5})
+}
diff --git a/pkg/kube/translation/translator.go b/pkg/kube/translation/translator.go
index 701d7dc..9742d2a 100644
--- a/pkg/kube/translation/translator.go
+++ b/pkg/kube/translation/translator.go
@@ -66,6 +66,9 @@ type Translator interface {
TranslateRouteV2alpha1(*configv2alpha1.ApisixRoute) (*TranslateContext, error)
// TranslateSSL translates the configv2alpha1.ApisixTls object into the APISIX SSL resource.
TranslateSSL(*configv1.ApisixTls) (*apisixv1.Ssl, error)
+ // TranslateClusterConfig translates the configv2alpha1.ApisixClusterConfig object into the APISIX
+ // Global Rule resource.
+ TranslateClusterConfig(config *configv2alpha1.ApisixClusterConfig) (*apisixv1.GlobalRule, error)
}
// TranslatorOptions contains options to help Translator
diff --git a/pkg/types/apisix/v1/types.go b/pkg/types/apisix/v1/types.go
index 7acbf92..846182b 100644
--- a/pkg/types/apisix/v1/types.go
+++ b/pkg/types/apisix/v1/types.go
@@ -25,7 +25,7 @@ import (
const (
// HashOnVars means the hash scope is variable.
HashOnVars = "vars"
- // HashVarsCombination means the hash scope is the
+ // HashOnVarsCombination means the hash scope is the
// variable combination.
HashOnVarsCombination = "vars_combinations"
// HashOnHeader means the hash scope is HTTP request
diff --git a/samples/deploy/rbac/apisix_view_clusterrole.yaml b/samples/deploy/rbac/apisix_view_clusterrole.yaml
index 572b063..a5f75be 100644
--- a/samples/deploy/rbac/apisix_view_clusterrole.yaml
+++ b/samples/deploy/rbac/apisix_view_clusterrole.yaml
@@ -138,6 +138,7 @@ rules:
- apisixroutes
- apisixupstreams
- apisixtlses
+ - apisixclusterconfigs
verbs:
- get
- list
diff --git a/test/e2e/features/global_rule.go b/test/e2e/features/global_rule.go
new file mode 100644
index 0000000..02b4440
--- /dev/null
+++ b/test/e2e/features/global_rule.go
@@ -0,0 +1,73 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package features
+
+import (
+ "time"
+
+ "github.com/apache/apisix-ingress-controller/pkg/id"
+ "github.com/apache/apisix-ingress-controller/test/e2e/scaffold"
+ "github.com/onsi/ginkgo"
+ "github.com/stretchr/testify/assert"
+)
+
+var _ = ginkgo.Describe("ApisixClusterConfig", func() {
+ opts := &scaffold.Options{
+ Name: "default",
+ Kubeconfig: scaffold.GetKubeconfig(),
+ APISIXConfigPath: "testdata/apisix-gw-config.yaml",
+ APISIXDefaultConfigPath: "testdata/apisix-gw-config-default.yaml",
+ IngressAPISIXReplicas: 1,
+ HTTPBinServicePort: 80,
+ APISIXRouteVersion: "apisix.apache.org/v2alpha1",
+ }
+ s := scaffold.NewScaffold(opts)
+ ginkgo.It("enable prometheus", func() {
+ acc := `
+apiVersion: apisix.apache.org/v2alpha1
+kind: ApisixClusterConfig
+metadata:
+ name: default
+spec:
+ monitoring:
+ prometheus:
+ enable: true
+`
+ err := s.CreateResourceFromString(acc)
+ assert.Nil(ginkgo.GinkgoT(), err, "creating ApisixClusterConfig")
+
+ defer func() {
+ err := s.RemoveResourceByString(acc)
+ assert.Nil(ginkgo.GinkgoT(), err)
+ }()
+
+ // Wait until the ApisixClusterConfig create event was delivered.
+ time.Sleep(3 * time.Second)
+
+ grs, err := s.ListApisixGlobalRules()
+ assert.Nil(ginkgo.GinkgoT(), err, "listing global_rules")
+ assert.Len(ginkgo.GinkgoT(), grs, 1)
+ assert.Equal(ginkgo.GinkgoT(), grs[0].ID, id.GenID("default"))
+ assert.Len(ginkgo.GinkgoT(), grs[0].Plugins, 1)
+ _, ok := grs[0].Plugins["prometheus"]
+ assert.Equal(ginkgo.GinkgoT(), ok, true)
+
+ resp := s.NewAPISIXClient().GET("/apisix/prometheus/metrics").Expect()
+ resp.Status(200)
+ resp.Body().Contains("# HELP apisix_etcd_modify_indexes Etcd modify index for APISIX keys")
+ resp.Body().Contains("# HELP apisix_etcd_reachable Config server etcd reachable from APISIX, 0 is unreachable")
+ resp.Body().Contains("# HELP apisix_node_info Info of APISIX node")
+ })
+})
diff --git a/test/e2e/scaffold/ingress.go b/test/e2e/scaffold/ingress.go
index 67f9343..3fdf36d 100644
--- a/test/e2e/scaffold/ingress.go
+++ b/test/e2e/scaffold/ingress.go
@@ -155,6 +155,7 @@ rules:
- apisixupstreams
- apisixservices
- apisixtlses
+ - apisixclusterconfigs
verbs:
- get
- list
@@ -243,7 +244,9 @@ spec:
- stdout
- --http-listen
- :8080
- - --apisix-base-url
+ - --default-apisix-cluster-name
+ - default
+ - --default-apisix-cluster-base-url
- http://apisix-service-e2e-test:9180/apisix/admin
- --app-namespace
- %s
diff --git a/test/e2e/scaffold/k8s.go b/test/e2e/scaffold/k8s.go
index 37f8868..f165d8a 100644
--- a/test/e2e/scaffold/k8s.go
+++ b/test/e2e/scaffold/k8s.go
@@ -193,6 +193,26 @@ func (s *Scaffold) ListApisixUpstreams() ([]*v1.Upstream, error) {
return cli.Cluster("").Upstream().List(context.TODO())
}
+// ListApisixGlobalRules list all global_rules from APISIX
+func (s *Scaffold) ListApisixGlobalRules() ([]*v1.GlobalRule, error) {
+ u := url.URL{
+ Scheme: "http",
+ Host: s.apisixAdminTunnel.Endpoint(),
+ Path: "/apisix/admin",
+ }
+ cli, err := apisix.NewClient()
+ if err != nil {
+ return nil, err
+ }
+ err = cli.AddCluster(&apisix.ClusterOptions{
+ BaseURL: u.String(),
+ })
+ if err != nil {
+ return nil, err
+ }
+ return cli.Cluster("").GlobalRule().List(context.TODO())
+}
+
// ListApisixRoutes list all routes from APISIX.
func (s *Scaffold) ListApisixRoutes() ([]*v1.Route, error) {
u := url.URL{