You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@apisix.apache.org by zh...@apache.org on 2022/06/28 10:06:07 UTC
[apisix-ingress-controller] branch master updated: feat: support gateway TLSRoute (#1087)
This is an automated email from the ASF dual-hosted git repository.
zhangjintao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix-ingress-controller.git
The following commit(s) were added to refs/heads/master by this push:
new b33d70c4 feat: support gateway TLSRoute (#1087)
b33d70c4 is described below
commit b33d70c429d32db6d2a065463d37a73ff59ab4ad
Author: Sarasa Kisaragi <li...@gmail.com>
AuthorDate: Tue Jun 28 18:05:59 2022 +0800
feat: support gateway TLSRoute (#1087)
---
pkg/ingress/gateway/gateway.go | 80 ++++++--
pkg/ingress/gateway/gateway_class.go | 50 +++--
pkg/ingress/gateway/gateway_httproute.go | 8 +-
.../{gateway_httproute.go => gateway_tlsroute.go} | 113 ++++++-----
pkg/ingress/gateway/provider.go | 97 ++++++++--
pkg/ingress/gateway/translation/gateway.go | 208 +++++++++++++++++++++
.../gateway/translation/gateway_httproute.go | 8 +
.../gateway/translation/gateway_tlsroute.go | 132 +++++++++++++
pkg/ingress/gateway/translation/translator.go | 5 +
.../{translation/translator.go => types/types.go} | 35 ++--
pkg/types/apisix/v1/types.go | 1 +
test/e2e/scaffold/apisix.go | 4 +
test/e2e/scaffold/ingress.go | 2 +
test/e2e/scaffold/k8s.go | 36 ++--
test/e2e/scaffold/scaffold.go | 39 +++-
test/e2e/suite-gateway/gateway_tlsroute.go | 206 ++++++++++++++++++++
test/e2e/testdata/apisix-gw-config.yaml | 2 +
17 files changed, 901 insertions(+), 125 deletions(-)
diff --git a/pkg/ingress/gateway/gateway.go b/pkg/ingress/gateway/gateway.go
index 58bea6bb..171ea14a 100644
--- a/pkg/ingress/gateway/gateway.go
+++ b/pkg/ingress/gateway/gateway.go
@@ -85,18 +85,26 @@ func (c *gatewayController) sync(ctx context.Context, ev *types.Event) error {
key := ev.Object.(string)
namespace, name, err := cache.SplitMetaNamespaceKey(key)
if err != nil {
- log.Errorf("found Gateway resource with invalid meta namespace key %s: %s", key, err)
+ log.Errorw("found Gateway resource with invalid meta namespace key",
+ zap.Error(err),
+ zap.String("key", key),
+ )
return err
}
gateway, err := c.controller.gatewayLister.Gateways(namespace).Get(name)
if err != nil {
if !k8serrors.IsNotFound(err) {
- log.Errorf("failed to get Gateway %s: %s", key, err)
+ log.Errorw("failed to get Gateway",
+ zap.Error(err),
+ zap.String("key", key),
+ )
return err
}
if ev.Type != types.EventDelete {
- log.Warnf("Gateway %s was deleted before it can be delivered", key)
+ log.Warnw("Gateway was deleted before it can be delivered",
+ zap.String("key", key),
+ )
// Don't need to retry.
return nil
}
@@ -107,14 +115,30 @@ func (c *gatewayController) sync(ctx context.Context, ev *types.Event) error {
// We still find the resource while we are processing the DELETE event,
// that means object with same namespace and name was created, discarding
// this stale DELETE event.
- log.Warnf("discard the stale Gateway delete event since the %s exists", key)
+ log.Warnw("discard the stale Gateway delete event since it exists",
+ zap.String("key", key),
+ )
return nil
}
gateway = ev.Tombstone.(*gatewayv1alpha2.Gateway)
- //} else {
- //if c.controller.HasGatewayClass(string(gateway.Spec.GatewayClassName)) {
- // // TODO: Translate listeners
- //}
+
+ err = c.controller.RemoveListeners(gateway.Namespace, gateway.Name)
+ if err != nil {
+ return err
+ }
+ } else {
+ if c.controller.HasGatewayClass(string(gateway.Spec.GatewayClassName)) {
+ // TODO: handle listeners
+ listeners, err := c.controller.translator.TranslateGatewayV1Alpha2(gateway)
+ if err != nil {
+ return err
+ }
+
+ err = c.controller.AddListeners(gateway.Namespace, gateway.Name, listeners)
+ if err != nil {
+ return err
+ }
+ }
}
// TODO The current implementation does not fully support the definition of Gateway.
@@ -152,7 +176,10 @@ func (c *gatewayController) handleSyncErr(obj interface{}, err error) {
func (c *gatewayController) onAdd(obj interface{}) {
key, err := cache.MetaNamespaceKeyFunc(obj)
if err != nil {
- log.Errorf("found gateway resource with bad meta namespace key: %s", err)
+ log.Errorw("found gateway resource with bad meta namespace key",
+ zap.Error(err),
+ zap.Any("obj", obj),
+ )
return
}
if !c.controller.NamespaceProvider.IsWatchingNamespace(key) {
@@ -167,8 +194,39 @@ func (c *gatewayController) onAdd(obj interface{}) {
Object: key,
})
}
-func (c *gatewayController) onUpdate(oldObj, newObj interface{}) {}
-func (c *gatewayController) OnDelete(obj interface{}) {}
+func (c *gatewayController) onUpdate(oldObj, newObj interface{}) {
+
+}
+
+func (c *gatewayController) OnDelete(obj interface{}) {
+ key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj)
+ if err != nil {
+ log.Errorw("failed to handle deletion Gateway meta key",
+ zap.Error(err),
+ zap.Any("obj", obj),
+ )
+ return
+ }
+
+ gateway, ok := obj.(*gatewayv1alpha2.Gateway)
+ if !ok {
+ tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
+ if !ok {
+ log.Errorw("Gateway in bad tombstone state",
+ zap.String("key", key),
+ zap.Any("obj", obj),
+ )
+ return
+ }
+ gateway = tombstone.Obj.(*gatewayv1alpha2.Gateway)
+ }
+
+ c.workqueue.Add(&types.Event{
+ Type: types.EventDelete,
+ Object: key,
+ Tombstone: gateway,
+ })
+}
// recordStatus record resources status
func (c *gatewayController) recordStatus(v *gatewayv1alpha2.Gateway, reason string, status metav1.ConditionStatus, generation int64) {
diff --git a/pkg/ingress/gateway/gateway_class.go b/pkg/ingress/gateway/gateway_class.go
index 00d2a4fb..829fd8e1 100644
--- a/pkg/ingress/gateway/gateway_class.go
+++ b/pkg/ingress/gateway/gateway_class.go
@@ -32,7 +32,7 @@ import (
)
const (
- GatewayClassName = "apisix-ingress-controller"
+ GatewayClassName = "apisix.apache.org/gateway-controller"
)
type gatewayClassController struct {
@@ -127,12 +127,12 @@ func (c *gatewayClassController) markAsUpdated(gatewayClass *v1alpha2.GatewayCla
}
func (c *gatewayClassController) run(ctx context.Context) {
- log.Info("gateway HTTPRoute controller started")
- defer log.Info("gateway HTTPRoute controller exited")
+ log.Info("GatewayClass controller started")
+ defer log.Info("GatewayClass controller exited")
defer c.workqueue.ShutDown()
if !cache.WaitForCacheSync(ctx.Done(), c.controller.gatewayClassInformer.HasSynced) {
- log.Error("sync Gateway HTTPRoute cache failed")
+ log.Error("sync GatewayClass cache failed")
return
}
@@ -155,8 +155,8 @@ func (c *gatewayClassController) runWorker(ctx context.Context) {
}
func (c *gatewayClassController) sync(ctx context.Context, ev *types.Event) error {
+ key := ev.Object.(string)
if ev.Type == types.EventAdd {
- key := ev.Object.(string)
gatewayClass, err := c.controller.gatewayClassLister.Get(key)
if err != nil {
return err
@@ -166,8 +166,7 @@ func (c *gatewayClassController) sync(ctx context.Context, ev *types.Event) erro
return c.markAsUpdated(gatewayClass)
}
} else if ev.Type == types.EventDelete {
- key := ev.Object.(string)
- c.controller.RemoveGatewayClass(key)
+ c.controller.RemoveGatewayClass(ev.Tombstone.(*v1alpha2.GatewayClass).Name)
}
return nil
@@ -181,14 +180,14 @@ func (c *gatewayClassController) handleSyncErr(obj interface{}, err error) {
}
event := obj.(*types.Event)
if k8serrors.IsNotFound(err) && event.Type != types.EventDelete {
- log.Infow("sync gateway HTTPRoute but not found, ignore",
+ log.Infow("sync gateway class but not found, ignore",
zap.String("event_type", event.Type.String()),
- zap.String("HTTPRoute ", event.Object.(string)),
+ zap.String("GatewayClass", event.Object.(string)),
)
c.workqueue.Forget(event)
return
}
- log.Warnw("sync gateway HTTPRoute failed, will retry",
+ log.Warnw("sync gateway class failed, will retry",
zap.Any("object", obj),
zap.Error(err),
)
@@ -199,13 +198,15 @@ func (c *gatewayClassController) handleSyncErr(obj interface{}, err error) {
func (c *gatewayClassController) onAdd(obj interface{}) {
key, err := cache.MetaNamespaceKeyFunc(obj)
if err != nil {
- log.Errorf("found gateway HTTPRoute resource with bad meta namespace key: %s", err)
+ log.Errorw("found gateway class resource with bad meta namespace key",
+ zap.Error(err),
+ )
return
}
if !c.controller.NamespaceProvider.IsWatchingNamespace(key) {
return
}
- log.Debugw("gateway HTTPRoute add event arrived",
+ log.Debugw("gateway class add event arrived",
zap.Any("object", obj),
)
@@ -220,10 +221,31 @@ func (c *gatewayClassController) onUpdate(oldObj, newObj interface{}) {
}
func (c *gatewayClassController) onDelete(obj interface{}) {
- gatewayClass := obj.(*v1alpha2.GatewayClass)
+ key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj)
+ if err != nil {
+ log.Errorw("failed to handle deletion GatewayClass meta key",
+ zap.Error(err),
+ zap.Any("obj", obj),
+ )
+ return
+ }
+
+ gatewayClass, ok := obj.(*v1alpha2.GatewayClass)
+ if !ok {
+ tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
+ if !ok {
+ log.Errorw("GatewayClass in bad tombstone state",
+ zap.String("key", key),
+ zap.Any("obj", obj),
+ )
+ return
+ }
+ gatewayClass = tombstone.Obj.(*v1alpha2.GatewayClass)
+ }
+
c.workqueue.Add(&types.Event{
Type: types.EventDelete,
- Object: gatewayClass.Name,
+ Object: key,
Tombstone: gatewayClass,
})
}
diff --git a/pkg/ingress/gateway/gateway_httproute.go b/pkg/ingress/gateway/gateway_httproute.go
index a7109240..3fb60ba5 100644
--- a/pkg/ingress/gateway/gateway_httproute.go
+++ b/pkg/ingress/gateway/gateway_httproute.go
@@ -115,7 +115,9 @@ func (c *gatewayHTTPRouteController) sync(ctx context.Context, ev *types.Event)
// We still find the resource while we are processing the DELETE event,
// that means object with same namespace and name was created, discarding
// this stale DELETE event.
- log.Warnf("discard the stale Gateway delete event since the %s exists", key)
+ log.Warnw("discard the stale Gateway delete event since it exists",
+ zap.String("key", key),
+ )
return nil
}
httpRoute = ev.Tombstone.(*gatewayv1alpha2.HTTPRoute)
@@ -200,7 +202,9 @@ func (c *gatewayHTTPRouteController) handleSyncErr(obj interface{}, err error) {
func (c *gatewayHTTPRouteController) onAdd(obj interface{}) {
key, err := cache.MetaNamespaceKeyFunc(obj)
if err != nil {
- log.Errorf("found gateway HTTPRoute resource with bad meta namespace key: %s", err)
+ log.Errorw("found gateway HTTPRoute resource with bad meta namespace key",
+ zap.Error(err),
+ )
return
}
if !c.controller.NamespaceProvider.IsWatchingNamespace(key) {
diff --git a/pkg/ingress/gateway/gateway_httproute.go b/pkg/ingress/gateway/gateway_tlsroute.go
similarity index 52%
copy from pkg/ingress/gateway/gateway_httproute.go
copy to pkg/ingress/gateway/gateway_tlsroute.go
index a7109240..52f6af05 100644
--- a/pkg/ingress/gateway/gateway_httproute.go
+++ b/pkg/ingress/gateway/gateway_tlsroute.go
@@ -1,3 +1,20 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
@@ -5,7 +22,7 @@
// (the "License"); you may not use this file except in compliance with
// the License. You may obtain a copy of the License at
//
-// http://www.apache.org/licenses/LICENSE-2.0
+// tls://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
@@ -30,20 +47,20 @@ import (
"github.com/apache/apisix-ingress-controller/pkg/types"
)
-type gatewayHTTPRouteController struct {
+type gatewayTLSRouteController struct {
controller *Provider
workqueue workqueue.RateLimitingInterface
workers int
}
-func newGatewayHTTPRouteController(c *Provider) *gatewayHTTPRouteController {
- ctrl := &gatewayHTTPRouteController{
+func newGatewayTLSRouteController(c *Provider) *gatewayTLSRouteController {
+ ctrl := &gatewayTLSRouteController{
controller: c,
- workqueue: workqueue.NewNamedRateLimitingQueue(workqueue.NewItemFastSlowRateLimiter(1*time.Second, 60*time.Second, 5), "GatewayHTTPRoute"),
+ workqueue: workqueue.NewNamedRateLimitingQueue(workqueue.NewItemFastSlowRateLimiter(1*time.Second, 60*time.Second, 5), "GatewayTLSRoute"),
workers: 1,
}
- ctrl.controller.gatewayHTTPRouteInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
+ ctrl.controller.gatewayTLSRouteInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: ctrl.onAdd,
UpdateFunc: ctrl.onUpdate,
DeleteFunc: ctrl.OnDelete,
@@ -51,13 +68,13 @@ func newGatewayHTTPRouteController(c *Provider) *gatewayHTTPRouteController {
return ctrl
}
-func (c *gatewayHTTPRouteController) run(ctx context.Context) {
- log.Info("gateway HTTPRoute controller started")
- defer log.Info("gateway HTTPRoute controller exited")
+func (c *gatewayTLSRouteController) run(ctx context.Context) {
+ log.Info("gateway TLSRoute controller started")
+ defer log.Info("gateway TLSRoute controller exited")
defer c.workqueue.ShutDown()
- if !cache.WaitForCacheSync(ctx.Done(), c.controller.gatewayHTTPRouteInformer.HasSynced) {
- log.Error("sync Gateway HTTPRoute cache failed")
+ if !cache.WaitForCacheSync(ctx.Done(), c.controller.gatewayTLSRouteInformer.HasSynced) {
+ log.Error("sync Gateway TLSRoute cache failed")
return
}
@@ -67,7 +84,7 @@ func (c *gatewayHTTPRouteController) run(ctx context.Context) {
<-ctx.Done()
}
-func (c *gatewayHTTPRouteController) runWorker(ctx context.Context) {
+func (c *gatewayTLSRouteController) runWorker(ctx context.Context) {
for {
obj, quit := c.workqueue.Get()
if quit {
@@ -79,30 +96,30 @@ func (c *gatewayHTTPRouteController) runWorker(ctx context.Context) {
}
}
-func (c *gatewayHTTPRouteController) sync(ctx context.Context, ev *types.Event) error {
+func (c *gatewayTLSRouteController) sync(ctx context.Context, ev *types.Event) error {
key := ev.Object.(string)
namespace, name, err := cache.SplitMetaNamespaceKey(key)
if err != nil {
- log.Errorw("found Gateway HTTPRoute resource with invalid key",
+ log.Errorw("found Gateway TLSRoute resource with invalid key",
zap.Error(err),
zap.String("key", key),
)
return err
}
- log.Debugw("sync HTTPRoute", zap.String("key", key))
+ log.Debugw("sync TLSRoute", zap.String("key", key))
- httpRoute, err := c.controller.gatewayHTTPRouteLister.HTTPRoutes(namespace).Get(name)
+ tlsRoute, err := c.controller.gatewayTLSRouteLister.TLSRoutes(namespace).Get(name)
if err != nil {
if !k8serrors.IsNotFound(err) {
- log.Errorw("failed to get Gateway HTTPRoute",
+ log.Errorw("failed to get Gateway TLSRoute",
zap.Error(err),
zap.String("key", key),
)
return err
}
if ev.Type != types.EventDelete {
- log.Warnw("Gateway HTTPRoute was deleted before process",
+ log.Warnw("Gateway TLSRoute was deleted before process",
zap.String("key", key),
)
// Don't need to retry.
@@ -111,33 +128,35 @@ func (c *gatewayHTTPRouteController) sync(ctx context.Context, ev *types.Event)
}
if ev.Type == types.EventDelete {
- if httpRoute != nil {
+ if tlsRoute != nil {
// We still find the resource while we are processing the DELETE event,
// that means object with same namespace and name was created, discarding
// this stale DELETE event.
- log.Warnf("discard the stale Gateway delete event since the %s exists", key)
+ log.Warnw("discard the stale Gateway delete event since it exists",
+ zap.String("key", key),
+ )
return nil
}
- httpRoute = ev.Tombstone.(*gatewayv1alpha2.HTTPRoute)
+ tlsRoute = ev.Tombstone.(*gatewayv1alpha2.TLSRoute)
}
- tctx, err := c.controller.translator.TranslateGatewayHTTPRouteV1Alpha2(httpRoute)
+ tctx, err := c.controller.translator.TranslateGatewayTLSRouteV1Alpha2(tlsRoute)
if err != nil {
- log.Errorw("failed to translate gateway HTTPRoute",
+ log.Warnw("failed to translate gateway TLSRoute",
zap.Error(err),
- zap.Any("object", httpRoute),
+ zap.Any("object", tlsRoute),
)
return err
}
- log.Debugw("translated HTTPRoute",
- zap.Any("routes", tctx.Routes),
+ log.Debugw("translated TLSRoute",
+ zap.Any("stream_routes", tctx.StreamRoutes),
zap.Any("upstreams", tctx.Upstreams),
)
m := &utils.Manifest{
- Routes: tctx.Routes,
- Upstreams: tctx.Upstreams,
+ StreamRoutes: tctx.StreamRoutes,
+ Upstreams: tctx.Upstreams,
}
var (
@@ -152,21 +171,21 @@ func (c *gatewayHTTPRouteController) sync(ctx context.Context, ev *types.Event)
added = m
} else {
var oldCtx *translation.TranslateContext
- oldObj := ev.OldObject.(*gatewayv1alpha2.HTTPRoute)
- oldCtx, err = c.controller.translator.TranslateGatewayHTTPRouteV1Alpha2(oldObj)
+ oldObj := ev.OldObject.(*gatewayv1alpha2.TLSRoute)
+ oldCtx, err = c.controller.translator.TranslateGatewayTLSRouteV1Alpha2(oldObj)
if err != nil {
- log.Errorw("failed to translate old HTTPRoute",
+ log.Errorw("failed to translate old TLSRoute",
zap.String("version", oldObj.APIVersion),
zap.String("event_type", "update"),
- zap.Any("HTTPRoute", oldObj),
+ zap.Any("TLSRoute", oldObj),
zap.Error(err),
)
return err
}
om := &utils.Manifest{
- Routes: oldCtx.Routes,
- Upstreams: oldCtx.Upstreams,
+ StreamRoutes: oldCtx.StreamRoutes,
+ Upstreams: oldCtx.Upstreams,
}
added, updated, deleted = m.Diff(om)
}
@@ -174,47 +193,49 @@ func (c *gatewayHTTPRouteController) sync(ctx context.Context, ev *types.Event)
return utils.SyncManifests(ctx, c.controller.APISIX, c.controller.APISIXClusterName, added, updated, deleted)
}
-func (c *gatewayHTTPRouteController) handleSyncErr(obj interface{}, err error) {
+func (c *gatewayTLSRouteController) handleSyncErr(obj interface{}, err error) {
if err == nil {
c.workqueue.Forget(obj)
- c.controller.MetricsCollector.IncrSyncOperation("gateway_httproute", "success")
+ c.controller.MetricsCollector.IncrSyncOperation("gateway_tlsroute", "success")
return
}
event := obj.(*types.Event)
if k8serrors.IsNotFound(err) && event.Type != types.EventDelete {
- log.Infow("sync gateway HTTPRoute but not found, ignore",
+ log.Infow("sync gateway TLSRoute but not found, ignore",
zap.String("event_type", event.Type.String()),
- zap.String("HTTPRoute ", event.Object.(string)),
+ zap.String("TLSRoute ", event.Object.(string)),
)
c.workqueue.Forget(event)
return
}
- log.Warnw("sync gateway HTTPRoute failed, will retry",
+ log.Warnw("sync gateway TLSRoute failed, will retry",
zap.Any("object", obj),
zap.Error(err),
)
c.workqueue.AddRateLimited(obj)
- c.controller.MetricsCollector.IncrSyncOperation("gateway_httproute", "failure")
+ c.controller.MetricsCollector.IncrSyncOperation("gateway_tlsroute", "failure")
}
-func (c *gatewayHTTPRouteController) onAdd(obj interface{}) {
+func (c *gatewayTLSRouteController) onAdd(obj interface{}) {
key, err := cache.MetaNamespaceKeyFunc(obj)
if err != nil {
- log.Errorf("found gateway HTTPRoute resource with bad meta namespace key: %s", err)
+ log.Errorw("found gateway TLSRoute resource with bad meta namespace key",
+ zap.Error(err),
+ )
return
}
if !c.controller.NamespaceProvider.IsWatchingNamespace(key) {
return
}
- log.Debugw("gateway HTTPRoute add event arrived",
+ log.Debugw("gateway TLSRoute add event arrived",
zap.Any("object", obj),
)
- log.Debugw("add HTTPRoute", zap.String("key", key))
+ log.Debugw("add TLSRoute", zap.String("key", key))
c.workqueue.Add(&types.Event{
Type: types.EventAdd,
Object: key,
})
}
-func (c *gatewayHTTPRouteController) onUpdate(oldObj, newObj interface{}) {}
-func (c *gatewayHTTPRouteController) OnDelete(obj interface{}) {}
+func (c *gatewayTLSRouteController) onUpdate(oldObj, newObj interface{}) {}
+func (c *gatewayTLSRouteController) OnDelete(obj interface{}) {}
diff --git a/pkg/ingress/gateway/provider.go b/pkg/ingress/gateway/provider.go
index 0e87a82c..ee3cb1b0 100644
--- a/pkg/ingress/gateway/provider.go
+++ b/pkg/ingress/gateway/provider.go
@@ -19,11 +19,13 @@ package gateway
import (
"context"
+ "fmt"
"sync"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/cache"
+ gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
gatewayclientset "sigs.k8s.io/gateway-api/pkg/client/clientset/gateway/versioned"
gatewayexternalversions "sigs.k8s.io/gateway-api/pkg/client/informers/gateway/externalversions"
gatewaylistersv1alpha2 "sigs.k8s.io/gateway-api/pkg/client/listers/gateway/apis/v1alpha2"
@@ -31,6 +33,7 @@ import (
"github.com/apache/apisix-ingress-controller/pkg/apisix"
"github.com/apache/apisix-ingress-controller/pkg/config"
gatewaytranslation "github.com/apache/apisix-ingress-controller/pkg/ingress/gateway/translation"
+ "github.com/apache/apisix-ingress-controller/pkg/ingress/gateway/types"
"github.com/apache/apisix-ingress-controller/pkg/ingress/namespace"
"github.com/apache/apisix-ingress-controller/pkg/ingress/utils"
"github.com/apache/apisix-ingress-controller/pkg/kube"
@@ -45,8 +48,14 @@ const (
type Provider struct {
name string
- gatewayNamesLock sync.RWMutex
- gatewayNames map[string]struct{}
+ gatewayClassesLock sync.RWMutex
+ // key is "name" of GatewayClass
+ gatewayClasses map[string]struct{}
+
+ listenersLock sync.RWMutex
+ // meta key ("ns/name") of Gateway -> section name -> ListenerConf
+ listeners map[string]map[string]*types.ListenerConf
+ portListeners map[gatewayv1alpha2.PortNumber]*types.ListenerConf
*ProviderOptions
gatewayClient gatewayclientset.Interface
@@ -64,6 +73,10 @@ type Provider struct {
gatewayHTTPRouteController *gatewayHTTPRouteController
gatewayHTTPRouteInformer cache.SharedIndexInformer
gatewayHTTPRouteLister gatewaylistersv1alpha2.HTTPRouteLister
+
+ gatewayTLSRouteController *gatewayTLSRouteController
+ gatewayTLSRouteInformer cache.SharedIndexInformer
+ gatewayTLSRouteLister gatewaylistersv1alpha2.TLSRouteLister
}
type ProviderOptions struct {
@@ -95,6 +108,11 @@ func NewGatewayProvider(opts *ProviderOptions) (*Provider, error) {
p := &Provider{
name: ProviderName,
+ gatewayClasses: make(map[string]struct{}),
+
+ listeners: make(map[string]map[string]*types.ListenerConf),
+ portListeners: make(map[gatewayv1alpha2.PortNumber]*types.ListenerConf),
+
ProviderOptions: opts,
gatewayClient: gatewayKubeClient,
@@ -114,6 +132,9 @@ func NewGatewayProvider(opts *ProviderOptions) (*Provider, error) {
p.gatewayHTTPRouteLister = gatewayFactory.Gateway().V1alpha2().HTTPRoutes().Lister()
p.gatewayHTTPRouteInformer = gatewayFactory.Gateway().V1alpha2().HTTPRoutes().Informer()
+ p.gatewayTLSRouteLister = gatewayFactory.Gateway().V1alpha2().TLSRoutes().Lister()
+ p.gatewayTLSRouteInformer = gatewayFactory.Gateway().V1alpha2().TLSRoutes().Informer()
+
p.gatewayController = newGatewayController(p)
p.gatewayClassController, err = newGatewayClassController(p)
@@ -122,6 +143,7 @@ func NewGatewayProvider(opts *ProviderOptions) (*Provider, error) {
}
p.gatewayHTTPRouteController = newGatewayHTTPRouteController(p)
+ p.gatewayTLSRouteController = newGatewayTLSRouteController(p)
return p, nil
}
@@ -141,6 +163,10 @@ func (p *Provider) Run(ctx context.Context) {
p.gatewayHTTPRouteInformer.Run(ctx.Done())
})
+ e.Add(func() {
+ p.gatewayTLSRouteInformer.Run(ctx.Done())
+ })
+
e.Add(func() {
p.gatewayController.run(ctx)
})
@@ -153,27 +179,74 @@ func (p *Provider) Run(ctx context.Context) {
p.gatewayHTTPRouteController.run(ctx)
})
+ e.Add(func() {
+ p.gatewayTLSRouteController.run(ctx)
+ })
+
e.Wait()
}
func (p *Provider) AddGatewayClass(name string) {
- p.gatewayNamesLock.Lock()
- defer p.gatewayNamesLock.Unlock()
-
- p.gatewayNames[name] = struct{}{}
+ p.gatewayClassesLock.Lock()
+ defer p.gatewayClassesLock.Unlock()
+ p.gatewayClasses[name] = struct{}{}
}
+
func (p *Provider) RemoveGatewayClass(name string) {
- p.gatewayNamesLock.Lock()
- defer p.gatewayNamesLock.Unlock()
+ p.gatewayClassesLock.Lock()
+ defer p.gatewayClassesLock.Unlock()
- delete(p.gatewayNames, name)
+ delete(p.gatewayClasses, name)
}
func (p *Provider) HasGatewayClass(name string) bool {
- p.gatewayNamesLock.RLock()
- defer p.gatewayNamesLock.RUnlock()
+ p.gatewayClassesLock.RLock()
+ defer p.gatewayClassesLock.RUnlock()
- _, ok := p.gatewayNames[name]
+ _, ok := p.gatewayClasses[name]
return ok
}
+
+func (p *Provider) AddListeners(ns, name string, listeners map[string]*types.ListenerConf) error {
+ p.listenersLock.Lock()
+ defer p.listenersLock.Unlock()
+
+ key := ns + "/" + name
+
+ // Check port conflicts
+ for _, listenerConf := range listeners {
+ if allocated, found := p.portListeners[listenerConf.Port]; found {
+ // TODO: support multi-error
+ return fmt.Errorf("port %d already allocated by %s/%s section %s",
+ listenerConf.Port, allocated.Namespace, allocated.Name, allocated.SectionName)
+ }
+ }
+
+ previousListeners, ok := p.listeners[key]
+ if ok {
+ // remove previous listeners
+ for _, listenerConf := range previousListeners {
+ delete(p.portListeners, listenerConf.Port)
+ }
+ }
+
+ // save data
+ p.listeners[key] = listeners
+
+ for _, listenerConf := range listeners {
+ p.portListeners[listenerConf.Port] = listenerConf
+ }
+
+ return nil
+}
+
+func (p *Provider) RemoveListeners(ns, name string) error {
+
+ return nil
+}
+
+func (p *Provider) FindListener(ns, name, sectionName string) (*types.ListenerConf, error) {
+
+ return nil, nil
+}
diff --git a/pkg/ingress/gateway/translation/gateway.go b/pkg/ingress/gateway/translation/gateway.go
new file mode 100644
index 00000000..edc79854
--- /dev/null
+++ b/pkg/ingress/gateway/translation/gateway.go
@@ -0,0 +1,208 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+package gateway_translation
+
+import (
+ "errors"
+
+ "go.uber.org/zap"
+ gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
+
+ "github.com/apache/apisix-ingress-controller/pkg/ingress/gateway/types"
+ "github.com/apache/apisix-ingress-controller/pkg/log"
+)
+
+const (
+ kindTCPRoute gatewayv1alpha2.Kind = "TCPRoute"
+ kindTLSRoute gatewayv1alpha2.Kind = "TLSRoute"
+ kindHTTPRoute gatewayv1alpha2.Kind = "HTTPRoute"
+)
+
+func (t *translator) TranslateGatewayV1Alpha2(gateway *gatewayv1alpha2.Gateway) (map[string]*types.ListenerConf, error) {
+ listeners := make(map[string]*types.ListenerConf)
+
+ for i, listener := range gateway.Spec.Listeners {
+ allowedKinds, err := getAllowedKinds(listener)
+ if err != nil {
+ return nil, err
+ }
+ if len(allowedKinds) == 0 {
+ log.Warnw("listener allowed kinds is empty",
+ zap.String("gateway", gateway.Name),
+ zap.String("namespace", gateway.Namespace),
+ zap.Int("listener_index", i),
+ )
+ continue
+ }
+
+ err = validateListenerConfigurations(gateway, i, allowedKinds, listener)
+ if err != nil {
+ // TODO: Update CRD status
+ log.Warnw("invalid listener conf",
+ zap.Error(err),
+ zap.String("gateway", gateway.Name),
+ zap.String("namespace", gateway.Namespace),
+ zap.Int("listener_index", i),
+ )
+ continue
+ }
+
+ conf := &types.ListenerConf{
+ Namespace: gateway.Namespace,
+ Name: gateway.Name,
+ SectionName: string(listener.Name),
+ Protocol: listener.Protocol,
+ Port: listener.Port,
+ RouteNamespace: nil,
+ AllowedKinds: allowedKinds,
+ }
+
+ if listener.AllowedRoutes.Namespaces != nil {
+ conf.RouteNamespace = listener.AllowedRoutes.Namespaces
+ }
+
+ listeners[conf.SectionName] = conf
+ }
+
+ return listeners, nil
+}
+
+func validateListenerConfigurations(gateway *gatewayv1alpha2.Gateway, idx int, allowedKinds []gatewayv1alpha2.RouteGroupKind,
+ listener gatewayv1alpha2.Listener) error {
+ // Check protocols and allowedKinds
+ protocol := listener.Protocol
+ if protocol == gatewayv1alpha2.HTTPProtocolType || protocol == gatewayv1alpha2.TCPProtocolType {
+ // Non-TLS
+ if listener.TLS != nil {
+ return errors.New("non-empty TLS conf for protocol " + string(protocol))
+ }
+ if protocol == gatewayv1alpha2.HTTPProtocolType {
+ if len(allowedKinds) != 1 || allowedKinds[0].Kind != kindHTTPRoute {
+ return errors.New("HTTP protocol must allow route type HTTPRoute")
+ }
+ } else if protocol == gatewayv1alpha2.TCPProtocolType {
+ if len(allowedKinds) != 1 || allowedKinds[0].Kind != kindTCPRoute {
+ return errors.New("TCP protocol must allow route type TCPRoute")
+ }
+ }
+ } else if protocol == gatewayv1alpha2.HTTPSProtocolType || protocol == gatewayv1alpha2.TLSProtocolType {
+ // TLS
+ if listener.TLS == nil {
+ return errors.New("empty TLS conf for protocol " + string(protocol))
+ }
+
+ if *listener.TLS.Mode == gatewayv1alpha2.TLSModeTerminate {
+ if len(listener.TLS.CertificateRefs) == 0 {
+ return errors.New("TLS mode Terminate requires CertificateRefs")
+ }
+
+ if len(listener.TLS.CertificateRefs) > 1 {
+ log.Warnw("only the first CertificateRefs take effect",
+ zap.String("gateway", gateway.Name),
+ zap.String("namespace", gateway.Namespace),
+ zap.Int("listener_index", idx),
+ )
+ }
+ } else {
+ if len(listener.TLS.CertificateRefs) != 0 {
+ log.Warnw("no CertificateRefs will take effect in non-terminate TLS mode",
+ zap.String("gateway", gateway.Name),
+ zap.String("namespace", gateway.Namespace),
+ zap.Int("listener_index", idx),
+ )
+ }
+ }
+
+ if protocol == gatewayv1alpha2.HTTPSProtocolType {
+ if *listener.TLS.Mode != gatewayv1alpha2.TLSModeTerminate {
+ return errors.New("TLS mode for HTTPS protocol must be Terminate")
+ }
+ if len(allowedKinds) != 1 || allowedKinds[0].Kind != kindHTTPRoute {
+ return errors.New("HTTP protocol must allow route type HTTPRoute")
+ }
+ } else if protocol == gatewayv1alpha2.TLSProtocolType {
+ for _, kind := range allowedKinds {
+ if kind.Kind != kindTLSRoute && kind.Kind != kindTCPRoute {
+ return errors.New("TLS protocol only support route type TLSRoute and TCPRoute")
+ }
+ }
+ }
+ }
+
+ return nil
+}
+
+func getAllowedKinds(listener gatewayv1alpha2.Listener) ([]gatewayv1alpha2.RouteGroupKind, error) {
+ var expectedKinds []gatewayv1alpha2.RouteGroupKind
+ group := gatewayv1alpha2.Group(gatewayv1alpha2.GroupName)
+ switch listener.Protocol {
+ case gatewayv1alpha2.HTTPProtocolType, gatewayv1alpha2.HTTPSProtocolType:
+ expectedKinds = []gatewayv1alpha2.RouteGroupKind{
+ {
+ Group: &group,
+ Kind: kindHTTPRoute,
+ },
+ }
+ case gatewayv1alpha2.TLSProtocolType:
+ expectedKinds = []gatewayv1alpha2.RouteGroupKind{
+ {
+ Group: &group,
+ Kind: kindTLSRoute,
+ },
+ {
+ Group: &group,
+ Kind: kindTCPRoute,
+ },
+ }
+ case gatewayv1alpha2.TCPProtocolType:
+ expectedKinds = []gatewayv1alpha2.RouteGroupKind{
+ {
+ Group: &group,
+ Kind: kindTCPRoute,
+ },
+ }
+ default:
+ return nil, errors.New("unknown protocol " + string(listener.Protocol))
+ }
+
+ if listener.AllowedRoutes == nil || len(listener.AllowedRoutes.Kinds) == 0 {
+ return expectedKinds, nil
+ }
+
+ uniqueAllowedKinds := make(map[gatewayv1alpha2.Kind]struct{})
+ var allowedKinds []gatewayv1alpha2.RouteGroupKind
+
+ for _, kind := range listener.AllowedRoutes.Kinds {
+ expected := false
+ for _, expectedKind := range expectedKinds {
+ if kind.Kind == expectedKind.Kind &&
+ kind.Group != nil && *kind.Group == *expectedKind.Group {
+ expected = true
+ break
+ }
+ }
+ if expected {
+ if _, ok := uniqueAllowedKinds[kind.Kind]; !ok {
+ uniqueAllowedKinds[kind.Kind] = struct{}{}
+ allowedKinds = append(allowedKinds, kind)
+ }
+ }
+ }
+
+ return allowedKinds, nil
+}
diff --git a/pkg/ingress/gateway/translation/gateway_httproute.go b/pkg/ingress/gateway/translation/gateway_httproute.go
index 3868bafb..ed73a993 100644
--- a/pkg/ingress/gateway/translation/gateway_httproute.go
+++ b/pkg/ingress/gateway/translation/gateway_httproute.go
@@ -38,6 +38,14 @@ func (t *translator) TranslateGatewayHTTPRouteV1Alpha2(httpRoute *gatewayv1alpha
var hosts []string
for _, hostname := range httpRoute.Spec.Hostnames {
hosts = append(hosts, string(hostname))
+
+ // TODO: See the document of gatewayv1alpha2.Listener.Hostname
+ _ = gatewayv1alpha2.Listener{}.Hostname
+ // For HTTPRoute and TLSRoute resources, there is an interaction with the
+ // `spec.hostnames` array. When both listener and route specify hostnames,
+ // there MUST be an intersection between the values for a Route to be
+ // accepted. For more information, refer to the Route specific Hostnames
+ // documentation.
}
rules := httpRoute.Spec.Rules
diff --git a/pkg/ingress/gateway/translation/gateway_tlsroute.go b/pkg/ingress/gateway/translation/gateway_tlsroute.go
new file mode 100644
index 00000000..085cb92b
--- /dev/null
+++ b/pkg/ingress/gateway/translation/gateway_tlsroute.go
@@ -0,0 +1,132 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+package gateway_translation
+
+import (
+ "fmt"
+ "strings"
+
+ "github.com/pkg/errors"
+ "go.uber.org/zap"
+ gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
+
+ "github.com/apache/apisix-ingress-controller/pkg/id"
+ "github.com/apache/apisix-ingress-controller/pkg/ingress/utils"
+ "github.com/apache/apisix-ingress-controller/pkg/kube/translation"
+ "github.com/apache/apisix-ingress-controller/pkg/log"
+ apisixv1 "github.com/apache/apisix-ingress-controller/pkg/types/apisix/v1"
+)
+
+func (t *translator) TranslateGatewayTLSRouteV1Alpha2(tlsRoute *gatewayv1alpha2.TLSRoute) (*translation.TranslateContext, error) {
+ ctx := translation.DefaultEmptyTranslateContext()
+
+ // TODO: Handle ParentRefs
+
+ var hosts []string
+ for _, hostname := range tlsRoute.Spec.Hostnames {
+ // TODO: calculate intersection of listeners
+ hosts = append(hosts, string(hostname))
+ }
+
+ rules := tlsRoute.Spec.Rules
+
+ for i, rule := range rules {
+ backends := rule.BackendRefs
+ if len(backends) == 0 {
+ continue
+ }
+
+ var ruleUpstreams []*apisixv1.Upstream
+
+ for j, backend := range backends {
+ //TODO: Support filters
+ //filters := backend.Filters
+ var kind string
+ if backend.Kind == nil {
+ kind = "service"
+ } else {
+ kind = strings.ToLower(string(*backend.Kind))
+ }
+ if kind != "service" {
+ log.Warnw(fmt.Sprintf("ignore non-service kind at Rules[%v].BackendRefs[%v]", i, j),
+ zap.String("kind", kind),
+ )
+ continue
+ }
+
+ var ns string
+ if backend.Namespace == nil {
+ ns = tlsRoute.Namespace
+ } else {
+ ns = string(*backend.Namespace)
+ }
+ //if ns != tlsRoute.Namespace {
+ // TODO: check gatewayv1alpha2.ReferencePolicy
+ //}
+
+ if backend.Port == nil {
+ log.Warnw(fmt.Sprintf("ignore nil port at Rules[%v].BackendRefs[%v]", i, j),
+ zap.String("kind", kind),
+ )
+ continue
+ }
+
+ ups, err := t.KubeTranslator.TranslateUpstream(ns, string(backend.Name), "", int32(*backend.Port))
+ if err != nil {
+ return nil, errors.Wrap(err, fmt.Sprintf("failed to translate Rules[%v].BackendRefs[%v]", i, j))
+ }
+ name := apisixv1.ComposeUpstreamName(ns, string(backend.Name), "", int32(*backend.Port))
+
+ ups.Labels["meta_namespace"] = utils.TruncateString(ns, 64)
+ ups.Labels["meta_backend"] = utils.TruncateString(string(backend.Name), 64)
+ ups.Labels["meta_port"] = fmt.Sprintf("%v", int32(*backend.Port))
+
+ ups.ID = id.GenID(name)
+ ctx.AddUpstream(ups)
+ ruleUpstreams = append(ruleUpstreams, ups)
+ }
+ if len(ruleUpstreams) == 0 {
+ log.Warnw(fmt.Sprintf("ignore all-failed backend refs at Rules[%v]", i),
+ zap.Any("BackendRefs", rule.BackendRefs),
+ )
+ continue
+ }
+
+ for _, host := range hosts {
+ route := apisixv1.NewDefaultStreamRoute()
+ name := apisixv1.ComposeRouteName(tlsRoute.Namespace, tlsRoute.Name, fmt.Sprintf("%d-%s", i, host))
+ route.ID = id.GenID(name)
+
+ route.Labels["meta_namespace"] = utils.TruncateString(tlsRoute.Namespace, 64)
+ route.Labels["meta_tlsroute"] = utils.TruncateString(tlsRoute.Name, 64)
+
+ route.SNI = host
+
+ route.UpstreamId = ruleUpstreams[0].ID
+ if len(ruleUpstreams) > 1 {
+ log.Warnw("ignore backends which is not the first one",
+ zap.String("namespace", tlsRoute.Namespace),
+ zap.String("tlsroute", tlsRoute.Name),
+ )
+ }
+ ctx.AddStreamRoute(route)
+ }
+ }
+
+ return ctx, nil
+}
diff --git a/pkg/ingress/gateway/translation/translator.go b/pkg/ingress/gateway/translation/translator.go
index 3a8a5fc3..4ef17e9e 100644
--- a/pkg/ingress/gateway/translation/translator.go
+++ b/pkg/ingress/gateway/translation/translator.go
@@ -20,6 +20,7 @@ package gateway_translation
import (
gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
+ "github.com/apache/apisix-ingress-controller/pkg/ingress/gateway/types"
"github.com/apache/apisix-ingress-controller/pkg/kube/translation"
)
@@ -32,8 +33,12 @@ type translator struct {
}
type Translator interface {
+ // TranslateGatewayV1Alpha2 translates Gateway to internal configurations
+ TranslateGatewayV1Alpha2(gateway *gatewayv1alpha2.Gateway) (map[string]*types.ListenerConf, error)
// TranslateGatewayHTTPRouteV1Alpha2 translates Gateway API HTTPRoute to APISIX resources
TranslateGatewayHTTPRouteV1Alpha2(httpRoute *gatewayv1alpha2.HTTPRoute) (*translation.TranslateContext, error)
+ // TranslateGatewayTLSRouteV1Alpha2 translates Gateway API TLSRoute to APISIX resources
+ TranslateGatewayTLSRouteV1Alpha2(tlsRoute *gatewayv1alpha2.TLSRoute) (*translation.TranslateContext, error)
}
// NewTranslator initializes a APISIX CRD resources Translator.
diff --git a/pkg/ingress/gateway/translation/translator.go b/pkg/ingress/gateway/types/types.go
similarity index 53%
copy from pkg/ingress/gateway/translation/translator.go
copy to pkg/ingress/gateway/types/types.go
index 3a8a5fc3..bdf014ef 100644
--- a/pkg/ingress/gateway/translation/translator.go
+++ b/pkg/ingress/gateway/types/types.go
@@ -15,30 +15,21 @@
// specific language governing permissions and limitations
// under the License.
//
-package gateway_translation
+package types
-import (
- gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
+import gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
- "github.com/apache/apisix-ingress-controller/pkg/kube/translation"
-)
+type ListenerConf struct {
+ // Gateway namespace
+ Namespace string
+ // Gateway name
+ Name string
-type TranslatorOptions struct {
- KubeTranslator translation.Translator
-}
-
-type translator struct {
- *TranslatorOptions
-}
-
-type Translator interface {
- // TranslateGatewayHTTPRouteV1Alpha2 translates Gateway API HTTPRoute to APISIX resources
- TranslateGatewayHTTPRouteV1Alpha2(httpRoute *gatewayv1alpha2.HTTPRoute) (*translation.TranslateContext, error)
-}
+ SectionName string
+ Protocol gatewayv1alpha2.ProtocolType
+ Port gatewayv1alpha2.PortNumber
-// NewTranslator initializes a APISIX CRD resources Translator.
-func NewTranslator(opts *TranslatorOptions) Translator {
- return &translator{
- TranslatorOptions: opts,
- }
+ // namespace selector of AllowedRoutes
+ RouteNamespace *gatewayv1alpha2.RouteNamespaces
+ AllowedKinds []gatewayv1alpha2.RouteGroupKind
}
diff --git a/pkg/types/apisix/v1/types.go b/pkg/types/apisix/v1/types.go
index 8bd7425a..3da9d462 100644
--- a/pkg/types/apisix/v1/types.go
+++ b/pkg/types/apisix/v1/types.go
@@ -334,6 +334,7 @@ type StreamRoute struct {
Desc string `json:"desc,omitempty" yaml:"desc,omitempty"`
Labels map[string]string `json:"labels,omitempty" yaml:"labels,omitempty"`
ServerPort int32 `json:"server_port,omitempty" yaml:"server_port,omitempty"`
+ SNI string `json:"sni,omitempty" yaml:"sni,omitempty"`
UpstreamId string `json:"upstream_id,omitempty" yaml:"upstream_id,omitempty"`
Upstream *Upstream `json:"upstream,omitempty" yaml:"upstream,omitempty"`
}
diff --git a/test/e2e/scaffold/apisix.go b/test/e2e/scaffold/apisix.go
index f72c7943..2487f9ab 100644
--- a/test/e2e/scaffold/apisix.go
+++ b/test/e2e/scaffold/apisix.go
@@ -119,6 +119,10 @@ spec:
port: 9100
protocol: TCP
targetPort: 9100
+ - name: tcp-tls
+ port: 9110
+ protocol: TCP
+ targetPort: 9110
- name: udp
port: 9200
protocol: UDP
diff --git a/test/e2e/scaffold/ingress.go b/test/e2e/scaffold/ingress.go
index 13cd350b..d9511be3 100644
--- a/test/e2e/scaffold/ingress.go
+++ b/test/e2e/scaffold/ingress.go
@@ -189,7 +189,9 @@ rules:
- gateway.networking.k8s.io
resources:
- httproutes
+ - tlsroutes
- gateways
+ - gatewayclasses
verbs:
- get
- list
diff --git a/test/e2e/scaffold/k8s.go b/test/e2e/scaffold/k8s.go
index e9c6cbb1..14a5dceb 100644
--- a/test/e2e/scaffold/k8s.go
+++ b/test/e2e/scaffold/k8s.go
@@ -497,18 +497,20 @@ func (s *Scaffold) ListApisixPluginConfig() ([]*v1.PluginConfig, error) {
func (s *Scaffold) newAPISIXTunnels() error {
var (
- adminNodePort int
- httpNodePort int
- httpsNodePort int
- tcpNodePort int
- udpNodePort int
- controlNodePort int
- adminPort int
- httpPort int
- httpsPort int
- tcpPort int
- udpPort int
- controlPort int
+ adminNodePort int
+ httpNodePort int
+ httpsNodePort int
+ tcpNodePort int
+ tlsOverTcpNodePort int
+ udpNodePort int
+ controlNodePort int
+ adminPort int
+ httpPort int
+ httpsPort int
+ tcpPort int
+ tlsOverTcpPort int
+ udpPort int
+ controlPort int
)
for _, port := range s.apisixService.Spec.Ports {
if port.Name == "http" {
@@ -523,6 +525,9 @@ func (s *Scaffold) newAPISIXTunnels() error {
} else if port.Name == "tcp" {
tcpNodePort = int(port.NodePort)
tcpPort = int(port.Port)
+ } else if port.Name == "tcp-tls" {
+ tlsOverTcpNodePort = int(port.NodePort)
+ tlsOverTcpPort = int(port.Port)
} else if port.Name == "udp" {
udpNodePort = int(port.NodePort)
udpPort = int(port.Port)
@@ -540,6 +545,8 @@ func (s *Scaffold) newAPISIXTunnels() error {
httpsNodePort, httpsPort)
s.apisixTCPTunnel = k8s.NewTunnel(s.kubectlOptions, k8s.ResourceTypeService, "apisix-service-e2e-test",
tcpNodePort, tcpPort)
+ s.apisixTLSOverTCPTunnel = k8s.NewTunnel(s.kubectlOptions, k8s.ResourceTypeService, "apisix-service-e2e-test",
+ tlsOverTcpNodePort, tlsOverTcpPort)
s.apisixUDPTunnel = k8s.NewTunnel(s.kubectlOptions, k8s.ResourceTypeService, "apisix-service-e2e-test",
udpNodePort, udpPort)
s.apisixControlTunnel = k8s.NewTunnel(s.kubectlOptions, k8s.ResourceTypeService, "apisix-service-e2e-test",
@@ -561,6 +568,10 @@ func (s *Scaffold) newAPISIXTunnels() error {
return err
}
s.addFinalizers(s.apisixTCPTunnel.Close)
+ if err := s.apisixTLSOverTCPTunnel.ForwardPortE(s.t); err != nil {
+ return err
+ }
+ s.addFinalizers(s.apisixTLSOverTCPTunnel.Close)
if err := s.apisixUDPTunnel.ForwardPortE(s.t); err != nil {
return err
}
@@ -577,6 +588,7 @@ func (s *Scaffold) shutdownApisixTunnel() {
s.apisixHttpTunnel.Close()
s.apisixHttpsTunnel.Close()
s.apisixTCPTunnel.Close()
+ s.apisixTLSOverTCPTunnel.Close()
s.apisixUDPTunnel.Close()
s.apisixControlTunnel.Close()
}
diff --git a/test/e2e/scaffold/scaffold.go b/test/e2e/scaffold/scaffold.go
index a9b0bbcc..1d03d809 100644
--- a/test/e2e/scaffold/scaffold.go
+++ b/test/e2e/scaffold/scaffold.go
@@ -77,12 +77,13 @@ type Scaffold struct {
testBackendService *corev1.Service
finializers []func()
- apisixAdminTunnel *k8s.Tunnel
- apisixHttpTunnel *k8s.Tunnel
- apisixHttpsTunnel *k8s.Tunnel
- apisixTCPTunnel *k8s.Tunnel
- apisixUDPTunnel *k8s.Tunnel
- apisixControlTunnel *k8s.Tunnel
+ apisixAdminTunnel *k8s.Tunnel
+ apisixHttpTunnel *k8s.Tunnel
+ apisixHttpsTunnel *k8s.Tunnel
+ apisixTCPTunnel *k8s.Tunnel
+ apisixTLSOverTCPTunnel *k8s.Tunnel
+ apisixUDPTunnel *k8s.Tunnel
+ apisixControlTunnel *k8s.Tunnel
// Used for template rendering.
EtcdServiceFQDN string
@@ -254,6 +255,32 @@ func (s *Scaffold) NewAPISIXClientWithTCPProxy() *httpexpect.Expect {
})
}
+// NewAPISIXClientWithTLSOverTCP creates a TSL over TCP client
+func (s *Scaffold) NewAPISIXClientWithTLSOverTCP(host string) *httpexpect.Expect {
+ u := url.URL{
+ Scheme: "https",
+ Host: s.apisixTLSOverTCPTunnel.Endpoint(),
+ }
+ return httpexpect.WithConfig(httpexpect.Config{
+ BaseURL: u.String(),
+ Client: &http.Client{
+ Transport: &http.Transport{
+ TLSClientConfig: &tls.Config{
+ // accept any certificate; for testing only!
+ InsecureSkipVerify: true,
+ ServerName: host,
+ },
+ },
+ CheckRedirect: func(req *http.Request, via []*http.Request) error {
+ return http.ErrUseLastResponse
+ },
+ },
+ Reporter: httpexpect.NewAssertReporter(
+ httpexpect.NewAssertReporter(ginkgo.GinkgoT()),
+ ),
+ })
+}
+
func (s *Scaffold) DNSResolver() *net.Resolver {
return &net.Resolver{
PreferGo: false,
diff --git a/test/e2e/suite-gateway/gateway_tlsroute.go b/test/e2e/suite-gateway/gateway_tlsroute.go
new file mode 100644
index 00000000..a0f38563
--- /dev/null
+++ b/test/e2e/suite-gateway/gateway_tlsroute.go
@@ -0,0 +1,206 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+package gateway
+
+import (
+ "fmt"
+ "net/http"
+ "time"
+
+ "github.com/onsi/ginkgo/v2"
+ "github.com/stretchr/testify/assert"
+
+ "github.com/apache/apisix-ingress-controller/test/e2e/scaffold"
+)
+
+func createSSL(s *scaffold.Scaffold) {
+ secretName := "test-apisix-tls"
+ cert := `-----BEGIN CERTIFICATE-----
+MIIFcjCCA1qgAwIBAgIJALDqPppBVXQ3MA0GCSqGSIb3DQEBCwUAMGUxCzAJBgNV
+BAYTAkNOMRAwDgYDVQQIDAdKaWFuZ3N1MQ8wDQYDVQQHDAZTdXpob3UxEDAOBgNV
+BAoMB2FwaTcuYWkxEDAOBgNVBAsMB2FwaTcuYWkxDzANBgNVBAMMBmp3LmNvbTAg
+Fw0yMTA0MDkwNzEyMDBaGA8yMDUxMDQwMjA3MTIwMFowZTELMAkGA1UEBhMCQ04x
+EDAOBgNVBAgMB0ppYW5nc3UxDzANBgNVBAcMBlN1emhvdTEQMA4GA1UECgwHYXBp
+Ny5haTEQMA4GA1UECwwHYXBpNy5haTEPMA0GA1UEAwwGancuY29tMIICIjANBgkq
+hkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAuEPPnUMlSw41CTdxUNxkQ4gAZ7cPotwY
+Y6sVLGtWoR8gKFSZImQIor3UF+8HhN/ZOFRv5tSeeQ/MTE72cs2T5mp6eRU8OrSV
+0npk/TpZfaCx7zobsfXB4YU1NZcVWKthFF5X8p//l5gxUMU8V4a01P0aDTmZ67wG
+3Fhr0AC067GVYvdwp1yRt6TUUk8ha7JsiySchUIFhX5QMWmrSNhc1bDnHetejMFl
+itFFPZkeYG89O/7Ca1K3ca/VVu+/IJ4h7fbF3rt4uP182cJdHl1L94dQSKCJukaW
+v+xauWnm4hxOzBK7ImpYB/2eP2D33tmuCLeSv4S+bTG1l7hIN9C/xrYPzfun415h
+M2jMK69aAkQL71xa+66kVxioJyNYogYz3ss5ruzDL8K/7bkdO0Zzqldd2+j8lkTl
+X4csA+aMHF3v/U7hL/4Wdwi8ziwToRMq9KK9vuh+mPgcdtFGFml+sU+NQfJNm/BN
+7fRMZKDIHLacSPE0GUkfW+x3dXOP2lWSZe/iOBZ0NOGNdrOnxTRTr7IH7DYU8aXF
+w2GqfAFEQbD4wazCh1AI8lkZr6mPGB1q+HnF2IF7kkgXBHtI5U2KErgcX5BirIVe
+v0Yg/OxbbymeTh/hNCcY1kJ1YUCbm9U3U6ZV+d8lj7dQHtugcAzWxSTwpBLVUPrO
+eFuhSMLVblUCAwEAAaMjMCEwHwYDVR0RBBgwFoIIYXBpNi5jb22CCiouYXBpNi5j
+b20wDQYJKoZIhvcNAQELBQADggIBAFgeSuMPpriYUrQByO3taiY1+56s+KoAmsyA
+LH15n2+thAgorusX2g1Zd7UNBHBtVO8c+ihpQb+2PnmgTTGT70ndpRbV5F6124Mt
+Hui/X0kjm76RYd1QKN1VFp0Zo+JVdRa+VhDsXWjO0VetINmFhNINFEJctyeHB8oi
+aaDL0wZrevHh47hBqtnrmLl+QVG34aLBRhZ5953leiNvXHUJNaT0nLgf0j9p4esS
+b2bx9uP4pFI1T9wcv/TE3K0rQbu/uqGY6MgznXHyi4qIK/I+WCa3fF2UZ5P/5EUM
+k2ptQneYkLLUVwwmj8C04bYhYe7Z6jkYYp17ojxIP+ejOY1eiE8SYKNjKinmphrM
+5aceqIyfbE4TPqvicNzIggA4yEMPbTA0PC/wqbCf61oMc15hwacdeIaQGiwsM+pf
+DTk+GBxp3megx/+0XwTQbguleTlHnaaES+ma0vbl6a1rUK0YAUDUrcfFLv6EFtGD
+6EHxFf7gH9sTfc2RiGhKxUhRbyEree+6INAvXy+AymVYmQmKuAFqkDQJ+09bTfm8
+bDs+00FijzHFBvC8cIhNffj0qqiv35g+9FTwnE9qpunlrtKG/sMgEXX6m8kL1YQ8
+m5DPGhyEZyt5Js2kzzo8TyINPKmrqriYuiD4p4EH13eSRs3ayanQh6ckC7lb+WXq
+3IrSc5hO
+-----END CERTIFICATE-----`
+ key := `-----BEGIN RSA PRIVATE KEY-----
+MIIJKAIBAAKCAgEAuEPPnUMlSw41CTdxUNxkQ4gAZ7cPotwYY6sVLGtWoR8gKFSZ
+ImQIor3UF+8HhN/ZOFRv5tSeeQ/MTE72cs2T5mp6eRU8OrSV0npk/TpZfaCx7zob
+sfXB4YU1NZcVWKthFF5X8p//l5gxUMU8V4a01P0aDTmZ67wG3Fhr0AC067GVYvdw
+p1yRt6TUUk8ha7JsiySchUIFhX5QMWmrSNhc1bDnHetejMFlitFFPZkeYG89O/7C
+a1K3ca/VVu+/IJ4h7fbF3rt4uP182cJdHl1L94dQSKCJukaWv+xauWnm4hxOzBK7
+ImpYB/2eP2D33tmuCLeSv4S+bTG1l7hIN9C/xrYPzfun415hM2jMK69aAkQL71xa
++66kVxioJyNYogYz3ss5ruzDL8K/7bkdO0Zzqldd2+j8lkTlX4csA+aMHF3v/U7h
+L/4Wdwi8ziwToRMq9KK9vuh+mPgcdtFGFml+sU+NQfJNm/BN7fRMZKDIHLacSPE0
+GUkfW+x3dXOP2lWSZe/iOBZ0NOGNdrOnxTRTr7IH7DYU8aXFw2GqfAFEQbD4wazC
+h1AI8lkZr6mPGB1q+HnF2IF7kkgXBHtI5U2KErgcX5BirIVev0Yg/OxbbymeTh/h
+NCcY1kJ1YUCbm9U3U6ZV+d8lj7dQHtugcAzWxSTwpBLVUPrOeFuhSMLVblUCAwEA
+AQKCAgApTupoMvlVTiYNnuREYGQJz59noN5cgELndR8WCiotjLDE2dJKp2pYMX4u
+r2NcImKsAiHj+Z5dPXFrWfhd3EBf01cJdf0+m+VKfi3NpxsQ0smQ+9Hhn1qLmDVJ
+gklCy4jD7DKDLeM6tN+5X74bUROQ+/yvIk6jTk+rbhcdVks422LGAPq8SkBQjx8a
+JKs1XZZ/ywFbzmU2fA62RR4lAnwtW680QeO8Yk7FRAzltkHdFJMBtCcZsD13uxd0
+meKbCVhJ5JyPRi/WKN2oY65EdF3na+pPnc3CeLiq5e2gy2D7J6VyknBpUrXRdMXZ
+J3/p8ZrWUXEQhk26ZP50uNdXy/Bx1jYe+U8mpkTMYVYxgu5K4Zea3yJyRn2piiE/
+9LnKNy/KsINt/0QE55ldvtciyP8RDA/08eQX0gvtKWWC/UFVRZCeL48bpqLmdAfE
+cMwlk1b0Lmo2PxULFLMAjaTKmcMAbwl53YRit0MtvaIOwiZBUVHE0blRiKC2DMKi
+SA6xLbaYJVDaMcfix8kZkKbC0xBNmL4123qX4RF6IUTPufyUTz/tpjzH6eKDw/88
+LmSx227d7/qWp5gROCDhZOPhtr4rj70JKNqcXUX9iFga+dhloOwfHYjdKndKOLUI
+Gp3K9YkPT/fCfesrguUx8BoleO5pC6RQJhRsemkRGlSY8U1ZsQKCAQEA5FepCn1f
+A46GsBSQ+/pbaHlbsR2syN3J5RmAFLFozYUrqyHE/cbNUlaYq397Ax7xwQkiN3F2
+z/whTxOh4Sk/HdDF4d+I0PZeoxINxgfzyYkx8Xpzn2loxsRO8fb3g+mOxZidjjXv
+vxqUBaj3Y01Ig0UXuw7YqCwys+xg3ELtvcGLBW/7NWMo8zqk2nUjhfcwp4e4AwBt
+Xcsc2aShUlr/RUrJH4adjha6Yaqc/8xTXHW8gZi5L2lucwB0LA+CBe4ES9BZLZdX
+N575/ohXRdjadHKYceYHiamt2326DzaxVJu2EIXU8dgdgOZ/6krITzuePRQHLPHX
+6bDfdg/WSpFrtwKCAQEAzpVqBcJ1fAI7bOkt89j2zZb1r5uD2f9sp/lA/Dj65QKV
+ShWR7Y6Jr4ShXmFvIenWtjwsl86PflMgtsJefOmLyv8o7PL154XD8SnNbBlds6IM
+MyNKkOJFa5NOrsal7TitaTvtYdKq8Zpqtxe+2kg80wi+tPVQNQS/drOpR3rDiLIE
+La/ty8XDYZsSowlzBX+uoFq7GuMct1Uh2T0/I4Kf0ZLXwYjkRlRk4LrU0BRPhRMu
+MHugOTYFKXShE2a3OEcxqCgvQ/3pn2TV92pPVKBIBGL6uKUwmXQYKaV3G4u10pJ4
+axq/avBOErcKZOReb0SNiOjiIsth8o+hnpYPU5COUwKCAQBksVNV0NtpUhyK4Ube
+FxTgCUQp4pAjM8qoQIp+lY1FtAgBuy6HSneYa59/YQP56FdrbH+uO1bNeL2nhVzJ
+UcsHdt0MMeq/WyV4e6mfPjp/EQT5G6qJDY6quD6n7ORRQ1k2QYqY/6fteeb0aAJP
+w/DKElnYnz9jSbpCJWbBOrJkD0ki6LK6ZDPWrnGr9CPqG4tVFUBL8pBH4B2kzDhn
+fME86TGvuUkZM2SVVQtOsefAyhqKe7KN+cw+4mBYXa5UtxUl6Yap2CcZ2/0aBT2X
+C32qBC69a1a/mheUxuiZdODWEqRCvQGedFLuWLbntnqGlh+9h2tyomM4JkskYO96
+io4ZAoIBAFouLW9AOUseKlTb4dx+DRcoXC4BpGhIsWUOUQkJ0rSgEQ2bJu3d+Erv
+igYKYJocW0eIMys917Qck75UUS0UQpsmEfaGBUTBRw0C45LZ6+abydmVAVsH+6f/
+USzIuOw6frDeoTy/2zHG5+jva7gcKrkxKxcRs6bBYNdvjGkQtUT5+Qr8rsDyntz/
+9f3IBTcUSuXjVaRiGkoJ1tHfg617u0qgYKEyofv1oWfdB0Oiaig8fEBb51CyPUSg
+jiRLBZaCtbGjgSacNB0JxsHP3buikG2hy7NJIVMLs/SSL9GNhpzapciTj5YeOua+
+ksICUxsdgO+QQg9QW3yoqLPy69Pd2dMCggEBANDLoUf3ZE7Dews6cfIa0WC33NCV
+FkyECaF2MNOp5Q9y/T35FyeA7UeDsTZ6Dy++XGW4uNStrSI6dCyQARqJw+i7gCst
+2m5lIde01ptzDQ9iO1Dv1XasxX/589CyLq6CxLfRgPMJPDeUEg0X7+a0lBT5Hpwk
+gNnZmws4l3i7RlVMtACCenmof9VtOcMK/9Qr502WHEoGkQR1r6HZFb25841cehL2
+do+oXlr8db++r87a8QQUkizzc6wXD9JffBNo9AO9Ed4HVOukpEA0gqVGBu85N3xW
+jW4KB95bGOTa7r7DM1Up0MbAIwWoeLBGhOIXk7inurZGg+FNjZMA5Lzm6qo=
+-----END RSA PRIVATE KEY-----`
+ // create kube secret
+ err := s.NewKubeTlsSecret(secretName, cert, key)
+ assert.Nil(ginkgo.GinkgoT(), err, "create secret error")
+ // create ApisixTls resource
+ tlsName := "tls-name"
+ host := "api6.com"
+ err = s.NewApisixTls(tlsName, host, secretName)
+ assert.Nil(ginkgo.GinkgoT(), err, "create tls error")
+
+ time.Sleep(10 * time.Second)
+ tls, err := s.ListApisixSsl()
+ assert.Nil(ginkgo.GinkgoT(), err, "list tls error")
+ assert.Len(ginkgo.GinkgoT(), tls, 1, "tls number not expect")
+ assert.Equal(ginkgo.GinkgoT(), cert, tls[0].Cert, "tls cert not expect")
+}
+
+var _ = ginkgo.Describe("suite-gateway: TLSRoute", func() {
+ s := scaffold.NewDefaultScaffold()
+
+ ginkgo.It("Basic with 1 Hosts 1 Rule 1 Match 1 BackendRef", func() {
+ createSSL(s)
+
+ host := "api6.com"
+
+ backendSvc, backendPorts := s.DefaultHTTPBackend()
+ time.Sleep(time.Second * 6)
+ route := fmt.Sprintf(`
+apiVersion: gateway.networking.k8s.io/v1alpha2
+kind: TLSRoute
+metadata:
+ name: basic-tls-route
+spec:
+ hostnames: ["%s"]
+ rules:
+ - backendRefs:
+ - name: %s
+ port: %d
+`, host, backendSvc, backendPorts[0])
+
+ assert.Nil(ginkgo.GinkgoT(), s.CreateResourceFromString(route), "creating TLSRoute")
+ time.Sleep(time.Second * 6)
+ assert.Nil(ginkgo.GinkgoT(), s.EnsureNumApisixStreamRoutesCreated(1), "Checking number of routes")
+ assert.Nil(ginkgo.GinkgoT(), s.EnsureNumApisixUpstreamsCreated(1), "Checking number of upstreams")
+
+ sr, err := s.ListApisixStreamRoutes()
+ assert.Nil(ginkgo.GinkgoT(), err)
+ assert.Len(ginkgo.GinkgoT(), sr, 1)
+ assert.Equal(ginkgo.GinkgoT(), host, sr[0].SNI)
+
+ client := s.NewAPISIXClientWithTLSOverTCP(host)
+ _ = client.GET("/ip").
+ Expect().
+ Status(http.StatusOK)
+ _ = client.GET("/notfound").
+ Expect().
+ Status(http.StatusNotFound)
+ })
+
+ ginkgo.It("Basic with 2 Hosts 1 Rule 1 Match 1 BackendRef", func() {
+ createSSL(s)
+
+ host := "api6.com"
+ backendSvc, backendPorts := s.DefaultHTTPBackend()
+ time.Sleep(time.Second * 6)
+ route := fmt.Sprintf(`
+apiVersion: gateway.networking.k8s.io/v1alpha2
+kind: TLSRoute
+metadata:
+ name: basic-tls-route
+spec:
+ hostnames: ["anydomain.com", "%s"]
+ rules:
+ - backendRefs:
+ - name: %s
+ port: %d
+`, host, backendSvc, backendPorts[0])
+
+ assert.Nil(ginkgo.GinkgoT(), s.CreateResourceFromString(route), "creating TLSRoute")
+ time.Sleep(time.Second * 6)
+ assert.Nil(ginkgo.GinkgoT(), s.EnsureNumApisixStreamRoutesCreated(2), "Checking number of routes")
+ assert.Nil(ginkgo.GinkgoT(), s.EnsureNumApisixUpstreamsCreated(1), "Checking number of upstreams")
+
+ client := s.NewAPISIXClientWithTLSOverTCP(host)
+ _ = client.GET("/ip").
+ Expect().
+ Status(http.StatusOK)
+ _ = client.GET("/notfound").
+ Expect().
+ Status(http.StatusNotFound)
+ })
+})
diff --git a/test/e2e/testdata/apisix-gw-config.yaml b/test/e2e/testdata/apisix-gw-config.yaml
index 39b35ae5..89921799 100644
--- a/test/e2e/testdata/apisix-gw-config.yaml
+++ b/test/e2e/testdata/apisix-gw-config.yaml
@@ -30,6 +30,8 @@ apisix:
only: false
tcp: # TCP proxy port list
- 9100
+ - addr: 9110
+ tls: true
udp:
- 9200
etcd: