You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ke...@apache.org on 2021/10/22 04:57:12 UTC
[skywalking-infra-e2e] branch main updated: Enhance `KinD`
environment (#56)
This is an automated email from the ASF dual-hosted git repository.
kezhenxu94 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/skywalking-infra-e2e.git
The following commit(s) were added to refs/heads/main by this push:
new 0bab0f6 Enhance `KinD` environment (#56)
0bab0f6 is described below
commit 0bab0f6e3ea7fe7e220bec32fa17df328d06ea26
Author: mrproliu <74...@qq.com>
AuthorDate: Fri Oct 22 12:57:08 2021 +0800
Enhance `KinD` environment (#56)
---
commands/run/run.go | 1 +
commands/setup/setup.go | 15 ++
docs/en/setup/Configuration-File.md | 68 ++++++--
examples/kind/e2e.yaml | 11 +-
go.sum | 1 +
internal/components/cleanup/kind.go | 6 +-
internal/components/setup/kind.go | 303 +++++++++++++++++++++++++++++++++++-
internal/config/e2eConfig.go | 22 ++-
internal/util/k8s.go | 3 +-
9 files changed, 394 insertions(+), 36 deletions(-)
diff --git a/commands/run/run.go b/commands/run/run.go
index 259cab8..fef97d3 100644
--- a/commands/run/run.go
+++ b/commands/run/run.go
@@ -110,6 +110,7 @@ func doCleanup(stopAction func()) {
if stopAction != nil {
stopAction()
}
+ setup.DoStopSetup()
if err := cleanup.DoCleanupAccordingE2E(); err != nil {
logger.Log.Errorf("cleanup part error: %s", err)
} else {
diff --git a/commands/setup/setup.go b/commands/setup/setup.go
index 564448f..4dc6adf 100644
--- a/commands/setup/setup.go
+++ b/commands/setup/setup.go
@@ -20,6 +20,7 @@ package setup
import (
"fmt"
+ "sync"
"github.com/apache/skywalking-infra-e2e/internal/components/setup"
"github.com/apache/skywalking-infra-e2e/internal/config"
@@ -40,6 +41,15 @@ var Setup = &cobra.Command{
if err := DoSetupAccordingE2E(); err != nil {
return fmt.Errorf("[Setup] %s", err)
}
+
+ if config.GlobalConfig.E2EConfig.Setup.Env == constant.Kind && setup.KindShouldWaitSignal() {
+ wg := sync.WaitGroup{}
+ wg.Add(1)
+ util.AddShutDownHook(wg.Done)
+ wg.Wait()
+
+ setup.KindCleanNotify()
+ }
return nil
},
}
@@ -68,3 +78,8 @@ func DoSetupAccordingE2E() error {
return nil
}
+
+func DoStopSetup() {
+ // notify clean up
+ setup.KindCleanNotify()
+}
diff --git a/docs/en/setup/Configuration-File.md b/docs/en/setup/Configuration-File.md
index eefc5d8..f890599 100644
--- a/docs/en/setup/Configuration-File.md
+++ b/docs/en/setup/Configuration-File.md
@@ -24,24 +24,66 @@ Support two kinds of the environment to set up the system.
```yaml
setup:
env: kind
- file: path/to/kind.yaml # Specified kinD manifest file path
- timeout: 1200 # timeout second
- steps: # customize steps for prepare the environment
- - name: customize setups # step name
+ file: path/to/kind.yaml # Specified kinD manifest file path
+ timeout: 1200 # timeout second
+ init-system-environment: path/to/env # Import environment file
+ steps: # customize steps for prepare the environment
+ - name: customize setups # step name
# one of command line or kinD manifest file
- command: command lines # use command line to setup
- path: /path/to/manifest.yaml # the manifest file path
- wait: # how to verify the manifest is set up finish
- - namespace: # The pod namespace
- resource: # The pod resource name
- label-selector: # The resource label selector
- for: # The wait condition
+ command: command lines # use command line to setup
+ path: /path/to/manifest.yaml # the manifest file path
+ wait: # how to verify the manifest is set up finish
+ - namespace: # The pod namespace
+ resource: # The pod resource name
+ label-selector: # The resource label selector
+ for: # The wait condition
+ kind:
+ import-images: # import docker images to KinD
+ - image:version # support using env to expand image, such as `${env_key}` or `$env_key`
+ expose-ports: # Expose resource for host access
+ - namespace: # The resource namespace
+ resource: # The resource name, such as `pod/foo` or `service/foo`
+ port: # Want to expose port from resource
```
The `KinD` environment follow these steps:
-1. Start the `KinD` cluster according to the config file.
+1. Start the `KinD` cluster according to the config file, expose `KUBECONFIG` to environment for help execute `kubectl` in the steps.
+1. Load docker images from `kind.import-images` if needed.
1. Apply the resources files (`--manifests`) or/and run the custom init command (`--commands`) by steps.
1. Wait until all steps are finished and all services are ready with the timeout(second).
+1. Expose all resource ports for host access.
+
+#### Import docker image
+
+If you want to import docker image from private registries, there are several ways to do this:
+1. Using `imagePullSecrets` to pull images, [please take reference from document](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#registry-secret-existing-credentials).
+2. Using `kind.import-images` to load images from host.
+ ```yaml
+ kind:
+ import-images:
+ - skywalking/oap:${OAP_HASH} # support using environment to expand the image name
+ ```
+
+#### Resource Export
+
+If you want to access the resource from host, should follow these steps:
+1. Declare which resource and ports need to be accessible from host.
+ ```yaml
+ setup:
+ kind:
+ expose-ports:
+ - namespace: default # Need to expose resource namespace
+ resource: pod/foo # Resource description, such as `pod/foo` or `service/foo`
+ port: 8080 # Resource port want to expose, support `<resource_port>`, `<bind_to_host_port>:<resource_port>`
+ ```
+2. Follow this format to get the host and port mapping by the environment, and it's available in steps(trigger, verify).
+ ```yaml
+ trigger:
+ # trigger with specified mapped port, the resource name replace all `/` or `-` as `_`
+ # host format: <resource_name>_host
+ # port format: <resource_name>_<container_port>
+ url: http://${pod_foo_host}:${pod_foo_8080}/
+ ```
### Compose
@@ -78,7 +120,7 @@ If you want to get the service host and port mapping, should follow these steps:
```yaml
trigger:
# trigger with specified mappinged port
- url: http://${oap.host}:${oap_8080}/
+ url: http://${oap_host}:${oap_8080}/
```
## Trigger
diff --git a/examples/kind/e2e.yaml b/examples/kind/e2e.yaml
index dfa017e..581fb72 100644
--- a/examples/kind/e2e.yaml
+++ b/examples/kind/e2e.yaml
@@ -42,6 +42,11 @@ setup:
- namespace: default
resource: deployment/nginx1
for: condition=Available
+ kind:
+ expose-ports:
+ - namespace: default
+ resource: deployment/nginx1
+ port: 80
timeout: 1200
cleanup:
@@ -52,7 +57,7 @@ trigger:
action: http
interval: 3s
times: 5
- url: http://localhost:9090/user
+ url: http://${deployment_nginx1_host}:${deployment_nginx1_80}/
method: GET
verify:
@@ -67,7 +72,3 @@ verify:
expected: ../../test/verify/1.expected.yaml
- actual: ../../test/verify/2.actual.yaml
expected: ../../test/verify/2.expected.yaml
- - actual: ../../test/verify/1.actual.yaml
- expected: ../../test/verify/2.expected.yaml
- - query: swctl --display yaml service ls
- expected: ../../test/verify/3.expected.yaml
diff --git a/go.sum b/go.sum
index a7be77a..302043d 100644
--- a/go.sum
+++ b/go.sum
@@ -247,6 +247,7 @@ github.com/evanphx/json-patch/v5 v5.1.0 h1:B0aXl1o/1cP8NbviYiBMkcHBtUjIJ1/Ccg6b+
github.com/evanphx/json-patch/v5 v5.1.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4=
github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM=
github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
+github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8=
github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
diff --git a/internal/components/cleanup/kind.go b/internal/components/cleanup/kind.go
index 3e5f8f1..9e2abf6 100644
--- a/internal/components/cleanup/kind.go
+++ b/internal/components/cleanup/kind.go
@@ -21,15 +21,13 @@ package cleanup
import (
"io/ioutil"
"os"
-
- "github.com/apache/skywalking-infra-e2e/internal/config"
+ "strings"
"gopkg.in/yaml.v2"
kind "sigs.k8s.io/kind/cmd/kind/app"
kindcmd "sigs.k8s.io/kind/pkg/cmd"
- "strings"
-
+ "github.com/apache/skywalking-infra-e2e/internal/config"
"github.com/apache/skywalking-infra-e2e/internal/constant"
"github.com/apache/skywalking-infra-e2e/internal/logger"
)
diff --git a/internal/components/setup/kind.go b/internal/components/setup/kind.go
index e50ccd9..eebce1e 100644
--- a/internal/components/setup/kind.go
+++ b/internal/components/setup/kind.go
@@ -19,34 +19,60 @@
package setup
import (
+ "bufio"
+ "bytes"
"fmt"
+ "io/ioutil"
+ "net/http"
"os"
+ "strconv"
"strings"
+ "time"
+ apiv1 "k8s.io/api/admission/v1"
+ v1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/runtime"
+ "k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/cli-runtime/pkg/genericclioptions"
+ "k8s.io/cli-runtime/pkg/resource"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/kubernetes"
+ "k8s.io/client-go/rest"
+ "k8s.io/client-go/tools/portforward"
+ "k8s.io/client-go/transport/spdy"
ctlwait "k8s.io/kubectl/pkg/cmd/wait"
-
- "github.com/apache/skywalking-infra-e2e/internal/config"
-
- apiv1 "k8s.io/api/admission/v1"
-
- "github.com/apache/skywalking-infra-e2e/internal/util"
+ "k8s.io/kubectl/pkg/polymorphichelpers"
+ "k8s.io/kubectl/pkg/scheme"
+ ctlutil "k8s.io/kubectl/pkg/util"
kind "sigs.k8s.io/kind/cmd/kind/app"
kindcmd "sigs.k8s.io/kind/pkg/cmd"
+ "github.com/apache/skywalking-infra-e2e/internal/config"
"github.com/apache/skywalking-infra-e2e/internal/constant"
-
"github.com/apache/skywalking-infra-e2e/internal/logger"
+ "github.com/apache/skywalking-infra-e2e/internal/util"
)
var (
kindConfigPath string
kubeConfigPath string
+
+ portForwardContext *kindPortForwardContext
)
+type kindPortForwardContext struct {
+ stopChannel chan struct{}
+ resourceCount int
+ resourceFinishedChannel chan struct{}
+}
+
+type kindPort struct {
+ inputPort string // User input port
+ realPort int // Real remote port, deference with input when resource is service or use port name
+ waitExpose string // Need to use when expose
+}
+
// KindSetup sets up environment according to e2e.yaml.
func KindSetup(e2eConfig *config.E2EConfig) error {
kindConfigPath = e2eConfig.Setup.GetFile()
@@ -62,10 +88,29 @@ func KindSetup(e2eConfig *config.E2EConfig) error {
return nil
}
+ // export env file
+ if e2eConfig.Setup.InitSystemEnvironment != "" {
+ profilePath := util.ResolveAbs(e2eConfig.Setup.InitSystemEnvironment)
+ util.ExportEnvVars(profilePath)
+ }
+
if err := createKindCluster(kindConfigPath); err != nil {
return err
}
+ // import images
+ if len(e2eConfig.Setup.Kind.ImportImages) > 0 {
+ for _, image := range e2eConfig.Setup.Kind.ImportImages {
+ image = os.ExpandEnv(image)
+ args := []string{"load", "docker-image", image}
+
+ logger.Log.Infof("import docker images: %s", image)
+ if err := kind.Run(kindcmd.NewLogger(), kindcmd.StandardIOStreams(), args); err != nil {
+ return err
+ }
+ }
+ }
+
cluster, err := util.ConnectToK8sCluster(kubeConfigPath)
if err != nil {
logger.Log.Errorf("connect to k8s cluster failed according to config file: %s", kubeConfigPath)
@@ -79,9 +124,30 @@ func KindSetup(e2eConfig *config.E2EConfig) error {
return err
}
+ // expose ports
+ err = exposeKindService(e2eConfig.Setup.Kind.ExposePorts, e2eConfig.Setup.Timeout, kubeConfigPath)
+ if err != nil {
+ logger.Log.Errorf("export ports error: %v", err)
+ return err
+ }
return nil
}
+func KindShouldWaitSignal() bool {
+ return portForwardContext != nil && portForwardContext.resourceCount > 0
+}
+
+// KindCleanNotify notify when clean up
+func KindCleanNotify() {
+ if portForwardContext != nil {
+ portForwardContext.stopChannel <- struct{}{}
+ // wait all stopped
+ for i := 0; i < portForwardContext.resourceCount; i++ {
+ <-portForwardContext.resourceFinishedChannel
+ }
+ }
+}
+
func createKindCluster(kindConfigPath string) error {
// the config file name of the k8s cluster that kind create
kubeConfigPath = constant.K8sClusterConfigFilePath
@@ -93,6 +159,13 @@ func createKindCluster(kindConfigPath string) error {
return err
}
logger.Log.Info("create kind cluster succeeded")
+
+ // export kubeconfig path for command line
+ err := os.Setenv("KUBECONFIG", kubeConfigPath)
+ if err != nil {
+ return fmt.Errorf("could not export kubeconfig file path, %v", err)
+ }
+ logger.Log.Infof("export KUBECONFIG=%s", kubeConfigPath)
return nil
}
@@ -160,3 +233,219 @@ func concurrentlyWait(wait *config.Wait, options *ctlwait.WaitOptions, waitSet *
}
logger.Log.Infof("wait %+v condition met", wait)
}
+
+// buildKindPort for help find real pod remote port
+func buildKindPort(port string, ro runtime.Object, pod *v1.Pod) (*kindPort, error) {
+ var needExpose, remotePort string
+ if strings.Contains(port, ":") {
+ needExpose = port
+ remotePort = strings.Split(port, ":")[1]
+ } else {
+ needExpose = fmt.Sprintf(":%s", port)
+ remotePort = port
+ }
+
+ service, isService := ro.(*v1.Service)
+ if !isService {
+ remotePortInt, err := strconv.Atoi(remotePort)
+ if err != nil {
+ containerPort, err := ctlutil.LookupContainerPortNumberByName(*pod, remotePort)
+ if err != nil {
+ return nil, err
+ }
+
+ remotePortInt = int(containerPort)
+ }
+ return &kindPort{
+ inputPort: remotePort,
+ realPort: remotePortInt,
+ waitExpose: needExpose,
+ }, nil
+ }
+
+ portnum64, err := strconv.ParseInt(remotePort, 10, 32)
+ var portnum int32
+ if err != nil {
+ svcPort, err1 := ctlutil.LookupServicePortNumberByName(*service, remotePort)
+ if err1 != nil {
+ return nil, err1
+ }
+ portnum = svcPort
+ } else {
+ portnum = int32(portnum64)
+ }
+ containerPort, err := ctlutil.LookupContainerPortNumberByServicePort(*service, *pod, portnum)
+ if err != nil {
+ // can't resolve a named port, or Service did not declare this port, return an error
+ return nil, err
+ }
+
+ // convert the resolved target port back to a string
+ realPort := int(containerPort)
+ if strconv.Itoa(realPort) != remotePort {
+ var localPort string
+ if strings.Contains(port, ":") {
+ localPort = strings.Split(port, ":")[0]
+ }
+ needExpose = fmt.Sprintf("%s:%d", localPort, realPort)
+ }
+
+ return &kindPort{
+ inputPort: remotePort,
+ realPort: realPort,
+ waitExpose: needExpose,
+ }, nil
+}
+
+func exposePerKindService(port config.KindExposePort, timeout time.Duration, clientGetter *util.SimpleRESTClientGetter,
+ client *rest.RESTClient, roundTripper http.RoundTripper, upgrader spdy.Upgrader, forward *kindPortForwardContext) error {
+ // find resource
+ builder := resource.NewBuilder(clientGetter).
+ WithScheme(scheme.Scheme, scheme.Scheme.PrioritizedVersionsAllGroups()...).
+ ContinueOnError().
+ NamespaceParam(port.Namespace).DefaultNamespace()
+ builder.ResourceNames("pods", port.Resource)
+ obj, err := builder.Do().Object()
+ if err != nil {
+ return err
+ }
+ forwardablePod, err := polymorphichelpers.AttachablePodForObjectFn(clientGetter, obj, timeout)
+ if err != nil {
+ return err
+ }
+
+ // build port forward request
+ req := client.Post().
+ Resource("pods").
+ Namespace(forwardablePod.Namespace).
+ Name(forwardablePod.Name).
+ SubResource("portforward")
+
+ dialer := spdy.NewDialer(upgrader, &http.Client{Transport: roundTripper}, http.MethodPost, req.URL())
+
+ // build ports
+ ports := strings.Split(port.Port, ",")
+ convertedPorts := make([]*kindPort, len(ports))
+ exposePorts := make([]string, len(ports))
+ for i, p := range ports {
+ if convertedPorts[i], err = buildKindPort(p, obj, forwardablePod); err != nil {
+ return err
+ }
+ exposePorts[i] = convertedPorts[i].waitExpose
+ }
+
+ var stdout bytes.Buffer
+ var stderr bytes.Buffer
+ readyChannel := make(chan struct{}, 1)
+ forwardErrorChannel := make(chan error, 1)
+
+ forwarder, err := portforward.New(dialer, exposePorts, forward.stopChannel, readyChannel,
+ bufio.NewWriter(&stdout), bufio.NewWriter(&stderr))
+ if err != nil {
+ return err
+ }
+
+ // start forward
+ go func() {
+ if err = forwarder.ForwardPorts(); err != nil {
+ forwardErrorChannel <- err
+ }
+ forward.resourceFinishedChannel <- struct{}{}
+ }()
+
+ // wait port forward result
+ select {
+ case <-readyChannel:
+ exportedPorts, err1 := forwarder.GetPorts()
+ if err1 != nil {
+ return err1
+ }
+
+ // format: <resource>_host
+ resourceName := port.Resource
+ resourceName = strings.ReplaceAll(resourceName, "/", "_")
+ resourceName = strings.ReplaceAll(resourceName, "-", "_")
+ if err1 := exportKindEnv(fmt.Sprintf("%s_host", resourceName),
+ "localhost", port.Resource); err1 != nil {
+ return err1
+ }
+
+ // format: <resource>_<need_export_port>
+ for _, p := range exportedPorts {
+ for _, kp := range convertedPorts {
+ if int(p.Remote) == kp.realPort {
+ if err1 := exportKindEnv(fmt.Sprintf("%s_%s", resourceName, kp.inputPort),
+ fmt.Sprintf("%d", p.Local), port.Resource); err1 != nil {
+ return err1
+ }
+ }
+ }
+ }
+
+ case err = <-forwardErrorChannel:
+ return fmt.Errorf("create forward error, %s : %v", stderr.String(), err)
+ }
+ return nil
+}
+
+func exposeKindService(exports []config.KindExposePort, timeout int, kubeConfig string) error {
+ // round tripper
+ kubeConfigYaml, err := ioutil.ReadFile(kubeConfig)
+ if err != nil {
+ return err
+ }
+ clientGetter := util.NewSimpleRESTClientGetter("", string(kubeConfigYaml))
+
+ restConf, err := clientGetter.ToRESTConfig()
+ if err != nil {
+ return err
+ }
+ restConf.NegotiatedSerializer = scheme.Codecs.WithoutConversion()
+ tripperFor, upgrader, err := spdy.RoundTripperFor(restConf)
+ if err != nil {
+ return err
+ }
+
+ // rest client
+ if restConf.GroupVersion == nil {
+ restConf.GroupVersion = &schema.GroupVersion{Version: "v1"}
+ }
+ restConf.APIPath = "/api"
+ client, err := rest.RESTClientFor(restConf)
+ if err != nil {
+ return err
+ }
+
+ // timeout
+ var waitTimeout time.Duration
+ if timeout <= 0 {
+ waitTimeout = constant.DefaultWaitTimeout
+ } else {
+ waitTimeout = time.Duration(timeout) * time.Second
+ }
+
+ // stop port-forward channel
+ forwardContext := &kindPortForwardContext{
+ stopChannel: make(chan struct{}, 1),
+ resourceFinishedChannel: make(chan struct{}, len(exports)),
+ resourceCount: len(exports),
+ }
+ for _, p := range exports {
+ if err := exposePerKindService(p, waitTimeout, clientGetter, client, tripperFor, upgrader, forwardContext); err != nil {
+ return err
+ }
+ }
+
+ // bind context
+ portForwardContext = forwardContext
+ return nil
+}
+
+func exportKindEnv(key, value, res string) error {
+ err := os.Setenv(key, value)
+ if err != nil {
+ return fmt.Errorf("could not set env for %s, %v", res, err)
+ }
+ logger.Log.Infof("export %s=%s", key, value)
+ return nil
+}
diff --git a/internal/config/e2eConfig.go b/internal/config/e2eConfig.go
index 6b8b974..e313573 100644
--- a/internal/config/e2eConfig.go
+++ b/internal/config/e2eConfig.go
@@ -29,11 +29,12 @@ type E2EConfig struct {
}
type Setup struct {
- Env string `yaml:"env"`
- File string `yaml:"file"`
- Steps []Step `yaml:"steps"`
- Timeout int `yaml:"timeout"`
- InitSystemEnvironment string `yaml:"init-system-environment"`
+ Env string `yaml:"env"`
+ File string `yaml:"file"`
+ Steps []Step `yaml:"steps"`
+ Timeout int `yaml:"timeout"`
+ InitSystemEnvironment string `yaml:"init-system-environment"`
+ Kind KindSetup `yaml:"kind"`
}
type Cleanup struct {
@@ -47,6 +48,17 @@ type Step struct {
Waits []Wait `yaml:"wait"`
}
+type KindSetup struct {
+ ImportImages []string `yaml:"import-images"`
+ ExposePorts []KindExposePort `yaml:"expose-ports"`
+}
+
+type KindExposePort struct {
+ Namespace string `yaml:"namespace"`
+ Resource string `yaml:"resource"`
+ Port string `yaml:"port"`
+}
+
type Verify struct {
RetryStrategy VerifyRetryStrategy `yaml:"retry"`
Cases []VerifyCase `yaml:"cases"`
diff --git a/internal/util/k8s.go b/internal/util/k8s.go
index 45aa3f8..d16537c 100644
--- a/internal/util/k8s.go
+++ b/internal/util/k8s.go
@@ -102,8 +102,7 @@ func GetManifests(manifests string) (files []string, err error) {
case mode.IsRegular():
filename := fi.Name()
if strings.HasSuffix(filename, ".yml") || strings.HasSuffix(filename, ".yaml") {
- filename = ResolveAbs(filename)
- s = append(s, filename)
+ s = append(s, f)
}
}
}