You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ha...@apache.org on 2020/12/18 03:00:58 UTC
[skywalking-swck] branch master updated: Add unit test case for
operator (#16)
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/skywalking-swck.git
The following commit(s) were added to refs/heads/master by this push:
new e13b753 Add unit test case for operator (#16)
e13b753 is described below
commit e13b753a88adaaa70a8cc2af47f42ac4c04a747a
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Fri Dec 18 11:00:32 2020 +0800
Add unit test case for operator (#16)
* Add test case for operator
Signed-off-by: Gao Hongtao <ha...@gmail.com>
* Fix lint issues
Signed-off-by: Gao Hongtao <ha...@gmail.com>
* Install kubebuilder for ci
Signed-off-by: Gao Hongtao <ha...@gmail.com>
* change shell file mode
Signed-off-by: Gao Hongtao <ha...@gmail.com>
* Update check name
Signed-off-by: Gao Hongtao <ha...@gmail.com>
* Restore check name
Signed-off-by: Gao Hongtao <ha...@gmail.com>
---
.asf.yaml | 2 +-
.github/workflows/go.yml | 47 ++++++--
CHANGES.md | 2 +
controllers/operator/oapserver_controller.go | 55 +++++----
controllers/operator/oapserver_controller_test.go | 130 ++++++++++++++++++++++
controllers/operator/suite_test.go | 69 ++++++++++++
go.mod | 1 +
.asf.yaml => hack/install-kubebuilder.sh | 32 ++----
pkg/kubernetes/kubernetes.go | 16 +++
9 files changed, 297 insertions(+), 57 deletions(-)
diff --git a/.asf.yaml b/.asf.yaml
index 5dadefc..53f174d 100644
--- a/.asf.yaml
+++ b/.asf.yaml
@@ -34,7 +34,7 @@ github:
required_status_checks:
strict: true
contexts:
- - build
+ - build
required_pull_request_reviews:
dismiss_stale_reviews: true
required_approving_review_count: 1
diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
index eb558f4..2f1c139 100644
--- a/.github/workflows/go.yml
+++ b/.github/workflows/go.yml
@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-name: Build
+name: Continuous Integration
on:
pull_request:
@@ -23,11 +23,29 @@ on:
- master
jobs:
+ check:
+ name: Check
+ runs-on: ubuntu-20.04
+ steps:
+ - name: Install Go
+ uses: actions/setup-go@v2
+ with:
+ go-version: 1.14
+ id: go
+ - name: Check out code into the Go module directory
+ uses: actions/checkout@v2
+ - name: Update dependencies
+ run: GOPROXY=https://proxy.golang.org go mod download
+ - name: Lint
+ run: make lint
+ - name: Check
+ run: make check
build:
+ name: Build
strategy:
matrix:
go-version: [ 1.14.x, 1.15.x ]
- runs-on: ubuntu-latest
+ runs-on: ubuntu-20.04
steps:
- name: Install Go
uses: actions/setup-go@v2
@@ -37,17 +55,30 @@ jobs:
uses: actions/checkout@v2
- name: Update dependencies
run: GOPROXY=https://proxy.golang.org go mod download
- - name: Lint
- run: make lint
- - name: Check
- run: make check
- name: Build
run: make
- name: Build docker image
run: make docker-build
+ unit-tests:
+ name: Unit tests
+ runs-on: ubuntu-20.04
+ steps:
+ - name: Install Go
+ uses: actions/setup-go@v2
+ with:
+ go-version: 1.14
+ id: go
+ - name: Check out code into the Go module directory
+ uses: actions/checkout@v2
+ - name: Update dependencies
+ run: GOPROXY=https://proxy.golang.org go mod download
+ - name: "install kubebuilder"
+ run: ./hack/install-kubebuilder.sh
+ - name: tests
+ run: make test
checks:
name: build
- runs-on: ubuntu-latest
- needs: [build]
+ runs-on: ubuntu-20.04
+ needs: [check, build, unit-tests]
steps:
- run: echo 'success'
diff --git a/CHANGES.md b/CHANGES.md
index 54cd700..8787733 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -7,9 +7,11 @@ Release Notes.
#### Features
- Introduce custom metrics adapter to SkyWalking OAP cluster for Kubernetes HPA autoscaling.
+- Add RBAC files and service account to support Kubernetes coordination.
#### Chores
- Transform project layers to support multiple applications.
+- Introduce unit test to verify the operator.
0.1.0
------------------
diff --git a/controllers/operator/oapserver_controller.go b/controllers/operator/oapserver_controller.go
index 76d2d26..0d55c04 100644
--- a/controllers/operator/oapserver_controller.go
+++ b/controllers/operator/oapserver_controller.go
@@ -24,9 +24,11 @@ import (
"time"
"github.com/go-logr/logr"
+ l "github.com/sirupsen/logrus"
apps "k8s.io/api/apps/v1"
core "k8s.io/api/core/v1"
apiequal "k8s.io/apimachinery/pkg/api/equality"
+ apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
@@ -38,7 +40,6 @@ import (
const annotationKeyIstioSetup = "istio-setup-command"
var schedDuration, _ = time.ParseDuration("1m")
-var rushModeSchedDuration, _ = time.ParseDuration("5s")
// OAPServerReconciler reconciles a OAPServer object
type OAPServerReconciler struct {
@@ -82,9 +83,17 @@ func (r *OAPServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
}
}
- r.istio(ctx, log, oapServer.Name, &oapServer)
+ if err := r.istio(ctx, log, oapServer.Name, &oapServer); err != nil {
+ l.Error(err, "failed to sync istio annotation")
+ return ctrl.Result{}, err
+ }
- return ctrl.Result{RequeueAfter: r.checkState(ctx, log, &oapServer)}, nil
+ if err := r.checkState(ctx, log, &oapServer); err != nil {
+ l.Error(err, "failed to check sub resources state")
+ return ctrl.Result{}, err
+ }
+
+ return ctrl.Result{RequeueAfter: schedDuration}, nil
}
func tmplFunc(oapServer *operatorv1alpha1.OAPServer) template.FuncMap {
@@ -110,50 +119,48 @@ func tmplFunc(oapServer *operatorv1alpha1.OAPServer) template.FuncMap {
}
}
-func (r *OAPServerReconciler) checkState(ctx context.Context, log logr.Logger, oapServer *operatorv1alpha1.OAPServer) time.Duration {
+func (r *OAPServerReconciler) checkState(ctx context.Context, log logr.Logger, oapServer *operatorv1alpha1.OAPServer) error {
overlay := operatorv1alpha1.OAPServerStatus{}
deployment := apps.Deployment{}
- nextSchedule := schedDuration
- if err := r.Client.Get(ctx, client.ObjectKey{Namespace: oapServer.Namespace, Name: oapServer.Name}, &deployment); err != nil {
- nextSchedule = rushModeSchedDuration
+ errCol := new(kubernetes.ErrorCollector)
+ if err := r.Client.Get(ctx, client.ObjectKey{Namespace: oapServer.Namespace, Name: oapServer.Name}, &deployment); err != nil && !apierrors.IsNotFound(err) {
+ errCol.Collect(fmt.Errorf("failed to get deployment: %w", err))
} else {
overlay.Conditions = deployment.Status.Conditions
overlay.AvailableReplicas = deployment.Status.AvailableReplicas
- if oapServer.Spec.Instances != overlay.AvailableReplicas {
- nextSchedule = rushModeSchedDuration
- }
if oapServer.Spec.Image != deployment.Spec.Template.Spec.Containers[0].Image {
oapServer.Spec.Image = deployment.Spec.Template.Spec.Containers[0].Image
if err := r.Update(ctx, oapServer); err != nil {
- log.Error(err, "failed to update OAPServer Image field")
+ errCol.Collect(fmt.Errorf("failed to update image field: %w", err))
+ return errCol.Error()
}
log.Info("updated OAPServer Image")
- nextSchedule = rushModeSchedDuration
}
}
service := core.Service{}
- if err := r.Client.Get(ctx, client.ObjectKey{Namespace: oapServer.Namespace, Name: oapServer.Name}, &service); err != nil {
- nextSchedule = rushModeSchedDuration
+ if err := r.Client.Get(ctx, client.ObjectKey{Namespace: oapServer.Namespace, Name: oapServer.Name}, &service); err != nil && !apierrors.IsNotFound(err) {
+ errCol.Collect(fmt.Errorf("failed to get service: %w", err))
} else {
overlay.Address = fmt.Sprintf("%s.%s", service.Name, service.Namespace)
}
if apiequal.Semantic.DeepDerivative(overlay, oapServer.Status) {
log.Info("Status keeps the same as before")
- return nextSchedule
}
oapServer.Status = overlay
+ oapServer.Kind = "OAPServer"
if err := kubernetes.ApplyOverlay(oapServer, &operatorv1alpha1.OAPServer{Status: overlay}); err != nil {
- log.Error(err, "failed to overlay OAPServer")
- return rushModeSchedDuration
+ errCol.Collect(fmt.Errorf("failed to apply overlay: %w", err))
+ return errCol.Error()
}
if err := r.Status().Update(ctx, oapServer); err != nil {
- return rushModeSchedDuration
+ errCol.Collect(fmt.Errorf("failed to update status of OAPServer: %w", err))
}
log.Info("updated Status sub resource")
- return nextSchedule
+
+ return errCol.Error()
}
-func (r *OAPServerReconciler) istio(ctx context.Context, log logr.Logger, serviceName string, oapServer *operatorv1alpha1.OAPServer) {
+func (r *OAPServerReconciler) istio(ctx context.Context, log logr.Logger, serviceName string, oapServer *operatorv1alpha1.OAPServer) error {
for _, envVar := range oapServer.Spec.Config {
if envVar.Name == "SW_ENVOY_METRIC_ALS_HTTP_ANALYSIS" &&
oapServer.ObjectMeta.Annotations[annotationKeyIstioSetup] == "" {
@@ -161,17 +168,19 @@ func (r *OAPServerReconciler) istio(ctx context.Context, log logr.Logger, servic
"--set meshConfig.defaultConfig.envoyAccessLogService.address=%s.%s:11800 "+
"--set meshConfig.enableEnvoyAccessLogService=true", serviceName, oapServer.Namespace)
if err := r.Update(ctx, oapServer); err != nil {
- log.Error(err, "unable to patch Istio setup command to annotation")
- return
+ return err
}
log.Info("patched Istio annotation")
- return
+ return nil
}
}
+ return nil
}
func (r *OAPServerReconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
For(&operatorv1alpha1.OAPServer{}).
+ Owns(&apps.Deployment{}).
+ Owns(&core.Service{}).
Complete(r)
}
diff --git a/controllers/operator/oapserver_controller_test.go b/controllers/operator/oapserver_controller_test.go
new file mode 100644
index 0000000..96c459a
--- /dev/null
+++ b/controllers/operator/oapserver_controller_test.go
@@ -0,0 +1,130 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package controllers_test
+
+import (
+ "context"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+ appsv1 "k8s.io/api/apps/v1"
+ corev1 "k8s.io/api/core/v1"
+ rbacv1 "k8s.io/api/rbac/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/types"
+ "sigs.k8s.io/controller-runtime/pkg/client"
+ logf "sigs.k8s.io/controller-runtime/pkg/log"
+ k8sreconcile "sigs.k8s.io/controller-runtime/pkg/reconcile"
+
+ "github.com/apache/skywalking-swck/apis/operator/v1alpha1"
+ controllers "github.com/apache/skywalking-swck/controllers/operator"
+ "github.com/apache/skywalking-swck/pkg/operator/repo"
+)
+
+var logger = logf.Log.WithName("unit-tests")
+var fileRepo = repo.NewRepo("oapserver")
+
+func TestNewObjectsOnReconciliation(t *testing.T) {
+ // prepare
+ nsn := types.NamespacedName{Name: "my-instance", Namespace: "default"}
+ reconciler := controllers.OAPServerReconciler{
+ Client: k8sClient,
+ Log: logger,
+ Scheme: testScheme,
+ FileRepo: fileRepo,
+ }
+ created := &v1alpha1.OAPServer{
+ TypeMeta: metav1.TypeMeta{
+ Kind: "OAPServer",
+ APIVersion: v1alpha1.GroupVersion.Version,
+ },
+ ObjectMeta: metav1.ObjectMeta{
+ Name: nsn.Name,
+ Namespace: nsn.Namespace,
+ },
+ Spec: v1alpha1.OAPServerSpec{
+ Instances: 1,
+ },
+ }
+ err := k8sClient.Create(context.Background(), created)
+ require.NoError(t, err)
+
+ // test
+ req := k8sreconcile.Request{
+ NamespacedName: nsn,
+ }
+ _, err = reconciler.Reconcile(context.Background(), req)
+
+ // verify
+ require.NoError(t, err)
+
+ // the base query for the underlying objects
+ opts := []client.ListOption{
+ client.InNamespace(nsn.Namespace),
+ client.MatchingLabels(map[string]string{
+ "operator.skywalking.apache.org/oap-server-name": nsn.Name,
+ "operator.skywalking.apache.org/application": "oapserver",
+ }),
+ }
+
+ // verify that we have at least one object for each of the types we create
+ // whether we have the right ones is up to the specific tests for each type
+ {
+ list := &corev1.ServiceAccountList{}
+ err = k8sClient.List(context.Background(), list, opts...)
+ assert.NoError(t, err)
+ assert.NotEmpty(t, list.Items)
+ }
+ {
+ list := &corev1.ServiceList{}
+ err = k8sClient.List(context.Background(), list, opts...)
+ assert.NoError(t, err)
+ assert.NotEmpty(t, list.Items)
+ }
+ {
+ list := &appsv1.DeploymentList{}
+ err = k8sClient.List(context.Background(), list, opts...)
+ assert.NoError(t, err)
+ assert.NotEmpty(t, list.Items)
+ }
+
+ // the base query for the underlying objects
+ rbacOpts := []client.ListOption{
+ client.MatchingLabels(map[string]string{
+ "operator.skywalking.apache.org/application": "oapserver",
+ "operator.skywalking.apache.org/component": "rbac",
+ }),
+ }
+ {
+ list := &rbacv1.ClusterRoleBindingList{}
+ err = k8sClient.List(context.Background(), list, rbacOpts...)
+ assert.NoError(t, err)
+ assert.NotEmpty(t, list.Items)
+ }
+ {
+ list := &rbacv1.ClusterRoleList{}
+ err = k8sClient.List(context.Background(), list, rbacOpts...)
+ assert.NoError(t, err)
+ assert.NotEmpty(t, list.Items)
+ }
+
+ // cleanup
+ require.NoError(t, k8sClient.Delete(context.Background(), created))
+
+}
diff --git a/controllers/operator/suite_test.go b/controllers/operator/suite_test.go
new file mode 100644
index 0000000..7b6c5e7
--- /dev/null
+++ b/controllers/operator/suite_test.go
@@ -0,0 +1,69 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package controllers_test
+
+import (
+ "fmt"
+ "os"
+ "path/filepath"
+ "testing"
+
+ "k8s.io/apimachinery/pkg/runtime"
+ "k8s.io/client-go/kubernetes/scheme"
+ "sigs.k8s.io/controller-runtime/pkg/client"
+ "sigs.k8s.io/controller-runtime/pkg/envtest"
+
+ "github.com/apache/skywalking-swck/apis/operator/v1alpha1"
+)
+
+var k8sClient client.Client
+var testEnv *envtest.Environment
+var testScheme *runtime.Scheme = scheme.Scheme
+
+func TestMain(m *testing.M) {
+ testEnv = &envtest.Environment{
+ CRDDirectoryPaths: []string{filepath.Join("../..", "config", "operator", "crd", "bases")},
+ }
+
+ cfg, err := testEnv.Start()
+ if err != nil {
+ fmt.Printf("failed to start testEnv: %v", err)
+ os.Exit(1)
+ }
+
+ if errAddScheme := v1alpha1.AddToScheme(testScheme); errAddScheme != nil {
+ fmt.Printf("failed to register scheme: %v", errAddScheme)
+ os.Exit(1)
+ }
+
+ k8sClient, err = client.New(cfg, client.Options{Scheme: testScheme})
+ if err != nil {
+ fmt.Printf("failed to setup a Kubernetes client: %v", err)
+ os.Exit(1)
+ }
+
+ code := m.Run()
+
+ err = testEnv.Stop()
+ if err != nil {
+ fmt.Printf("failed to stop testEnv: %v", err)
+ os.Exit(1)
+ }
+
+ os.Exit(code)
+}
diff --git a/go.mod b/go.mod
index 9877ff3..0d7d8ab 100644
--- a/go.mod
+++ b/go.mod
@@ -12,6 +12,7 @@ require (
github.com/machinebox/graphql v0.2.2
github.com/sirupsen/logrus v1.7.0
github.com/spf13/cobra v1.0.0
+ github.com/stretchr/testify v1.6.1
github.com/urfave/cli v1.22.1
k8s.io/api v0.19.3
k8s.io/apiextensions-apiserver v0.19.3 // indirect
diff --git a/.asf.yaml b/hack/install-kubebuilder.sh
old mode 100644
new mode 100755
similarity index 59%
copy from .asf.yaml
copy to hack/install-kubebuilder.sh
index 5dadefc..c5fdcc5
--- a/.asf.yaml
+++ b/hack/install-kubebuilder.sh
@@ -1,3 +1,5 @@
+#!/usr/bin/env bash
+
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
@@ -13,29 +15,9 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-#
-
-github:
- description: Apache SkyWalking Cloud on Kubernetes
- homepage: https://skywalking.apache.org/
- labels:
- - skywalking
- - observability
- - apm
- - distributed-tracing
- - kubernetes
- - operator
- enabled_merge_buttons:
- squash: true
- merge: false
- rebase: false
- protected_branches:
- master:
- required_status_checks:
- strict: true
- contexts:
- - build
- required_pull_request_reviews:
- dismiss_stale_reviews: true
- required_approving_review_count: 1
+os=$(go env GOOS)
+arch=$(go env GOARCH)
+curl -L https://go.kubebuilder.io/dl/2.3.1/${os}/${arch} | tar -xz -C /tmp/
+sudo mv /tmp/kubebuilder_2.3.1_${os}_${arch} /usr/local/kubebuilder
+export PATH=$PATH:/usr/local/kubebuilder/bin
diff --git a/pkg/kubernetes/kubernetes.go b/pkg/kubernetes/kubernetes.go
index 91a65cf..fd56e65 100644
--- a/pkg/kubernetes/kubernetes.go
+++ b/pkg/kubernetes/kubernetes.go
@@ -19,6 +19,7 @@ package kubernetes
import (
"bytes"
+ "fmt"
"text/template"
"github.com/Masterminds/sprig/v3"
@@ -63,3 +64,18 @@ func LoadTemplate(manifest string, values interface{}, funcMap template.FuncMap,
}
return yaml.Unmarshal(buf.Bytes(), spec)
}
+
+type ErrorCollector []error
+
+func (c *ErrorCollector) Collect(e error) { *c = append(*c, e) }
+
+func (c *ErrorCollector) Error() error {
+ if len(*c) < 1 {
+ return nil
+ }
+ err := "Collected errors:\n"
+ for i, e := range *c {
+ err += fmt.Sprintf("\tError %d: %s\n", i, e.Error())
+ }
+ return fmt.Errorf(err)
+}