You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@submarine.apache.org by ka...@apache.org on 2021/07/04 12:12:38 UTC
[submarine] branch master updated: SUBMARINE-850. Add operator e2e
test framework
This is an automated email from the ASF dual-hosted git repository.
kaihsun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new 72b8b10 SUBMARINE-850. Add operator e2e test framework
72b8b10 is described below
commit 72b8b10273a920b19b5164f17be181f358c0a91e
Author: Kenchu123 <k8...@gmail.com>
AuthorDate: Sun Jul 4 18:56:15 2021 +0800
SUBMARINE-850. Add operator e2e test framework
### What is this PR for?
Setup operator e2e test framework for further test case
Reference: [spark-on-k8s-operator](https://github.com/GoogleCloudPlatform/spark-on-k8s-operator/tree/master/test/e2e)
This PR will test the following:
- Create submarine-operator
- Delete submarine-operator
### What type of PR is it?
Feature
### Todos
* [ ] Operator state maintain
Test case:
* [ ] Create submarine
* [ ] Delete submarine
* [ ] Scale submarine
### What is the Jira issue?
https://issues.apache.org/jira/browse/SUBMARINE-850
### How should this be tested?
```bash
# Step1: Build image "submarine-operator" to minikube's Docker
eval $(minikube docker-env)
make image
# Step2: Register Custom Resource Definition
kubectl apply -f artifacts/examples/crd.yaml
# Step3: Run test
go test ./test/e2e
```
### Screenshots (if appropriate)
https://user-images.githubusercontent.com/17617373/121552095-c2377a00-ca42-11eb-967e-263e6a7734a9.mov
### Questions:
* Do the license files need updating? No
* Are there breaking changes for older versions? No
* Does this need new documentation? No
Author: Kenchu123 <k8...@gmail.com>
Signed-off-by: Kai-Hsun Chen <ka...@apache.org>
Closes #603 from Kenchu123/SUBMARINE-850 and squashes the following commits:
2ff00376 [Kenchu123] SUBMARINE-850. fix e2e import path
efaa2260 [Kenchu123] SUBMARINE-850. fix framework submarine bug and add licence
7f84dbc3 [Kenchu123] SUBMARINE-850. Add submarine framework functions
ac1057b5 [Kenchu123] SUBMARINE-850. change submarine namespace to submarine-user-test
6086f7a8 [Kenchu123] SUBMARINE-850. Add go mod command in operator e2e test readme
2ae3fe3f [Kenchu123] SUBMARINE-850. Add operator e2e test framework
---
submarine-cloud-v2/README.md | 16 ++
submarine-cloud-v2/go.mod | 2 +
.../test/e2e/framework/cluster_role.go | 102 +++++++++++++
.../test/e2e/framework/cluster_role_binding.go | 104 +++++++++++++
submarine-cloud-v2/test/e2e/framework/context.go | 76 ++++++++++
.../test/e2e/framework/deployment.go | 87 +++++++++++
submarine-cloud-v2/test/e2e/framework/framework.go | 167 +++++++++++++++++++++
submarine-cloud-v2/test/e2e/framework/helpers.go | 74 +++++++++
submarine-cloud-v2/test/e2e/framework/namespace.go | 65 ++++++++
submarine-cloud-v2/test/e2e/framework/operator.go | 42 ++++++
.../test/e2e/framework/service_account.go | 90 +++++++++++
submarine-cloud-v2/test/e2e/framework/submarine.go | 74 +++++++++
submarine-cloud-v2/test/e2e/main_test.go | 58 +++++++
13 files changed, 957 insertions(+)
diff --git a/submarine-cloud-v2/README.md b/submarine-cloud-v2/README.md
index 558cae3..48f6886 100644
--- a/submarine-cloud-v2/README.md
+++ b/submarine-cloud-v2/README.md
@@ -193,3 +193,19 @@ Examples:
```
./hack/run_frontend_e2e.sh loginIT
```
+
+# Run Operator E2E tests
+
+Reference: [spark-on-k8s-operator e2e test](https://github.com/GoogleCloudPlatform/spark-on-k8s-operator/tree/master/test/e2e)
+
+```bash
+# Step1: Build image "submarine-operator" to minikube's Docker
+eval $(minikube docker-env)
+make image
+
+# Step2: Register Custom Resource Definition
+kubectl apply -f artifacts/examples/crd.yaml
+
+# Step3: Run Test
+go ./test/e2e
+```
diff --git a/submarine-cloud-v2/go.mod b/submarine-cloud-v2/go.mod
index 94c5fe0..96634f0 100644
--- a/submarine-cloud-v2/go.mod
+++ b/submarine-cloud-v2/go.mod
@@ -6,7 +6,9 @@ require (
github.com/fatih/color v1.7.0
github.com/gofrs/flock v0.8.0
github.com/pkg/errors v0.9.1
+ github.com/stretchr/testify v1.7.0
github.com/traefik/traefik/v2 v2.4.8
+ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a
gopkg.in/yaml.v2 v2.4.0
helm.sh/helm/v3 v3.5.3
k8s.io/api v0.20.4
diff --git a/submarine-cloud-v2/test/e2e/framework/cluster_role.go b/submarine-cloud-v2/test/e2e/framework/cluster_role.go
new file mode 100644
index 0000000..3348519
--- /dev/null
+++ b/submarine-cloud-v2/test/e2e/framework/cluster_role.go
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package framework
+
+import (
+ "context"
+ "encoding/json"
+ "io"
+ "os"
+
+ rbacv1 "k8s.io/api/rbac/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+ "k8s.io/apimachinery/pkg/util/yaml"
+ "k8s.io/client-go/kubernetes"
+)
+
+func CreateClusterRole(kubeClient kubernetes.Interface, relativePath string) error {
+ clusterRole, err := parseClusterRoleYaml(relativePath)
+ if err != nil {
+ return err
+ }
+
+ _, err = kubeClient.RbacV1().ClusterRoles().Get(context.TODO(), clusterRole.Name, metav1.GetOptions{})
+
+ if err == nil {
+ // ClusterRole already exists -> Update
+ _, err = kubeClient.RbacV1().ClusterRoles().Update(context.TODO(), clusterRole, metav1.UpdateOptions{})
+ if err != nil {
+ return err
+ }
+
+ } else {
+ // ClusterRole doesn't exists -> Create
+ _, err = kubeClient.RbacV1().ClusterRoles().Create(context.TODO(), clusterRole, metav1.CreateOptions{})
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func DeleteClusterRole(kubeClient kubernetes.Interface, relativePath string) error {
+ clusterRole, err := parseClusterRoleYaml(relativePath)
+ if err != nil {
+ return err
+ }
+
+ if err := kubeClient.RbacV1().ClusterRoles().Delete(context.TODO(), clusterRole.Name, metav1.DeleteOptions{}); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func parseClusterRoleYaml(relativePath string) (*rbacv1.ClusterRole, error) {
+ var manifest *os.File
+ var err error
+
+ var clusterRole rbacv1.ClusterRole
+ if manifest, err = PathToOSFile(relativePath); err != nil {
+ return nil, err
+ }
+
+ decoder := yaml.NewYAMLOrJSONDecoder(manifest, 100)
+ for {
+ var out unstructured.Unstructured
+ err = decoder.Decode(&out)
+ if err != nil {
+ // this would indicate it's malformed YAML.
+ break
+ }
+
+ if out.GetKind() == "ClusterRole" {
+ var marshaled []byte
+ marshaled, err = out.MarshalJSON()
+ json.Unmarshal(marshaled, &clusterRole)
+ break
+ }
+ }
+
+ if err != io.EOF && err != nil {
+ return nil, err
+ }
+ return &clusterRole, nil
+}
\ No newline at end of file
diff --git a/submarine-cloud-v2/test/e2e/framework/cluster_role_binding.go b/submarine-cloud-v2/test/e2e/framework/cluster_role_binding.go
new file mode 100644
index 0000000..309c3ed
--- /dev/null
+++ b/submarine-cloud-v2/test/e2e/framework/cluster_role_binding.go
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package framework
+
+import (
+ "context"
+ "encoding/json"
+ "io"
+ "os"
+
+ rbacv1 "k8s.io/api/rbac/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+ "k8s.io/apimachinery/pkg/util/yaml"
+ "k8s.io/client-go/kubernetes"
+)
+
+func CreateClusterRoleBinding(kubeClient kubernetes.Interface, relativePath string) (finalizerFn, error) {
+ finalizerFn := func() error {
+ return DeleteClusterRoleBinding(kubeClient, relativePath)
+ }
+ clusterRoleBinding, err := parseClusterRoleBindingYaml(relativePath)
+ if err != nil {
+ return finalizerFn, err
+ }
+
+ _, err = kubeClient.RbacV1().ClusterRoleBindings().Get(context.TODO(), clusterRoleBinding.Name, metav1.GetOptions{})
+
+ if err == nil {
+ // ClusterRoleBinding already exists -> Update
+ _, err = kubeClient.RbacV1().ClusterRoleBindings().Update(context.TODO(), clusterRoleBinding, metav1.UpdateOptions{})
+ if err != nil {
+ return finalizerFn, err
+ }
+ } else {
+ // ClusterRoleBinding doesn't exists -> Create
+ _, err = kubeClient.RbacV1().ClusterRoleBindings().Create(context.TODO(), clusterRoleBinding, metav1.CreateOptions{})
+ if err != nil {
+ return finalizerFn, err
+ }
+ }
+
+ return finalizerFn, err
+}
+
+func DeleteClusterRoleBinding(kubeClient kubernetes.Interface, relativePath string) error {
+ clusterRoleBinding, err := parseClusterRoleYaml(relativePath)
+ if err != nil {
+ return err
+ }
+
+ if err := kubeClient.RbacV1().ClusterRoleBindings().Delete(context.TODO(), clusterRoleBinding.Name, metav1.DeleteOptions{}); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func parseClusterRoleBindingYaml(relativePath string) (*rbacv1.ClusterRoleBinding, error) {
+ var manifest *os.File
+ var err error
+
+ var clusterRoleBinding rbacv1.ClusterRoleBinding
+ if manifest, err = PathToOSFile(relativePath); err != nil {
+ return nil, err
+ }
+
+ decoder := yaml.NewYAMLOrJSONDecoder(manifest, 100)
+ for {
+ var out unstructured.Unstructured
+ err = decoder.Decode(&out)
+ if err != nil {
+ // this would indicate it's malformed YAML.
+ break
+ }
+
+ if out.GetKind() == "ClusterRoleBinding" {
+ var marshaled []byte
+ marshaled, err = out.MarshalJSON()
+ json.Unmarshal(marshaled, &clusterRoleBinding)
+ break
+ }
+ }
+
+ if err != io.EOF && err != nil {
+ return nil, err
+ }
+ return &clusterRoleBinding, nil
+}
\ No newline at end of file
diff --git a/submarine-cloud-v2/test/e2e/framework/context.go b/submarine-cloud-v2/test/e2e/framework/context.go
new file mode 100644
index 0000000..8e25db5
--- /dev/null
+++ b/submarine-cloud-v2/test/e2e/framework/context.go
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package framework
+
+import (
+ "strconv"
+ "strings"
+ "testing"
+ "time"
+
+ "golang.org/x/sync/errgroup"
+)
+
+type TestCtx struct {
+ ID string
+ cleanUpFns []finalizerFn
+}
+
+type finalizerFn func() error
+
+func (f *Framework) NewTestCtx(t *testing.T) TestCtx {
+ // TestCtx is used among others for namespace names where '/' is forbidden
+ prefix := strings.TrimPrefix(
+ strings.Replace(
+ strings.ToLower(t.Name()),
+ "/",
+ "-",
+ -1,
+ ),
+ "test",
+ )
+
+ id := prefix + "-" + strconv.FormatInt(time.Now().Unix(), 36)
+ return TestCtx{
+ ID: id,
+ }
+}
+
+// GetObjID returns an ascending ID based on the length of cleanUpFns. It is
+// based on the premise that every new object also appends a new finalizerFn on
+// cleanUpFns. This can e.g. be used to create multiple namespaces in the same
+// test context.
+func (ctx *TestCtx) GetObjID() string {
+ return ctx.ID + "-" + strconv.Itoa(len(ctx.cleanUpFns))
+}
+
+func (ctx *TestCtx) Cleanup(t *testing.T) {
+ var eg errgroup.Group
+
+ for i := len(ctx.cleanUpFns) - 1; i >= 0; i-- {
+ eg.Go(ctx.cleanUpFns[i])
+ }
+
+ if err := eg.Wait(); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func (ctx *TestCtx) AddFinalizerFn(fn finalizerFn) {
+ ctx.cleanUpFns = append(ctx.cleanUpFns, fn)
+}
\ No newline at end of file
diff --git a/submarine-cloud-v2/test/e2e/framework/deployment.go b/submarine-cloud-v2/test/e2e/framework/deployment.go
new file mode 100644
index 0000000..8f21698
--- /dev/null
+++ b/submarine-cloud-v2/test/e2e/framework/deployment.go
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package framework
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "github.com/pkg/errors"
+ appsv1 "k8s.io/api/apps/v1"
+ apierrors "k8s.io/apimachinery/pkg/api/errors"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/util/wait"
+ "k8s.io/apimachinery/pkg/util/yaml"
+ "k8s.io/client-go/kubernetes"
+)
+
+func MakeDeployment(pathToYaml string) (*appsv1.Deployment, error) {
+ manifest, err := PathToOSFile(pathToYaml)
+ if err != nil {
+ return nil, err
+ }
+ deployment := appsv1.Deployment{}
+ if err := yaml.NewYAMLOrJSONDecoder(manifest, 100).Decode(&deployment); err != nil {
+ return nil, errors.Wrap(err, fmt.Sprintf("failed to decode file %s", pathToYaml))
+ }
+
+ return &deployment, nil
+}
+
+func CreateDeployment(kubeClient kubernetes.Interface, namespace string, d *appsv1.Deployment) error {
+ _, err := kubeClient.AppsV1().Deployments(namespace).Create(context.TODO(), d, metav1.CreateOptions{})
+ if err != nil {
+ return errors.Wrap(err, fmt.Sprintf("failed to create deployment %s", d.Name))
+ }
+ return nil
+}
+
+func DeleteDeployment(kubeClient kubernetes.Interface, namespace, name string) error {
+ d, err := kubeClient.AppsV1().Deployments(namespace).Get(context.TODO(), name, metav1.GetOptions{})
+ if err != nil {
+ return err
+ }
+
+ zero := int32(0)
+ d.Spec.Replicas = &zero
+
+ d, err = kubeClient.AppsV1().Deployments(namespace).Update(context.TODO(), d, metav1.UpdateOptions{})
+ if err != nil {
+ return err
+ }
+ return kubeClient.AppsV1().Deployments(namespace).Delete(context.TODO(), d.Name, metav1.DeleteOptions{})
+}
+
+func WaitUntilDeploymentGone(kubeClient kubernetes.Interface, namespace, name string, timeout time.Duration) error {
+ return wait.Poll(time.Second, timeout, func() (bool, error) {
+ _, err := kubeClient.
+ AppsV1().Deployments(namespace).
+ Get(context.TODO(), name, metav1.GetOptions{})
+
+ if err != nil {
+ if apierrors.IsNotFound(err) {
+ return true, nil
+ }
+
+ return false, err
+ }
+
+ return false, nil
+ })
+}
\ No newline at end of file
diff --git a/submarine-cloud-v2/test/e2e/framework/framework.go b/submarine-cloud-v2/test/e2e/framework/framework.go
new file mode 100644
index 0000000..94fa383
--- /dev/null
+++ b/submarine-cloud-v2/test/e2e/framework/framework.go
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package framework
+
+import (
+ "context"
+ "time"
+
+ clientset "github.com/apache/submarine/submarine-cloud-v2/pkg/client/clientset/versioned"
+
+ "github.com/pkg/errors"
+ corev1 "k8s.io/api/core/v1"
+ apierrors "k8s.io/apimachinery/pkg/api/errors"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/fields"
+ "k8s.io/client-go/kubernetes"
+ "k8s.io/client-go/tools/clientcmd"
+)
+
+type Framework struct {
+ KubeClient kubernetes.Interface
+ SubmarineClient clientset.Interface
+ Namespace *corev1.Namespace
+ OperatorPod *corev1.Pod
+ MasterHost string
+ DefaultTimeout time.Duration
+}
+
+var SubmarineTestNamespace = "submarine-user-test"
+
+
+func New(ns, submarineNs, kubeconfig, opImage, opImagePullPolicy string) (*Framework, error) {
+
+ cfg, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
+ if err != nil {
+ return nil, errors.Wrap(err, "build config failed")
+ }
+
+ kubeClient, err := kubernetes.NewForConfig(cfg)
+ if err != nil {
+ return nil, errors.Wrap(err, "creating new kube-client fail")
+ }
+
+ // create submarine-operator namespace
+ namespace, err := kubeClient.CoreV1().Namespaces().Create(context.TODO(), &corev1.Namespace{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: ns,
+ },
+ },
+ metav1.CreateOptions{})
+ if apierrors.IsAlreadyExists(err) {
+ namespace, err = kubeClient.CoreV1().Namespaces().Get(context.TODO(), ns, metav1.GetOptions{})
+ } else {
+ return nil, errors.Wrap(err, "create submarine operator namespace fail")
+ }
+
+ submarineClient, err := clientset.NewForConfig(cfg)
+ if err != nil {
+ return nil, errors.Wrap(err, "creating new submarine-client fail")
+ }
+
+ f := &Framework {
+ MasterHost: cfg.Host,
+ KubeClient: kubeClient,
+ SubmarineClient: submarineClient,
+ Namespace: namespace,
+ DefaultTimeout: time.Minute,
+ }
+ err = f.Setup(submarineNs, opImage, opImagePullPolicy)
+ if err != nil {
+ return nil, errors.Wrap(err, "setup test environment failed")
+ }
+
+ return f, nil
+}
+
+func (f *Framework) Setup(submarineNs, opImage, opImagePullPolicy string) error {
+ if err := f.setupOperator(submarineNs, opImage, opImagePullPolicy); err != nil {
+ return errors.Wrap(err, "setup operator failed")
+ }
+ return nil
+}
+
+func (f* Framework) setupOperator(submarineNs, opImage, opImagePullPolicy string) error {
+
+ // setup RBAC (ClusterRole, ClusterRoleBinding, and ServiceAccount)
+ if _, err := CreateServiceAccount(f.KubeClient, f.Namespace.Name, "../../artifacts/examples/submarine-operator-service-account.yaml"); err != nil && !apierrors.IsAlreadyExists(err) {
+ return errors.Wrap(err, "failed to create operator service account")
+ }
+
+ if err := CreateClusterRole(f.KubeClient, "../../artifacts/examples/submarine-operator-service-account.yaml"); err != nil && !apierrors.IsAlreadyExists(err) {
+ return errors.Wrap(err, "failed to create cluster role")
+ }
+
+ if _, err := CreateClusterRoleBinding(f.KubeClient, "../../artifacts/examples/submarine-operator-service-account.yaml"); err != nil && !apierrors.IsAlreadyExists(err) {
+ return errors.Wrap(err, "failed to create cluster role binding")
+ }
+ // Deploy a submarine-operator
+ deploy, err := MakeDeployment("../../artifacts/examples/submarine-operator.yaml")
+ if err != nil {
+ return err
+ }
+
+ if opImage != "" {
+ // Override operator image used, if specified when running tests.
+ deploy.Spec.Template.Spec.Containers[0].Image = opImage
+ }
+
+ for _, container := range deploy.Spec.Template.Spec.Containers {
+ container.ImagePullPolicy = corev1.PullPolicy(opImagePullPolicy)
+ }
+
+ err = CreateDeployment(f.KubeClient, f.Namespace.Name, deploy)
+ if err != nil {
+ return err
+ }
+
+ opts := metav1.ListOptions{LabelSelector: fields.SelectorFromSet(fields.Set(deploy.Spec.Template.ObjectMeta.Labels)).String()}
+ err = WaitForPodsReady(f.KubeClient, f.Namespace.Name, f.DefaultTimeout, 1, opts)
+ if err != nil {
+ return errors.Wrap(err, "failed to wait for operator to become ready")
+ }
+
+ pl, err := f.KubeClient.CoreV1().Pods(f.Namespace.Name).List(context.TODO(), opts)
+ if err != nil {
+ return err
+ }
+ f.OperatorPod = &pl.Items[0]
+
+ return nil
+}
+
+// Teardown ters down a previously initialized test environment
+func (f *Framework) Teardown() error {
+ if err := DeleteClusterRole(f.KubeClient, "../../artifacts/examples/submarine-operator-service-account.yaml"); err != nil && !apierrors.IsAlreadyExists(err) {
+ return errors.Wrap(err, "failed to delete operator cluster role")
+ }
+
+ if err := DeleteClusterRoleBinding(f.KubeClient, "../../artifacts/examples/submarine-operator-service-account.yaml"); err != nil && !apierrors.IsAlreadyExists(err) {
+ return errors.Wrap(err, "failed to delete operator cluster role binding")
+ }
+
+ if err := f.KubeClient.AppsV1().Deployments(f.Namespace.Name).Delete(context.TODO(), "submarine-operator-demo", metav1.DeleteOptions{}); err != nil {
+ return err
+ }
+
+ if err := DeleteNamespace(f.KubeClient, f.Namespace.Name); err != nil && !apierrors.IsForbidden(err) {
+ return err
+ }
+
+ return nil
+}
\ No newline at end of file
diff --git a/submarine-cloud-v2/test/e2e/framework/helpers.go b/submarine-cloud-v2/test/e2e/framework/helpers.go
new file mode 100644
index 0000000..f104a55
--- /dev/null
+++ b/submarine-cloud-v2/test/e2e/framework/helpers.go
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package framework
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "path/filepath"
+ "time"
+
+ "github.com/pkg/errors"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/util/wait"
+ "k8s.io/client-go/kubernetes"
+)
+
+// PathToOSFile gets the absolute path from relative path.
+func PathToOSFile(relativePath string) (*os.File, error) {
+ path, err := filepath.Abs(relativePath)
+ if err != nil {
+ return nil, errors.Wrap(err, fmt.Sprintf("failed generate absolute file path of %s", relativePath))
+ }
+
+ manifest, err := os.Open(path)
+ if err != nil {
+ return nil, errors.Wrap(err, fmt.Sprintf("failed to open file %s", path))
+ }
+
+ return manifest, nil
+}
+
+// WaitForPodsReady waits for a selection of Pods to be running and each
+// container to pass its readiness check.
+func WaitForPodsReady(kubeClient kubernetes.Interface, namespace string, timeout time.Duration, expectedReplicas int, opts metav1.ListOptions) error {
+ return wait.Poll(time.Second, timeout, func() (bool, error) {
+ pl, err := kubeClient.CoreV1().Pods(namespace).List(context.TODO(), opts)
+ if err != nil {
+ return false, err
+ }
+
+ runningAndReady := 0
+ for _, p := range pl.Items {
+ isRunningAndReady, err := PodRunningAndReady(p)
+ if err != nil {
+ return false, err
+ }
+
+ if isRunningAndReady {
+ runningAndReady++
+ }
+ }
+
+ if runningAndReady == expectedReplicas {
+ return true, nil
+ }
+ return false, nil
+ })
+}
diff --git a/submarine-cloud-v2/test/e2e/framework/namespace.go b/submarine-cloud-v2/test/e2e/framework/namespace.go
new file mode 100644
index 0000000..76ae147
--- /dev/null
+++ b/submarine-cloud-v2/test/e2e/framework/namespace.go
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package framework
+
+import (
+ "context"
+ "fmt"
+ "testing"
+
+ "github.com/pkg/errors"
+ v1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/client-go/kubernetes"
+)
+
+func CreateNamespace(kubeClient kubernetes.Interface, name string) (*v1.Namespace, error) {
+ namespace, err := kubeClient.CoreV1().Namespaces().Create(context.TODO(), &v1.Namespace{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: name,
+ },
+ },
+ metav1.CreateOptions{},
+ )
+ if err != nil {
+ return nil, errors.Wrap(err, fmt.Sprintf("failed to create namespace with name %v", name))
+ }
+ return namespace, nil
+}
+
+func (ctx *TestCtx) CreateNamespace(t *testing.T, kubeClient kubernetes.Interface) string {
+ name := ctx.GetObjID()
+ if _, err := CreateNamespace(kubeClient, name); err != nil {
+ t.Fatal(err)
+ }
+
+ namespaceFinalizerFn := func() error {
+ if err := DeleteNamespace(kubeClient, name); err != nil {
+ return err
+ }
+ return nil
+ }
+
+ ctx.AddFinalizerFn(namespaceFinalizerFn)
+
+ return name
+}
+
+func DeleteNamespace(kubeClient kubernetes.Interface, name string) error {
+ return kubeClient.CoreV1().Namespaces().Delete(context.TODO(), name, metav1.DeleteOptions{})
+}
diff --git a/submarine-cloud-v2/test/e2e/framework/operator.go b/submarine-cloud-v2/test/e2e/framework/operator.go
new file mode 100644
index 0000000..d56765a
--- /dev/null
+++ b/submarine-cloud-v2/test/e2e/framework/operator.go
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package framework
+
+import (
+ "fmt"
+
+ corev1 "k8s.io/api/core/v1"
+)
+
+// PodRunningAndReady returns whether a pod is running and each container has
+// passed it's ready state.
+func PodRunningAndReady(pod corev1.Pod) (bool, error) {
+ switch pod.Status.Phase {
+ case corev1.PodFailed, corev1.PodSucceeded:
+ return false, fmt.Errorf("pod completed")
+ case corev1.PodRunning:
+ for _, cond := range pod.Status.Conditions {
+ if cond.Type != corev1.PodReady {
+ continue
+ }
+ return cond.Status == corev1.ConditionTrue, nil
+ }
+ return false, fmt.Errorf("pod ready condition not found")
+ }
+ return false, nil
+}
\ No newline at end of file
diff --git a/submarine-cloud-v2/test/e2e/framework/service_account.go b/submarine-cloud-v2/test/e2e/framework/service_account.go
new file mode 100644
index 0000000..b118ff7
--- /dev/null
+++ b/submarine-cloud-v2/test/e2e/framework/service_account.go
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package framework
+
+import (
+ "context"
+ "encoding/json"
+ "io"
+ "os"
+
+ v1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+ "k8s.io/apimachinery/pkg/util/yaml"
+ "k8s.io/client-go/kubernetes"
+)
+
+func CreateServiceAccount(kubeClient kubernetes.Interface, namespace string, relativePath string) (finalizerFn, error) {
+ finalizerFn := func() error {
+ return DeleteServiceAccount(kubeClient, namespace, relativePath)
+ }
+
+ serviceAccount, err := parseServiceAccountYaml(relativePath)
+ if err != nil {
+ return finalizerFn, err
+ }
+ serviceAccount.Namespace = namespace
+ _, err = kubeClient.CoreV1().ServiceAccounts(namespace).Create(context.TODO(), serviceAccount, metav1.CreateOptions{})
+ if err != nil {
+ return finalizerFn, err
+ }
+
+ return finalizerFn, nil
+}
+
+func parseServiceAccountYaml(relativePath string) (*v1.ServiceAccount, error) {
+ var manifest *os.File
+ var err error
+
+ var serviceAccount v1.ServiceAccount
+ if manifest, err = PathToOSFile(relativePath); err != nil {
+ return nil, err
+ }
+
+ decoder := yaml.NewYAMLOrJSONDecoder(manifest, 100)
+ for {
+ var out unstructured.Unstructured
+ err = decoder.Decode(&out)
+ if err != nil {
+ // this would indicate it's malformed YAML.
+ break
+ }
+
+ if out.GetKind() == "ServiceAccount" {
+ var marshaled []byte
+ marshaled, err = out.MarshalJSON()
+ json.Unmarshal(marshaled, &serviceAccount)
+ break
+ }
+ }
+
+ if err != io.EOF && err != nil {
+ return nil, err
+ }
+ return &serviceAccount, nil
+}
+
+func DeleteServiceAccount(kubeClient kubernetes.Interface, namespace string, relativePath string) error {
+ serviceAccount, err := parseServiceAccountYaml(relativePath)
+ if err != nil {
+ return err
+ }
+
+ return kubeClient.CoreV1().ServiceAccounts(namespace).Delete(context.TODO(), serviceAccount.Name, metav1.DeleteOptions{})
+}
\ No newline at end of file
diff --git a/submarine-cloud-v2/test/e2e/framework/submarine.go b/submarine-cloud-v2/test/e2e/framework/submarine.go
new file mode 100644
index 0000000..386105f
--- /dev/null
+++ b/submarine-cloud-v2/test/e2e/framework/submarine.go
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package framework
+
+import (
+ "context"
+ "fmt"
+
+ v1alpha1 "github.com/apache/submarine/submarine-cloud-v2/pkg/apis/submarine/v1alpha1"
+ clientset "github.com/apache/submarine/submarine-cloud-v2/pkg/client/clientset/versioned"
+
+ "github.com/pkg/errors"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/util/yaml"
+)
+
+func MakeSubmarineFromYaml(pathToYaml string) (*v1alpha1.Submarine, error) {
+ manifest, err := PathToOSFile(pathToYaml)
+ if err != nil {
+ return nil, err
+ }
+ tmp := v1alpha1.Submarine{}
+ if err := yaml.NewYAMLOrJSONDecoder(manifest, 100).Decode(&tmp); err != nil {
+ return nil, errors.Wrap(err, fmt.Sprintf("failed to decode file %s", pathToYaml))
+ }
+ return &tmp, err
+}
+
+func CreateSubmarine(clientset clientset.Interface, namespace string, submarine *v1alpha1.Submarine) error {
+ _, err := clientset.SubmarineV1alpha1().Submarines(namespace).Create(context.TODO(), submarine, metav1.CreateOptions{})
+ if err != nil {
+ return errors.Wrap(err, fmt.Sprintf("failed to create Submarine %s", submarine.Name))
+ }
+ return nil
+}
+
+func UpdateSubmarine(clientset clientset.Interface, namespace string, submarine *v1alpha1.Submarine) error {
+ _, err := clientset.SubmarineV1alpha1().Submarines(namespace).Update(context.TODO(), submarine, metav1.UpdateOptions{})
+ if err != nil {
+ return errors.Wrap(err, fmt.Sprintf("failed to update Submarine %s", submarine.Name))
+ }
+ return nil
+}
+
+func GetSubmarine(clientset clientset.Interface, namespace string, name string) (*v1alpha1.Submarine, error) {
+ submarine, err := clientset.SubmarineV1alpha1().Submarines(namespace).Get(context.TODO(), name, metav1.GetOptions{})
+ if err != nil {
+ return nil, err
+ }
+ return submarine, nil
+}
+
+func DeleteSubmarine(clientset clientset.Interface, namespace string, name string) error {
+ err := clientset.SubmarineV1alpha1().Submarines(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{})
+ if err != nil {
+ return err
+ }
+ return nil
+}
diff --git a/submarine-cloud-v2/test/e2e/main_test.go b/submarine-cloud-v2/test/e2e/main_test.go
new file mode 100644
index 0000000..40697af
--- /dev/null
+++ b/submarine-cloud-v2/test/e2e/main_test.go
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package e2e
+
+import (
+ "flag"
+ "log"
+ "os"
+ "testing"
+
+ operatorFramework "github.com/apache/submarine/submarine-cloud-v2/test/e2e/framework"
+)
+
+var (
+ framework *operatorFramework.Framework
+)
+
+func TestMain(m *testing.M) {
+ kubeconfig := flag.String("kubeconfig", os.Getenv("HOME")+"/.kube/config", "Path to a kubeconfig. Only required if out-of-cluster.")
+ opImage := flag.String("operator-image", "", "operator image, e.g. image:tag")
+ opImagePullPolicy := flag.String("operator-image-pullPolicy", "Never", "pull policy, e.g. Always")
+ ns := flag.String("namespace", "default", "e2e test operator namespace")
+ submarineTestNamespace := flag.String("submarine-test-namespace", "submarine-user-test", "e2e test submarine namespace")
+ flag.Parse()
+
+ var (
+ err error
+ exitCode int
+ )
+
+ if framework, err = operatorFramework.New(*ns, *submarineTestNamespace, *kubeconfig, *opImage, *opImagePullPolicy); err != nil {
+ log.Fatalf("Error setting up framework: %+v", err)
+ }
+
+ operatorFramework.SubmarineTestNamespace = *submarineTestNamespace
+
+ exitCode = m.Run()
+
+ if err := framework.Teardown(); err != nil {
+ log.Fatalf("Failed to tear down framework :%v\n", err)
+ }
+ os.Exit(exitCode)
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@submarine.apache.org
For additional commands, e-mail: dev-help@submarine.apache.org