You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@submarine.apache.org by ka...@apache.org on 2021/04/24 08:42:10 UTC
[submarine] branch master updated: SUBMARINE-801. Create RBAC for
submarine server via submarine-operator
This is an automated email from the ASF dual-hosted git repository.
kaihsun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new 2b3903e SUBMARINE-801. Create RBAC for submarine server via submarine-operator
2b3903e is described below
commit 2b3903e84468d28256cfa86979dfaff1aaebb32e
Author: kevin85421 <b0...@ntu.edu.tw>
AuthorDate: Sat Apr 24 16:42:03 2021 +0800
SUBMARINE-801. Create RBAC for submarine server via submarine-operator
### What is this PR for?
Create RBAC for submarine server via submarine-operator
Reference: https://github.com/apache/submarine/blob/master/helm-charts/submarine/templates/rbac.yaml
### What type of PR is it?
[Feature]
### Todos
### What is the Jira issue?
https://issues.apache.org/jira/browse/SUBMARINE-801
### How should this be tested?
```
# initialization
# Step1: Register Custom Resource Definition
kubectl apply -f artifacts/examples/crd.yaml
# Step2: Create a Custom Resource
kubectl apply -f artifacts/examples/example-submarine.yaml
# helm
cd submarine
helm install submarine ./helm-charts/submarine
kubectl describe clusterrole submarine-server
kubectl describe clusterrolebinding submarine-server
# in-cluster
cd submarine/submarine-cloud-v2
eval $(minikube docker-env)
make image
kubectl apply -f artifacts/examples/submarine-operator-service-account.yaml
kubectl apply -f artifacts/examples/submarine-operator.yaml
kubectl describe clusterrole submarine-server
kubectl describe clusterrolebinding submarine-server
# out-of-cluster
go build -o submarine-operator
./submarine-operator
```
### Screenshots (if appropriate)
* ClusterRole (helm)
<img width="966" alt="helm_clusterrole" src="https://user-images.githubusercontent.com/20109646/115945169-03081d80-a4ed-11eb-8310-f7cdbb161cec.png">
* ClusterRole (out-of-cluster submarine-operator)
<img width="971" alt="out_of_cluster_clusterrole" src="https://user-images.githubusercontent.com/20109646/115945288-a5c09c00-a4ed-11eb-9958-61b5200df60d.png">
* ClusterRole (in-cluster submarine-operator)
<img width="967" alt="in_cluster_clusterrole" src="https://user-images.githubusercontent.com/20109646/115945285-9fcabb00-a4ed-11eb-8cab-d8e7adb9840d.png">
* ClusterRoleBinding (helm)
<img width="829" alt="helm_clusterrolebinding" src="https://user-images.githubusercontent.com/20109646/115945221-39de3380-a4ed-11eb-9b5b-f6b4f19a8598.png">
* ClusterRoleBinding (out-of-cluster submarine-operator)
<img width="427" alt="out_of_cluster_clusterrolebinding" src="https://user-images.githubusercontent.com/20109646/115945193-187d4780-a4ed-11eb-9394-fc8da3819f3b.png">
* ClusterRoleBinding (in-cluster submarine-operator)
<img width="942" alt="in_cluster_clusterrolebinding" src="https://user-images.githubusercontent.com/20109646/115945268-8295ec80-a4ed-11eb-8b2d-0aa4d154e36d.png">
### Questions:
* Do the license files need updating? No
* Are there breaking changes for older versions? No
* Does this need new documentation? No
---
.../submarine-operator-service-account.yaml | 7 ++
submarine-cloud-v2/controller.go | 134 ++++++++++++++++++---
submarine-cloud-v2/main.go | 2 +
3 files changed, 129 insertions(+), 14 deletions(-)
diff --git a/submarine-cloud-v2/artifacts/examples/submarine-operator-service-account.yaml b/submarine-cloud-v2/artifacts/examples/submarine-operator-service-account.yaml
index 0160207..4624ea2 100644
--- a/submarine-cloud-v2/artifacts/examples/submarine-operator-service-account.yaml
+++ b/submarine-cloud-v2/artifacts/examples/submarine-operator-service-account.yaml
@@ -52,6 +52,13 @@ rules:
- ingresses
verbs:
- '*'
+- apiGroups:
+ - "rbac.authorization.k8s.io"
+ resources:
+ - clusterroles
+ - clusterrolebindings
+ verbs:
+ - '*'
---
apiVersion: v1
kind: ServiceAccount
diff --git a/submarine-cloud-v2/controller.go b/submarine-cloud-v2/controller.go
index c985977..8d60daf 100644
--- a/submarine-cloud-v2/controller.go
+++ b/submarine-cloud-v2/controller.go
@@ -31,6 +31,7 @@ import (
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
extensionsv1beta1 "k8s.io/api/extensions/v1beta1"
+ rbacv1 "k8s.io/api/rbac/v1"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/util/intstr"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -38,12 +39,14 @@ import (
appsinformers "k8s.io/client-go/informers/apps/v1"
coreinformers "k8s.io/client-go/informers/core/v1"
extinformers "k8s.io/client-go/informers/extensions/v1beta1"
+ rbacinformers "k8s.io/client-go/informers/rbac/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/scheme"
typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
appslisters "k8s.io/client-go/listers/apps/v1"
corelisters "k8s.io/client-go/listers/core/v1"
extlisters "k8s.io/client-go/listers/extensions/v1beta1"
+ rbaclisters "k8s.io/client-go/listers/rbac/v1"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/tools/record"
"k8s.io/client-go/util/workqueue"
@@ -62,10 +65,12 @@ type Controller struct {
submarinesLister listers.SubmarineLister
submarinesSynced cache.InformerSynced
- deploymentLister appslisters.DeploymentLister
- serviceaccountLister corelisters.ServiceAccountLister
- serviceLister corelisters.ServiceLister
- ingressLister extlisters.IngressLister
+ deploymentLister appslisters.DeploymentLister
+ serviceaccountLister corelisters.ServiceAccountLister
+ serviceLister corelisters.ServiceLister
+ ingressLister extlisters.IngressLister
+ clusterroleLister rbaclisters.ClusterRoleLister
+ clusterrolebindingLister rbaclisters.ClusterRoleBindingLister
// workqueue is a rate limited work queue. This is used to queue work to be
// processed instead of performing it as soon as a change happens. This
// means we can ensure we only process a fixed amount of resources at a
@@ -85,6 +90,8 @@ func NewController(
serviceInformer coreinformers.ServiceInformer,
serviceaccountInformer coreinformers.ServiceAccountInformer,
ingressInformer extinformers.IngressInformer,
+ clusterroleInformer rbacinformers.ClusterRoleInformer,
+ clusterrolebindingInformer rbacinformers.ClusterRoleBindingInformer,
submarineInformer informers.SubmarineInformer) *Controller {
// TODO: Create event broadcaster
@@ -99,16 +106,18 @@ func NewController(
// Initialize controller
controller := &Controller{
- kubeclientset: kubeclientset,
- submarineclientset: submarineclientset,
- submarinesLister: submarineInformer.Lister(),
- submarinesSynced: submarineInformer.Informer().HasSynced,
- deploymentLister: deploymentInformer.Lister(),
- serviceLister: serviceInformer.Lister(),
- serviceaccountLister: serviceaccountInformer.Lister(),
- ingressLister: ingressInformer.Lister(),
- workqueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "Submarines"),
- recorder: recorder,
+ kubeclientset: kubeclientset,
+ submarineclientset: submarineclientset,
+ submarinesLister: submarineInformer.Lister(),
+ submarinesSynced: submarineInformer.Informer().HasSynced,
+ deploymentLister: deploymentInformer.Lister(),
+ serviceLister: serviceInformer.Lister(),
+ serviceaccountLister: serviceaccountInformer.Lister(),
+ ingressLister: ingressInformer.Lister(),
+ clusterroleLister: clusterroleInformer.Lister(),
+ clusterrolebindingLister: clusterrolebindingInformer.Lister(),
+ workqueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "Submarines"),
+ recorder: recorder,
}
// Setting up event handler for Submarine
@@ -373,6 +382,96 @@ func (c *Controller) newIngress(namespace string) error {
if ingress_err != nil {
return ingress_err
}
+
+ // TODO: (sample-controller) controller.go:287 ~ 293
+
+ return nil
+}
+
+// newSubmarineServerRBAC is a function to create RBAC for submarine-server.
+// Reference: https://github.com/apache/submarine/blob/master/helm-charts/submarine/templates/rbac.yaml
+func (c *Controller) newSubmarineServerRBAC(serviceaccount_namespace string) error {
+ klog.Info("[newSubmarineServerRBAC]")
+ serverName := "submarine-server"
+ // Step1: Create ClusterRole
+ clusterrole, clusterrole_err := c.clusterroleLister.Get(serverName)
+ // If the resource doesn't exist, we'll create it
+ if errors.IsNotFound(clusterrole_err) {
+ clusterrole, clusterrole_err = c.kubeclientset.RbacV1().ClusterRoles().Create(context.TODO(),
+ &rbacv1.ClusterRole{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: serverName,
+ },
+ Rules: []rbacv1.PolicyRule{
+ {
+ Verbs: []string{"get", "list", "watch", "create", "delete", "deletecollection", "patch", "update"},
+ APIGroups: []string{"kubeflow.org"},
+ Resources: []string{"tfjobs", "tfjobs/status", "pytorchjobs", "pytorchjobs/status", "notebooks", "notebooks/status"},
+ },
+ {
+ Verbs: []string{"get", "list", "watch", "create", "delete", "deletecollection", "patch", "update"},
+ APIGroups: []string{"traefik.containo.us"},
+ Resources: []string{"ingressroutes"},
+ },
+ {
+ Verbs: []string{"*"},
+ APIGroups: []string{""},
+ Resources: []string{"pods", "pods/log", "services", "persistentvolumes", "persistentvolumeclaims"},
+ },
+ {
+ Verbs: []string{"*"},
+ APIGroups: []string{"apps"},
+ Resources: []string{"deployments", "deployments/status"},
+ },
+ },
+ },
+ metav1.CreateOptions{})
+ klog.Info(" Create ClusterRole: ", clusterrole.Name)
+ }
+
+ // If an error occurs during Get/Create, we'll requeue the item so we can
+ // attempt processing again later. This could have been caused by a
+ // temporary network failure, or any other transient reason.
+ if clusterrole_err != nil {
+ return clusterrole_err
+ }
+
+ // TODO: (sample-controller) controller.go:287 ~ 293
+
+ clusterrolebinding, clusterrolebinding_err := c.clusterrolebindingLister.Get(serverName)
+ // If the resource doesn't exist, we'll create it
+ if errors.IsNotFound(clusterrolebinding_err) {
+ clusterrolebinding, clusterrolebinding_err = c.kubeclientset.RbacV1().ClusterRoleBindings().Create(context.TODO(),
+ &rbacv1.ClusterRoleBinding{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: serverName,
+ },
+ Subjects: []rbacv1.Subject{
+ rbacv1.Subject{
+ Kind: "ServiceAccount",
+ Namespace: serviceaccount_namespace,
+ Name: serverName,
+ },
+ },
+ RoleRef: rbacv1.RoleRef{
+ Kind: "ClusterRole",
+ Name: serverName,
+ APIGroup: "rbac.authorization.k8s.io",
+ },
+ },
+ metav1.CreateOptions{})
+ klog.Info(" Create ClusterRoleBinding: ", clusterrolebinding.Name)
+ }
+
+ // If an error occurs during Get/Create, we'll requeue the item so we can
+ // attempt processing again later. This could have been caused by a
+ // temporary network failure, or any other transient reason.
+ if clusterrolebinding_err != nil {
+ return clusterrolebinding_err
+ }
+
+ // TODO: (sample-controller) controller.go:287 ~ 293
+
return nil
}
@@ -413,6 +512,7 @@ func (c *Controller) syncHandler(key string) error {
serverImage = "apache/submarine:server-" + submarine.Spec.Version
}
+ // Create Submarine Server
err = c.newSubmarineServer(serverImage, serverReplicas, namespace)
if err != nil {
return err
@@ -424,6 +524,12 @@ func (c *Controller) syncHandler(key string) error {
return err
}
+ // Create RBAC
+ err = c.newSubmarineServerRBAC(namespace)
+ if err != nil {
+ return err
+ }
+
return nil
}
diff --git a/submarine-cloud-v2/main.go b/submarine-cloud-v2/main.go
index 0766d87..0d23c5b 100644
--- a/submarine-cloud-v2/main.go
+++ b/submarine-cloud-v2/main.go
@@ -79,6 +79,8 @@ func main() {
kubeInformerFactory.Core().V1().Services(),
kubeInformerFactory.Core().V1().ServiceAccounts(),
kubeInformerFactory.Extensions().V1beta1().Ingresses(),
+ kubeInformerFactory.Rbac().V1().ClusterRoles(),
+ kubeInformerFactory.Rbac().V1().ClusterRoleBindings(),
submarineInformerFactory.Submarine().V1alpha1().Submarines())
// notice that there is no need to run Start methods in a separate goroutine. (i.e. go kubeInformerFactory.Start(stopCh)
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@submarine.apache.org
For additional commands, e-mail: dev-help@submarine.apache.org