You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by as...@apache.org on 2021/03/26 13:13:23 UTC
[camel-k] 01/03: feat(operator): toleration install flag
This is an automated email from the ASF dual-hosted git repository.
astefanutti pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/camel-k.git
commit 21e7ceb37da0262fb5ad4cf401cb52629ee839f3
Author: Pasquale Congiusti <pa...@gmail.com>
AuthorDate: Thu Mar 11 14:34:33 2021 +0100
feat(operator): toleration install flag
* Refactored toleration trait to create Tolerations in a util method
* Refactored OperatorOrCollect method to ease readability
* Added toleration configuration in OLM and non OLM installations
---
pkg/cmd/install.go | 8 +-
pkg/cmd/install_test.go | 10 +++
pkg/install/operator.go | 180 ++++++++++++++++++++++++++------------------
pkg/trait/toleration.go | 40 +---------
pkg/util/kubernetes/util.go | 38 ++++++++++
pkg/util/olm/operator.go | 19 ++++-
6 files changed, 180 insertions(+), 115 deletions(-)
diff --git a/pkg/cmd/install.go b/pkg/cmd/install.go
index a116c50..a6b7468 100644
--- a/pkg/cmd/install.go
+++ b/pkg/cmd/install.go
@@ -125,6 +125,9 @@ func newCmdInstall(rootCmdOptions *RootCmdOptions) (*cobra.Command, *installCmdO
cmd.Flags().Bool("monitoring", false, "To enable or disable the operator monitoring")
cmd.Flags().Int("monitoring-port", 8080, "The port of the metrics endpoint")
+ // Pod settings
+ cmd.Flags().StringArrayP("toleration", "", nil, "Add a Toleration to the operator Pod")
+
// save
cmd.Flags().Bool("save", false, "Save the install parameters into the default kamel configuration file (kamel-config.yaml)")
@@ -169,6 +172,7 @@ type installCmdOptions struct {
MonitoringPort int32 `mapstructure:"monitoring-port"`
Properties []string `mapstructure:"properties"`
TraitProfile string `mapstructure:"trait-profile"`
+ Tolerations []string `mapstructure:"tolerations"`
HTTPProxySecret string `mapstructure:"http-proxy-secret"`
registry v1.IntegrationPlatformRegistrySpec
@@ -199,7 +203,6 @@ func (o *installCmdOptions) install(cobraCmd *cobra.Command, _ []string) error {
if olmAvailable, err = olm.IsAPIAvailable(o.Context, olmClient, o.Namespace); err != nil {
return errors.Wrap(err, "error while checking OLM availability. Run with '--olm=false' to skip this check")
}
-
if olmAvailable {
if installViaOLM, err = olm.HasPermissionToInstall(o.Context, olmClient, o.Namespace, o.Global, o.olmOptions); err != nil {
return errors.Wrap(err, "error while checking permissions to install operator via OLM. Run with '--olm=false' to skip this check")
@@ -216,7 +219,7 @@ func (o *installCmdOptions) install(cobraCmd *cobra.Command, _ []string) error {
if installViaOLM {
fmt.Fprintln(cobraCmd.OutOrStdout(), "OLM is available in the cluster")
var installed bool
- if installed, err = olm.Install(o.Context, olmClient, o.Namespace, o.Global, o.olmOptions, collection); err != nil {
+ if installed, err = olm.Install(o.Context, olmClient, o.Namespace, o.Global, o.olmOptions, collection, o.Tolerations); err != nil {
return err
}
if !installed {
@@ -267,6 +270,7 @@ func (o *installCmdOptions) install(cobraCmd *cobra.Command, _ []string) error {
Enabled: o.Monitoring,
Port: o.MonitoringPort,
},
+ Tolerations: o.Tolerations,
}
err = install.OperatorOrCollect(o.Context, c, cfg, collection, o.Force)
if err != nil {
diff --git a/pkg/cmd/install_test.go b/pkg/cmd/install_test.go
index 157430c..b26c3eb 100644
--- a/pkg/cmd/install_test.go
+++ b/pkg/cmd/install_test.go
@@ -371,3 +371,13 @@ func TestDecodeMavenSettings(t *testing.T) {
_, err = decodeMavenSettings("secret")
assert.NotNil(t, err)
}
+
+func TestInstallTolerationFlag(t *testing.T) {
+ installCmdOptions, rootCmd, _ := initializeInstallCmdOptions(t)
+ _, err := test.ExecuteCommand(rootCmd, cmdInstall,
+ "--toleration", "key1=value1:NoSchedule",
+ "--toleration", "key2=value2:NoExecute")
+ assert.Nil(t, err)
+ assert.Equal(t, "key1=value1:NoSchedule", installCmdOptions.Tolerations[0])
+ assert.Equal(t, "key2=value2:NoExecute", installCmdOptions.Tolerations[1])
+}
diff --git a/pkg/install/operator.go b/pkg/install/operator.go
index 2a650f8..397047b 100644
--- a/pkg/install/operator.go
+++ b/pkg/install/operator.go
@@ -31,6 +31,7 @@ import (
k8serrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/intstr"
ctrl "sigs.k8s.io/controller-runtime/pkg/client"
@@ -54,6 +55,7 @@ type OperatorConfiguration struct {
ClusterType string
Health OperatorHealthConfiguration
Monitoring OperatorMonitoringConfiguration
+ Tolerations []string
}
// OperatorHealthConfiguration --
@@ -70,81 +72,15 @@ type OperatorMonitoringConfiguration struct {
// OperatorOrCollect installs the operator resources or adds them to the collector if present
func OperatorOrCollect(ctx context.Context, c client.Client, cfg OperatorConfiguration, collection *kubernetes.Collection, force bool) error {
customizer := func(o ctrl.Object) ctrl.Object {
- if cfg.CustomImage != "" {
- if d, ok := o.(*appsv1.Deployment); ok {
- if d.Labels["camel.apache.org/component"] == "operator" {
- d.Spec.Template.Spec.Containers[0].Image = cfg.CustomImage
- }
- }
- }
-
- if cfg.CustomImagePullPolicy != "" {
- if d, ok := o.(*appsv1.Deployment); ok {
- if d.Labels["camel.apache.org/component"] == "operator" {
- d.Spec.Template.Spec.Containers[0].ImagePullPolicy = corev1.PullPolicy(cfg.CustomImagePullPolicy)
- }
- }
- }
-
- if d, ok := o.(*appsv1.Deployment); ok {
- if d.Labels["camel.apache.org/component"] == "operator" {
- // Metrics endpoint port
- d.Spec.Template.Spec.Containers[0].Args = append(d.Spec.Template.Spec.Containers[0].Args,
- fmt.Sprintf("--monitoring-port=%d", cfg.Monitoring.Port))
- d.Spec.Template.Spec.Containers[0].Ports[0].ContainerPort = cfg.Monitoring.Port
- // Health endpoint port
- d.Spec.Template.Spec.Containers[0].Args = append(d.Spec.Template.Spec.Containers[0].Args,
- fmt.Sprintf("--health-port=%d", cfg.Health.Port))
- d.Spec.Template.Spec.Containers[0].LivenessProbe.HTTPGet.Port = intstr.FromInt(int(cfg.Health.Port))
- }
- }
-
- if cfg.Global {
- if d, ok := o.(*appsv1.Deployment); ok {
- if d.Labels["camel.apache.org/component"] == "operator" {
- // Make the operator watch all namespaces
- envvar.SetVal(&d.Spec.Template.Spec.Containers[0].Env, "WATCH_NAMESPACE", "")
- }
- }
-
- // Turn Role & RoleBinding into their equivalent cluster types
- if r, ok := o.(*rbacv1.Role); ok {
- if strings.HasPrefix(r.Name, "camel-k-operator") {
- o = &rbacv1.ClusterRole{
- ObjectMeta: metav1.ObjectMeta{
- Namespace: cfg.Namespace,
- Name: r.Name,
- Labels: map[string]string{
- "app": "camel-k",
- },
- },
- Rules: r.Rules,
- }
- }
- }
-
- if rb, ok := o.(*rbacv1.RoleBinding); ok {
- if strings.HasPrefix(rb.Name, "camel-k-operator") {
- rb.Subjects[0].Namespace = cfg.Namespace
-
- o = &rbacv1.ClusterRoleBinding{
- ObjectMeta: metav1.ObjectMeta{
- Namespace: cfg.Namespace,
- Name: rb.Name,
- Labels: map[string]string{
- "app": "camel-k",
- },
- },
- Subjects: rb.Subjects,
- RoleRef: rbacv1.RoleRef{
- APIGroup: rb.RoleRef.APIGroup,
- Kind: "ClusterRole",
- Name: rb.RoleRef.Name,
- },
- }
- }
- }
+ operatorDeployment := operatorDeployment(o)
+ if operatorDeployment == nil {
+ return o
}
+ maybeSetCustomImage(cfg, operatorDeployment)
+ maybeSetCustomImagePullPolicy(cfg, operatorDeployment)
+ setPorts(cfg, operatorDeployment)
+ maybeSetGlobal(cfg, o)
+ maybeSetTolerations(cfg, operatorDeployment)
return o
}
@@ -237,6 +173,102 @@ func OperatorOrCollect(ctx context.Context, c client.Client, cfg OperatorConfigu
return nil
}
+func operatorDeployment(o runtime.Object) *appsv1.Deployment {
+ if d, ok := o.(*appsv1.Deployment); ok {
+ if d.Labels["camel.apache.org/component"] == "operator" {
+ return d
+ }
+ }
+ return nil
+}
+
+func maybeSetCustomImage(cfg OperatorConfiguration, d *appsv1.Deployment) error {
+ if cfg.CustomImage != "" {
+ d.Spec.Template.Spec.Containers[0].Image = cfg.CustomImage
+ }
+ return nil
+}
+
+func maybeSetCustomImagePullPolicy(cfg OperatorConfiguration, d *appsv1.Deployment) error {
+ if cfg.CustomImagePullPolicy != "" {
+ d.Spec.Template.Spec.Containers[0].ImagePullPolicy = corev1.PullPolicy(cfg.CustomImagePullPolicy)
+ }
+ return nil
+}
+
+func maybeSetTolerations(cfg OperatorConfiguration, d *appsv1.Deployment) error {
+ if cfg.Tolerations != nil {
+ tolerations, err := kubernetes.GetTolerations(cfg.Tolerations)
+ if err != nil {
+ return err
+ }
+ d.Spec.Template.Spec.Tolerations = tolerations
+ }
+ return nil
+}
+
+func maybeSetGlobal(cfg OperatorConfiguration, o runtime.Object) error {
+ if cfg.Global {
+ if d, ok := o.(*appsv1.Deployment); ok {
+ if d.Labels["camel.apache.org/component"] == "operator" {
+ // Make the operator watch all namespaces
+ envvar.SetVal(&d.Spec.Template.Spec.Containers[0].Env, "WATCH_NAMESPACE", "")
+ }
+ }
+
+ // Turn Role & RoleBinding into their equivalent cluster types
+ if r, ok := o.(*rbacv1.Role); ok {
+ if strings.HasPrefix(r.Name, "camel-k-operator") {
+ o = &rbacv1.ClusterRole{
+ ObjectMeta: metav1.ObjectMeta{
+ Namespace: cfg.Namespace,
+ Name: r.Name,
+ Labels: map[string]string{
+ "app": "camel-k",
+ },
+ },
+ Rules: r.Rules,
+ }
+ }
+ }
+
+ if rb, ok := o.(*rbacv1.RoleBinding); ok {
+ if strings.HasPrefix(rb.Name, "camel-k-operator") {
+ rb.Subjects[0].Namespace = cfg.Namespace
+
+ o = &rbacv1.ClusterRoleBinding{
+ ObjectMeta: metav1.ObjectMeta{
+ Namespace: cfg.Namespace,
+ Name: rb.Name,
+ Labels: map[string]string{
+ "app": "camel-k",
+ },
+ },
+ Subjects: rb.Subjects,
+ RoleRef: rbacv1.RoleRef{
+ APIGroup: rb.RoleRef.APIGroup,
+ Kind: "ClusterRole",
+ Name: rb.RoleRef.Name,
+ },
+ }
+ }
+ }
+ }
+ return nil
+}
+
+func setPorts(cfg OperatorConfiguration, d *appsv1.Deployment) error {
+ // Metrics endpoint port
+ d.Spec.Template.Spec.Containers[0].Args = append(d.Spec.Template.Spec.Containers[0].Args,
+ fmt.Sprintf("--monitoring-port=%d", cfg.Monitoring.Port))
+ d.Spec.Template.Spec.Containers[0].Ports[0].ContainerPort = cfg.Monitoring.Port
+ // Health endpoint port
+ d.Spec.Template.Spec.Containers[0].Args = append(d.Spec.Template.Spec.Containers[0].Args,
+ fmt.Sprintf("--health-port=%d", cfg.Health.Port))
+ d.Spec.Template.Spec.Containers[0].LivenessProbe.HTTPGet.Port = intstr.FromInt(int(cfg.Health.Port))
+ return nil
+}
+
func installOpenShiftClusterRoleBinding(ctx context.Context, c client.Client, collection *kubernetes.Collection, namespace string) error {
var target *rbacv1.ClusterRoleBinding
existing, err := c.RbacV1().ClusterRoleBindings().Get(ctx, "camel-k-operator-openshift", metav1.GetOptions{})
diff --git a/pkg/trait/toleration.go b/pkg/trait/toleration.go
index c62bb7a..9b97c6e 100644
--- a/pkg/trait/toleration.go
+++ b/pkg/trait/toleration.go
@@ -19,13 +19,12 @@ package trait
import (
"fmt"
- "regexp"
- "strconv"
corev1 "k8s.io/api/core/v1"
v1 "github.com/apache/camel-k/pkg/apis/camel/v1"
"github.com/apache/camel-k/pkg/util"
+ "github.com/apache/camel-k/pkg/util/kubernetes"
)
// This trait sets Tolerations over Integration pods. Tolerations allow (but do not require) the pods to schedule onto nodes with matching taints.
@@ -48,8 +47,6 @@ type tolerationTrait struct {
Taints []string `property:"taints" json:"taints,omitempty"`
}
-var validTaintRegexp = regexp.MustCompile(`^([\w\/_\-\.]+)(=)?([\w_\-\.]+)?:(NoSchedule|NoExecute|PreferNoSchedule):?(\d*)?$`)
-
func newTolerationTrait() Trait {
return &tolerationTrait{
BaseTrait: NewBaseTrait("toleration", 1200),
@@ -69,7 +66,7 @@ func (t *tolerationTrait) Configure(e *Environment) (bool, error) {
}
func (t *tolerationTrait) Apply(e *Environment) (err error) {
- tolerations, err := t.getTolerations()
+ tolerations, err := kubernetes.GetTolerations(t.Taints)
if err != nil {
return err
}
@@ -84,36 +81,3 @@ func (t *tolerationTrait) Apply(e *Environment) (err error) {
podSpec.Tolerations = append(podSpec.Tolerations, tolerations...)
return nil
}
-
-func (t *tolerationTrait) getTolerations() ([]corev1.Toleration, error) {
- tolerations := make([]corev1.Toleration, 0)
- for _, t := range t.Taints {
- if !validTaintRegexp.MatchString(t) {
- return nil, fmt.Errorf("could not match taint %v", t)
- }
- toleration := corev1.Toleration{}
- // Parse the regexp groups
- groups := validTaintRegexp.FindStringSubmatch(t)
- toleration.Key = groups[1]
- if groups[2] != "" {
- toleration.Operator = corev1.TolerationOpEqual
- } else {
- toleration.Operator = corev1.TolerationOpExists
- }
- if groups[3] != "" {
- toleration.Value = groups[3]
- }
- toleration.Effect = corev1.TaintEffect(groups[4])
-
- if groups[5] != "" {
- tolerationSeconds, err := strconv.ParseInt(groups[5], 10, 64)
- if err != nil {
- return nil, err
- }
- toleration.TolerationSeconds = &tolerationSeconds
- }
- tolerations = append(tolerations, toleration)
- }
-
- return tolerations, nil
-}
diff --git a/pkg/util/kubernetes/util.go b/pkg/util/kubernetes/util.go
index 0ce1517..26bba6a 100644
--- a/pkg/util/kubernetes/util.go
+++ b/pkg/util/kubernetes/util.go
@@ -20,6 +20,8 @@ package kubernetes
import (
"context"
"fmt"
+ "regexp"
+ "strconv"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -32,6 +34,8 @@ import (
"github.com/apache/camel-k/pkg/util"
)
+var validTaintRegexp = regexp.MustCompile(`^([\w\/_\-\.]+)(=)?([\w_\-\.]+)?:(NoSchedule|NoExecute|PreferNoSchedule):?(\d*)?$`)
+
// ToJSON --
func ToJSON(value runtime.Object) ([]byte, error) {
return json.Marshal(value)
@@ -228,3 +232,37 @@ func ResolveValueSource(ctx context.Context, client k8sclient.Reader, namespace
return "", nil
}
+
+// GetTolerations build an array of Tolerations from an array of string
+func GetTolerations(taints []string) ([]corev1.Toleration, error) {
+ tolerations := make([]corev1.Toleration, 0)
+ for _, t := range taints {
+ if !validTaintRegexp.MatchString(t) {
+ return nil, fmt.Errorf("could not match taint %v", t)
+ }
+ toleration := corev1.Toleration{}
+ // Parse the regexp groups
+ groups := validTaintRegexp.FindStringSubmatch(t)
+ toleration.Key = groups[1]
+ if groups[2] != "" {
+ toleration.Operator = corev1.TolerationOpEqual
+ } else {
+ toleration.Operator = corev1.TolerationOpExists
+ }
+ if groups[3] != "" {
+ toleration.Value = groups[3]
+ }
+ toleration.Effect = corev1.TaintEffect(groups[4])
+
+ if groups[5] != "" {
+ tolerationSeconds, err := strconv.ParseInt(groups[5], 10, 64)
+ if err != nil {
+ return nil, err
+ }
+ toleration.TolerationSeconds = &tolerationSeconds
+ }
+ tolerations = append(tolerations, toleration)
+ }
+
+ return tolerations, nil
+}
diff --git a/pkg/util/olm/operator.go b/pkg/util/olm/operator.go
index a01a2f0..360cc15 100644
--- a/pkg/util/olm/operator.go
+++ b/pkg/util/olm/operator.go
@@ -138,7 +138,7 @@ func HasPermissionToInstall(ctx context.Context, client client.Client, namespace
}
// Install creates a subscription for the OLM package
-func Install(ctx context.Context, client client.Client, namespace string, global bool, options Options, collection *kubernetes.Collection) (bool, error) {
+func Install(ctx context.Context, client client.Client, namespace string, global bool, options Options, collection *kubernetes.Collection, tolerations []string) (bool, error) {
options = fillDefaults(options)
if installed, err := IsOperatorInstalled(ctx, client, namespace, global, options); err != nil {
return false, err
@@ -166,6 +166,12 @@ func Install(ctx context.Context, client client.Client, namespace string, global
InstallPlanApproval: operatorsv1alpha1.ApprovalAutomatic,
},
}
+ // Additional configuration
+ err := maybeSetTolerations(&sub, tolerations)
+ if err != nil {
+ return false, errors.Wrap(err, fmt.Sprintf("could not set tolerations"))
+ }
+
if collection != nil {
collection.Add(&sub)
} else if err := client.Create(ctx, &sub); err != nil {
@@ -199,6 +205,17 @@ func Install(ctx context.Context, client client.Client, namespace string, global
return true, nil
}
+func maybeSetTolerations(sub *operatorsv1alpha1.Subscription, tolArray []string) error {
+ if tolArray != nil {
+ tolerations, err := kubernetes.GetTolerations(tolArray)
+ if err != nil {
+ return err
+ }
+ sub.Spec.Config.Tolerations = tolerations
+ }
+ return nil
+}
+
// Uninstall removes CSV and subscription from the namespace
func Uninstall(ctx context.Context, client client.Client, namespace string, global bool, options Options) error {
sub, err := findSubscription(ctx, client, namespace, global, options)