You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by lb...@apache.org on 2019/02/15 06:44:03 UTC
[camel-k] branch master updated: Add garbage collector trait
This is an automated email from the ASF dual-hosted git repository.
lburgazzoli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/camel-k.git
The following commit(s) were added to refs/heads/master by this push:
new cc7735d Add garbage collector trait
cc7735d is described below
commit cc7735dc18de33d5ecccdd577d3e4538318a06da
Author: Antonin Stefanutti <an...@stefanutti.fr>
AuthorDate: Thu Feb 14 12:26:56 2019 +0100
Add garbage collector trait
Fixes #211
---
docs/traits.adoc | 7 ++
pkg/controller/integration/deploy.go | 8 +-
pkg/controller/integration/initialize.go | 1 -
pkg/trait/catalog.go | 82 ++++++++-------
pkg/trait/gc.go | 165 +++++++++++++++++++++++++++++++
pkg/trait/trait.go | 7 +-
pkg/trait/types.go | 14 +--
7 files changed, 234 insertions(+), 50 deletions(-)
diff --git a/docs/traits.adoc b/docs/traits.adoc
index 0ef4863..774b96d 100644
--- a/docs/traits.adoc
+++ b/docs/traits.adoc
@@ -247,4 +247,11 @@ There are also platform traits that **normally should not be configured** by the
!===
+| gc
+| All
+| Garbage collect resources that are no longer necessary upon integration updates.
+ +
+ +
+ It's enabled by default.
+
|=======================
diff --git a/pkg/controller/integration/deploy.go b/pkg/controller/integration/deploy.go
index 7329e27..ff78ea8 100644
--- a/pkg/controller/integration/deploy.go
+++ b/pkg/controller/integration/deploy.go
@@ -64,12 +64,18 @@ func (action *deployAction) Handle(ctx context.Context, integration *v1alpha1.In
return err
}
- // TODO we should look for objects that are no longer present in the collection and remove them
err = kubernetes.ReplaceResources(ctx, action.client, env.Resources.Items())
if err != nil {
return err
}
+ for _, postAction := range env.PostActions {
+ err := postAction(env)
+ if err != nil {
+ action.L.Errorf(err, "error executing deployment post action")
+ }
+ }
+
target := integration.DeepCopy()
target.Status.Phase = v1alpha1.IntegrationPhaseRunning
diff --git a/pkg/controller/integration/initialize.go b/pkg/controller/integration/initialize.go
index ff89589..636b0f6 100644
--- a/pkg/controller/integration/initialize.go
+++ b/pkg/controller/integration/initialize.go
@@ -95,7 +95,6 @@ func (action *initializeAction) Handle(ctx context.Context, integration *v1alpha
return err
}
- // TODO we should look for objects that are no longer present in the collection and remove them
err = kubernetes.ReplaceResources(ctx, action.client, env.Resources.Items())
if err != nil {
return err
diff --git a/pkg/trait/catalog.go b/pkg/trait/catalog.go
index defa0e1..f38dc3b 100644
--- a/pkg/trait/catalog.go
+++ b/pkg/trait/catalog.go
@@ -30,49 +30,51 @@ import (
// Catalog collects all information about traits in one place
type Catalog struct {
- L log.Logger
- tDebug Trait
- tDependencies Trait
- tDeployment Trait
- tKnativeService Trait
- tKnative Trait
- tService Trait
- tRoute Trait
- tIngress Trait
- tJolokia Trait
- tPrometheus Trait
- tOwner Trait
- tImages Trait
- tBuilder Trait
- tSpringBoot Trait
- tIstio Trait
- tEnvironment Trait
- tClasspath Trait
- tRest Trait
+ L log.Logger
+ tDebug Trait
+ tDependencies Trait
+ tDeployment Trait
+ tGarbageCollector Trait
+ tKnativeService Trait
+ tKnative Trait
+ tService Trait
+ tRoute Trait
+ tIngress Trait
+ tJolokia Trait
+ tPrometheus Trait
+ tOwner Trait
+ tImages Trait
+ tBuilder Trait
+ tSpringBoot Trait
+ tIstio Trait
+ tEnvironment Trait
+ tClasspath Trait
+ tRest Trait
}
// NewCatalog creates a new trait Catalog
func NewCatalog(ctx context.Context, c client.Client) *Catalog {
catalog := Catalog{
- L: log.Log.WithName("trait"),
- tDebug: newDebugTrait(),
- tRest: newRestTrait(),
- tKnative: newKnativeTrait(),
- tDependencies: newDependenciesTrait(),
- tDeployment: newDeploymentTrait(),
- tKnativeService: newKnativeServiceTrait(),
- tService: newServiceTrait(),
- tRoute: newRouteTrait(),
- tIngress: newIngressTrait(),
- tJolokia: newJolokiaTrait(),
- tPrometheus: newPrometheusTrait(),
- tOwner: newOwnerTrait(),
- tImages: newImagesTrait(),
- tBuilder: newBuilderTrait(),
- tSpringBoot: newSpringBootTrait(),
- tIstio: newIstioTrait(),
- tEnvironment: newEnvironmentTrait(),
- tClasspath: newClasspathTrait(),
+ L: log.Log.WithName("trait"),
+ tDebug: newDebugTrait(),
+ tRest: newRestTrait(),
+ tKnative: newKnativeTrait(),
+ tDependencies: newDependenciesTrait(),
+ tDeployment: newDeploymentTrait(),
+ tGarbageCollector: newGarbageCollectorTrait(),
+ tKnativeService: newKnativeServiceTrait(),
+ tService: newServiceTrait(),
+ tRoute: newRouteTrait(),
+ tIngress: newIngressTrait(),
+ tJolokia: newJolokiaTrait(),
+ tPrometheus: newPrometheusTrait(),
+ tOwner: newOwnerTrait(),
+ tImages: newImagesTrait(),
+ tBuilder: newBuilderTrait(),
+ tSpringBoot: newSpringBootTrait(),
+ tIstio: newIstioTrait(),
+ tEnvironment: newEnvironmentTrait(),
+ tClasspath: newClasspathTrait(),
}
for _, t := range catalog.allTraits() {
@@ -93,6 +95,7 @@ func (c *Catalog) allTraits() []Trait {
c.tKnative,
c.tDependencies,
c.tDeployment,
+ c.tGarbageCollector,
c.tKnativeService,
c.tService,
c.tRoute,
@@ -115,6 +118,7 @@ func (c *Catalog) traitsFor(environment *Environment) []Trait {
switch environment.DetermineProfile() {
case v1alpha1.TraitProfileOpenShift:
return []Trait{
+ c.tGarbageCollector,
c.tDebug,
c.tRest,
c.tDependencies,
@@ -132,6 +136,7 @@ func (c *Catalog) traitsFor(environment *Environment) []Trait {
}
case v1alpha1.TraitProfileKubernetes:
return []Trait{
+ c.tGarbageCollector,
c.tDebug,
c.tRest,
c.tDependencies,
@@ -149,6 +154,7 @@ func (c *Catalog) traitsFor(environment *Environment) []Trait {
}
case v1alpha1.TraitProfileKnative:
return []Trait{
+ c.tGarbageCollector,
c.tDebug,
c.tRest,
c.tKnative,
diff --git a/pkg/trait/gc.go b/pkg/trait/gc.go
new file mode 100644
index 0000000..4a7bd22
--- /dev/null
+++ b/pkg/trait/gc.go
@@ -0,0 +1,165 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package trait
+
+import (
+ "context"
+ "fmt"
+ "strconv"
+
+ k8serrors "k8s.io/apimachinery/pkg/api/errors"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+ "k8s.io/apimachinery/pkg/labels"
+ k8sclient "sigs.k8s.io/controller-runtime/pkg/client"
+
+ "github.com/apache/camel-k/pkg/apis/camel/v1alpha1"
+ "github.com/apache/camel-k/pkg/client"
+)
+
+type garbageCollectorTrait struct {
+ BaseTrait `property:",squash"`
+}
+
+func newGarbageCollectorTrait() *garbageCollectorTrait {
+ return &garbageCollectorTrait{
+ BaseTrait: newBaseTrait("gc"),
+ }
+}
+
+func (t *garbageCollectorTrait) Configure(e *Environment) (bool, error) {
+ if t.Enabled != nil && !*t.Enabled {
+ return false, nil
+ }
+
+ return e.IntegrationInPhase(v1alpha1.IntegrationPhaseInitial) ||
+ e.IntegrationInPhase(v1alpha1.IntegrationPhaseDeploying), nil
+}
+
+func (t *garbageCollectorTrait) Apply(e *Environment) error {
+ // Register a post processor that adds the required labels to the new resources
+ e.PostProcessors = append(e.PostProcessors, func(env *Environment) error {
+ env.Resources.VisitMetaObject(func(resource metav1.Object) {
+ labels := resource.GetLabels()
+ if labels == nil {
+ labels = map[string]string{}
+ }
+ // Label the resource with the current integration generation
+ labels["camel.apache.org/generation"] = strconv.FormatInt(env.Integration.GetGeneration(), 10)
+ // Make sure the integration label is set
+ labels["camel.apache.org/integration"] = env.Integration.Name
+ resource.SetLabels(labels)
+ })
+ return nil
+ })
+
+ // Let's run garbage collection during the integration deploying phase
+ if !e.IntegrationInPhase(v1alpha1.IntegrationPhaseDeploying) {
+ return nil
+ }
+ // Register a post action that deletes the existing resources that are labelled
+ // with the previous integration generations.
+ e.PostActions = append(e.PostActions, func(environment *Environment) error {
+ // Retrieve older generation resources that may can enlisted for garbage collection
+ resources, err := getOldGenerationResources(e)
+ if err != nil {
+ return err
+ }
+ // And delete them
+ for _, resource := range resources {
+ err = e.Client.Delete(context.TODO(), resource)
+ if err != nil {
+ // The resource may have already been deleted
+ if !k8serrors.IsNotFound(err) {
+ t.L.ForIntegration(e.Integration).Errorf(err, "cannot delete child resource: %s/%s", resource.GetKind(), resource.GetName())
+ }
+ } else {
+ t.L.ForIntegration(e.Integration).Debugf("child resource deleted: %s/%s", resource.GetKind(), resource.GetName())
+ }
+ }
+
+ return nil
+ })
+
+ return nil
+}
+
+func getOldGenerationResources(e *Environment) ([]*unstructured.Unstructured, error) {
+ // We rely on the discovery API to retrieve all the resources group and kind.
+ // That results in an unbounded collection that can be a bit slow (a couple of seconds).
+ // We may want to refine that step by white-listing or enlisting types to speed-up
+ // the collection duration.
+ types, err := getDiscoveryTypes(e.Client)
+ if err != nil {
+ return nil, err
+ }
+
+ selector, err := labels.Parse(fmt.Sprintf("camel.apache.org/integration=%s,camel.apache.org/generation,camel.apache.org/generation notin (%d)", e.Integration.Name, e.Integration.GetGeneration()))
+ if err != nil {
+ return nil, err
+ }
+
+ res := make([]*unstructured.Unstructured, 0)
+
+ for _, t := range types {
+ options := k8sclient.ListOptions{
+ Namespace: e.Integration.Namespace,
+ LabelSelector: selector,
+ Raw: &metav1.ListOptions{
+ TypeMeta: t,
+ },
+ }
+ list := unstructured.UnstructuredList{
+ Object: map[string]interface{}{
+ "apiVersion": t.APIVersion,
+ "kind": t.Kind,
+ },
+ }
+ if err := e.Client.List(context.TODO(), &options, &list); err != nil {
+ if k8serrors.IsNotFound(err) ||
+ k8serrors.IsForbidden(err) ||
+ k8serrors.IsMethodNotSupported(err) {
+ continue
+ }
+ return nil, err
+ }
+ for _, item := range list.Items {
+ res = append(res, &item)
+ }
+ }
+ return res, nil
+}
+
+func getDiscoveryTypes(client client.Client) ([]metav1.TypeMeta, error) {
+ resources, err := client.Discovery().ServerPreferredNamespacedResources()
+ if err != nil {
+ return nil, err
+ }
+
+ types := make([]metav1.TypeMeta, 0)
+ for _, resource := range resources {
+ for _, r := range resource.APIResources {
+ types = append(types, metav1.TypeMeta{
+ Kind: r.Kind,
+ APIVersion: resource.GroupVersion,
+ })
+ }
+ }
+
+ return types, nil
+}
diff --git a/pkg/trait/trait.go b/pkg/trait/trait.go
index 0556753..633b805 100644
--- a/pkg/trait/trait.go
+++ b/pkg/trait/trait.go
@@ -20,15 +20,15 @@ package trait
import (
"context"
- "github.com/apache/camel-k/pkg/util/camel"
+ corev1 "k8s.io/api/core/v1"
"github.com/apache/camel-k/pkg/apis/camel/v1alpha1"
"github.com/apache/camel-k/pkg/client"
"github.com/apache/camel-k/pkg/platform"
+ "github.com/apache/camel-k/pkg/util/camel"
"github.com/apache/camel-k/pkg/util/kubernetes"
- "github.com/pkg/errors"
- corev1 "k8s.io/api/core/v1"
+ "github.com/pkg/errors"
)
// True --
@@ -87,6 +87,7 @@ func newEnvironment(ctx context.Context, c client.Client, integration *v1alpha1.
return &Environment{
Platform: pl,
CamelCatalog: catalog,
+ Client: c,
Context: ictx,
Integration: integration,
ExecutedTraits: make([]Trait, 0),
diff --git a/pkg/trait/types.go b/pkg/trait/types.go
index 4284ef9..c65cd2e 100644
--- a/pkg/trait/types.go
+++ b/pkg/trait/types.go
@@ -20,20 +20,18 @@ package trait
import (
"context"
- "github.com/apache/camel-k/pkg/util/camel"
-
- "github.com/apache/camel-k/pkg/util/log"
-
- "github.com/apache/camel-k/pkg/util/source"
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/api/core/v1"
"github.com/apache/camel-k/pkg/apis/camel/v1alpha1"
"github.com/apache/camel-k/pkg/builder"
"github.com/apache/camel-k/pkg/client"
"github.com/apache/camel-k/pkg/metadata"
"github.com/apache/camel-k/pkg/platform"
+ "github.com/apache/camel-k/pkg/util/camel"
"github.com/apache/camel-k/pkg/util/kubernetes"
- corev1 "k8s.io/api/core/v1"
- v1 "k8s.io/api/core/v1"
+ "github.com/apache/camel-k/pkg/util/log"
+ "github.com/apache/camel-k/pkg/util/source"
)
// Identifiable represent an identifiable type
@@ -98,10 +96,12 @@ func (trait *BaseTrait) InjectContext(ctx context.Context) {
type Environment struct {
CamelCatalog *camel.RuntimeCatalog
Catalog *Catalog
+ Client client.Client
Platform *v1alpha1.IntegrationPlatform
Context *v1alpha1.IntegrationContext
Integration *v1alpha1.Integration
Resources *kubernetes.Collection
+ PostActions []func(*Environment) error
PostProcessors []func(*Environment) error
Steps []builder.Step
BuildDir string