You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by pc...@apache.org on 2024/01/12 11:42:26 UTC
(camel-k) 01/03: feat(pipeline): filter and sort tasks execution
This is an automated email from the ASF dual-hosted git repository.
pcongiusti pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel-k.git
commit 4e2bc7959185de72f32230a1f14255c3860d3999
Author: Pasquale Congiusti <pa...@gmail.com>
AuthorDate: Thu Dec 7 13:29:02 2023 +0100
feat(pipeline): filter and sort tasks execution
---
.../bases/camel.apache.org_integrationkits.yaml | 7 ++
.../camel.apache.org_integrationplatforms.yaml | 14 +++
.../crd/bases/camel.apache.org_integrations.yaml | 7 ++
.../bases/camel.apache.org_kameletbindings.yaml | 8 ++
config/crd/bases/camel.apache.org_pipes.yaml | 8 ++
docs/modules/ROOT/partials/apis/camel-k-crds.adoc | 9 ++
docs/modules/traits/pages/builder.adoc | 6 ++
helm/camel-k/crds/crd-integration-kit.yaml | 7 ++
helm/camel-k/crds/crd-integration-platform.yaml | 14 +++
helm/camel-k/crds/crd-integration.yaml | 7 ++
helm/camel-k/crds/crd-kamelet-binding.yaml | 8 ++
helm/camel-k/crds/crd-pipe.yaml | 8 ++
pkg/apis/camel/v1/trait/builder.go | 4 +
pkg/trait/builder.go | 46 ++++++++++
pkg/trait/builder_test.go | 99 ++++++++++++++++++++--
resources/traits.yaml | 6 ++
16 files changed, 249 insertions(+), 9 deletions(-)
diff --git a/config/crd/bases/camel.apache.org_integrationkits.yaml b/config/crd/bases/camel.apache.org_integrationkits.yaml
index e3dece12f..3d4714344 100644
--- a/config/crd/bases/camel.apache.org_integrationkits.yaml
+++ b/config/crd/bases/camel.apache.org_integrationkits.yaml
@@ -284,6 +284,13 @@ spec:
items:
type: string
type: array
+ tasksFilter:
+ description: A list of tasks (available only when using `pod`
+ strategy), sorted by the order of execution in a csv format,
+ ie, `<taskName1>,<taskName2>,...`. Mind that you must include
+ also the operator tasks (`builder`, `quarkus-native`, `package`,
+ `jib`, `spectrum`, `s2i`) if you need to execute them.
+ type: string
tasksLimitCPU:
description: A list of limit cpu configuration for the specific
task with format `<task-name>:<limit-cpu-conf>`.
diff --git a/config/crd/bases/camel.apache.org_integrationplatforms.yaml b/config/crd/bases/camel.apache.org_integrationplatforms.yaml
index f548fe157..240fb4143 100644
--- a/config/crd/bases/camel.apache.org_integrationplatforms.yaml
+++ b/config/crd/bases/camel.apache.org_integrationplatforms.yaml
@@ -589,6 +589,13 @@ spec:
items:
type: string
type: array
+ tasksFilter:
+ description: A list of tasks (available only when using `pod`
+ strategy), sorted by the order of execution in a csv format,
+ ie, `<taskName1>,<taskName2>,...`. Mind that you must include
+ also the operator tasks (`builder`, `quarkus-native`, `package`,
+ `jib`, `spectrum`, `s2i`) if you need to execute them.
+ type: string
tasksLimitCPU:
description: A list of limit cpu configuration for the specific
task with format `<task-name>:<limit-cpu-conf>`.
@@ -2472,6 +2479,13 @@ spec:
items:
type: string
type: array
+ tasksFilter:
+ description: A list of tasks (available only when using `pod`
+ strategy), sorted by the order of execution in a csv format,
+ ie, `<taskName1>,<taskName2>,...`. Mind that you must include
+ also the operator tasks (`builder`, `quarkus-native`, `package`,
+ `jib`, `spectrum`, `s2i`) if you need to execute them.
+ type: string
tasksLimitCPU:
description: A list of limit cpu configuration for the specific
task with format `<task-name>:<limit-cpu-conf>`.
diff --git a/config/crd/bases/camel.apache.org_integrations.yaml b/config/crd/bases/camel.apache.org_integrations.yaml
index fd9263a05..fa9fd5c0c 100644
--- a/config/crd/bases/camel.apache.org_integrations.yaml
+++ b/config/crd/bases/camel.apache.org_integrations.yaml
@@ -6502,6 +6502,13 @@ spec:
items:
type: string
type: array
+ tasksFilter:
+ description: A list of tasks (available only when using `pod`
+ strategy), sorted by the order of execution in a csv format,
+ ie, `<taskName1>,<taskName2>,...`. Mind that you must include
+ also the operator tasks (`builder`, `quarkus-native`, `package`,
+ `jib`, `spectrum`, `s2i`) if you need to execute them.
+ type: string
tasksLimitCPU:
description: A list of limit cpu configuration for the specific
task with format `<task-name>:<limit-cpu-conf>`.
diff --git a/config/crd/bases/camel.apache.org_kameletbindings.yaml b/config/crd/bases/camel.apache.org_kameletbindings.yaml
index 0739d217f..ad82cf8c1 100644
--- a/config/crd/bases/camel.apache.org_kameletbindings.yaml
+++ b/config/crd/bases/camel.apache.org_kameletbindings.yaml
@@ -6786,6 +6786,14 @@ spec:
items:
type: string
type: array
+ tasksFilter:
+ description: A list of tasks (available only when using
+ `pod` strategy), sorted by the order of execution in
+ a csv format, ie, `<taskName1>,<taskName2>,...`. Mind
+ that you must include also the operator tasks (`builder`,
+ `quarkus-native`, `package`, `jib`, `spectrum`, `s2i`)
+ if you need to execute them.
+ type: string
tasksLimitCPU:
description: A list of limit cpu configuration for the
specific task with format `<task-name>:<limit-cpu-conf>`.
diff --git a/config/crd/bases/camel.apache.org_pipes.yaml b/config/crd/bases/camel.apache.org_pipes.yaml
index 8063ab981..d018e4349 100644
--- a/config/crd/bases/camel.apache.org_pipes.yaml
+++ b/config/crd/bases/camel.apache.org_pipes.yaml
@@ -6784,6 +6784,14 @@ spec:
items:
type: string
type: array
+ tasksFilter:
+ description: A list of tasks (available only when using
+ `pod` strategy), sorted by the order of execution in
+ a csv format, ie, `<taskName1>,<taskName2>,...`. Mind
+ that you must include also the operator tasks (`builder`,
+ `quarkus-native`, `package`, `jib`, `spectrum`, `s2i`)
+ if you need to execute them.
+ type: string
tasksLimitCPU:
description: A list of limit cpu configuration for the
specific task with format `<task-name>:<limit-cpu-conf>`.
diff --git a/docs/modules/ROOT/partials/apis/camel-k-crds.adoc b/docs/modules/ROOT/partials/apis/camel-k-crds.adoc
index 4d7208467..29cc535a7 100644
--- a/docs/modules/ROOT/partials/apis/camel-k-crds.adoc
+++ b/docs/modules/ROOT/partials/apis/camel-k-crds.adoc
@@ -5887,6 +5887,15 @@ Syntax: [configmap{vbar}secret]:name[/key], where name represents the resource n
A list of tasks to be executed (available only when using `pod` strategy) with format `<name>;<container-image>;<container-command>`.
+|`tasksFilter` +
+string
+|
+
+
+A list of tasks (available only when using `pod` strategy), sorted by the order of execution in a csv format, ie, `<taskName1>,<taskName2>,...`.
+Mind that you must include also the operator tasks (`builder`, `quarkus-native`, `package`, `jib`, `spectrum`, `s2i`)
+if you need to execute them.
+
|`tasksRequestCPU` +
[]string
|
diff --git a/docs/modules/traits/pages/builder.adoc b/docs/modules/traits/pages/builder.adoc
index f86b3edab..6330dea1e 100755
--- a/docs/modules/traits/pages/builder.adoc
+++ b/docs/modules/traits/pages/builder.adoc
@@ -82,6 +82,12 @@ Syntax: [configmap\|secret]:name[/key], where name represents the resource name,
| []string
| A list of tasks to be executed (available only when using `pod` strategy) with format `<name>;<container-image>;<container-command>`.
+| builder.tasks-filter
+| string
+| A list of tasks (available only when using `pod` strategy), sorted by the order of execution in a csv format, ie, `<taskName1>,<taskName2>,...`.
+Mind that you must include also the operator tasks (`builder`, `quarkus-native`, `package`, `jib`, `spectrum`, `s2i`)
+if you need to execute them.
+
| builder.tasks-request-cpu
| []string
| A list of request cpu configuration for the specific task with format `<task-name>:<request-cpu-conf>`.
diff --git a/helm/camel-k/crds/crd-integration-kit.yaml b/helm/camel-k/crds/crd-integration-kit.yaml
index e3dece12f..3d4714344 100644
--- a/helm/camel-k/crds/crd-integration-kit.yaml
+++ b/helm/camel-k/crds/crd-integration-kit.yaml
@@ -284,6 +284,13 @@ spec:
items:
type: string
type: array
+ tasksFilter:
+ description: A list of tasks (available only when using `pod`
+ strategy), sorted by the order of execution in a csv format,
+ ie, `<taskName1>,<taskName2>,...`. Mind that you must include
+ also the operator tasks (`builder`, `quarkus-native`, `package`,
+ `jib`, `spectrum`, `s2i`) if you need to execute them.
+ type: string
tasksLimitCPU:
description: A list of limit cpu configuration for the specific
task with format `<task-name>:<limit-cpu-conf>`.
diff --git a/helm/camel-k/crds/crd-integration-platform.yaml b/helm/camel-k/crds/crd-integration-platform.yaml
index f548fe157..240fb4143 100644
--- a/helm/camel-k/crds/crd-integration-platform.yaml
+++ b/helm/camel-k/crds/crd-integration-platform.yaml
@@ -589,6 +589,13 @@ spec:
items:
type: string
type: array
+ tasksFilter:
+ description: A list of tasks (available only when using `pod`
+ strategy), sorted by the order of execution in a csv format,
+ ie, `<taskName1>,<taskName2>,...`. Mind that you must include
+ also the operator tasks (`builder`, `quarkus-native`, `package`,
+ `jib`, `spectrum`, `s2i`) if you need to execute them.
+ type: string
tasksLimitCPU:
description: A list of limit cpu configuration for the specific
task with format `<task-name>:<limit-cpu-conf>`.
@@ -2472,6 +2479,13 @@ spec:
items:
type: string
type: array
+ tasksFilter:
+ description: A list of tasks (available only when using `pod`
+ strategy), sorted by the order of execution in a csv format,
+ ie, `<taskName1>,<taskName2>,...`. Mind that you must include
+ also the operator tasks (`builder`, `quarkus-native`, `package`,
+ `jib`, `spectrum`, `s2i`) if you need to execute them.
+ type: string
tasksLimitCPU:
description: A list of limit cpu configuration for the specific
task with format `<task-name>:<limit-cpu-conf>`.
diff --git a/helm/camel-k/crds/crd-integration.yaml b/helm/camel-k/crds/crd-integration.yaml
index fd9263a05..fa9fd5c0c 100644
--- a/helm/camel-k/crds/crd-integration.yaml
+++ b/helm/camel-k/crds/crd-integration.yaml
@@ -6502,6 +6502,13 @@ spec:
items:
type: string
type: array
+ tasksFilter:
+ description: A list of tasks (available only when using `pod`
+ strategy), sorted by the order of execution in a csv format,
+ ie, `<taskName1>,<taskName2>,...`. Mind that you must include
+ also the operator tasks (`builder`, `quarkus-native`, `package`,
+ `jib`, `spectrum`, `s2i`) if you need to execute them.
+ type: string
tasksLimitCPU:
description: A list of limit cpu configuration for the specific
task with format `<task-name>:<limit-cpu-conf>`.
diff --git a/helm/camel-k/crds/crd-kamelet-binding.yaml b/helm/camel-k/crds/crd-kamelet-binding.yaml
index 0739d217f..ad82cf8c1 100644
--- a/helm/camel-k/crds/crd-kamelet-binding.yaml
+++ b/helm/camel-k/crds/crd-kamelet-binding.yaml
@@ -6786,6 +6786,14 @@ spec:
items:
type: string
type: array
+ tasksFilter:
+ description: A list of tasks (available only when using
+ `pod` strategy), sorted by the order of execution in
+ a csv format, ie, `<taskName1>,<taskName2>,...`. Mind
+ that you must include also the operator tasks (`builder`,
+ `quarkus-native`, `package`, `jib`, `spectrum`, `s2i`)
+ if you need to execute them.
+ type: string
tasksLimitCPU:
description: A list of limit cpu configuration for the
specific task with format `<task-name>:<limit-cpu-conf>`.
diff --git a/helm/camel-k/crds/crd-pipe.yaml b/helm/camel-k/crds/crd-pipe.yaml
index 8063ab981..d018e4349 100644
--- a/helm/camel-k/crds/crd-pipe.yaml
+++ b/helm/camel-k/crds/crd-pipe.yaml
@@ -6784,6 +6784,14 @@ spec:
items:
type: string
type: array
+ tasksFilter:
+ description: A list of tasks (available only when using
+ `pod` strategy), sorted by the order of execution in
+ a csv format, ie, `<taskName1>,<taskName2>,...`. Mind
+ that you must include also the operator tasks (`builder`,
+ `quarkus-native`, `package`, `jib`, `spectrum`, `s2i`)
+ if you need to execute them.
+ type: string
tasksLimitCPU:
description: A list of limit cpu configuration for the
specific task with format `<task-name>:<limit-cpu-conf>`.
diff --git a/pkg/apis/camel/v1/trait/builder.go b/pkg/apis/camel/v1/trait/builder.go
index 69441859f..9dc6c361a 100644
--- a/pkg/apis/camel/v1/trait/builder.go
+++ b/pkg/apis/camel/v1/trait/builder.go
@@ -58,6 +58,10 @@ type BuilderTrait struct {
MavenProfiles []string `property:"maven-profiles" json:"mavenProfiles,omitempty"`
// A list of tasks to be executed (available only when using `pod` strategy) with format `<name>;<container-image>;<container-command>`.
Tasks []string `property:"tasks" json:"tasks,omitempty"`
+ // A list of tasks sorted by the order of execution in a csv format, ie, `<taskName1>,<taskName2>,...`.
+ // Mind that you must include also the operator tasks (`builder`, `quarkus-native`, `package`, `jib`, `spectrum`, `s2i`)
+ // if you need to execute them. Useful only wih `pod` strategy.
+ TasksFilter string `property:"tasks-filter" json:"tasksFilter,omitempty"`
// A list of request cpu configuration for the specific task with format `<task-name>:<request-cpu-conf>`.
TasksRequestCPU []string `property:"tasks-request-cpu" json:"tasksRequestCPU,omitempty"`
// A list of request memory configuration for the specific task with format `<task-name>:<request-memory-conf>`.
diff --git a/pkg/trait/builder.go b/pkg/trait/builder.go
index fef01bcd8..87c3e484d 100644
--- a/pkg/trait/builder.go
+++ b/pkg/trait/builder.go
@@ -313,6 +313,13 @@ func (t *builderTrait) Apply(e *Environment) error {
}})
}
+ // filter only those tasks required by the user
+ if t.TasksFilter != "" {
+ flt := strings.Split(t.TasksFilter, ",")
+ if pipelineTasks, err = filter(pipelineTasks, flt); err != nil {
+ return err
+ }
+ }
// add local pipeline tasks to env pipeline
e.Pipeline = append(e.Pipeline, pipelineTasks...)
return nil
@@ -559,3 +566,42 @@ func splitContainerCommand(command string) []string {
return removeQuotes
}
+
+func filter(tasks []v1.Task, filterTasks []string) ([]v1.Task, error) {
+ var filteredTasks []v1.Task
+ for _, f := range filterTasks {
+ found := false
+ for _, t := range tasks {
+ if t.Builder != nil && t.Builder.Name == f {
+ filteredTasks = append(filteredTasks, t)
+ found = true
+ } else if t.Custom != nil && t.Custom.Name == f {
+ filteredTasks = append(filteredTasks, t)
+ found = true
+ } else if t.Package != nil && t.Package.Name == f {
+ filteredTasks = append(filteredTasks, t)
+ found = true
+ } else if t.Spectrum != nil && t.Spectrum.Name == f {
+ filteredTasks = append(filteredTasks, t)
+ found = true
+ } else if t.S2i != nil && t.S2i.Name == f {
+ filteredTasks = append(filteredTasks, t)
+ found = true
+ } else if t.Jib != nil && t.Jib.Name == f {
+ filteredTasks = append(filteredTasks, t)
+ found = true
+ } else if t.Buildah != nil && t.Buildah.Name == f {
+ filteredTasks = append(filteredTasks, t)
+ found = true
+ } else if t.Kaniko != nil && t.Kaniko.Name == f {
+ filteredTasks = append(filteredTasks, t)
+ found = true
+ }
+ }
+ if !found {
+ // If we reach this point it means no tasks exists for the name
+ return nil, fmt.Errorf("no task exist for %s name", f)
+ }
+ }
+ return filteredTasks, nil
+}
diff --git a/pkg/trait/builder_test.go b/pkg/trait/builder_test.go
index 129c4505b..343dc5e31 100644
--- a/pkg/trait/builder_test.go
+++ b/pkg/trait/builder_test.go
@@ -251,15 +251,6 @@ func TestCustomTaskBuilderTraitInvalidStrategyOverride(t *testing.T) {
assert.Equal(t, env.IntegrationKit.Status.Conditions[0].Type, v1.IntegrationKitConditionType("IntegrationKitTasksValid"))
}
-func findCustomTaskByName(tasks []v1.Task, name string) v1.Task {
- for _, t := range tasks {
- if t.Custom != nil && t.Custom.Name == name {
- return t
- }
- }
- return v1.Task{}
-}
-
func TestMavenProfilesBuilderTrait(t *testing.T) {
env := createBuilderTestEnv(v1.IntegrationPlatformClusterKubernetes, v1.IntegrationPlatformBuildPublishStrategyKaniko, v1.BuildStrategyRoutine)
builderTrait := createNominalBuilderTraitTest()
@@ -486,3 +477,93 @@ func TestBuilderWithNodeSelector(t *testing.T) {
assert.Equal(t, map[string]string{"size": "large"}, env.Pipeline[0].Builder.Configuration.NodeSelector)
assert.Equal(t, map[string]string{"size": "large"}, builderTrait.NodeSelector)
}
+
+func TestBuilderNoTasksFilter(t *testing.T) {
+ env := createBuilderTestEnv(v1.IntegrationPlatformClusterKubernetes, v1.IntegrationPlatformBuildPublishStrategyJib, v1.BuildStrategyPod)
+ builderTrait := createNominalBuilderTraitTest()
+
+ err := builderTrait.Apply(env)
+ assert.Nil(t, err)
+
+ pipelineTasks := tasksByName(env.Pipeline)
+ assert.Equal(t, []string{"builder", "package", "jib"}, pipelineTasks)
+}
+
+func TestBuilderTasksFilterNotExistingTasks(t *testing.T) {
+ env := createBuilderTestEnv(v1.IntegrationPlatformClusterKubernetes, v1.IntegrationPlatformBuildPublishStrategyJib, v1.BuildStrategyPod)
+ builderTrait := createNominalBuilderTraitTest()
+ builderTrait.TasksFilter = "builder,missing-task"
+
+ err := builderTrait.Apply(env)
+ assert.NotNil(t, err)
+ assert.Equal(t, "no task exist for missing-task name", err.Error())
+}
+
+func TestBuilderTasksFilterOperatorTasks(t *testing.T) {
+ env := createBuilderTestEnv(v1.IntegrationPlatformClusterKubernetes, v1.IntegrationPlatformBuildPublishStrategyJib, v1.BuildStrategyPod)
+ builderTrait := createNominalBuilderTraitTest()
+ builderTrait.TasksFilter = "builder,package"
+
+ err := builderTrait.Apply(env)
+ assert.Nil(t, err)
+ pipelineTasks := tasksByName(env.Pipeline)
+ assert.Equal(t, []string{"builder", "package"}, pipelineTasks)
+}
+
+func TestBuilderTasksFilterAndReorderOperatorTasks(t *testing.T) {
+ env := createBuilderTestEnv(v1.IntegrationPlatformClusterKubernetes, v1.IntegrationPlatformBuildPublishStrategyJib, v1.BuildStrategyPod)
+ builderTrait := createNominalBuilderTraitTest()
+ builderTrait.TasksFilter = "package,builder"
+
+ err := builderTrait.Apply(env)
+ assert.Nil(t, err)
+ pipelineTasks := tasksByName(env.Pipeline)
+ assert.Equal(t, []string{"package", "builder"}, pipelineTasks)
+}
+
+func TestBuilderTasksFilterAndReorderCustomTasks(t *testing.T) {
+ env := createBuilderTestEnv(v1.IntegrationPlatformClusterKubernetes, v1.IntegrationPlatformBuildPublishStrategyJib, v1.BuildStrategyPod)
+ builderTrait := createNominalBuilderTraitTest()
+ builderTrait.Tasks = append(builderTrait.Tasks, `my-custom-publish;alpine;mvn test`)
+ builderTrait.Tasks = append(builderTrait.Tasks, "my-custom-task;alpine;ls")
+ builderTrait.TasksFilter = "builder,my-custom-task,package,my-custom-publish"
+
+ err := builderTrait.Apply(env)
+ assert.Nil(t, err)
+ pipelineTasks := tasksByName(env.Pipeline)
+ assert.Equal(t, []string{"builder", "my-custom-task", "package", "my-custom-publish"}, pipelineTasks)
+}
+
+func findCustomTaskByName(tasks []v1.Task, name string) v1.Task {
+ for _, t := range tasks {
+ if t.Custom != nil && t.Custom.Name == name {
+ return t
+ }
+ }
+ return v1.Task{}
+}
+
+func tasksByName(tasks []v1.Task) []string {
+ pipelineTasks := make([]string, len(tasks))
+ for i, t := range tasks {
+ if t.Builder != nil {
+ pipelineTasks[i] = t.Builder.Name
+ }
+ if t.Custom != nil {
+ pipelineTasks[i] = t.Custom.Name
+ }
+ if t.Package != nil {
+ pipelineTasks[i] = t.Package.Name
+ }
+ if t.S2i != nil {
+ pipelineTasks[i] = t.S2i.Name
+ }
+ if t.Spectrum != nil {
+ pipelineTasks[i] = t.Spectrum.Name
+ }
+ if t.Jib != nil {
+ pipelineTasks[i] = t.Jib.Name
+ }
+ }
+ return pipelineTasks
+}
diff --git a/resources/traits.yaml b/resources/traits.yaml
index 0f9243967..dac366094 100755
--- a/resources/traits.yaml
+++ b/resources/traits.yaml
@@ -283,6 +283,12 @@ traits:
type: '[]string'
description: A list of tasks to be executed (available only when using `pod` strategy)
with format `<name>;<container-image>;<container-command>`.
+ - name: tasks-filter
+ type: string
+ description: A list of tasks (available only when using `pod` strategy), sorted
+ by the order of execution in a csv format, ie, `<taskName1>,<taskName2>,...`.
+ Mind that you must include also the operator tasks (`builder`, `quarkus-native`,
+ `package`, `jib`, `spectrum`, `s2i`) if you need to execute them.
- name: tasks-request-cpu
type: '[]string'
description: A list of request cpu configuration for the specific task with format