You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by nf...@apache.org on 2021/06/09 10:10:05 UTC
[camel-k] 01/03: Fix #2361: allow multiline properties by correctly
encoding them
This is an automated email from the ASF dual-hosted git repository.
nferraro pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel-k.git
commit 41e7ac59188abb51e010e6fe991e245ee026924e
Author: nicolaferraro <ni...@gmail.com>
AuthorDate: Mon Jun 7 15:23:12 2021 +0200
Fix #2361: allow multiline properties by correctly encoding them
---
.../properties-binding.yaml | 19 ++++++
.../properties.feature | 26 ++++++++
.../timer-source.kamelet.yaml | 69 ++++++++++++++++++++++
.../yaks-config.yaml | 27 +++++++++
go.mod | 2 +-
go.sum | 2 +
pkg/cmd/run.go | 33 ++++++++---
pkg/cmd/run_test.go | 18 +++---
pkg/controller/kameletbinding/common.go | 24 +++++++-
pkg/trait/builder.go | 2 +-
pkg/trait/container.go | 12 +++-
pkg/trait/trait_types.go | 22 ++++---
pkg/trait/trait_types_test.go | 16 +++++
13 files changed, 242 insertions(+), 30 deletions(-)
diff --git a/e2e/yaks/common/kamelet-binding-property-encoding/properties-binding.yaml b/e2e/yaks/common/kamelet-binding-property-encoding/properties-binding.yaml
new file mode 100644
index 0000000..f482ece
--- /dev/null
+++ b/e2e/yaks/common/kamelet-binding-property-encoding/properties-binding.yaml
@@ -0,0 +1,19 @@
+kind: KameletBinding
+apiVersion: camel.apache.org/v1alpha1
+metadata:
+ name: properties-binding
+spec:
+ source:
+ ref:
+ apiVersion: camel.apache.org/v1alpha1
+ kind: Kamelet
+ name: timer-source
+ properties:
+ message: |
+ {
+ "content": "thecontent",
+ "key2": "val2"
+ }
+ contentType: "application/json"
+ sink:
+ uri: http://probe-service/events
diff --git a/e2e/yaks/common/kamelet-binding-property-encoding/properties.feature b/e2e/yaks/common/kamelet-binding-property-encoding/properties.feature
new file mode 100644
index 0000000..1d7f795
--- /dev/null
+++ b/e2e/yaks/common/kamelet-binding-property-encoding/properties.feature
@@ -0,0 +1,26 @@
+Feature: Ensure that Kamelets support multiline configuration
+
+ Background:
+ Given Disable auto removal of Kamelet resources
+ Given Disable auto removal of Kubernetes resources
+ Given Camel-K resource polling configuration
+ | maxAttempts | 60 |
+ | delayBetweenAttempts | 3000 |
+
+ Scenario: Wait for binding to start
+ Given create Kubernetes service probe-service with target port 8080
+ Then Camel-K integration properties-binding should be running
+
+ Scenario: Verify binding
+ Given HTTP server "probe-service"
+ And HTTP server timeout is 300000 ms
+ Then expect HTTP request body
+ """
+ {
+ "content": "thecontent",
+ "key2": "val2"
+ }
+ """
+ And expect HTTP request header: Content-Type="application/json;charset=UTF-8"
+ And receive POST /events
+ And delete KameletBinding properties-binding
diff --git a/e2e/yaks/common/kamelet-binding-property-encoding/timer-source.kamelet.yaml b/e2e/yaks/common/kamelet-binding-property-encoding/timer-source.kamelet.yaml
new file mode 100644
index 0000000..a9342e2
--- /dev/null
+++ b/e2e/yaks/common/kamelet-binding-property-encoding/timer-source.kamelet.yaml
@@ -0,0 +1,69 @@
+# ---------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ---------------------------------------------------------------------------
+
+apiVersion: camel.apache.org/v1alpha1
+kind: Kamelet
+metadata:
+ name: timer-source
+ annotations:
+ camel.apache.org/kamelet.support.level: "Preview"
+ camel.apache.org/catalog.version: "main-SNAPSHOT"
+ camel.apache.org/kamelet.icon: data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0idXRmLTgiPz4NCjwhLS0gU3ZnIFZlY3RvciBJY29ucyA6IGh0dHA6Ly93d3cub25saW5ld2ViZm9udHMuY29tL2ljb24gLS0+DQo8IURPQ1RZUEUgc3ZnIFBVQkxJQyAiLS8vVzNDLy9EVEQgU1ZHIDEuMS8vRU4iICJodHRwOi8vd3d3LnczLm9yZy9HcmFwaGljcy9TVkcvMS4xL0RURC9zdmcxMS5kdGQiPg0KPHN2ZyB2ZXJzaW9uPSIxLjEiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgeG1sbnM6eGxpbms9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkveGxpbmsiIHg9IjBweCIgeT0iMHB4IiB2aWV3Qm9 [...]
+ camel.apache.org/provider: "Apache Software Foundation"
+ camel.apache.org/kamelet.group: "Timer"
+ labels:
+ camel.apache.org/kamelet.type: source
+ camel.apache.org/kamelet.verified: "true"
+spec:
+ definition:
+ title: Timer Source
+ description: Produces periodic events with a custom payload.
+ required:
+ - message
+ type: object
+ properties:
+ period:
+ title: Period
+ description: The interval between two events in milliseconds
+ type: integer
+ default: 1000
+ message:
+ title: Message
+ description: The message to generate
+ type: string
+ example: hello world
+ contentType:
+ title: Content Type
+ description: The content type of the message being generated
+ type: string
+ default: text/plain
+ dependencies:
+ - "camel:core"
+ - "camel:timer"
+ - "camel:kamelet"
+ flow:
+ from:
+ uri: timer:tick
+ parameters:
+ period: "{{period}}"
+ steps:
+ - set-body:
+ constant: "{{message}}"
+ - set-header:
+ name: "Content-Type"
+ constant: "{{contentType}}"
+ - to: kamelet:sink
diff --git a/e2e/yaks/common/kamelet-binding-property-encoding/yaks-config.yaml b/e2e/yaks/common/kamelet-binding-property-encoding/yaks-config.yaml
new file mode 100644
index 0000000..650df12
--- /dev/null
+++ b/e2e/yaks/common/kamelet-binding-property-encoding/yaks-config.yaml
@@ -0,0 +1,27 @@
+# ---------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ---------------------------------------------------------------------------
+
+config:
+ namespace:
+ temporary: true
+pre:
+- name: installation
+ run: |
+ kamel install -n $YAKS_NAMESPACE
+
+ kubectl apply -f timer-source.kamelet.yaml -n $YAKS_NAMESPACE
+ kubectl apply -f properties-binding.yaml -n $YAKS_NAMESPACE
diff --git a/go.mod b/go.mod
index 9312782..8c2e45f 100644
--- a/go.mod
+++ b/go.mod
@@ -18,7 +18,7 @@ require (
github.com/google/go-github/v32 v32.1.0
github.com/google/uuid v1.2.0
github.com/jpillora/backoff v1.0.0
- github.com/magiconair/properties v1.8.1
+ github.com/magiconair/properties v1.8.5
github.com/mitchellh/go-homedir v1.1.0
github.com/mitchellh/mapstructure v1.1.2
github.com/onsi/gomega v1.10.3
diff --git a/go.sum b/go.sum
index b633fdf..5185b94 100644
--- a/go.sum
+++ b/go.sum
@@ -660,6 +660,8 @@ github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0Q
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls=
+github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
diff --git a/pkg/cmd/run.go b/pkg/cmd/run.go
index b097cf1..5973dd4 100644
--- a/pkg/cmd/run.go
+++ b/pkg/cmd/run.go
@@ -18,6 +18,7 @@ limitations under the License.
package cmd
import (
+ "bytes"
"context"
"encoding/json"
"fmt"
@@ -584,9 +585,13 @@ func (o *runCmdOptions) updateIntegrationCode(c client.Client, sources []string,
return nil, err
}
for _, k := range props.Keys() {
- v, ok := props.Get(k)
+ _, ok := props.Get(k)
if ok {
- o.Traits = append(o.Traits, fmt.Sprintf("builder.properties=%s", escapePropertyFileItem(k)+"="+escapePropertyFileItem(v)))
+ entry, err := toPropertyEntry(props, k)
+ if err != nil {
+ return nil, err
+ }
+ o.Traits = append(o.Traits, fmt.Sprintf("builder.properties=%s", entry))
} else {
return nil, err
}
@@ -788,10 +793,13 @@ func isLocalAndFileExists(fileName string) (bool, error) {
func addIntegrationProperties(props *properties.Properties, spec *v1.IntegrationSpec) error {
for _, k := range props.Keys() {
- v, _ := props.Get(k)
+ entry, err := toPropertyEntry(props, k)
+ if err != nil {
+ return err
+ }
spec.AddConfiguration(
"property",
- escapePropertyFileItem(k)+"="+escapePropertyFileItem(v),
+ entry,
)
}
return nil
@@ -809,10 +817,19 @@ func loadPropertyFile(fileName string) (*properties.Properties, error) {
return p, nil
}
-func escapePropertyFileItem(item string) string {
- item = strings.ReplaceAll(item, `=`, `\=`)
- item = strings.ReplaceAll(item, `:`, `\:`)
- return item
+func toPropertyEntry(props *properties.Properties, key string) (string, error) {
+ value, _ := props.Get(key)
+ p := properties.NewProperties()
+ p.DisableExpansion = true
+ if _, _, err := p.Set(key, value); err != nil {
+ return "", err
+ }
+ buf := new(bytes.Buffer)
+ if _, err := p.Write(buf, properties.UTF8); err != nil {
+ return "", err
+ }
+ pair := strings.TrimSuffix(buf.String(), "\n")
+ return pair, nil
}
func resolvePodTemplate(ctx context.Context, templateSrc string, spec *v1.IntegrationSpec) (err error) {
diff --git a/pkg/cmd/run_test.go b/pkg/cmd/run_test.go
index 1d9192f..36242de 100644
--- a/pkg/cmd/run_test.go
+++ b/pkg/cmd/run_test.go
@@ -245,10 +245,12 @@ func TestRunPropertyFileFlagMissingFileExtension(t *testing.T) {
const TestPropertyFileContent = `
a=b
-c\=d=e
-d=c\=e
+# There's an issue in the properties lib: https://github.com/magiconair/properties/issues/59
+# so the following cases have been commented. Btw, we don't use equal sign in keys
+#c\=d=e
+#d=c\=e
#ignore=me
-f=g\:h
+f=g:h
`
func TestAddPropertyFile(t *testing.T) {
@@ -265,11 +267,11 @@ func TestAddPropertyFile(t *testing.T) {
properties, err := extractProperties("file:" + tmpFile.Name())
assert.Nil(t, err)
assert.Nil(t, addIntegrationProperties(properties, &spec))
- assert.Equal(t, 4, len(spec.Configuration))
- assert.Equal(t, `a=b`, spec.Configuration[0].Value)
- assert.Equal(t, `c\=d=e`, spec.Configuration[1].Value)
- assert.Equal(t, `d=c\=e`, spec.Configuration[2].Value)
- assert.Equal(t, `f=g\:h`, spec.Configuration[3].Value)
+ assert.Equal(t, 2, len(spec.Configuration))
+ assert.Equal(t, `a = b`, spec.Configuration[0].Value)
+ //assert.Equal(t, `c\=d=e`, spec.Configuration[1].Value)
+ //assert.Equal(t, `d=c\=e`, spec.Configuration[2].Value)
+ assert.Equal(t, `f = g:h`, spec.Configuration[1].Value)
}
func TestRunPropertyFileFlag(t *testing.T) {
diff --git a/pkg/controller/kameletbinding/common.go b/pkg/controller/kameletbinding/common.go
index 4ad94ba..1615364 100644
--- a/pkg/controller/kameletbinding/common.go
+++ b/pkg/controller/kameletbinding/common.go
@@ -18,10 +18,11 @@ limitations under the License.
package kameletbinding
import (
+ "bytes"
"context"
"encoding/json"
- "fmt"
"sort"
+ "strings"
v1 "github.com/apache/camel-k/pkg/apis/camel/v1"
"github.com/apache/camel-k/pkg/apis/camel/v1alpha1"
@@ -31,6 +32,7 @@ import (
"github.com/apache/camel-k/pkg/util/bindings"
"github.com/apache/camel-k/pkg/util/knative"
"github.com/apache/camel-k/pkg/util/kubernetes"
+ "github.com/magiconair/properties"
"github.com/pkg/errors"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -127,7 +129,11 @@ func createIntegrationFor(ctx context.Context, c client.Client, kameletbinding *
it.Spec.Traits[k] = v
}
for k, v := range b.ApplicationProperties {
- propList = append(propList, fmt.Sprintf("%s=%s", k, v))
+ entry, err := toPropertyEntry(k, v)
+ if err != nil {
+ return nil, err
+ }
+ propList = append(propList, entry)
}
}
@@ -192,3 +198,17 @@ func determineProfile(ctx context.Context, c client.Client, binding *v1alpha1.Ka
}
return v1.DefaultTraitProfile, nil
}
+
+func toPropertyEntry(key, value string) (string, error) {
+ p := properties.NewProperties()
+ p.DisableExpansion = true
+ if _, _, err := p.Set(key, value); err != nil {
+ return "", err
+ }
+ buf := new(bytes.Buffer)
+ if _, err := p.Write(buf, properties.UTF8); err != nil {
+ return "", err
+ }
+ pair := strings.TrimSuffix(buf.String(), "\n")
+ return pair, nil
+}
diff --git a/pkg/trait/builder.go b/pkg/trait/builder.go
index a7e354a..639a55c 100644
--- a/pkg/trait/builder.go
+++ b/pkg/trait/builder.go
@@ -156,7 +156,7 @@ func (t *builderTrait) builderTask(e *Environment) (*v1.BuilderTask, error) {
// User provided Maven properties
if t.Properties != nil {
for _, v := range t.Properties {
- split := strings.Split(v, "=")
+ split := strings.SplitN(v, "=", 2)
if len(split) != 2 {
return nil, fmt.Errorf("maven property must have key=value format, it was %v", v)
}
diff --git a/pkg/trait/container.go b/pkg/trait/container.go
index 65781bb..60bac5a 100644
--- a/pkg/trait/container.go
+++ b/pkg/trait/container.go
@@ -252,7 +252,9 @@ func (t *containerTrait) configureContainer(e *Environment) error {
for _, envVar := range e.EnvVars {
envvar.SetVar(&container.Env, envVar)
}
- if props := e.computeApplicationProperties(); props != nil {
+ if props, err := e.computeApplicationProperties(); err != nil {
+ return err
+ } else if props != nil {
e.Resources.Add(props)
}
@@ -291,7 +293,9 @@ func (t *containerTrait) configureContainer(e *Environment) error {
envvar.SetVar(&container.Env, env)
}
}
- if props := e.computeApplicationProperties(); props != nil {
+ if props, err := e.computeApplicationProperties(); err != nil {
+ return err
+ } else if props != nil {
e.Resources.Add(props)
}
@@ -318,7 +322,9 @@ func (t *containerTrait) configureContainer(e *Environment) error {
for _, envVar := range e.EnvVars {
envvar.SetVar(&container.Env, envVar)
}
- if props := e.computeApplicationProperties(); props != nil {
+ if props, err := e.computeApplicationProperties(); err != nil {
+ return err
+ } else if props != nil {
e.Resources.Add(props)
}
diff --git a/pkg/trait/trait_types.go b/pkg/trait/trait_types.go
index 2ca1095..af884d5 100644
--- a/pkg/trait/trait_types.go
+++ b/pkg/trait/trait_types.go
@@ -18,6 +18,7 @@ limitations under the License.
package trait
import (
+ "bytes"
"context"
"fmt"
"path"
@@ -25,6 +26,8 @@ import (
"strconv"
"strings"
+ "github.com/magiconair/properties"
+ "github.com/pkg/errors"
appsv1 "k8s.io/api/apps/v1"
"k8s.io/api/batch/v1beta1"
corev1 "k8s.io/api/core/v1"
@@ -363,13 +366,17 @@ func (e *Environment) DetermineCatalogNamespace() string {
return ""
}
-func (e *Environment) computeApplicationProperties() *corev1.ConfigMap {
+func (e *Environment) computeApplicationProperties() (*corev1.ConfigMap, error) {
// application properties
- applicationProperties := ""
-
- for key, val := range e.ApplicationProperties {
- applicationProperties += fmt.Sprintf("%s=%s\n", key, val)
+ props := properties.LoadMap(e.ApplicationProperties)
+ props.DisableExpansion = true
+ props.Sort()
+ buf := new(bytes.Buffer)
+ _, err := props.Write(buf, properties.UTF8)
+ if err != nil {
+ return nil, errors.Wrapf(err, "could not compute application properties")
}
+ applicationProperties := buf.String()
if applicationProperties != "" {
return &corev1.ConfigMap{
@@ -388,10 +395,10 @@ func (e *Environment) computeApplicationProperties() *corev1.ConfigMap {
Data: map[string]string{
"application.properties": applicationProperties,
},
- }
+ }, nil
}
- return nil
+ return nil, nil
}
func (e *Environment) computeConfigMaps() []ctrl.Object {
@@ -403,6 +410,7 @@ func (e *Environment) computeConfigMaps() []ctrl.Object {
userProperties := ""
for _, prop := range e.collectConfigurationPairs("property") {
+ // properties in resource configuration are expected to be pre-encoded using properties format
userProperties += fmt.Sprintf("%s=%s\n", prop.Name, prop.Value)
}
diff --git a/pkg/trait/trait_types_test.go b/pkg/trait/trait_types_test.go
index 137f0f9..2a68853 100644
--- a/pkg/trait/trait_types_test.go
+++ b/pkg/trait/trait_types_test.go
@@ -18,6 +18,9 @@ limitations under the License.
package trait
import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
serving "knative.dev/serving/pkg/apis/serving/v1"
appsv1 "k8s.io/api/apps/v1"
@@ -29,6 +32,19 @@ import (
"github.com/apache/camel-k/pkg/util/kubernetes"
)
+func TestMultilinePropertiesHandled(t *testing.T) {
+ e := Environment{
+ ApplicationProperties: map[string]string{
+ "prop": "multi\nline",
+ },
+ Integration: &v1.Integration{},
+ }
+ cm, err := e.computeApplicationProperties()
+ assert.NoError(t, err)
+ assert.NotNil(t, cm)
+ assert.Equal(t, "prop = multi\\nline\n", cm.Data["application.properties"])
+}
+
func createNominalDeploymentTraitTest() (*Environment, *appsv1.Deployment) {
deployment := &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{