You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by as...@apache.org on 2021/02/03 12:12:28 UTC
[camel-k] 01/06: feat(trait): leverage ConfigMap binary data for
resources
This is an automated email from the ASF dual-hosted git repository.
astefanutti pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/camel-k.git
commit 8e97ba7a2e7cdd2ec808198c52dc5b8a7e288121
Author: Pasquale Congiusti <pa...@gmail.com>
AuthorDate: Fri Jan 29 15:25:38 2021 +0100
feat(trait): leverage ConfigMap binary data for resources
* Any binary data will be using BinaryData ConfiMap instead of Data. We let the cluster to encode/decode the resource
* Any text resource will be still using the Data (plain) ConfigMap
* The `compression` feature can be run both on binary and text resources, providing a base64 encoded file.
* Added unit test to check all the possible scenarios
Close #1946, close #1881
---
pkg/apis/camel/v1/common_types.go | 2 +
pkg/cmd/run.go | 46 +++++++---
pkg/cmd/run_test.go | 44 +++++++++
pkg/cmd/util_content.go | 37 +++++---
pkg/cmd/util_content_test.go | 99 ++++++++++++++++++++
pkg/cmd/util_dependencies.go | 2 +-
pkg/trait/trait_test.go | 185 ++++++++++++++++++++++++++------------
pkg/trait/trait_types.go | 15 +++-
8 files changed, 344 insertions(+), 86 deletions(-)
diff --git a/pkg/apis/camel/v1/common_types.go b/pkg/apis/camel/v1/common_types.go
index b940b2f..2d50208 100644
--- a/pkg/apis/camel/v1/common_types.go
+++ b/pkg/apis/camel/v1/common_types.go
@@ -207,8 +207,10 @@ const (
type DataSpec struct {
Name string `json:"name,omitempty"`
Content string `json:"content,omitempty"`
+ RawContent []byte `json:"rawContent,omitempty"`
ContentRef string `json:"contentRef,omitempty"`
ContentKey string `json:"contentKey,omitempty"`
+ ContentType string `json:"contentType,omitempty"`
Compression bool `json:"compression,omitempty"`
}
diff --git a/pkg/cmd/run.go b/pkg/cmd/run.go
index 695f01f..8ac482e 100644
--- a/pkg/cmd/run.go
+++ b/pkg/cmd/run.go
@@ -495,23 +495,20 @@ func (o *runCmdOptions) updateIntegrationCode(c client.Client, sources []string,
}
for _, resource := range o.Resources {
- data, compressed, err := loadContent(resource, o.Compression, o.CompressBinary)
+ rawData, contentType, err := loadRawContent(resource)
if err != nil {
return nil, err
}
- integration.Spec.AddResources(v1.ResourceSpec{
- DataSpec: v1.DataSpec{
- Name: path.Base(resource),
- Content: data,
- Compression: compressed,
- },
- Type: v1.ResourceTypeData,
- })
+ resourceSpec, err := binaryOrTextResource(path.Base(resource), rawData, contentType, o.Compression)
+ if err != nil {
+ return nil, err
+ }
+ integration.Spec.AddResources(resourceSpec)
}
for _, resource := range o.OpenAPIs {
- data, compressed, err := loadContent(resource, o.Compression, o.CompressBinary)
+ data, _, compressed, err := loadTextContent(resource, o.Compression)
if err != nil {
return nil, err
}
@@ -623,6 +620,35 @@ func (o *runCmdOptions) updateIntegrationCode(c client.Client, sources []string,
return &integration, nil
}
+func binaryOrTextResource(fileName string, data []byte, contentType string, base64Compression bool) (v1.ResourceSpec, error) {
+ resourceSpec := v1.ResourceSpec{
+ DataSpec: v1.DataSpec{
+ Name: fileName,
+ ContentKey: fileName,
+ ContentType: contentType,
+ Compression: false,
+ },
+ Type: v1.ResourceTypeData,
+ }
+
+ if !base64Compression && isBinary(contentType) {
+ resourceSpec.RawContent = data
+ return resourceSpec, nil
+ }
+ // either is a text resource or base64 compression is enabled
+ if base64Compression {
+ content, err := compressToString(data)
+ if err != nil {
+ return resourceSpec, err
+ }
+ resourceSpec.Content = content
+ resourceSpec.Compression = true
+ } else {
+ resourceSpec.Content = string(data)
+ }
+ return resourceSpec, nil
+}
+
func (o *runCmdOptions) GetIntegrationName(sources []string) string {
name := ""
if o.IntegrationName != "" {
diff --git a/pkg/cmd/run_test.go b/pkg/cmd/run_test.go
index ce9527f..2dda764 100644
--- a/pkg/cmd/run_test.go
+++ b/pkg/cmd/run_test.go
@@ -477,3 +477,47 @@ func TestRunWithSavedValues(t *testing.T) {
assert.Nil(t, err)
assert.Len(t, runCmdOptions.Sources, 2)
}*/
+
+func TestRunBinaryResource(t *testing.T) {
+ binaryResourceSpec, err := binaryOrTextResource("file.ext", []byte{1, 2, 3, 4}, "application/octet-stream", false)
+ assert.Nil(t, err)
+ assert.Equal(t, "", binaryResourceSpec.Content)
+ assert.NotNil(t, binaryResourceSpec.RawContent)
+ assert.Equal(t, "file.ext", binaryResourceSpec.Name)
+ assert.Equal(t, "application/octet-stream", binaryResourceSpec.ContentType)
+ assert.False(t, binaryResourceSpec.Compression)
+}
+
+func TestRunBinaryCompressedResource(t *testing.T) {
+ data := []byte{1, 2, 3, 4}
+ base64Compressed, _ := compressToString(data)
+ binaryResourceSpec, err := binaryOrTextResource("file.ext", data, "application/octet-stream", true)
+ assert.Nil(t, err)
+ assert.Equal(t, base64Compressed, binaryResourceSpec.Content)
+ assert.Nil(t, binaryResourceSpec.RawContent)
+ assert.Equal(t, "file.ext", binaryResourceSpec.Name)
+ assert.Equal(t, "application/octet-stream", binaryResourceSpec.ContentType)
+ assert.True(t, binaryResourceSpec.Compression)
+}
+
+func TestRunTextResource(t *testing.T) {
+ textResourceSpec, err := binaryOrTextResource("file.ext", []byte("hello world"), "text/plain", false)
+ assert.Nil(t, err)
+ assert.Equal(t, "hello world", textResourceSpec.Content)
+ assert.Nil(t, textResourceSpec.RawContent)
+ assert.Equal(t, "file.ext", textResourceSpec.Name)
+ assert.Equal(t, "text/plain", textResourceSpec.ContentType)
+ assert.False(t, textResourceSpec.Compression)
+}
+
+func TestRunTextCompressedResource(t *testing.T) {
+ data := []byte("hello horld")
+ base64Compressed, _ := compressToString(data)
+ textResourceSpec, err := binaryOrTextResource("file.ext", []byte("hello horld"), "text/plain", true)
+ assert.Nil(t, err)
+ assert.Equal(t, base64Compressed, textResourceSpec.Content)
+ assert.Nil(t, textResourceSpec.RawContent)
+ assert.Equal(t, "file.ext", textResourceSpec.Name)
+ assert.Equal(t, "text/plain", textResourceSpec.ContentType)
+ assert.True(t, textResourceSpec.Compression)
+}
diff --git a/pkg/cmd/util_content.go b/pkg/cmd/util_content.go
index c210870..7950e44 100644
--- a/pkg/cmd/util_content.go
+++ b/pkg/cmd/util_content.go
@@ -26,7 +26,7 @@ import (
"strings"
)
-func loadContent(source string, compress bool, compressBinary bool) (string, bool, error) {
+func loadRawContent(source string) ([]byte, string, error) {
var content []byte
var err error
@@ -35,7 +35,7 @@ func loadContent(source string, compress bool, compressBinary bool) (string, boo
} else {
u, err := url.Parse(source)
if err != nil {
- return "", false, err
+ return nil, "", err
}
switch u.Scheme {
@@ -46,27 +46,36 @@ func loadContent(source string, compress bool, compressBinary bool) (string, boo
case "https":
content, err = loadContentHTTP(u)
default:
- return "", false, fmt.Errorf("unsupported scheme %s", u.Scheme)
+ return nil, "", fmt.Errorf("unsupported scheme %s", u.Scheme)
}
}
if err != nil {
- return "", false, err
+ return nil, "", err
}
- doCompress := compress
- if !doCompress && compressBinary {
- contentType := http.DetectContentType(content)
- if strings.HasPrefix(contentType, "application/octet-stream") {
- doCompress = true
- }
+
+ contentType := http.DetectContentType(content)
+ return content, contentType, nil
+}
+
+func isBinary(contentType string) bool {
+ // According the http.DetectContentType method
+ // also json and other "text" application mime types would be reported as text
+ return !strings.HasPrefix(contentType, "text")
+}
+
+func loadTextContent(source string, base64Compression bool) (string, string, bool, error) {
+ content, contentType, err := loadRawContent(source)
+ if err != nil {
+ return "", "", false, err
}
- if doCompress {
- answer, err := compressToString(content)
- return answer, true, err
+ if base64Compression {
+ base64Compressed, err := compressToString(content)
+ return base64Compressed, contentType, true, err
}
- return string(content), false, nil
+ return string(content), contentType, false, nil
}
func loadContentHTTP(u *url.URL) ([]byte, error) {
diff --git a/pkg/cmd/util_content_test.go b/pkg/cmd/util_content_test.go
new file mode 100644
index 0000000..10d6610
--- /dev/null
+++ b/pkg/cmd/util_content_test.go
@@ -0,0 +1,99 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cmd
+
+import (
+ "io/ioutil"
+ "os"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestRawContentFileMissing(t *testing.T) {
+ _, _, err := loadRawContent("dsadas")
+ assert.NotNil(t, err)
+}
+
+func TestRawBinaryContentType(t *testing.T) {
+ var tmpFile *os.File
+ var err error
+ if tmpFile, err = ioutil.TempFile("", "camel-k-*.json"); err != nil {
+ t.Error(err)
+ }
+ assert.Nil(t, tmpFile.Close())
+ assert.Nil(t, ioutil.WriteFile(tmpFile.Name(), []byte{1, 2, 3, 4, 5, 6}, 0644))
+
+ data, contentType, err := loadRawContent(tmpFile.Name())
+ assert.Nil(t, err)
+ assert.Equal(t, []byte{1, 2, 3, 4, 5, 6}, data)
+ assert.True(t, isBinary(contentType))
+}
+
+func TestRawApplicationContentType(t *testing.T) {
+ var tmpFile *os.File
+ var err error
+ if tmpFile, err = ioutil.TempFile("", "camel-k-*.json"); err != nil {
+ t.Error(err)
+ }
+ assert.Nil(t, tmpFile.Close())
+ assert.Nil(t, ioutil.WriteFile(tmpFile.Name(), []byte(`{"hello":"world"}`), 0644))
+
+ data, contentType, err := loadRawContent(tmpFile.Name())
+ assert.Nil(t, err)
+ assert.Equal(t, `{"hello":"world"}`, string(data))
+ assert.False(t, isBinary(contentType))
+}
+
+func TestTextContentType(t *testing.T) {
+ var tmpFile *os.File
+ var err error
+ if tmpFile, err = ioutil.TempFile("", "camel-k-*.json"); err != nil {
+ t.Error(err)
+ }
+ assert.Nil(t, tmpFile.Close())
+ assert.Nil(t, ioutil.WriteFile(tmpFile.Name(), []byte(`{"hello":"world"}`), 0644))
+
+ data, contentType, compressed, err := loadTextContent(tmpFile.Name(), false)
+ assert.Nil(t, err)
+ assert.Equal(t, `{"hello":"world"}`, string(data))
+ assert.False(t, isBinary(contentType))
+ assert.False(t, compressed)
+}
+
+func TestTextCompressed(t *testing.T) {
+ var tmpFile *os.File
+ var err error
+ if tmpFile, err = ioutil.TempFile("", "camel-k-*.json"); err != nil {
+ t.Error(err)
+ }
+ assert.Nil(t, tmpFile.Close())
+ assert.Nil(t, ioutil.WriteFile(tmpFile.Name(), []byte(`{"hello":"world"}`), 0644))
+
+ data, contentType, compressed, err := loadTextContent(tmpFile.Name(), true)
+ assert.Nil(t, err)
+ assert.NotEqual(t, `{"hello":"world"}`, string(data))
+ assert.False(t, isBinary(contentType))
+ assert.True(t, compressed)
+}
+
+func TestIsBinary(t *testing.T) {
+ assert.True(t, isBinary("image/jpeg"))
+ assert.True(t, isBinary("application/zip"))
+ assert.False(t, isBinary("text/plain"))
+}
diff --git a/pkg/cmd/util_dependencies.go b/pkg/cmd/util_dependencies.go
index 94cc757..f987aa7 100644
--- a/pkg/cmd/util_dependencies.go
+++ b/pkg/cmd/util_dependencies.go
@@ -81,7 +81,7 @@ func getTopLevelDependencies(catalog *camel.RuntimeCatalog, args []string) ([]st
// Invoke the dependency inspector code for each source file
for _, source := range args {
- data, _, err := loadContent(source, false, false)
+ data, _, _, err := loadTextContent(source, false)
if err != nil {
return []string{}, err
}
diff --git a/pkg/trait/trait_test.go b/pkg/trait/trait_test.go
index 6336fd7..2b4f2a0 100644
--- a/pkg/trait/trait_test.go
+++ b/pkg/trait/trait_test.go
@@ -207,65 +207,7 @@ func TestTraitHierarchyDecode(t *testing.T) {
assert.Equal(t, 15, *kns.Target)
}
-func TestConfigureVolumesAndMountsSources(t *testing.T) {
- env := Environment{
- Resources: kubernetes.NewCollection(),
- Integration: &v1.Integration{
- ObjectMeta: metav1.ObjectMeta{
- Name: TestDeploymentName,
- Namespace: "ns",
- },
- Spec: v1.IntegrationSpec{
- Sources: []v1.SourceSpec{
- {
- DataSpec: v1.DataSpec{
- Name: "source1.java",
- ContentRef: "my-cm1",
- ContentKey: "source1.java",
- },
- Type: "data",
- },
- {
- DataSpec: v1.DataSpec{
- Name: "source2.java",
- ContentRef: "my-cm2",
- },
- Type: "data",
- },
- },
- },
- },
- }
-
- vols := make([]corev1.Volume, 0)
- mnts := make([]corev1.VolumeMount, 0)
-
- env.Resources.AddAll(env.ComputeConfigMaps())
- env.ConfigureVolumesAndMounts(&vols, &mnts)
-
- assert.Len(t, vols, 2)
- assert.Len(t, mnts, 2)
-
- v := findVolume(vols, func(v corev1.Volume) bool { return v.ConfigMap.Name == "my-cm1" })
- assert.NotNil(t, v)
- assert.NotNil(t, v.VolumeSource.ConfigMap)
- assert.Len(t, v.VolumeSource.ConfigMap.Items, 1)
- assert.Equal(t, "source1.java", v.VolumeSource.ConfigMap.Items[0].Key)
-
- m := findVVolumeMount(mnts, func(m corev1.VolumeMount) bool { return m.Name == v.Name })
- assert.NotNil(t, m)
-
- v = findVolume(vols, func(v corev1.Volume) bool { return v.ConfigMap.Name == "my-cm2" })
- assert.NotNil(t, v)
- assert.NotNil(t, v.VolumeSource.ConfigMap)
- assert.Len(t, v.VolumeSource.ConfigMap.Items, 1)
- assert.Equal(t, "content", v.VolumeSource.ConfigMap.Items[0].Key)
-
- m = findVVolumeMount(mnts, func(m corev1.VolumeMount) bool { return m.Name == v.Name })
- assert.NotNil(t, m)
-}
-
-func TestConfigureVolumesAndMountsResourcesAndProperties(t *testing.T) {
+func TestConfigureVolumesAndMountsTextResourcesAndProperties(t *testing.T) {
env := Environment{
Resources: kubernetes.NewCollection(),
Integration: &v1.Integration{
@@ -425,6 +367,131 @@ func TestConfigureVolumesAndMountsResourcesAndProperties(t *testing.T) {
assert.Equal(t, "/foo/bar", m.MountPath)
}
+func TestConfigureVolumesAndMountsSources(t *testing.T) {
+ env := Environment{
+ Resources: kubernetes.NewCollection(),
+ Integration: &v1.Integration{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: TestDeploymentName,
+ Namespace: "ns",
+ },
+ Spec: v1.IntegrationSpec{
+ Sources: []v1.SourceSpec{
+ {
+ DataSpec: v1.DataSpec{
+ Name: "source1.java",
+ ContentRef: "my-cm1",
+ ContentKey: "source1.java",
+ },
+ Type: "data",
+ },
+ {
+ DataSpec: v1.DataSpec{
+ Name: "source2.java",
+ ContentRef: "my-cm2",
+ },
+ Type: "data",
+ },
+ },
+ },
+ },
+ }
+
+ vols := make([]corev1.Volume, 0)
+ mnts := make([]corev1.VolumeMount, 0)
+
+ env.Resources.AddAll(env.ComputeConfigMaps())
+ env.ConfigureVolumesAndMounts(&vols, &mnts)
+
+ assert.Len(t, vols, 2)
+ assert.Len(t, mnts, 2)
+
+ v := findVolume(vols, func(v corev1.Volume) bool { return v.ConfigMap.Name == "my-cm1" })
+ assert.NotNil(t, v)
+ assert.NotNil(t, v.VolumeSource.ConfigMap)
+ assert.Len(t, v.VolumeSource.ConfigMap.Items, 1)
+ assert.Equal(t, "source1.java", v.VolumeSource.ConfigMap.Items[0].Key)
+
+ m := findVVolumeMount(mnts, func(m corev1.VolumeMount) bool { return m.Name == v.Name })
+ assert.NotNil(t, m)
+
+ v = findVolume(vols, func(v corev1.Volume) bool { return v.ConfigMap.Name == "my-cm2" })
+ assert.NotNil(t, v)
+ assert.NotNil(t, v.VolumeSource.ConfigMap)
+ assert.Len(t, v.VolumeSource.ConfigMap.Items, 1)
+ assert.Equal(t, "content", v.VolumeSource.ConfigMap.Items[0].Key)
+
+ m = findVVolumeMount(mnts, func(m corev1.VolumeMount) bool { return m.Name == v.Name })
+ assert.NotNil(t, m)
+}
+
+func TestConfigureVolumesAndMountsBinaryAndTextResources(t *testing.T) {
+ env := Environment{
+ Resources: kubernetes.NewCollection(),
+ Integration: &v1.Integration{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: TestDeploymentName,
+ Namespace: "ns",
+ },
+ Spec: v1.IntegrationSpec{
+ Resources: []v1.ResourceSpec{
+ {
+ DataSpec: v1.DataSpec{
+ Name: "res1.bin",
+ RawContent: []byte{1, 2, 3, 4},
+ ContentRef: "my-cm1",
+ ContentKey: "my-binary",
+ ContentType: "application/octet-stream",
+ },
+ Type: "data",
+ },
+ {
+ DataSpec: v1.DataSpec{
+ Name: "res2.txt",
+ ContentRef: "my-cm2",
+ Content: "hello",
+ ContentKey: "my-text",
+ ContentType: "text/plain",
+ },
+ Type: "data",
+ },
+ },
+ },
+ },
+ }
+
+ vols := make([]corev1.Volume, 0)
+ mnts := make([]corev1.VolumeMount, 0)
+
+ env.Resources.AddAll(env.ComputeConfigMaps())
+ env.ConfigureVolumesAndMounts(&vols, &mnts)
+
+ assert.Len(t, vols, 2)
+ assert.Len(t, mnts, 2)
+
+ v := findVolume(vols, func(v corev1.Volume) bool { return v.ConfigMap.Name == "my-cm1" })
+ assert.NotNil(t, v)
+ assert.NotNil(t, v.VolumeSource.ConfigMap)
+ assert.Len(t, v.VolumeSource.ConfigMap.Items, 1)
+ assert.Equal(t, "my-binary", v.VolumeSource.ConfigMap.Items[0].Key)
+ assert.Equal(t, "res1.bin", v.VolumeSource.ConfigMap.Items[0].Path)
+
+ m := findVVolumeMount(mnts, func(m corev1.VolumeMount) bool { return m.Name == v.Name })
+ assert.NotNil(t, m)
+ assert.Equal(t, "/etc/camel/resources/i-resource-000", m.MountPath)
+
+ v = findVolume(vols, func(v corev1.Volume) bool { return v.ConfigMap.Name == "my-cm2" })
+ assert.NotNil(t, v)
+ assert.NotNil(t, v.VolumeSource.ConfigMap)
+ assert.Len(t, v.VolumeSource.ConfigMap.Items, 1)
+ assert.Equal(t, "my-text", v.VolumeSource.ConfigMap.Items[0].Key)
+ assert.Equal(t, "res2.txt", v.VolumeSource.ConfigMap.Items[0].Path)
+
+ m = findVVolumeMount(mnts, func(m corev1.VolumeMount) bool { return m.Name == v.Name })
+ assert.NotNil(t, m)
+ assert.Equal(t, "/etc/camel/resources/i-resource-001", m.MountPath)
+}
+
func TestOnlySomeTraitsInfluenceBuild(t *testing.T) {
c := NewTraitTestCatalog()
buildTraits := []string{"builder", "quarkus"}
diff --git a/pkg/trait/trait_types.go b/pkg/trait/trait_types.go
index e896b3b..10a99c0 100644
--- a/pkg/trait/trait_types.go
+++ b/pkg/trait/trait_types.go
@@ -464,9 +464,20 @@ func (e *Environment) ComputeConfigMaps() []runtime.Object {
"camel.apache.org/resource.compression": strconv.FormatBool(r.Compression),
},
},
- Data: map[string]string{
+ }
+
+ if r.ContentType != "" {
+ cm.Annotations["camel.apache.org/resource.content-type"] = r.ContentType
+ }
+
+ if r.RawContent != nil {
+ cm.BinaryData = map[string][]byte{
+ cmKey: []byte(r.RawContent),
+ }
+ } else {
+ cm.Data = map[string]string{
cmKey: r.Content,
- },
+ }
}
maps = append(maps, &cm)