[camel-k] 02/04: Added incremental publisher

[nf...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

nferraro pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/camel-k.git

commit 2c51dfd0d359885a8b298b35d77e50fbf0fb200f
Author: nferraro <ni...@gmail.com>
AuthorDate: Fri Sep 21 13:26:20 2018 +0200

    Added incremental publisher
---
 pkg/build/publish/s2i_incremental_publisher.go | 105 +++++++++++++++++++++++++
 pkg/build/publish/s2i_publisher.go             |  43 +++++++---
 2 files changed, 137 insertions(+), 11 deletions(-)

diff --git a/pkg/build/publish/s2i_incremental_publisher.go b/pkg/build/publish/s2i_incremental_publisher.go
new file mode 100644
index 0000000..42d7c81
--- /dev/null
+++ b/pkg/build/publish/s2i_incremental_publisher.go
@@ -0,0 +1,105 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package publish
+
+import (
+	"context"
+	"github.com/apache/camel-k/pkg/build"
+)
+
+type s2iIncrementalPublisher struct {
+	s2iPublisher *s2iPublisher
+	lister       PublishedImagesLister
+}
+
+// PublishedImage represent a base image that can be used as starting point
+type PublishedImage struct {
+	Image     string
+	Classpath []string
+}
+
+// PublishedImagesLister allows to list all images already published
+type PublishedImagesLister interface {
+	ListPublishedImages() ([]PublishedImage, error)
+}
+
+// NewS2IIncrementalPublisher creates a new publisher that is able to do a Openshift S2I binary builds on top of other builds
+func NewS2IIncrementalPublisher(ctx context.Context, namespace string, lister PublishedImagesLister) build.Publisher {
+	layeredPublisher := s2iIncrementalPublisher{
+		lister: lister,
+	}
+	layeredPublisher.s2iPublisher = newS2IPublisher(ctx, namespace, layeredPublisher.selectArtifactsToUpload)
+	return &layeredPublisher
+}
+
+func (p *s2iIncrementalPublisher) Publish(req build.Request, assembled build.AssembledOutput) <-chan build.PublishedOutput {
+	return p.s2iPublisher.Publish(req, assembled)
+}
+
+func (p *s2iIncrementalPublisher) selectArtifactsToUpload(entries []build.ClasspathEntry) (string, []build.ClasspathEntry, error) {
+	images, err := p.lister.ListPublishedImages()
+	if err != nil {
+		return "", nil, err
+	}
+
+	bestImage, commonLibs := p.findBestImage(images, entries)
+	if (bestImage != nil) {
+		selectedClasspath := make([]build.ClasspathEntry, 0)
+		for _, entry := range entries {
+			if _, isCommon := commonLibs[entry.ID]; !isCommon {
+				selectedClasspath = append(selectedClasspath, entry)
+			}
+		}
+
+		return bestImage.Image, selectedClasspath, nil
+	}
+
+	// return default selection
+	return baseImage, entries, nil
+}
+
+func (p *s2iIncrementalPublisher) findBestImage(images []PublishedImage, entries []build.ClasspathEntry) (*PublishedImage, map[string]bool) {
+	requiredLibs := make(map[string]bool, len(entries))
+	for _, entry := range entries {
+		requiredLibs[entry.ID] = true
+	}
+
+	var bestImage *PublishedImage
+	bestImageCommonLibs := make(map[string]bool, 0)
+	for _, image := range images {
+		common := make(map[string]bool)
+		for _, id := range image.Classpath {
+			if _, ok := requiredLibs[id]; ok {
+				common[id] = true
+			}
+		}
+		numCommonLibs := len(common)
+		surplus := len(image.Classpath) - numCommonLibs
+		if surplus >= numCommonLibs/3 {
+			// Heuristic approach: if there are too many unrelated libraries, just use the base image
+			continue
+		}
+
+		if (numCommonLibs > len(bestImageCommonLibs)) {
+			bestImage = &image
+			bestImageCommonLibs = common
+		}
+	}
+
+	return bestImage, bestImageCommonLibs
+}
diff --git a/pkg/build/publish/s2i_publisher.go b/pkg/build/publish/s2i_publisher.go
index 5548194..ee86a1e 100644
--- a/pkg/build/publish/s2i_publisher.go
+++ b/pkg/build/publish/s2i_publisher.go
@@ -40,11 +40,13 @@ import (
 
 const (
 	artifactDirPrefix = "s2i-"
+	baseImage         = "fabric8/s2i-java:2.3"
 )
 
 type s2iPublisher struct {
 	buffer    chan publishOperation
 	namespace string
+	uploadedArtifactsSelector
 }
 
 type publishOperation struct {
@@ -53,11 +55,20 @@ type publishOperation struct {
 	output    chan build.PublishedOutput
 }
 
+type uploadedArtifactsSelector func([]build.ClasspathEntry) (string, []build.ClasspathEntry, error)
+
 // NewS2IPublisher creates a new publisher doing a Openshift S2I binary build
 func NewS2IPublisher(ctx context.Context, namespace string) build.Publisher {
+	identitySelector := func(entries []build.ClasspathEntry) (string, []build.ClasspathEntry, error) { return baseImage, entries, nil }
+	return newS2IPublisher(ctx, namespace, identitySelector)
+}
+
+// NewS2IPublisher creates a new publisher doing a Openshift S2I binary build
+func newS2IPublisher(ctx context.Context, namespace string, uploadedArtifactsSelector uploadedArtifactsSelector) *s2iPublisher {
 	publisher := s2iPublisher{
-		buffer:    make(chan publishOperation, 100),
-		namespace: namespace,
+		buffer:                    make(chan publishOperation, 100),
+		namespace:                 namespace,
+		uploadedArtifactsSelector: uploadedArtifactsSelector,
 	}
 	go publisher.publishCycle(ctx)
 	return &publisher
@@ -98,12 +109,17 @@ func (b *s2iPublisher) publishCycle(ctx context.Context) {
 }
 
 func (b *s2iPublisher) execute(request build.Request, assembled build.AssembledOutput) build.PublishedOutput {
-	tarFile, err := b.createTar(assembled)
+	baseImageName, selectedArtifacts, err := b.uploadedArtifactsSelector(assembled.Classpath)
 	if err != nil {
 		return build.PublishedOutput{Error: err}
 	}
 
-	image, err := b.publish(tarFile, request)
+	tarFile, err := b.createTar(assembled, selectedArtifacts)
+	if err != nil {
+		return build.PublishedOutput{Error: err}
+	}
+
+	image, err := b.publish(tarFile, baseImageName, request)
 	if err != nil {
 		return build.PublishedOutput{Error: err}
 	}
@@ -111,7 +127,7 @@ func (b *s2iPublisher) execute(request build.Request, assembled build.AssembledO
 	return build.PublishedOutput{Image: image}
 }
 
-func (b *s2iPublisher) publish(tarFile string, source build.Request) (string, error) {
+func (b *s2iPublisher) publish(tarFile string, imageName string, source build.Request) (string, error) {
 
 	bc := buildv1.BuildConfig{
 		TypeMeta: metav1.TypeMeta{
@@ -131,7 +147,7 @@ func (b *s2iPublisher) publish(tarFile string, source build.Request) (string, er
 					SourceStrategy: &buildv1.SourceBuildStrategy{
 						From: v1.ObjectReference{
 							Kind: "DockerImage",
-							Name: "fabric8/s2i-java:2.3",
+							Name: imageName,
 						},
 					},
 				},
@@ -235,7 +251,7 @@ func (b *s2iPublisher) publish(tarFile string, source build.Request) (string, er
 	return is.Status.DockerImageRepository + ":" + source.Identifier.Qualifier, nil
 }
 
-func (b *s2iPublisher) createTar(assembled build.AssembledOutput) (string, error) {
+func (b *s2iPublisher) createTar(assembled build.AssembledOutput, selectedArtifacts []build.ClasspathEntry) (string, error) {
 	artifactDir, err := ioutil.TempDir("", artifactDirPrefix)
 	if err != nil {
 		return "", errors.Wrap(err, "could not create temporary dir for s2i artifacts")
@@ -248,19 +264,24 @@ func (b *s2iPublisher) createTar(assembled build.AssembledOutput) (string, error
 	}
 	defer tarAppender.Close()
 
-	cp := ""
-	for _, entry := range assembled.Classpath {
+	tarDir := "dependencies/"
+	for _, entry := range selectedArtifacts {
 		gav, err := maven.ParseGAV(entry.ID)
 		if err != nil {
 			return "", nil
 		}
 
-		tarPath := path.Join("dependencies/", gav.GroupID)
-		fileName, err := tarAppender.AddFile(entry.Location, tarPath)
+		tarPath := path.Join(tarDir, gav.GroupID)
+		_, err = tarAppender.AddFile(entry.Location, tarPath)
 		if err != nil {
 			return "", err
 		}
+	}
 
+	cp := ""
+	for _, entry := range assembled.Classpath {
+		_, fileName := path.Split(entry.Location)
+		fileName = path.Join(tarDir, fileName)
 		cp += fileName + "\n"
 	}