You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by nf...@apache.org on 2021/09/23 07:26:35 UTC

[camel-k] 01/02: Fix local builds for containerized and non-containerized uses.

This is an automated email from the ASF dual-hosted git repository.

nferraro pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel-k.git

commit 92836dc1a84f58621166d70154705e85096c6e3c
Author: Gheorghe Teodor Bercea <gh...@ibm.com>
AuthorDate: Tue Sep 21 16:30:52 2021 -0400

    Fix local builds for containerized and non-containerized uses.
---
 pkg/cmd/local_build.go           |   6 +-
 pkg/cmd/local_run.go             |  38 +++++++-
 pkg/cmd/util_containerization.go |  13 ++-
 pkg/cmd/util_dependencies.go     |  86 ++++++++++++++++++
 pkg/util/docker/docker.go        |  16 +++-
 pkg/util/docker/docker_base.go   |   8 ++
 pkg/util/util.go                 | 186 +++++++++++++++++++++++++++++++++++++++
 7 files changed, 346 insertions(+), 7 deletions(-)

diff --git a/pkg/cmd/local_build.go b/pkg/cmd/local_build.go
index 1a09adf..01f684a 100644
--- a/pkg/cmd/local_build.go
+++ b/pkg/cmd/local_build.go
@@ -221,9 +221,9 @@ func (command *localBuildCmdOptions) run(cmd *cobra.Command, args []string) erro
 
 	// Integration directory can only be used when building an integration image or when we just
 	// build the integration without also building the image. A local build of the integration is
-	// represented by all the files that define	 the integration: dependencies, properties and routes.
+	// represented by all the files that define the integration: dependencies, properties and routes.
 
-	// The only case where we should not execute the image integration creation is when we want to
+	// The only case in which we should not execute the integration image creation is when we want to
 	// just output the files that comprise the integration locally.
 	if command.IntegrationDirectory != "" && command.Image == "" {
 		return nil
@@ -231,7 +231,7 @@ func (command *localBuildCmdOptions) run(cmd *cobra.Command, args []string) erro
 
 	// Create and build integration image.
 	err := createAndBuildIntegrationImage(command.Context, command.ContainerRegistry, command.BaseImage,
-		command.Image, propertyFilesList, dependenciesList, routeFiles, cmd.OutOrStdout(), cmd.ErrOrStderr())
+		command.Image, propertyFilesList, dependenciesList, routeFiles, false, cmd.OutOrStdout(), cmd.ErrOrStderr())
 	if err != nil {
 		return err
 	}
diff --git a/pkg/cmd/local_run.go b/pkg/cmd/local_run.go
index 418830d..a808ad4 100644
--- a/pkg/cmd/local_run.go
+++ b/pkg/cmd/local_run.go
@@ -149,11 +149,40 @@ func (command *localRunCmdOptions) run(cmd *cobra.Command, args []string) error
 
 	var dependencies []string
 	if hasIntegrationDir {
+		// Fetch local dependencies
 		localBuildDependencies, err := getLocalBuildDependencies(command.IntegrationDirectory)
 		if err != nil {
 			return err
 		}
 		dependencies = localBuildDependencies
+
+		// Local dependencies directory
+		localDependenciesDirectory := getCustomDependenciesDir(command.IntegrationDirectory)
+
+		// The quarkus application files need to be at a specific location i.e.:
+		// <current_working_folder>/quarkus/quarkus-application.dat
+		// <current_working_folder>/quarkus/generated-bytecode.jar
+		localQuarkusDir, err := getCustomQuarkusDir()
+		if err != nil {
+			return err
+		}
+		util.CopyQuarkusAppFiles(localDependenciesDirectory, localQuarkusDir)
+
+		// The dependency jar files need to be at a specific location i.e.:
+		// <current_working_folder>/lib/main/*.jar
+		localLibDirectory, err := getCustomLibDir()
+		if err != nil {
+			return err
+		}
+		util.CopyLibFiles(localDependenciesDirectory, localLibDirectory)
+
+		// The Camel-K jar file needs to be at a specific location i.e.:
+		// <current_working_folder>/app/camel-k-integration-X.X.X{-SNAPSHOT}.jar
+		localAppDirectory, err := getCustomAppDir()
+		if err != nil {
+			return err
+		}
+		util.CopyAppFile(localDependenciesDirectory, localAppDirectory)
 	} else {
 		computedDependencies, err := getDependencies(command.Context, args, command.AdditionalDependencies, command.MavenRepositories, true)
 		if err != nil {
@@ -190,7 +219,7 @@ func (command *localRunCmdOptions) run(cmd *cobra.Command, args []string) error
 	// If this is a containerized local run, create, build and run the container image.
 	if command.Containerize {
 		// Create and build integration image.
-		err := createAndBuildIntegrationImage(command.Context, "", false, command.Image, propertyFiles, dependencies, routes, cmd.OutOrStdout(), cmd.ErrOrStderr())
+		err := createAndBuildIntegrationImage(command.Context, "", false, command.Image, propertyFiles, dependencies, routes, hasIntegrationDir, cmd.OutOrStdout(), cmd.ErrOrStderr())
 		if err != nil {
 			return err
 		}
@@ -229,5 +258,12 @@ func (command *localRunCmdOptions) deinit() error {
 		}
 	}
 
+	if command.IntegrationDirectory != "" {
+		err := deleteLocalIntegrationDirs()
+		if err != nil {
+			return err
+		}
+	}
+
 	return deleteMavenWorkingDirectory()
 }
diff --git a/pkg/cmd/util_containerization.go b/pkg/cmd/util_containerization.go
index f633a0e..42901c2 100644
--- a/pkg/cmd/util_containerization.go
+++ b/pkg/cmd/util_containerization.go
@@ -110,7 +110,7 @@ func createAndBuildBaseImage(ctx context.Context) error {
 }
 
 func createAndBuildIntegrationImage(ctx context.Context, containerRegistry string, justBaseImage bool, image string,
-	propertyFiles []string, dependencies []string, routes []string, stdout, stderr io.Writer) error {
+	propertyFiles []string, dependencies []string, routes []string, startsFromLocalFolder bool, stdout, stderr io.Writer) error {
 	// This ensures the Dockerfile for the base image will not end up in an undesired location.
 	if docker.BaseWorkingDirectory == "" {
 		return errors.New("base directory that holds the base image Dockerfile has not been set correctly")
@@ -156,6 +156,15 @@ func createAndBuildIntegrationImage(ctx context.Context, containerRegistry strin
 		return err
 	}
 
+	// Copy quarkus files in maven subdirectory
+	updateQuarkusDirectory()
+
+	// Copy app files in maven subdirectory
+	updateAppDirectory()
+
+	// Copy lib files in maven subdirectory
+	updateLibDirectory()
+
 	// Get integration run command to be run inside the container. This means the command
 	// has to be created with the paths which will be valid inside the container.
 	containerCmd, err := GetContainerIntegrationRunCommand(ctx, propertyFiles, dependencies, routes, stdout, stderr)
@@ -164,7 +173,7 @@ func createAndBuildIntegrationImage(ctx context.Context, containerRegistry strin
 	}
 
 	// Create the integration image Docker file.
-	err = docker.CreateIntegrationImageDockerFile(containerCmd)
+	err = docker.CreateIntegrationImageDockerFile(containerCmd, startsFromLocalFolder)
 	if err != nil {
 		return err
 	}
diff --git a/pkg/cmd/util_dependencies.go b/pkg/cmd/util_dependencies.go
index 89c1999..18efc4b 100644
--- a/pkg/cmd/util_dependencies.go
+++ b/pkg/cmd/util_dependencies.go
@@ -452,6 +452,39 @@ func updateIntegrationRoutes(routes []string) error {
 	return nil
 }
 
+func updateQuarkusDirectory() error {
+	err := util.CreateLocalQuarkusDirectory()
+	if err != nil {
+		return err
+	}
+
+	util.CopyQuarkusAppFiles(util.CustomQuarkusDirectoryName, util.GetLocalQuarkusDir())
+
+	return nil
+}
+
+func updateAppDirectory() error {
+	err := util.CreateLocalAppDirectory()
+	if err != nil {
+		return err
+	}
+
+	util.CopyAppFile(util.CustomAppDirectoryName, util.GetLocalAppDir())
+
+	return nil
+}
+
+func updateLibDirectory() error {
+	err := util.CreateLocalLibDirectory()
+	if err != nil {
+		return err
+	}
+
+	util.CopyLibFiles(util.CustomLibDirectoryName, util.GetLocalLibDir())
+
+	return nil
+}
+
 func createMavenWorkingDirectory() error {
 	// Create local Maven context
 	temporaryDirectory, err := ioutil.TempDir(os.TempDir(), "maven-")
@@ -483,3 +516,56 @@ func getCustomPropertiesDir(integrationDirectory string) string {
 func getCustomRoutesDir(integrationDirectory string) string {
 	return path.Join(integrationDirectory, "routes")
 }
+
+func getCustomQuarkusDir() (string, error) {
+	currentDir, err := os.Getwd()
+	if err != nil {
+		return "", err
+	}
+	return path.Join(currentDir, "quarkus"), nil
+}
+
+func getCustomLibDir() (string, error) {
+	currentDir, err := os.Getwd()
+	if err != nil {
+		return "", err
+	}
+	return path.Join(currentDir, "lib/main"), nil
+}
+
+func getCustomAppDir() (string, error) {
+	currentDir, err := os.Getwd()
+	if err != nil {
+		return "", err
+	}
+	return path.Join(currentDir, "app"), nil
+}
+
+func deleteLocalIntegrationDirs() error {
+	directory, err := getCustomQuarkusDir()
+	if err != nil {
+		return err
+	}
+
+	err = os.RemoveAll(directory)
+	if err != nil {
+		return err
+	}
+
+	err = os.RemoveAll("lib")
+	if err != nil {
+		return err
+	}
+
+	directory, err = getCustomAppDir()
+	if err != nil {
+		return err
+	}
+
+	err = os.RemoveAll(directory)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
diff --git a/pkg/util/docker/docker.go b/pkg/util/docker/docker.go
index 8cf2cc0..8c4ada8 100644
--- a/pkg/util/docker/docker.go
+++ b/pkg/util/docker/docker.go
@@ -48,7 +48,7 @@ func CreateBaseImageDockerFile() error {
 }
 
 // CreateIntegrationImageDockerFile --
-func CreateIntegrationImageDockerFile(integrationRunCmd *exec.Cmd) error {
+func CreateIntegrationImageDockerFile(integrationRunCmd *exec.Cmd, startsFromLocalFolder bool) error {
 	dockerFile := []string{}
 
 	// Start from the base image that contains the maven install: <RegistryName>/<BaseImageName>
@@ -62,6 +62,20 @@ func CreateIntegrationImageDockerFile(integrationRunCmd *exec.Cmd) error {
 	dockerFile = append(dockerFile, COPY(util.DefaultPropertiesDirectoryName, GetContainerPropertiesDir()))
 	dockerFile = append(dockerFile, COPY(util.DefaultDependenciesDirectoryName, GetContainerDependenciesDir()))
 
+	// Copy additional folder structure used for integrations built from a separately built local folder.
+	if startsFromLocalFolder {
+		dockerFile = append(dockerFile, RUNMakeDir(util.ContainerQuarkusDirectoryName))
+		dockerFile = append(dockerFile, RUNMakeDir(util.ContainerLibDirectoryName))
+		dockerFile = append(dockerFile, RUNMakeDir(util.ContainerAppDirectoryName))
+
+		dockerFile = append(dockerFile, WORKDIR("/workspace"))
+
+		dockerFile = append(dockerFile, COPY(util.CustomQuarkusDirectoryName, util.ContainerQuarkusDirectoryName))
+		dockerFile = append(dockerFile, COPY(util.CustomLibDirectoryName, util.ContainerLibDirectoryName))
+		dockerFile = append(dockerFile, COPY(util.CustomAppDirectoryName, util.ContainerAppDirectoryName))
+
+	}
+
 	// All Env variables the command requires need to be set in the container.
 	for _, keyValue := range integrationRunCmd.Env {
 		values := strings.Split(keyValue, "=")
diff --git a/pkg/util/docker/docker_base.go b/pkg/util/docker/docker_base.go
index 14eba9d..5f54574 100644
--- a/pkg/util/docker/docker_base.go
+++ b/pkg/util/docker/docker_base.go
@@ -129,6 +129,14 @@ func FullImageArg(dockerImage string) []string {
 	if len(imageComponents) == 2 {
 		// Image has a tag already.
 		return ImageArg(imageComponents[0], imageComponents[1])
+	} else if len(imageComponents) > 2 {
+		// Image has an image registry name with a colon inside it:
+		// localhost:5000
+		// and image also has a tag:
+		// localhost:5000/image:latest
+		// Assume tag is always included and is last in the imageComponents list.
+		last := len(imageComponents) - 1
+		return ImageArg(strings.Join(imageComponents[:last], ":"), imageComponents[last])
 	}
 
 	// Image has no tag, latest tag will be added automatically.
diff --git a/pkg/util/util.go b/pkg/util/util.go
index 40e559f..0a28eb9 100644
--- a/pkg/util/util.go
+++ b/pkg/util/util.go
@@ -54,6 +54,15 @@ const DefaultRoutesDirectoryName = "routes"
 // DefaultWorkingDirectoryName --
 const DefaultWorkingDirectoryName = "workspace"
 
+// CustomQuarkusDirectoryName --
+const CustomQuarkusDirectoryName = "quarkus"
+
+// CustomAppDirectoryName --
+const CustomAppDirectoryName = "app"
+
+// CustomLibDirectoryName --
+const CustomLibDirectoryName = "lib/main"
+
 // ContainerDependenciesDirectory --
 var ContainerDependenciesDirectory = "/deployments/dependencies"
 
@@ -66,6 +75,15 @@ var ContainerRoutesDirectory = "/etc/camel/sources"
 // ContainerResourcesDirectory --
 var ContainerResourcesDirectory = "/etc/camel/resources"
 
+// ContainerQuarkusDirectoryName --
+const ContainerQuarkusDirectoryName = "/quarkus"
+
+// ContainerAppDirectoryName --
+const ContainerAppDirectoryName = "/app"
+
+// ContainerLibDirectoryName --
+const ContainerLibDirectoryName = "/lib/main"
+
 // QuarkusDependenciesBaseDirectory --
 var QuarkusDependenciesBaseDirectory = "/quarkus-app"
 
@@ -415,6 +433,21 @@ func GetLocalRoutesDir() string {
 	return path.Join(MavenWorkingDirectory, DefaultRoutesDirectoryName)
 }
 
+// GetLocalQuarkusDir -- <mavenWorkingDirectory>/quarkus
+func GetLocalQuarkusDir() string {
+	return path.Join(MavenWorkingDirectory, CustomQuarkusDirectoryName)
+}
+
+// GetLocalAppDir -- <mavenWorkingDirectory>/app
+func GetLocalAppDir() string {
+	return path.Join(MavenWorkingDirectory, CustomAppDirectoryName)
+}
+
+// GetLocalLibDir -- <mavenWorkingDirectory>/lib/main
+func GetLocalLibDir() string {
+	return path.Join(MavenWorkingDirectory, CustomLibDirectoryName)
+}
+
 func CreateLocalPropertiesDirectory() error {
 	// Do not create a directory unless the maven directory contains a valid value.
 	if MavenWorkingDirectory == "" {
@@ -475,6 +508,66 @@ func CreateLocalRoutesDirectory() error {
 	return nil
 }
 
+func CreateLocalQuarkusDirectory() error {
+	// Do not create a directory unless the maven directory contains a valid value.
+	if MavenWorkingDirectory == "" {
+		return nil
+	}
+
+	directoryExists, err := DirectoryExists(GetLocalQuarkusDir())
+	if err != nil {
+		return err
+	}
+
+	if !directoryExists {
+		err := os.MkdirAll(GetLocalQuarkusDir(), 0777)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func CreateLocalAppDirectory() error {
+	// Do not create a directory unless the maven directory contains a valid value.
+	if MavenWorkingDirectory == "" {
+		return nil
+	}
+
+	directoryExists, err := DirectoryExists(GetLocalAppDir())
+	if err != nil {
+		return err
+	}
+
+	if !directoryExists {
+		err := os.MkdirAll(GetLocalAppDir(), 0777)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func CreateLocalLibDirectory() error {
+	// Do not create a directory unless the maven directory contains a valid value.
+	if MavenWorkingDirectory == "" {
+		return nil
+	}
+
+	directoryExists, err := DirectoryExists(GetLocalLibDir())
+	if err != nil {
+		return err
+	}
+
+	if !directoryExists {
+		err := os.MkdirAll(GetLocalLibDir(), 0777)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func GetEnvironmentVariable(variable string) (string, error) {
 	value, isPresent := os.LookupEnv(variable)
 	if !isPresent {
@@ -546,3 +639,96 @@ func CopyIntegrationFilesToDirectory(files []string, directory string) ([]string
 
 	return relocatedFilesList, nil
 }
+
+func CopyQuarkusAppFiles(localDependenciesDirectory string, localQuarkusDir string) error {
+	// Create directory if one does not already exist
+	err := CreateDirectory(localQuarkusDir)
+	if err != nil {
+		return err
+	}
+
+	source := path.Join(localDependenciesDirectory, "quarkus-application.dat")
+	destination := path.Join(localQuarkusDir, "quarkus-application.dat")
+	_, err = CopyFile(source, destination)
+	if err != nil {
+		return err
+	}
+
+	source = path.Join(localDependenciesDirectory, "generated-bytecode.jar")
+	destination = path.Join(localQuarkusDir, "generated-bytecode.jar")
+	_, err = CopyFile(source, destination)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func getRegularFileNamesInDir(directory string) ([]string, error) {
+	var dirFiles []string
+	files, err := ioutil.ReadDir(directory)
+	for _, file := range files {
+		fileName := file.Name()
+
+		// Do not include hidden files or sub-directories.
+		if !file.IsDir() && !strings.HasPrefix(fileName, ".") {
+			dirFiles = append(dirFiles, fileName)
+		}
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	return dirFiles, nil
+}
+
+func CopyLibFiles(localDependenciesDirectory string, localLibDirectory string) error {
+	// Create directory if one does not already exist
+	err := CreateDirectory(localLibDirectory)
+	if err != nil {
+		return err
+	}
+
+	fileNames, err := getRegularFileNamesInDir(localDependenciesDirectory)
+	if err != nil {
+		return err
+	}
+
+	for _, dependencyJar := range fileNames {
+		source := path.Join(localDependenciesDirectory, dependencyJar)
+		destination := path.Join(localLibDirectory, dependencyJar)
+		_, err = CopyFile(source, destination)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func CopyAppFile(localDependenciesDirectory string, localAppDirectory string) error {
+	// Create directory if one does not already exist
+	err := CreateDirectory(localAppDirectory)
+	if err != nil {
+		return err
+	}
+
+	fileNames, err := getRegularFileNamesInDir(localDependenciesDirectory)
+	if err != nil {
+		return err
+	}
+
+	for _, dependencyJar := range fileNames {
+		if strings.HasPrefix(dependencyJar, "camel-k-integration-") {
+			source := path.Join(localDependenciesDirectory, dependencyJar)
+			destination := path.Join(localAppDirectory, dependencyJar)
+			_, err = CopyFile(source, destination)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}