You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampark.apache.org by be...@apache.org on 2026/07/31 01:02:50 UTC
(streampark) branch dev updated: [Migrate] Migrate streampark-flink-packer from Scala to Java (#4463)
This is an automated email from the ASF dual-hosted git repository.
wolfboys pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/streampark.git
The following commit(s) were added to refs/heads/dev by this push:
new 8d3025683 [Migrate] Migrate streampark-flink-packer from Scala to Java (#4463)
8d3025683 is described below
commit 8d3025683ccd708280c31ee8000dbd355f843f19
Author: Assert <42...@users.noreply.github.com>
AuthorDate: Fri Jul 31 09:02:45 2026 +0800
[Migrate] Migrate streampark-flink-packer from Scala to Java (#4463)
* [Migrate] #4449 Migrate streampark-flink-packer from Scala to Java
Replace all 34 Scala sources in the packer module with Java equivalents,
preserving public API for console and flink-client callers. Tests migrated
to JUnit 5 with Surefire workspace configuration for Maven resolve tests.
Co-authored-by: Cursor <cu...@cursor.com>
* [CI] Fix Sonar reliability issues for #4463
Handle InterruptedException in BuildPipeline.launch and check uberJar.delete()
result in MavenTool.
Co-authored-by: Cursor <cu...@cursor.com>
* [CI] Fix DockerClientTest and Sonar reliability issues for #4463
Rebase onto dev after #4462 merge; prepare hello-world test image in CI,
replace Throwable catches with specific exception handling, and document
intentionally empty silent watcher callbacks.
Co-authored-by: Cursor <cu...@cursor.com>
* [CI] Fix Scala/Java Map interop after packer migration for #4463
DockerImageBuildResponse.podTemplatePaths() now returns a Java Map; convert
via JavaConverters before iterating in KubernetesNativeClientTrait.
Co-authored-by: Cursor <cu...@cursor.com>
* [Build] Use properties for packer dependency versions in #4463
Replace hard-coded jackson-annotations and httpclient versions with
property references aligned to the root POM.
Co-authored-by: Cursor <cu...@cursor.com>
---------
Co-authored-by: Cursor <cu...@cursor.com>
---
.../client/trait/KubernetesNativeClientTrait.scala | 4 +-
streampark-flink/streampark-flink-packer/pom.xml | 27 +-
.../streampark/flink/packer/PackerResourceGC.java | 126 +++++++
.../streampark/flink/packer/docker/DockerConf.java | 76 ++++
.../flink/packer/docker/DockerImageExist.java} | 34 +-
.../flink/packer/docker/DockerRetriever.java | 68 ++++
.../flink/packer/docker/DockerUtils.java | 78 +++++
.../packer/docker/FlinkDockerfileTemplate.java | 68 ++++
.../docker/FlinkDockerfileTemplateTrait.java | 118 +++++++
.../docker/FlinkHadoopDockerfileTemplate.java | 150 ++++++++
.../packer/docker/SparkDockerfileTemplate.java | 69 ++++
.../docker/SparkDockerfileTemplateTrait.java | 117 +++++++
.../docker/SparkHadoopDockerfileTemplate.java | 143 ++++++++
.../streampark/flink/packer/maven/Artifact.java | 86 +++++
.../flink/packer/maven/DependencyInfo.java | 83 +++++
.../streampark/flink/packer/maven/MavenTool.java | 321 +++++++++++++++++
.../flink/packer/pipeline/BuildParam.java} | 20 +-
.../flink/packer/pipeline/BuildPipeline.java | 267 ++++++++++++++
.../flink/packer/pipeline/BuildPipelineExpose.java | 47 +++
.../packer/pipeline/BuildPipelineProcess.java} | 28 +-
.../flink/packer/pipeline/BuildResult.java} | 24 +-
.../packer/pipeline/DockerBuildProgress.java} | 44 ++-
.../packer/pipeline/DockerBuildSnapshot.java} | 52 +--
.../packer/pipeline/DockerImageBuildResponse.java | 119 +++++++
.../flink/packer/pipeline/DockerLayerProgress.java | 70 ++++
.../packer/pipeline/DockerProgressWatcher.java} | 28 +-
.../flink/packer/pipeline/DockerPullProgress.java | 74 ++++
.../flink/packer/pipeline/DockerPullSnapshot.java | 75 ++++
.../flink/packer/pipeline/DockerPushProgress.java | 74 ++++
.../flink/packer/pipeline/DockerPushSnapshot.java | 75 ++++
.../packer/pipeline/DockerResolveProgress.java} | 42 +--
.../packer/pipeline/DockerResolvedSnapshot.java | 50 +++
.../flink/packer/pipeline/ErrorResult.java} | 34 +-
.../flink/packer/pipeline/FlinkBuildParam.java | 61 ++++
.../flink/packer/pipeline/FlinkBuildResult.java} | 18 +-
.../pipeline/FlinkK8sApplicationBuildRequest.java | 147 ++++++++
.../flink/packer/pipeline/FlinkK8sBuildParam.java} | 19 +-
.../pipeline/FlinkK8sSessionBuildRequest.java | 110 ++++++
.../pipeline/FlinkRemotePerJobBuildRequest.java | 101 ++++++
.../pipeline/FlinkYarnApplicationBuildRequest.java | 72 ++++
.../packer/pipeline/K8sAppModeBuildResponse.java | 111 ++++++
.../flink/packer/pipeline/PipeError.java | 101 ++++++
.../flink/packer/pipeline/PipeWatcher.java} | 34 +-
.../flink/packer/pipeline/PipelineSnapshot.java | 117 +++++++
.../flink/packer/pipeline/ShadedBuildResponse.java | 82 +++++
.../pipeline/SilentDockerProgressWatcher.java} | 31 +-
.../flink/packer/pipeline/SilentPipeWatcher.java} | 31 +-
.../flink/packer/pipeline/SimpleBuildResponse.java | 61 ++++
.../pipeline/SparkK8sApplicationBuildRequest.java | 125 +++++++
.../packer/pipeline/SparkYarnBuildRequest.java | 80 +++++
.../impl/FlinkK8sApplicationBuildPipeline.java | 385 +++++++++++++++++++++
.../impl/FlinkK8sSessionBuildPipeline.java | 90 +++++
.../pipeline/impl/FlinkRemoteBuildPipeline.java | 132 +++++++
.../impl/FlinkYarnApplicationBuildPipeline.java | 151 ++++++++
.../impl/SparkK8sApplicationBuildPipeline.java | 347 +++++++++++++++++++
.../pipeline/impl/SparkYarnBuildPipeline.java | 149 ++++++++
.../streampark/flink/packer/PackerResourceGC.scala | 72 ----
.../flink/packer/docker/DockerConf.scala | 57 ---
.../flink/packer/docker/DockerRetriever.scala | 70 ----
.../packer/docker/FlinkDockerfileTemplate.scala | 48 ---
.../docker/FlinkDockerfileTemplateTrait.scala | 121 -------
.../docker/FlinkHadoopDockerfileTemplate.scala | 128 -------
.../packer/docker/SparkDockerfileTemplate.scala | 48 ---
.../docker/SparkDockerfileTemplateTrait.scala | 121 -------
.../docker/SparkHadoopDockerfileTemplate.scala | 128 -------
.../streampark/flink/packer/docker/package.scala | 55 ---
.../streampark/flink/packer/maven/Artifact.scala | 60 ----
.../flink/packer/maven/DependencyInfo.scala | 55 ---
.../streampark/flink/packer/maven/MavenTool.scala | 268 --------------
.../flink/packer/pipeline/BuildPipeline.scala | 202 -----------
.../flink/packer/pipeline/BuildRequest.scala | 149 --------
.../flink/packer/pipeline/BuildResponse.scala | 84 -----
.../packer/pipeline/DockerResolveProgress.scala | 143 --------
.../packer/pipeline/DockerResolveSnapshot.scala | 74 ----
.../flink/packer/pipeline/PipelineError.scala | 53 ---
.../flink/packer/pipeline/PipelineSnapshot.scala | 65 ----
.../impl/FlinkK8sApplicationBuildPipeline.scala | 271 ---------------
.../impl/FlinkK8sSessionBuildPipeline.scala | 66 ----
.../pipeline/impl/FlinkRemoteBuildPipeline.scala | 99 ------
.../impl/FlinkYarnApplicationBuildPipeline.scala | 121 -------
.../impl/SparkK8sApplicationBuildPipeline.scala | 249 -------------
.../pipeline/impl/SparkYarnBuildPipeline.scala | 124 -------
.../streampark/flink/packer/DockerClientTest.java | 113 ++++++
.../flink/packer/FlinkDockerfileTemplateTest.java | 98 ++++++
.../streampark/flink/packer/MavenArtifactTest.java | 52 +++
.../streampark/flink/packer/MavenToolTest.java | 177 ++++++++++
.../streampark/flink/packer/PackerTestUtils.java | 82 +++++
.../streampark/flink/packer/DockerClientTest.scala | 61 ----
.../flink/packer/FlinkDockerfileTemplateSpec.scala | 77 -----
.../flink/packer/MavenArtifactSpec.scala | 57 ---
.../streampark/flink/packer/MavenToolSpec.scala | 140 --------
91 files changed, 5535 insertions(+), 3499 deletions(-)
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/trait/KubernetesNativeClientTrait.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/trait/KubernetesNativeClientTrait.scala
index 28d106ec2..c77ed3af3 100644
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/trait/KubernetesNativeClientTrait.scala
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/trait/KubernetesNativeClientTrait.scala
@@ -33,6 +33,8 @@ import org.apache.flink.kubernetes.configuration.KubernetesConfigOptions.Service
import javax.annotation.Nonnull
+import scala.collection.JavaConverters.mapAsScalaMapConverter
+
/** kubernetes native mode submit */
trait KubernetesNativeClientTrait extends FlinkClientTrait {
@@ -48,7 +50,7 @@ trait KubernetesNativeClientTrait extends FlinkClientTrait {
if (submitRequest.buildResult != null && submitRequest.deployMode == FlinkDeployMode.KUBERNETES_NATIVE_APPLICATION) {
val buildResult =
submitRequest.buildResult.asInstanceOf[DockerImageBuildResponse]
- buildResult.podTemplatePaths.foreach(p => {
+ buildResult.podTemplatePaths.asScala.foreach(p => {
if (PodTemplateTool.KUBERNETES_POD_TEMPLATE.key.equals(p._1)) {
flinkConfig.safeSet(KubernetesConfigOptions.KUBERNETES_POD_TEMPLATE, p._2)
} else if (PodTemplateTool.KUBERNETES_JM_POD_TEMPLATE.key.equals(p._1)) {
diff --git a/streampark-flink/streampark-flink-packer/pom.xml b/streampark-flink/streampark-flink-packer/pom.xml
index 163d5f735..bb3adf636 100644
--- a/streampark-flink/streampark-flink-packer/pom.xml
+++ b/streampark-flink/streampark-flink-packer/pom.xml
@@ -30,9 +30,9 @@
<properties>
<eclipse.aether.version>1.1.0</eclipse.aether.version>
<maven.aether.version>3.3.9</maven.aether.version>
- <maven.shade.version>3.2.4</maven.shade.version>
+ <maven.shade.version>${maven-shade-plugin.version}</maven.shade.version>
<docker.client.version>3.3.6</docker.client.version>
- <scalatest.skiptests>true</scalatest.skiptests>
+ <httpclient.version>4.5.14</httpclient.version>
</properties>
<dependencies>
@@ -140,10 +140,31 @@
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
- <version>2.12.0</version>
+ <version>${jackson.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>${httpclient.version}</version>
+ <scope>test</scope>
</dependency>
</dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <systemPropertyVariables>
+ <streampark.workspace.local>${project.build.directory}/streampark-workspace</streampark.workspace.local>
+ </systemPropertyVariables>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
<profiles>
<profile>
<id>apache-release</id>
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/PackerResourceGC.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/PackerResourceGC.java
new file mode 100644
index 000000000..4bd4c1b4a
--- /dev/null
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/PackerResourceGC.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.packer;
+
+import org.apache.streampark.common.conf.Workspace;
+import org.apache.streampark.common.constants.Constants;
+import org.apache.streampark.common.util.LoggerSupport;
+
+import org.apache.commons.io.FileUtils;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+/** Garbage resource collector during packing. */
+public final class PackerResourceGC extends LoggerSupport {
+
+ private static final PackerResourceGC INSTANCE = new PackerResourceGC();
+
+ private static final String APP_WORKSPACE_PATH = Workspace.local().APP_WORKSPACE();
+
+ private PackerResourceGC() {
+ }
+
+ /** Start a building legacy resources collection process. */
+ public static void startGc(Integer expiredHours) {
+ INSTANCE.doStartGc(expiredHours);
+ }
+
+ private void doStartGc(Integer expiredHours) {
+ File appWorkspace = new File(APP_WORKSPACE_PATH);
+ if (!appWorkspace.exists()) {
+ return;
+ }
+ long evictedBarrier = System.currentTimeMillis() - expiredHours * 3600L * 1000L;
+
+ File[] dirs = appWorkspace.listFiles(File::isDirectory);
+ if (dirs == null) {
+ return;
+ }
+ List<File> evictedFiles = new ArrayList<>();
+ for (File dir : dirs) {
+ if (!dir.getName().contains("@")) {
+ continue;
+ }
+ for (FileWithTime entry : findLastModifiedOfSubFile(dir)) {
+ if (entry.lastModified < evictedBarrier) {
+ evictedFiles.add(entry.file);
+ }
+ }
+ }
+
+ if (evictedFiles.isEmpty()) {
+ return;
+ }
+ StringBuilder sb = new StringBuilder();
+ for (File path : evictedFiles) {
+ if (sb.length() > 0) {
+ sb.append(", ");
+ }
+ sb.append(path.getAbsolutePath());
+ }
+ logInfo("Delete expired building resources, " + sb);
+ for (File path : evictedFiles) {
+ try {
+ FileUtils.deleteDirectory(path);
+ } catch (Exception ignored) {
+ // ignore delete failures
+ }
+ }
+ }
+
+ private static List<FileWithTime> findLastModifiedOfSubFile(File file) {
+ File[] children = file.listFiles();
+ if (children == null) {
+ return new ArrayList<>();
+ }
+ boolean isApplicationMode =
+ Arrays.stream(children).anyMatch(f -> f.getName().contains(Constants.JAR_SUFFIX));
+ List<FileWithTime> result = new ArrayList<>();
+ if (isApplicationMode) {
+ long max = Arrays.stream(children).mapToLong(File::lastModified).max().orElse(0L);
+ result.add(new FileWithTime(file, max));
+ } else {
+ for (File subFile : children) {
+ if (!subFile.isDirectory()) {
+ continue;
+ }
+ File[] subChildren = subFile.listFiles();
+ if (subChildren == null) {
+ continue;
+ }
+ long max = Arrays.stream(subChildren).mapToLong(File::lastModified).max().orElse(0L);
+ result.add(new FileWithTime(subFile, max));
+ }
+ }
+ return result;
+ }
+
+ private static final class FileWithTime {
+
+ private final File file;
+ private final long lastModified;
+
+ private FileWithTime(File file, long lastModified) {
+ this.file = file;
+ this.lastModified = lastModified;
+ }
+ }
+}
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/DockerConf.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/DockerConf.java
new file mode 100644
index 000000000..01ef8a9e5
--- /dev/null
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/DockerConf.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.packer.docker;
+
+import com.github.dockerjava.api.model.AuthConfig;
+
+import javax.annotation.Nullable;
+
+/** Authentication Configuration of Remote Docker Register */
+public final class DockerConf {
+
+ @Nullable
+ private final String registerAddress;
+ private final String imageNamespace;
+ private final String registerUsername;
+ private final String registerPassword;
+
+ public DockerConf(
+ @Nullable String registerAddress,
+ String imageNamespace,
+ String registerUsername,
+ String registerPassword) {
+ this.registerAddress = registerAddress;
+ this.imageNamespace = imageNamespace;
+ this.registerUsername = registerUsername;
+ this.registerPassword = registerPassword;
+ }
+
+ @Nullable
+ public String registerAddress() {
+ return registerAddress;
+ }
+
+ public String imageNamespace() {
+ return imageNamespace;
+ }
+
+ public String registerUsername() {
+ return registerUsername;
+ }
+
+ public String registerPassword() {
+ return registerPassword;
+ }
+
+ /** covert to com.github.docker.java.api.model.AuthConfig */
+ public AuthConfig toAuthConf() {
+ return new AuthConfig()
+ .withRegistryAddress(registerAddress)
+ .withUsername(registerUsername)
+ .withPassword(registerPassword);
+ }
+
+ public static DockerConf of(
+ @Nullable String registerAddress,
+ String imageNameSpace,
+ String registerUsername,
+ String registerPassword) {
+ return new DockerConf(registerAddress, imageNameSpace, registerUsername, registerPassword);
+ }
+}
diff --git a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/DockerImageExist.scala b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/DockerImageExist.java
similarity index 52%
copy from streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/DockerImageExist.scala
copy to streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/DockerImageExist.java
index 11c26e1d0..fb3f0dedd 100644
--- a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/DockerImageExist.scala
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/DockerImageExist.java
@@ -15,20 +15,26 @@
* limitations under the License.
*/
-package org.apache.streampark.flink.packer.docker
+package org.apache.streampark.flink.packer.docker;
-import com.github.dockerjava.api.exception.NotFoundException
+import com.github.dockerjava.api.exception.NotFoundException;
-class DockerImageExist {
- def doesDockerImageExist(imageName: String): Boolean = {
- usingDockerClient {
- dockerClient =>
- try {
- dockerClient.inspectImageCmd(imageName).exec()
- true
- } catch {
- case _: NotFoundException => false
- }
- }(err => throw new Exception(s"Check docker image failed, imageName=$imageName", err))
- }
+/** Check docker image existence. */
+public class DockerImageExist {
+
+ public boolean doesDockerImageExist(String imageName) {
+ return DockerUtils.usingDockerClient(
+ dockerClient -> {
+ try {
+ dockerClient.inspectImageCmd(imageName).exec();
+ return true;
+ } catch (NotFoundException e) {
+ return false;
+ }
+ },
+ err -> {
+ throw new RuntimeException(
+ "Check docker image failed, imageName=" + imageName, err);
+ });
+ }
}
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/DockerRetriever.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/DockerRetriever.java
new file mode 100644
index 000000000..ce1875895
--- /dev/null
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/DockerRetriever.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.packer.docker;
+
+import org.apache.streampark.common.conf.CommonConfig;
+import org.apache.streampark.common.conf.InternalConfigHolder;
+import org.apache.streampark.common.util.Utils;
+
+import com.github.dockerjava.api.DockerClient;
+import com.github.dockerjava.core.DefaultDockerClientConfig;
+import com.github.dockerjava.core.DockerClientConfig;
+import com.github.dockerjava.core.HackDockerClient;
+import com.github.dockerjava.httpclient5.ApacheDockerHttpClient;
+
+import java.net.URI;
+import java.time.Duration;
+
+/** Docker client factory. */
+public final class DockerRetriever {
+
+ public static final DockerClientConfig dockerClientConf =
+ DefaultDockerClientConfig.createDefaultConfigBuilder().build();
+
+ private static final ApacheDockerHttpClient.Builder dockerHttpClientBuilder =
+ new ApacheDockerHttpClient.Builder()
+ .dockerHost(dockerClientConf.getDockerHost())
+ .sslConfig(dockerClientConf.getSSLConfig())
+ .maxConnections(InternalConfigHolder.get(CommonConfig.DOCKER_MAX_CONNECTIONS()))
+ .connectionTimeout(
+ Duration.ofSeconds(
+ InternalConfigHolder.get(CommonConfig.DOCKER_CONNECTION_TIMEOUT_SEC())))
+ .responseTimeout(
+ Duration.ofSeconds(
+ InternalConfigHolder.get(CommonConfig.DOCKER_RESPONSE_TIMEOUT_SEC())));
+
+ private DockerRetriever() {
+ }
+
+ /** get new DockerClient instance */
+ public static DockerClient newDockerClient() {
+ setDockerHost();
+ return HackDockerClient.getInstance(dockerClientConf, dockerHttpClientBuilder.build());
+ }
+
+ /** set docker-host for kata */
+ private static void setDockerHost() {
+ String dockerHost = InternalConfigHolder.get(CommonConfig.DOCKER_HOST());
+ if (Utils.isNotEmpty(dockerHost)) {
+ URI dockerHostUri = URI.create(dockerHost);
+ dockerHttpClientBuilder.dockerHost(dockerHostUri);
+ }
+ }
+}
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/DockerUtils.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/DockerUtils.java
new file mode 100644
index 000000000..8d91fad94
--- /dev/null
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/DockerUtils.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.packer.docker;
+
+import com.github.dockerjava.api.DockerClient;
+import com.github.dockerjava.api.listener.BuildImageCallbackListener;
+import com.github.dockerjava.api.listener.PullImageCallbackListener;
+import com.github.dockerjava.api.listener.PushImageCallbackListener;
+import com.github.dockerjava.api.model.PullResponseItem;
+import com.github.dockerjava.api.model.PushResponseItem;
+
+import java.util.function.Consumer;
+import java.util.function.Function;
+
+/** Docker helper utilities. */
+public final class DockerUtils {
+
+ private DockerUtils() {
+ }
+
+ public static BuildImageCallbackListener watchDockerBuildStep(Consumer<String> func) {
+ return new BuildImageCallbackListener() {
+
+ @Override
+ public void watchBuildStep(String buildStepMsg) {
+ func.accept(buildStepMsg);
+ }
+ };
+ }
+
+ public static PullImageCallbackListener watchDockerPullProcess(Consumer<PullResponseItem> func) {
+ return new PullImageCallbackListener() {
+
+ @Override
+ public void watchPullProcess(PullResponseItem processDetail) {
+ func.accept(processDetail);
+ }
+ };
+ }
+
+ public static PushImageCallbackListener watchDockerPushProcess(Consumer<PushResponseItem> func) {
+ return new PushImageCallbackListener() {
+
+ @Override
+ public void watchPushProcess(PushResponseItem processDetail) {
+ func.accept(processDetail);
+ }
+ };
+ }
+
+ public static <R> R usingDockerClient(Function<DockerClient, R> process, Function<Throwable, R> handleException) {
+ try {
+ DockerClient client = DockerRetriever.newDockerClient();
+ try {
+ return process.apply(client);
+ } finally {
+ client.close();
+ }
+ } catch (Throwable e) {
+ return handleException.apply(e);
+ }
+ }
+}
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/FlinkDockerfileTemplate.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/FlinkDockerfileTemplate.java
new file mode 100644
index 000000000..16b890652
--- /dev/null
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/FlinkDockerfileTemplate.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.packer.docker;
+
+import java.util.Set;
+
+/** Base flink docker file image template. */
+public class FlinkDockerfileTemplate extends FlinkDockerfileTemplateTrait {
+
+ private final String workspacePath;
+ private final String flinkBaseImage;
+ private final String flinkMainJarPath;
+ private final Set<String> flinkExtraLibPaths;
+
+ public FlinkDockerfileTemplate(
+ String workspacePath,
+ String flinkBaseImage,
+ String flinkMainJarPath,
+ Set<String> flinkExtraLibPaths) {
+ this.workspacePath = workspacePath;
+ this.flinkBaseImage = flinkBaseImage;
+ this.flinkMainJarPath = flinkMainJarPath;
+ this.flinkExtraLibPaths = flinkExtraLibPaths;
+ }
+
+ @Override
+ public String workspacePath() {
+ return workspacePath;
+ }
+
+ @Override
+ public String flinkBaseImage() {
+ return flinkBaseImage;
+ }
+
+ @Override
+ public String flinkMainJarPath() {
+ return flinkMainJarPath;
+ }
+
+ @Override
+ public Set<String> flinkExtraLibPaths() {
+ return flinkExtraLibPaths;
+ }
+
+ @Override
+ public String offerDockerfileContent() {
+ return "FROM " + flinkBaseImage + "\n"
+ + "RUN mkdir -p " + FLINK_HOME + "/usrlib\n"
+ + "COPY " + extraLibName() + " " + FLINK_HOME + "/lib/\n"
+ + "COPY " + mainJarName() + " " + FLINK_HOME + "/usrlib/" + mainJarName() + "\n";
+ }
+}
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/FlinkDockerfileTemplateTrait.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/FlinkDockerfileTemplateTrait.java
new file mode 100644
index 000000000..a10509b15
--- /dev/null
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/FlinkDockerfileTemplateTrait.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.packer.docker;
+
+import org.apache.streampark.common.constants.Constants;
+import org.apache.streampark.common.fs.LfsOperator;
+
+import org.apache.commons.io.FileUtils;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Set;
+
+/** Flink image dockerfile template. */
+public abstract class FlinkDockerfileTemplateTrait {
+
+ protected static final String DEFAULT_DOCKER_FILE_NAME = "Dockerfile";
+ protected static final String FLINK_LIB_PATH = "lib";
+ protected static final String FLINK_HOME = "$FLINK_HOME";
+
+ private Path workspace;
+ private String mainJarName;
+ private String extraLibName;
+
+ public abstract String workspacePath();
+
+ public abstract String flinkBaseImage();
+
+ public abstract String flinkMainJarPath();
+
+ public abstract Set<String> flinkExtraLibPaths();
+
+ public abstract String offerDockerfileContent();
+
+ /** Startup flink main jar path inner Docker */
+ public String innerMainJarPath() {
+ return "local:///opt/flink/usrlib/" + mainJarName();
+ }
+
+ protected Path workspace() {
+ if (workspace == null) {
+ Path path = Paths.get(workspacePath()).toAbsolutePath();
+ if (!LfsOperator.exists(workspacePath())) {
+ LfsOperator.mkdirs(workspacePath());
+ }
+ workspace = path;
+ }
+ return workspace;
+ }
+
+ public String mainJarName() {
+ if (mainJarName == null) {
+ Path mainJarPath = Paths.get(flinkMainJarPath()).toAbsolutePath();
+ if (!mainJarPath.getParent().equals(workspace())) {
+ LfsOperator.copy(
+ mainJarPath.toString(),
+ workspace().toString() + "/" + mainJarPath.getFileName().toString());
+ }
+ mainJarName = mainJarPath.getFileName().toString();
+ }
+ return mainJarName;
+ }
+
+ public String extraLibName() {
+ if (extraLibName == null) {
+ LfsOperator.mkCleanDirs(workspace().toString() + "/" + FLINK_LIB_PATH);
+ for (String libPath : flinkExtraLibPaths()) {
+ File f = new File(libPath);
+ if (!f.exists() || !f.getName().endsWith(Constants.JAR_SUFFIX)) {
+ continue;
+ }
+ if (f.isDirectory()) {
+ File[] files = f.listFiles();
+ if (files != null) {
+ for (File file : files) {
+ if (file.isFile() && file.getName().endsWith(Constants.JAR_SUFFIX)) {
+ LfsOperator.copy(file.getAbsolutePath(), workspace().toString() + "/" + FLINK_LIB_PATH);
+ }
+ }
+ }
+ } else if (f.isFile()) {
+ LfsOperator.copy(f.getAbsolutePath(), workspace().toString() + "/" + FLINK_LIB_PATH);
+ }
+ }
+ extraLibName = FLINK_LIB_PATH;
+ }
+ return extraLibName;
+ }
+
+ public File writeDockerfile() throws IOException {
+ File output = new File(workspacePath() + "/" + DEFAULT_DOCKER_FILE_NAME);
+ FileUtils.write(output, offerDockerfileContent(), "UTF-8");
+ return output;
+ }
+
+ public File writeDockerfile(String dockerfileName) throws IOException {
+ File output = new File(workspacePath() + "/" + dockerfileName);
+ FileUtils.write(output, offerDockerfileContent(), "UTF-8");
+ return output;
+ }
+}
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/FlinkHadoopDockerfileTemplate.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/FlinkHadoopDockerfileTemplate.java
new file mode 100644
index 000000000..97f15d258
--- /dev/null
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/FlinkHadoopDockerfileTemplate.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.packer.docker;
+
+import org.apache.streampark.common.fs.LfsOperator;
+import org.apache.streampark.common.util.HadoopConfigUtils;
+
+import javax.annotation.Nullable;
+
+import java.nio.file.Paths;
+import java.util.Optional;
+import java.util.Set;
+
+/** flink-hadoop integration docker image template. */
+public class FlinkHadoopDockerfileTemplate extends FlinkDockerfileTemplateTrait {
+
+ private final String workspacePath;
+ private final String flinkBaseImage;
+ private final String flinkMainJarPath;
+ private final Set<String> flinkExtraLibPaths;
+ @Nullable
+ private final String hadoopConfDirPath;
+ @Nullable
+ private final String hiveConfDirPath;
+ private final String hadoopConfDir;
+ private final String hiveConfDir;
+
+ public FlinkHadoopDockerfileTemplate(
+ String workspacePath,
+ String flinkBaseImage,
+ String flinkMainJarPath,
+ Set<String> flinkExtraLibPaths,
+ @Nullable String hadoopConfDirPath,
+ @Nullable String hiveConfDirPath) {
+ this.workspacePath = workspacePath;
+ this.flinkBaseImage = flinkBaseImage;
+ this.flinkMainJarPath = flinkMainJarPath;
+ this.flinkExtraLibPaths = flinkExtraLibPaths;
+ this.hadoopConfDirPath = hadoopConfDirPath;
+ this.hiveConfDirPath = hiveConfDirPath;
+ this.hadoopConfDir =
+ workspace()
+ .relativize(Paths.get(hadoopConfDirPath == null ? "" : hadoopConfDirPath))
+ .toString();
+ this.hiveConfDir =
+ workspace()
+ .relativize(Paths.get(hiveConfDirPath == null ? "" : hiveConfDirPath))
+ .toString();
+ }
+
+ @Override
+ public String workspacePath() {
+ return workspacePath;
+ }
+
+ @Override
+ public String flinkBaseImage() {
+ return flinkBaseImage;
+ }
+
+ @Override
+ public String flinkMainJarPath() {
+ return flinkMainJarPath;
+ }
+
+ @Override
+ public Set<String> flinkExtraLibPaths() {
+ return flinkExtraLibPaths;
+ }
+
+ @Override
+ public String offerDockerfileContent() {
+ StringBuilder dockerfile =
+ new StringBuilder(
+ "FROM " + flinkBaseImage + "\n"
+ + "RUN mkdir -p " + FLINK_HOME + "/usrlib\n");
+ if (!hadoopConfDir.isEmpty()) {
+ dockerfile
+ .append("COPY ")
+ .append(hadoopConfDir)
+ .append(" /opt/hadoop-conf\n")
+ .append("ENV HADOOP_CONF_DIR /opt/hadoop-conf\n");
+ }
+ if (!hiveConfDir.isEmpty()) {
+ dockerfile
+ .append("COPY ")
+ .append(hiveConfDir)
+ .append(" /opt/hive-conf\n")
+ .append("ENV HIVE_CONF_DIR /opt/hive-conf\n");
+ }
+ dockerfile
+ .append("COPY ")
+ .append(extraLibName())
+ .append(" ")
+ .append(FLINK_HOME)
+ .append("/lib/\n")
+ .append("COPY ")
+ .append(mainJarName())
+ .append(" ")
+ .append(FLINK_HOME)
+ .append("/usrlib/")
+ .append(mainJarName())
+ .append("\n");
+ return dockerfile.toString();
+ }
+
+ /** Use relevant system variables as the value of hadoopConfDirPath, hiveConfDirPath. */
+ public static FlinkHadoopDockerfileTemplate fromSystemHadoopConf(
+ String workspacePath,
+ String flinkBaseImage,
+ String flinkMainJarPath,
+ Set<String> flinkExtraLibPaths) {
+ String hadoopConfDir = "";
+ Optional<String> systemHadoopConf = HadoopConfigUtils.getSystemHadoopConfDirAsJava();
+ if (systemHadoopConf.isPresent() && LfsOperator.exists(systemHadoopConf.get())) {
+ hadoopConfDir = workspacePath + "/hadoop-conf";
+ LfsOperator.mkCleanDirs(hadoopConfDir);
+ LfsOperator.copyDir(systemHadoopConf.get(), hadoopConfDir);
+ }
+ String hiveConfDir = "";
+ Optional<String> systemHiveConf = HadoopConfigUtils.getSystemHiveConfDirAsJava();
+ if (systemHiveConf.isPresent() && LfsOperator.exists(systemHiveConf.get())) {
+ hiveConfDir = workspacePath + "/hive-conf";
+ LfsOperator.mkCleanDirs(hiveConfDir);
+ LfsOperator.copyDir(systemHiveConf.get(), hiveConfDir);
+ }
+ return new FlinkHadoopDockerfileTemplate(
+ workspacePath,
+ flinkBaseImage,
+ flinkMainJarPath,
+ flinkExtraLibPaths,
+ hadoopConfDir,
+ hiveConfDir);
+ }
+}
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/SparkDockerfileTemplate.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/SparkDockerfileTemplate.java
new file mode 100644
index 000000000..f60f283b6
--- /dev/null
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/SparkDockerfileTemplate.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.packer.docker;
+
+import java.util.Set;
+
+/** Base spark docker file image template. */
+public class SparkDockerfileTemplate extends SparkDockerfileTemplateTrait {
+
+ private final String workspacePath;
+ private final String sparkBaseImage;
+ private final String sparkMainJarPath;
+ private final Set<String> sparkExtraLibPaths;
+
+ public SparkDockerfileTemplate(
+ String workspacePath,
+ String sparkBaseImage,
+ String sparkMainJarPath,
+ Set<String> sparkExtraLibPaths) {
+ this.workspacePath = workspacePath;
+ this.sparkBaseImage = sparkBaseImage;
+ this.sparkMainJarPath = sparkMainJarPath;
+ this.sparkExtraLibPaths = sparkExtraLibPaths;
+ }
+
+ @Override
+ public String workspacePath() {
+ return workspacePath;
+ }
+
+ @Override
+ public String sparkBaseImage() {
+ return sparkBaseImage;
+ }
+
+ @Override
+ public String sparkMainJarPath() {
+ return sparkMainJarPath;
+ }
+
+ @Override
+ public Set<String> sparkExtraLibPaths() {
+ return sparkExtraLibPaths;
+ }
+
+ @Override
+ public String offerDockerfileContent() {
+ return "FROM " + sparkBaseImage + "\n"
+ + "USER root\n"
+ + "RUN mkdir -p " + SPARK_HOME + "/usrlib\n"
+ + "COPY " + mainJarName() + " " + SPARK_HOME + "/usrlib/" + mainJarName() + "\n"
+ + "COPY " + extraLibName() + " " + SPARK_HOME + "/lib/\n";
+ }
+}
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/SparkDockerfileTemplateTrait.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/SparkDockerfileTemplateTrait.java
new file mode 100644
index 000000000..f311d4e41
--- /dev/null
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/SparkDockerfileTemplateTrait.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.packer.docker;
+
+import org.apache.streampark.common.constants.Constants;
+import org.apache.streampark.common.fs.LfsOperator;
+
+import org.apache.commons.io.FileUtils;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Set;
+
+/** Spark image dockerfile template. */
+public abstract class SparkDockerfileTemplateTrait {
+
+ protected static final String DEFAULT_DOCKER_FILE_NAME = "Dockerfile";
+ protected static final String SPARK_LIB_PATH = "lib";
+ protected static final String SPARK_HOME = "$SPARK_HOME";
+
+ private Path workspace;
+ private String mainJarName;
+ private String extraLibName;
+
+ public abstract String workspacePath();
+
+ public abstract String sparkBaseImage();
+
+ public abstract String sparkMainJarPath();
+
+ public abstract Set<String> sparkExtraLibPaths();
+
+ public abstract String offerDockerfileContent();
+
+ public String innerMainJarPath() {
+ return "local:///opt/spark/usrlib/" + mainJarName();
+ }
+
+ protected Path workspace() {
+ if (workspace == null) {
+ Path path = Paths.get(workspacePath()).toAbsolutePath();
+ if (!LfsOperator.exists(workspacePath())) {
+ LfsOperator.mkdirs(workspacePath());
+ }
+ workspace = path;
+ }
+ return workspace;
+ }
+
+ public String mainJarName() {
+ if (mainJarName == null) {
+ Path mainJarPath = Paths.get(sparkMainJarPath()).toAbsolutePath();
+ if (!mainJarPath.getParent().equals(workspace())) {
+ LfsOperator.copy(
+ mainJarPath.toString(),
+ workspace().toString() + "/" + mainJarPath.getFileName().toString());
+ }
+ mainJarName = mainJarPath.getFileName().toString();
+ }
+ return mainJarName;
+ }
+
+ public String extraLibName() {
+ if (extraLibName == null) {
+ LfsOperator.mkCleanDirs(workspace().toString() + "/" + SPARK_LIB_PATH);
+ for (String libPath : sparkExtraLibPaths()) {
+ File f = new File(libPath);
+ if (!f.exists() || !f.getName().endsWith(Constants.JAR_SUFFIX)) {
+ continue;
+ }
+ if (f.isDirectory()) {
+ File[] files = f.listFiles();
+ if (files != null) {
+ for (File file : files) {
+ if (file.isFile() && file.getName().endsWith(Constants.JAR_SUFFIX)) {
+ LfsOperator.copy(file.getAbsolutePath(), workspace().toString() + "/" + SPARK_LIB_PATH);
+ }
+ }
+ }
+ } else if (f.isFile()) {
+ LfsOperator.copy(f.getAbsolutePath(), workspace().toString() + "/" + SPARK_LIB_PATH);
+ }
+ }
+ extraLibName = SPARK_LIB_PATH;
+ }
+ return extraLibName;
+ }
+
+ public File writeDockerfile() throws IOException {
+ File output = new File(workspacePath() + "/" + DEFAULT_DOCKER_FILE_NAME);
+ FileUtils.write(output, offerDockerfileContent(), "UTF-8");
+ return output;
+ }
+
+ public File writeDockerfile(String dockerfileName) throws IOException {
+ File output = new File(workspacePath() + "/" + dockerfileName);
+ FileUtils.write(output, offerDockerfileContent(), "UTF-8");
+ return output;
+ }
+}
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/SparkHadoopDockerfileTemplate.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/SparkHadoopDockerfileTemplate.java
new file mode 100644
index 000000000..af166db82
--- /dev/null
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/SparkHadoopDockerfileTemplate.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.packer.docker;
+
+import org.apache.streampark.common.fs.LfsOperator;
+import org.apache.streampark.common.util.HadoopConfigUtils;
+
+import javax.annotation.Nullable;
+
+import java.nio.file.Paths;
+import java.util.Optional;
+import java.util.Set;
+
+/** spark-hadoop integration docker image template. */
+public class SparkHadoopDockerfileTemplate extends SparkDockerfileTemplateTrait {
+
+ private final String workspacePath;
+ private final String sparkBaseImage;
+ private final String sparkMainJarPath;
+ private final Set<String> sparkExtraLibPaths;
+ private final String hadoopConfDir;
+ private final String hiveConfDir;
+
+ public SparkHadoopDockerfileTemplate(
+ String workspacePath,
+ String sparkBaseImage,
+ String sparkMainJarPath,
+ Set<String> sparkExtraLibPaths,
+ @Nullable String hadoopConfDirPath,
+ @Nullable String hiveConfDirPath) {
+ this.workspacePath = workspacePath;
+ this.sparkBaseImage = sparkBaseImage;
+ this.sparkMainJarPath = sparkMainJarPath;
+ this.sparkExtraLibPaths = sparkExtraLibPaths;
+ this.hadoopConfDir =
+ workspace()
+ .relativize(Paths.get(hadoopConfDirPath == null ? "" : hadoopConfDirPath))
+ .toString();
+ this.hiveConfDir =
+ workspace()
+ .relativize(Paths.get(hiveConfDirPath == null ? "" : hiveConfDirPath))
+ .toString();
+ }
+
+ @Override
+ public String workspacePath() {
+ return workspacePath;
+ }
+
+ @Override
+ public String sparkBaseImage() {
+ return sparkBaseImage;
+ }
+
+ @Override
+ public String sparkMainJarPath() {
+ return sparkMainJarPath;
+ }
+
+ @Override
+ public Set<String> sparkExtraLibPaths() {
+ return sparkExtraLibPaths;
+ }
+
+ @Override
+ public String offerDockerfileContent() {
+ StringBuilder dockerfile =
+ new StringBuilder(
+ "FROM " + sparkBaseImage + "\n"
+ + "RUN mkdir -p " + SPARK_HOME + "/usrlib\n");
+ if (!hadoopConfDir.isEmpty()) {
+ dockerfile
+ .append("COPY ")
+ .append(hadoopConfDir)
+ .append(" /opt/hadoop-conf\n")
+ .append("ENV HADOOP_CONF_DIR /opt/hadoop-conf\n");
+ }
+ if (!hiveConfDir.isEmpty()) {
+ dockerfile
+ .append("COPY ")
+ .append(hiveConfDir)
+ .append(" /opt/hive-conf\n")
+ .append("ENV HIVE_CONF_DIR /opt/hive-conf\n");
+ }
+ dockerfile
+ .append("COPY ")
+ .append(extraLibName())
+ .append(" ")
+ .append(SPARK_HOME)
+ .append("/lib/\n")
+ .append("COPY ")
+ .append(mainJarName())
+ .append(" ")
+ .append(SPARK_HOME)
+ .append("/usrlib/")
+ .append(mainJarName())
+ .append("\n");
+ return dockerfile.toString();
+ }
+
+ public static SparkHadoopDockerfileTemplate fromSystemHadoopConf(
+ String workspacePath,
+ String sparkBaseImage,
+ String sparkMainJarPath,
+ Set<String> sparkExtraLibPaths) {
+ String hadoopConfDir = "";
+ Optional<String> systemHadoopConf = HadoopConfigUtils.getSystemHadoopConfDirAsJava();
+ if (systemHadoopConf.isPresent() && LfsOperator.exists(systemHadoopConf.get())) {
+ hadoopConfDir = workspacePath + "/hadoop-conf";
+ LfsOperator.mkCleanDirs(hadoopConfDir);
+ LfsOperator.copyDir(systemHadoopConf.get(), hadoopConfDir);
+ }
+ String hiveConfDir = "";
+ Optional<String> systemHiveConf = HadoopConfigUtils.getSystemHiveConfDirAsJava();
+ if (systemHiveConf.isPresent() && LfsOperator.exists(systemHiveConf.get())) {
+ hiveConfDir = workspacePath + "/hive-conf";
+ LfsOperator.mkCleanDirs(hiveConfDir);
+ LfsOperator.copyDir(systemHiveConf.get(), hiveConfDir);
+ }
+ return new SparkHadoopDockerfileTemplate(
+ workspacePath,
+ sparkBaseImage,
+ sparkMainJarPath,
+ sparkExtraLibPaths,
+ hadoopConfDir,
+ hiveConfDir);
+ }
+}
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/maven/Artifact.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/maven/Artifact.java
new file mode 100644
index 000000000..3cfd8a951
--- /dev/null
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/maven/Artifact.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.packer.maven;
+
+import javax.annotation.Nullable;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/** Maven artifact coordinates. */
+public final class Artifact {
+
+ private static final Pattern PATTERN = Pattern.compile("([^: ]+):([^: ]+):([^: ]+)");
+
+ private final String groupId;
+ private final String artifactId;
+ private final String version;
+ @Nullable
+ private final String classifier;
+
+ public Artifact(String groupId, String artifactId, String version) {
+ this(groupId, artifactId, version, null);
+ }
+
+ public Artifact(String groupId, String artifactId, String version, @Nullable String classifier) {
+ this.groupId = groupId;
+ this.artifactId = artifactId;
+ this.version = version;
+ this.classifier = classifier;
+ }
+
+ public String groupId() {
+ return groupId;
+ }
+
+ public String artifactId() {
+ return artifactId;
+ }
+
+ public String version() {
+ return version;
+ }
+
+ @Nullable
+ public String classifier() {
+ return classifier;
+ }
+
+ public boolean eq(org.eclipse.aether.artifact.Artifact artifact) {
+ if (!groupId.equals(artifact.getGroupId())) {
+ return false;
+ }
+ String a = artifact.getArtifactId();
+ if ("*".equals(a)) {
+ return true;
+ }
+ return artifactId.equals(a);
+ }
+
+ /** build from coords */
+ public static Artifact of(String coords) {
+ Matcher matcher = PATTERN.matcher(coords);
+ if (matcher.matches()) {
+ return new Artifact(matcher.group(1), matcher.group(2), matcher.group(3));
+ }
+ throw new IllegalArgumentException(
+ "Bad artifact coordinates "
+ + coords
+ + ", expected format is <groupId>:<artifactId>[:<extension>[:<classifier>]]:<version>");
+ }
+}
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/maven/DependencyInfo.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/maven/DependencyInfo.java
new file mode 100644
index 000000000..c6f04eeb9
--- /dev/null
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/maven/DependencyInfo.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.packer.maven;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Maven artifacts and external jar libraries for building a fat-jar.
+ *
+ * @param mavenArts collection of maven artifacts
+ * @param extJarLibs collection of jar lib paths, which elements can be a directory or file path.
+ */
+public final class DependencyInfo {
+
+ private final Set<Artifact> mavenArts;
+ private final Set<String> extJarLibs;
+
+ public DependencyInfo() {
+ this(Collections.emptySet(), Collections.emptySet());
+ }
+
+ public DependencyInfo(Set<Artifact> mavenArts, Set<String> extJarLibs) {
+ this.mavenArts = mavenArts == null ? Collections.emptySet() : new HashSet<>(mavenArts);
+ this.extJarLibs = extJarLibs == null ? Collections.emptySet() : new HashSet<>(extJarLibs);
+ }
+
+ public DependencyInfo(List<Artifact> mavenArts, List<String> extJarLibs) {
+ this(
+ mavenArts == null ? Collections.emptySet() : new HashSet<>(mavenArts),
+ extJarLibs == null ? Collections.emptySet() : new HashSet<>(extJarLibs));
+ }
+
+ public Set<Artifact> mavenArts() {
+ return mavenArts;
+ }
+
+ public Set<String> extJarLibs() {
+ return extJarLibs;
+ }
+
+ public DependencyInfo merge(Set<String> jarLibs) {
+ if (jarLibs != null) {
+ Set<String> merged = new HashSet<>(extJarLibs);
+ merged.addAll(jarLibs);
+ return new DependencyInfo(mavenArts, merged);
+ }
+ return new DependencyInfo(mavenArts, extJarLibs);
+ }
+
+ public DependencyInfo merge(List<Artifact> mvnPoms, List<String> jarLibs) {
+ Set<Artifact> mergedArts = new HashSet<>(mavenArts);
+ Set<String> mergedJars = new HashSet<>(extJarLibs);
+ if (mvnPoms != null) {
+ mergedArts.addAll(mvnPoms);
+ }
+ if (jarLibs != null) {
+ mergedJars.addAll(jarLibs);
+ }
+ return new DependencyInfo(mergedArts, mergedJars);
+ }
+
+ public static DependencyInfo empty() {
+ return new DependencyInfo();
+ }
+}
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/maven/MavenTool.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/maven/MavenTool.java
new file mode 100644
index 000000000..9cd28c745
--- /dev/null
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/maven/MavenTool.java
@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.packer.maven;
+
+import org.apache.streampark.common.conf.CommonConfig;
+import org.apache.streampark.common.conf.InternalConfigHolder;
+import org.apache.streampark.common.conf.Workspace;
+import org.apache.streampark.common.constants.Constants;
+import org.apache.streampark.common.util.AssertUtils;
+import org.apache.streampark.common.util.LoggerSupport;
+import org.apache.streampark.common.util.Utils;
+
+import org.apache.maven.plugins.shade.DefaultShader;
+import org.apache.maven.plugins.shade.ShadeRequest;
+import org.apache.maven.plugins.shade.filter.Filter;
+import org.apache.maven.plugins.shade.resource.ManifestResourceTransformer;
+import org.apache.maven.plugins.shade.resource.ResourceTransformer;
+import org.apache.maven.plugins.shade.resource.ServicesResourceTransformer;
+import org.apache.maven.repository.internal.MavenRepositorySystemUtils;
+
+import com.google.common.collect.Lists;
+import org.codehaus.plexus.logging.console.ConsoleLogger;
+import org.eclipse.aether.RepositorySystem;
+import org.eclipse.aether.RepositorySystemSession;
+import org.eclipse.aether.artifact.DefaultArtifact;
+import org.eclipse.aether.connector.basic.BasicRepositoryConnectorFactory;
+import org.eclipse.aether.repository.LocalRepository;
+import org.eclipse.aether.repository.RemoteRepository;
+import org.eclipse.aether.resolution.ArtifactDescriptorRequest;
+import org.eclipse.aether.resolution.ArtifactRequest;
+import org.eclipse.aether.spi.connector.RepositoryConnectorFactory;
+import org.eclipse.aether.spi.connector.transport.TransporterFactory;
+import org.eclipse.aether.transport.file.FileTransporterFactory;
+import org.eclipse.aether.transport.http.HttpTransporterFactory;
+import org.eclipse.aether.util.repository.AuthenticationBuilder;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/** Maven build utilities. */
+public final class MavenTool extends LoggerSupport {
+
+ private static final MavenTool INSTANCE = new MavenTool();
+
+ private static final org.codehaus.plexus.logging.Logger PLEXUS_LOG =
+ new ConsoleLogger(org.codehaus.plexus.logging.Logger.LEVEL_INFO, "streampark-maven");
+
+ private static final List<Artifact> EXCLUDE_ARTIFACT =
+ Arrays.asList(
+ Artifact.of("org.apache.flink:force-shading:*"),
+ Artifact.of("com.google.code.findbugs:jsr305:*"),
+ Artifact.of("org.apache.logging.log4j:*:*"));
+
+ private MavenTool() {
+ }
+
+ private static List<RemoteRepository> getRemoteRepos() {
+ RemoteRepository.Builder builder =
+ new RemoteRepository.Builder(
+ "central",
+ Constants.DEFAULT,
+ InternalConfigHolder.get(CommonConfig.MAVEN_REMOTE_URL()));
+ RemoteRepository remoteRepository;
+ if (InternalConfigHolder.get(CommonConfig.MAVEN_AUTH_USER()) == null
+ || InternalConfigHolder.get(CommonConfig.MAVEN_AUTH_PASSWORD()) == null) {
+ remoteRepository = builder.build();
+ } else {
+ String username = InternalConfigHolder.get(CommonConfig.MAVEN_AUTH_USER());
+ String password = InternalConfigHolder.get(CommonConfig.MAVEN_AUTH_PASSWORD());
+ org.eclipse.aether.repository.Authentication authentication =
+ new AuthenticationBuilder()
+ .addUsername(username)
+ .addPassword(password)
+ .build();
+ remoteRepository = builder.setAuthentication(authentication).build();
+ }
+ return Collections.singletonList(remoteRepository);
+ }
+
+ private static boolean isJarFile(File file) {
+ if (!file.isFile()) {
+ return false;
+ }
+ try {
+ Utils.requireCheckJarFile(file.toURI().toURL());
+ return true;
+ } catch (Exception e) {
+ return false;
+ }
+ }
+
+ /**
+ * Build a fat-jar with custom jar libraries.
+ *
+ * @param mainClass main class name
+ * @param jarLibs list of jar lib paths for building fat-jar
+ * @param outFatJarPath output paths of fat-jar
+ * @return File Object of output fat-jar
+ */
+ @Nonnull
+ public static File buildFatJar(
+ @Nullable String mainClass,
+ @Nonnull Set<String> jarLibs,
+ @Nonnull String outFatJarPath) throws Exception {
+ return INSTANCE.buildFatJarInternal(mainClass, jarLibs, outFatJarPath);
+ }
+
+ private File buildFatJarInternal(
+ @Nullable String mainClass,
+ @Nonnull Set<String> jarLibs,
+ @Nonnull String outFatJarPath) throws Exception {
+ File uberJar = new File(outFatJarPath);
+ if (!outFatJarPath.endsWith(Constants.JAR_SUFFIX) || uberJar.isDirectory()) {
+ throw new IllegalArgumentException(
+ "[StreamPark] streampark-packer: outFatJarPath("
+ + outFatJarPath
+ + ") should be a JAR file.");
+ }
+ if (uberJar.exists() && !uberJar.delete()) {
+ throw new java.io.IOException(
+ "[StreamPark] streampark-packer: failed to delete existing uber jar: "
+ + outFatJarPath);
+ }
+ Set<File> jarSet = new HashSet<>();
+ for (String lib : jarLibs) {
+ File libFile = new File(lib);
+ if (!libFile.exists()) {
+ continue;
+ }
+ if (isJarFile(libFile)) {
+ jarSet.add(libFile);
+ } else if (libFile.isDirectory()) {
+ File[] files = libFile.listFiles();
+ if (files != null) {
+ for (File f : files) {
+ if (isJarFile(f)) {
+ jarSet.add(f);
+ }
+ }
+ }
+ }
+ }
+ logInfo("start shaded fat-jar: " + String.join(",", jarLibs));
+ ShadeRequest shadeRequest = new ShadeRequest();
+ shadeRequest.setJars(jarSet);
+ shadeRequest.setUberJar(uberJar);
+ List<ResourceTransformer> transformer = new ArrayList<>();
+ transformer.add(new ServicesResourceTransformer());
+ if (mainClass != null) {
+ ManifestResourceTransformer manifest = new ManifestResourceTransformer();
+ manifest.setMainClass(mainClass);
+ transformer.add(manifest);
+ }
+ shadeRequest.setResourceTransformers(transformer);
+ shadeRequest.setFilters(Collections.singletonList(new ShadeFilter()));
+ shadeRequest.setRelocators(Lists.newArrayList());
+ DefaultShader shader = new DefaultShader();
+ shader.enableLogging(PLEXUS_LOG);
+ shader.shade(shadeRequest);
+ logInfo("finish build fat-jar: " + uberJar.getAbsolutePath());
+ return uberJar;
+ }
+
+ /**
+ * Build a fat-jar with custom jar libraries and maven artifacts.
+ *
+ * @param mainClass main class name
+ * @param dependencyInfo maven artifacts and jar libraries for building a fat-jar
+ * @param outFatJarPath output paths of fat-jar
+ */
+ @Nonnull
+ public static File buildFatJar(
+ @Nullable String mainClass,
+ @Nonnull DependencyInfo dependencyInfo,
+ @Nonnull String outFatJarPath) throws Exception {
+ Set<String> jarLibs = dependencyInfo.extJarLibs();
+ Set<Artifact> arts = dependencyInfo.mavenArts();
+ AssertUtils.required(
+ !(jarLibs.isEmpty() && arts.isEmpty()),
+ "[StreamPark] streampark-packer: empty artifacts.");
+ List<String> artFilePaths =
+ resolveArtifacts(arts).stream().map(File::getAbsolutePath).collect(Collectors.toList());
+ Set<String> allJars = new HashSet<>(jarLibs);
+ allJars.addAll(artFilePaths);
+ return INSTANCE.buildFatJarInternal(mainClass, allJars, outFatJarPath);
+ }
+
+ @Nonnull
+ public static List<File> resolveArtifacts(Artifact mavenArtifact) throws Exception {
+ return resolveArtifacts(Collections.singleton(mavenArtifact));
+ }
+
+ /**
+ * Resolve the collection of artifacts. Artifacts will be download to local maven repo if
+ * necessary. Only compile scope dependencies will be resolved.
+ */
+ @Nonnull
+ public static List<File> resolveArtifacts(@Nullable Set<Artifact> mavenArtifacts) throws Exception {
+ if (mavenArtifacts == null) {
+ return Collections.emptyList();
+ }
+ RepositorySystem repoSystem;
+ RepositorySystemSession session;
+ MavenEndpoint endpoint = getMavenEndpoint();
+ repoSystem = endpoint.repoSystem;
+ session = endpoint.session;
+
+ List<org.eclipse.aether.artifact.Artifact> artifacts = new ArrayList<>();
+ for (Artifact e : mavenArtifacts) {
+ org.eclipse.aether.artifact.Artifact artifact =
+ new DefaultArtifact(e.groupId(), e.artifactId(), e.classifier(), "jar", e.version());
+ artifacts.add(artifact);
+ }
+ INSTANCE.logInfo("start resolving dependencies: " + artifacts);
+
+ List<RemoteRepository> remoteRepos = getRemoteRepos();
+ List<org.eclipse.aether.artifact.Artifact> resolvedArtifacts = new ArrayList<>();
+ for (org.eclipse.aether.artifact.Artifact artifact : artifacts) {
+ ArtifactDescriptorRequest artDescReq =
+ new ArtifactDescriptorRequest(artifact, remoteRepos, null);
+ resolvedArtifacts.addAll(
+ repoSystem.readArtifactDescriptor(session, artDescReq).getDependencies().stream()
+ .filter(dep -> "compile".equals(dep.getScope()))
+ .filter(
+ dep -> EXCLUDE_ARTIFACT.stream()
+ .noneMatch(ex -> ex.eq(dep.getArtifact())))
+ .map(dep -> dep.getArtifact())
+ .collect(Collectors.toList()));
+ }
+
+ List<org.eclipse.aether.artifact.Artifact> mergedArtifacts = new ArrayList<>(artifacts);
+ mergedArtifacts.addAll(resolvedArtifacts);
+ INSTANCE.logInfo("resolved dependencies: " + mergedArtifacts);
+
+ List<ArtifactRequest> artReqs =
+ mergedArtifacts.stream()
+ .map(artifact -> new ArtifactRequest(artifact, remoteRepos, null))
+ .collect(Collectors.toList());
+ return repoSystem.resolveArtifacts(session, artReqs).stream()
+ .map(result -> result.getArtifact().getFile())
+ .collect(Collectors.toList());
+ }
+
+ private static MavenEndpoint getMavenEndpoint() {
+ org.eclipse.aether.impl.DefaultServiceLocator locator =
+ MavenRepositorySystemUtils.newServiceLocator();
+ LocalRepository localRepo = new LocalRepository(Workspace.MAVEN_LOCAL_PATH());
+
+ locator.addService(
+ RepositoryConnectorFactory.class, BasicRepositoryConnectorFactory.class);
+ locator.addService(TransporterFactory.class, FileTransporterFactory.class);
+ locator.addService(TransporterFactory.class, HttpTransporterFactory.class);
+ RepositorySystem repoSystem = locator.getService(RepositorySystem.class);
+
+ org.eclipse.aether.DefaultRepositorySystemSession session =
+ MavenRepositorySystemUtils.newSession();
+ session.setLocalRepositoryManager(
+ repoSystem.newLocalRepositoryManager(session, localRepo));
+ return new MavenEndpoint(repoSystem, session);
+ }
+
+ private static final class MavenEndpoint {
+
+ private final RepositorySystem repoSystem;
+ private final RepositorySystemSession session;
+
+ private MavenEndpoint(RepositorySystem repoSystem, RepositorySystemSession session) {
+ this.repoSystem = repoSystem;
+ this.session = session;
+ }
+ }
+
+ private static final class ShadeFilter implements Filter {
+
+ @Override
+ public boolean canFilter(File jar) {
+ return true;
+ }
+
+ @Override
+ public boolean isFiltered(String name) {
+ boolean isFilteredState =
+ name.startsWith("META-INF/") && name.endsWith(".SF")
+ || name.endsWith(".DSA")
+ || name.endsWith(".RSA");
+ if (isFilteredState) {
+ INSTANCE.logInfo("shade ignore file: " + name);
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public void finished() {
+ }
+ }
+}
diff --git a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/DockerImageExist.scala b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/BuildParam.java
similarity index 61%
copy from streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/DockerImageExist.scala
copy to streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/BuildParam.java
index 11c26e1d0..87494052d 100644
--- a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/DockerImageExist.scala
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/BuildParam.java
@@ -15,20 +15,12 @@
* limitations under the License.
*/
-package org.apache.streampark.flink.packer.docker
+package org.apache.streampark.flink.packer.pipeline;
-import com.github.dockerjava.api.exception.NotFoundException
+/** Params of a BuildPipeline instance. */
+public interface BuildParam {
-class DockerImageExist {
- def doesDockerImageExist(imageName: String): Boolean = {
- usingDockerClient {
- dockerClient =>
- try {
- dockerClient.inspectImageCmd(imageName).exec()
- true
- } catch {
- case _: NotFoundException => false
- }
- }(err => throw new Exception(s"Check docker image failed, imageName=$imageName", err))
- }
+ String appName();
+
+ String mainClass();
}
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/BuildPipeline.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/BuildPipeline.java
new file mode 100644
index 000000000..f0bb8a4f7
--- /dev/null
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/BuildPipeline.java
@@ -0,0 +1,267 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.packer.pipeline;
+
+import org.apache.streampark.common.util.LoggerSupport;
+import org.apache.streampark.common.util.ThreadUtils;
+
+import java.util.AbstractMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+/** Building pipeline abstract class. */
+public abstract class BuildPipeline extends LoggerSupport
+ implements
+ BuildPipelineProcess,
+ BuildPipelineExpose {
+
+ private static final ThreadPoolExecutor EXEC_POOL =
+ new ThreadPoolExecutor(
+ Runtime.getRuntime().availableProcessors() * 2,
+ 300,
+ 60L,
+ TimeUnit.SECONDS,
+ new LinkedBlockingQueue<>(2048),
+ ThreadUtils.threadFactory("streampark-pipeline-watcher-executor"),
+ new ThreadPoolExecutor.AbortPolicy());
+
+ protected PipelineStatusEnum pipeStatus = PipelineStatusEnum.pending;
+
+ protected PipeError error = PipeError.empty();
+
+ protected int curStep = 0;
+
+ protected final Map<Integer, Map.Entry<PipelineStepStatusEnum, Long>> stepsStatus =
+ new LinkedHashMap<>();
+
+ private PipeWatcher watcher = new SilentPipeWatcher();
+
+ protected BuildPipeline() {
+ pipeType()
+ .getSteps()
+ .forEach(
+ (seq, desc) -> stepsStatus.put(
+ seq,
+ new AbstractMap.SimpleEntry<>(
+ PipelineStepStatusEnum.waiting, System.currentTimeMillis())));
+ }
+
+ /** use to identify the log record that belongs to which pipeline instance */
+ private String logSuffix() {
+ return "appName=" + offerBuildParam().appName();
+ }
+
+ public BuildPipeline registerWatcher(PipeWatcher watcher) {
+ this.watcher = watcher;
+ return this;
+ }
+
+ private void notifyStart() {
+ try {
+ watcher.onStart(snapshot());
+ } catch (Exception e) {
+ logError("Pipeline watcher onStart callback failed", e);
+ }
+ }
+
+ private void notifyStepChange() {
+ try {
+ watcher.onStepStateChange(snapshot());
+ } catch (Exception e) {
+ logError("Pipeline watcher onStepStateChange callback failed", e);
+ }
+ }
+
+ private void notifyFinish(BuildResult result) {
+ try {
+ watcher.onFinish(snapshot(), result);
+ } catch (Exception e) {
+ logError("Pipeline watcher onFinish callback failed", e);
+ }
+ }
+
+ protected <R> java.util.Optional<R> execStep(int seq, Callable<R> process) {
+ try {
+ curStep = seq;
+ stepsStatus.put(
+ seq,
+ new AbstractMap.SimpleEntry<>(
+ PipelineStepStatusEnum.running, System.currentTimeMillis()));
+ logInfo(
+ "Building pipeline step["
+ + seq
+ + "/"
+ + allSteps()
+ + "] running => "
+ + pipeType().getSteps().get(seq));
+ notifyStepChange();
+ R result = process.call();
+ stepsStatus.put(
+ seq,
+ new AbstractMap.SimpleEntry<>(
+ PipelineStepStatusEnum.success, System.currentTimeMillis()));
+ logInfo("Building pipeline step[" + seq + "/" + allSteps() + "] success");
+ notifyStepChange();
+ return java.util.Optional.of(result);
+ } catch (Exception cause) {
+ stepsStatus.put(
+ seq,
+ new AbstractMap.SimpleEntry<>(
+ PipelineStepStatusEnum.failure, System.currentTimeMillis()));
+ pipeStatus = PipelineStatusEnum.failure;
+ error = PipeError.of(cause.getMessage(), cause);
+ logInfo(
+ "Building pipeline step["
+ + seq
+ + "/"
+ + allSteps()
+ + "] failure => "
+ + pipeType().getSteps().get(seq));
+ notifyStepChange();
+ return java.util.Optional.empty();
+ }
+ }
+
+ protected void skipStep(int step) {
+ curStep = step;
+ stepsStatus.put(
+ step,
+ new AbstractMap.SimpleEntry<>(
+ PipelineStepStatusEnum.skipped, System.currentTimeMillis()));
+ logInfo(
+ "Building pipeline step["
+ + step
+ + "/"
+ + allSteps()
+ + "] skipped => "
+ + pipeType().getSteps().get(step));
+ notifyStepChange();
+ }
+
+ /** Launch the building pipeline. */
+ @Override
+ public BuildResult launch() {
+ pipeStatus = PipelineStatusEnum.running;
+ try {
+ notifyStart();
+ logInfo("Building pipeline is launching, params=" + offerBuildParam());
+ BuildResult result =
+ EXEC_POOL.submit(this::buildProcess).get(20, TimeUnit.MINUTES);
+ pipeStatus = PipelineStatusEnum.success;
+ logInfo("Building pipeline has finished successfully.");
+ notifyFinish(result);
+ return result;
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ pipeStatus = PipelineStatusEnum.failure;
+ error = PipeError.of(e.getMessage(), e);
+ logError("Building pipeline has failed.", e);
+ BuildResult result = new ErrorResult();
+ notifyFinish(result);
+ return result;
+ } catch (ExecutionException e) {
+ pipeStatus = PipelineStatusEnum.failure;
+ Throwable cause = e.getCause();
+ if (cause == null) {
+ cause = e;
+ }
+ error = PipeError.of(cause.getMessage(), cause);
+ logError("Building pipeline has failed.", cause);
+ BuildResult result = new ErrorResult();
+ notifyFinish(result);
+ return result;
+ } catch (TimeoutException e) {
+ pipeStatus = PipelineStatusEnum.failure;
+ error = PipeError.of(e.getMessage(), e);
+ logError("Building pipeline has failed.", e);
+ BuildResult result = new ErrorResult();
+ notifyFinish(result);
+ return result;
+ }
+ }
+
+ @Override
+ public PipelineStatusEnum getPipeStatus() {
+ return pipeStatus;
+ }
+
+ @Override
+ public PipeError getError() {
+ return error.copy();
+ }
+
+ @Override
+ public Map<Integer, Map.Entry<PipelineStepStatusEnum, Long>> getStepsStatus() {
+ return new LinkedHashMap<>(stepsStatus);
+ }
+
+ @Override
+ public int getCurStep() {
+ return curStep;
+ }
+
+ @Override
+ public int allSteps() {
+ return pipeType().getSteps().size();
+ }
+
+ @Override
+ protected void logInfo(String msg) {
+ super.logInfo("[streampark-packer] " + msg + " | " + logSuffix());
+ }
+
+ @Override
+ protected void logError(String msg) {
+ super.logError("[streampark-packer] " + msg + " | " + logSuffix());
+ }
+
+ @Override
+ protected void logError(String msg, Throwable throwable) {
+ super.logError("[streampark-packer] " + msg + " | " + logSuffix(), throwable);
+ }
+
+ protected RuntimeException pipelineException() {
+ Throwable ex = getError().exception();
+ if (ex instanceof RuntimeException) {
+ return (RuntimeException) ex;
+ }
+ if (ex != null) {
+ return new RuntimeException(ex);
+ }
+ return new RuntimeException(getError().summary());
+ }
+
+ /** intercept snapshot */
+ public PipelineSnapshot snapshot() {
+ return new PipelineSnapshot(
+ offerBuildParam().appName(),
+ pipeType(),
+ getPipeStatus(),
+ getCurStep(),
+ allSteps(),
+ getStepsStatus(),
+ getError(),
+ System.currentTimeMillis());
+ }
+}
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/BuildPipelineExpose.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/BuildPipelineExpose.java
new file mode 100644
index 000000000..555c2c59a
--- /dev/null
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/BuildPipelineExpose.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.packer.pipeline;
+
+import java.util.Map;
+
+/** Callable methods exposed by BuildPipeline to the outside. */
+public interface BuildPipelineExpose {
+
+ /** get current state of the pipeline instance */
+ PipelineStatusEnum getPipeStatus();
+
+ /** get error of pipeline instance */
+ PipeError getError();
+
+ /** get all of the steps status StepSeq -> (PipeStepStatus -> status update timestamp) */
+ Map<Integer, Map.Entry<PipelineStepStatusEnum, Long>> getStepsStatus();
+
+ /** get current build step index */
+ int getCurStep();
+
+ /** get count of all build steps */
+ int allSteps();
+
+ /** launch the pipeline instance */
+ BuildResult launch();
+
+ @SuppressWarnings("unchecked")
+ default <T extends BuildPipeline> T as(Class<T> clz) {
+ return (T) this;
+ }
+}
diff --git a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/DockerImageExist.scala b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/BuildPipelineProcess.java
similarity index 61%
copy from streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/DockerImageExist.scala
copy to streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/BuildPipelineProcess.java
index 11c26e1d0..c4080c9dc 100644
--- a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/DockerImageExist.scala
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/BuildPipelineProcess.java
@@ -15,20 +15,20 @@
* limitations under the License.
*/
-package org.apache.streampark.flink.packer.docker
+package org.apache.streampark.flink.packer.pipeline;
-import com.github.dockerjava.api.exception.NotFoundException
+/** Behavior that BuildPipeline subclasses must inherit to implement. */
+public interface BuildPipelineProcess {
-class DockerImageExist {
- def doesDockerImageExist(imageName: String): Boolean = {
- usingDockerClient {
- dockerClient =>
- try {
- dockerClient.inspectImageCmd(imageName).exec()
- true
- } catch {
- case _: NotFoundException => false
- }
- }(err => throw new Exception(s"Check docker image failed, imageName=$imageName", err))
- }
+ /** the type of pipeline */
+ PipelineTypeEnum pipeType();
+
+ /**
+ * the actual build process. the effective steps progress should be implemented in multiple
+ * BuildPipeline.execStep() functions.
+ */
+ BuildResult buildProcess();
+
+ /** the build params of build process */
+ BuildParam offerBuildParam();
}
diff --git a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/DockerImageExist.scala b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/BuildResult.java
similarity index 61%
copy from streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/DockerImageExist.scala
copy to streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/BuildResult.java
index 11c26e1d0..56d9969fc 100644
--- a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/DockerImageExist.scala
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/BuildResult.java
@@ -15,20 +15,16 @@
* limitations under the License.
*/
-package org.apache.streampark.flink.packer.docker
+package org.apache.streampark.flink.packer.pipeline;
-import com.github.dockerjava.api.exception.NotFoundException
+/** Result of a BuildPipeline instance. */
+public interface BuildResult {
-class DockerImageExist {
- def doesDockerImageExist(imageName: String): Boolean = {
- usingDockerClient {
- dockerClient =>
- try {
- dockerClient.inspectImageCmd(imageName).exec()
- true
- } catch {
- case _: NotFoundException => false
- }
- }(err => throw new Exception(s"Check docker image failed, imageName=$imageName", err))
- }
+ /** is pass aka is successfully */
+ boolean pass();
+
+ @SuppressWarnings("unchecked")
+ default <T extends BuildResult> T as(Class<T> clz) {
+ return (T) this;
+ }
}
diff --git a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/DockerImageExist.scala b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/DockerBuildProgress.java
similarity index 50%
copy from streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/DockerImageExist.scala
copy to streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/DockerBuildProgress.java
index 11c26e1d0..ab397406a 100644
--- a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/DockerImageExist.scala
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/DockerBuildProgress.java
@@ -15,20 +15,34 @@
* limitations under the License.
*/
-package org.apache.streampark.flink.packer.docker
-
-import com.github.dockerjava.api.exception.NotFoundException
-
-class DockerImageExist {
- def doesDockerImageExist(imageName: String): Boolean = {
- usingDockerClient {
- dockerClient =>
- try {
- dockerClient.inspectImageCmd(imageName).exec()
- true
- } catch {
- case _: NotFoundException => false
+package org.apache.streampark.flink.packer.pipeline;
+
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class DockerBuildProgress {
+
+ private final List<String> steps = new ArrayList<>();
+ private long lastTime;
+
+ public DockerBuildProgress(long lastTime) {
+ this.lastTime = lastTime;
+ }
+
+ public void update(String buildStep) {
+ if (StringUtils.isNotBlank(buildStep)) {
+ steps.add(buildStep);
+ lastTime = System.currentTimeMillis();
}
- }(err => throw new Exception(s"Check docker image failed, imageName=$imageName", err))
- }
+ }
+
+ public DockerBuildSnapshot snapshot() {
+ return new DockerBuildSnapshot(new ArrayList<>(steps), lastTime);
+ }
+
+ public static DockerBuildProgress empty() {
+ return new DockerBuildProgress(System.currentTimeMillis());
+ }
}
diff --git a/streampark-flink/streampark-flink-packer/src/test/scala/org/apache/streampark/flink/packer/package.scala b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/DockerBuildSnapshot.java
similarity index 52%
copy from streampark-flink/streampark-flink-packer/src/test/scala/org/apache/streampark/flink/packer/package.scala
copy to streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/DockerBuildSnapshot.java
index d7fc18056..676262a84 100644
--- a/streampark-flink/streampark-flink-packer/src/test/scala/org/apache/streampark/flink/packer/package.scala
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/DockerBuildSnapshot.java
@@ -15,31 +15,33 @@
* limitations under the License.
*/
-package org.apache.streampark.flink
-
-import org.codehaus.plexus.util.IOUtil
-
-import java.io.{InputStream, IOException}
-import java.util
-import java.util.jar.JarFile
-
-package object packer {
-
- def path: String => String = (path: String) => getClass.getClassLoader.getResource(path).getFile
-
- def jarEquals(jar1: JarFile, jar2: JarFile, entry: String): Boolean = {
- var s1: InputStream = null
- var s2: InputStream = null
- try {
- s1 = jar1.getInputStream(jar1.getJarEntry(entry))
- s2 = jar2.getInputStream(jar2.getJarEntry(entry))
- util.Arrays.equals(IOUtil.toByteArray(s1), IOUtil.toByteArray(s2))
- } catch {
- case e: IOException => false
- } finally {
- if (s1 != null) s1.close()
- if (s2 != null) s2.close()
+package org.apache.streampark.flink.packer.pipeline;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
+import java.util.List;
+
+/** snapshot for building docker image progress. */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class DockerBuildSnapshot {
+
+ private final List<String> detail;
+ private final long emitTime;
+
+ public DockerBuildSnapshot(List<String> detail, long emitTime) {
+ this.detail = detail;
+ this.emitTime = emitTime;
}
- }
+ public List<String> detail() {
+ return detail;
+ }
+
+ public long emitTime() {
+ return emitTime;
+ }
+
+ public List<String> detailAsJava() {
+ return detail;
+ }
}
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/DockerImageBuildResponse.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/DockerImageBuildResponse.java
new file mode 100644
index 000000000..443df4bd5
--- /dev/null
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/DockerImageBuildResponse.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.packer.pipeline;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.Map;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class DockerImageBuildResponse implements FlinkBuildResult {
+
+ private String workspacePath;
+ private String flinkImageTag;
+ private Map<String, String> podTemplatePaths;
+ private String dockerInnerMainJarPath;
+ private boolean pass = true;
+
+ public DockerImageBuildResponse() {
+ }
+
+ public DockerImageBuildResponse(
+ String workspacePath,
+ String flinkImageTag,
+ Map<String, String> podTemplatePaths,
+ String dockerInnerMainJarPath) {
+ this(workspacePath, flinkImageTag, podTemplatePaths, dockerInnerMainJarPath, true);
+ }
+
+ public DockerImageBuildResponse(
+ String workspacePath,
+ String flinkImageTag,
+ Map<String, String> podTemplatePaths,
+ String dockerInnerMainJarPath,
+ boolean pass) {
+ this.workspacePath = workspacePath;
+ this.flinkImageTag = flinkImageTag;
+ this.podTemplatePaths = podTemplatePaths;
+ this.dockerInnerMainJarPath = dockerInnerMainJarPath;
+ this.pass = pass;
+ }
+
+ @Override
+ public String workspacePath() {
+ return workspacePath;
+ }
+
+ public String flinkImageTag() {
+ return flinkImageTag;
+ }
+
+ public Map<String, String> podTemplatePaths() {
+ return podTemplatePaths;
+ }
+
+ public String dockerInnerMainJarPath() {
+ return dockerInnerMainJarPath;
+ }
+
+ @Override
+ public boolean pass() {
+ return pass;
+ }
+
+ @JsonProperty("workspacePath")
+ public void setWorkspacePath(String workspacePath) {
+ this.workspacePath = workspacePath;
+ }
+
+ @JsonProperty("flinkImageTag")
+ public void setFlinkImageTag(String flinkImageTag) {
+ this.flinkImageTag = flinkImageTag;
+ }
+
+ @JsonProperty("podTemplatePaths")
+ public void setPodTemplatePaths(Map<String, String> podTemplatePaths) {
+ this.podTemplatePaths = podTemplatePaths;
+ }
+
+ @JsonProperty("dockerInnerMainJarPath")
+ public void setDockerInnerMainJarPath(String dockerInnerMainJarPath) {
+ this.dockerInnerMainJarPath = dockerInnerMainJarPath;
+ }
+
+ @JsonProperty("pass")
+ public void setPass(boolean pass) {
+ this.pass = pass;
+ }
+
+ @Override
+ public String toString() {
+ return "{ workspacePath: "
+ + workspacePath
+ + ", flinkImageTag: "
+ + flinkImageTag
+ + ", podTemplatePaths: "
+ + podTemplatePaths
+ + ", dockerInnerMainJarPath: "
+ + dockerInnerMainJarPath
+ + ", pass: "
+ + pass
+ + " }";
+ }
+}
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/DockerLayerProgress.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/DockerLayerProgress.java
new file mode 100644
index 000000000..3a8fd722d
--- /dev/null
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/DockerLayerProgress.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.packer.pipeline;
+
+import org.apache.streampark.common.util.Utils;
+
+/** push/pull progress of per docker layer. */
+public class DockerLayerProgress {
+
+ private final String layerId;
+ private final String status;
+ private final long current;
+ private final long total;
+
+ public DockerLayerProgress(String layerId, String status, long current, long total) {
+ this.layerId = layerId;
+ this.status = status;
+ this.current = current;
+ this.total = total;
+ }
+
+ public String layerId() {
+ return layerId;
+ }
+
+ public String status() {
+ return status;
+ }
+
+ public long current() {
+ return current;
+ }
+
+ public long total() {
+ return total;
+ }
+
+ public double percent() {
+ return Utils.calPercent(current, total);
+ }
+
+ public double currentMb() {
+ if (current == 0) {
+ return 0;
+ }
+ return Double.parseDouble(String.format("%.2f", current / (1024.0 * 1024.0)));
+ }
+
+ public double totalMb() {
+ if (total == 0) {
+ return 0;
+ }
+ return Double.parseDouble(String.format("%.2f", total / (1024.0 * 1024.0)));
+ }
+}
diff --git a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/DockerProgressWatcher.scala b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/DockerProgressWatcher.java
similarity index 52%
rename from streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/DockerProgressWatcher.scala
rename to streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/DockerProgressWatcher.java
index 3d77cd4df..a43a2da79 100644
--- a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/DockerProgressWatcher.scala
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/DockerProgressWatcher.java
@@ -15,32 +15,20 @@
* limitations under the License.
*/
-package org.apache.streampark.flink.packer.pipeline
+package org.apache.streampark.flink.packer.pipeline;
/**
* Trait for watching the change events of the Docker resolved progress for a BuildPipeline
* instance.
*/
-trait DockerProgressWatcher {
+public interface DockerProgressWatcher {
- /** async call when pulling docker image progress is changed. */
- def onDockerPullProgressChange(snapshot: DockerPullSnapshot): Unit
+ /** async call when pulling docker image progress is changed. */
+ void onDockerPullProgressChange(DockerPullSnapshot snapshot);
- /** async call when building docker image progress is changed. */
- def onDockerBuildProgressChange(snapshot: DockerBuildSnapshot): Unit
-
- /** async call when pushing docker image progress is changed. */
- def onDockerPushProgressChange(snapshot: DockerPushSnapshot): Unit
-
-}
-
-/** silent watcher */
-class SilentDockerProgressWatcher extends DockerProgressWatcher {
-
- override def onDockerPullProgressChange(snapshot: DockerPullSnapshot): Unit = {}
-
- override def onDockerBuildProgressChange(snapshot: DockerBuildSnapshot): Unit = {}
-
- override def onDockerPushProgressChange(snapshot: DockerPushSnapshot): Unit = {}
+ /** async call when building docker image progress is changed. */
+ void onDockerBuildProgressChange(DockerBuildSnapshot snapshot);
+ /** async call when pushing docker image progress is changed. */
+ void onDockerPushProgressChange(DockerPushSnapshot snapshot);
}
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/DockerPullProgress.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/DockerPullProgress.java
new file mode 100644
index 000000000..61bd90a75
--- /dev/null
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/DockerPullProgress.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.packer.pipeline;
+
+import org.apache.commons.lang3.StringUtils;
+
+import com.github.dockerjava.api.model.PullResponseItem;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+public class DockerPullProgress {
+
+ private final Map<String, DockerLayerProgress> layers = new LinkedHashMap<>();
+ private String error = "";
+ private long lastTime;
+
+ public DockerPullProgress(long lastTime) {
+ this.lastTime = lastTime;
+ }
+
+ public void update(PullResponseItem pullRsp) {
+ if (pullRsp == null
+ || StringUtils.isBlank(pullRsp.getId())
+ || StringUtils.isBlank(pullRsp.getStatus())) {
+ return;
+ }
+ if (pullRsp.getStatus().contains("complete")) {
+ layers.put(
+ pullRsp.getId(),
+ new DockerLayerProgress(pullRsp.getId(), pullRsp.getStatus(), 1, 1));
+ lastTime = System.currentTimeMillis();
+ } else {
+ long cur =
+ pullRsp.getProgressDetail() == null || pullRsp.getProgressDetail().getCurrent() == null
+ ? 0
+ : pullRsp.getProgressDetail().getCurrent();
+ long total =
+ pullRsp.getProgressDetail() == null || pullRsp.getProgressDetail().getTotal() == null
+ ? 0
+ : pullRsp.getProgressDetail().getTotal();
+ layers.put(
+ pullRsp.getId(),
+ new DockerLayerProgress(pullRsp.getId(), pullRsp.getStatus(), cur, total));
+ error =
+ pullRsp.getErrorDetail() == null ? "" : pullRsp.getErrorDetail().getMessage();
+ lastTime = System.currentTimeMillis();
+ }
+ }
+
+ public DockerPullSnapshot snapshot() {
+ return DockerPullSnapshot.of(new ArrayList<>(layers.values()), error, lastTime);
+ }
+
+ public static DockerPullProgress empty() {
+ return new DockerPullProgress(System.currentTimeMillis());
+ }
+}
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/DockerPullSnapshot.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/DockerPullSnapshot.java
new file mode 100644
index 000000000..32a242288
--- /dev/null
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/DockerPullSnapshot.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.packer.pipeline;
+
+import org.apache.streampark.common.util.Utils;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
+import java.util.List;
+
+/** snapshot for pulling docker image progress. */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class DockerPullSnapshot {
+
+ private final List<DockerLayerProgress> detail;
+ private final String error;
+ private final long emitTime;
+ private final double percent;
+
+ public DockerPullSnapshot(
+ List<DockerLayerProgress> detail,
+ String error,
+ long emitTime,
+ double percent) {
+ this.detail = detail;
+ this.error = error;
+ this.emitTime = emitTime;
+ this.percent = percent;
+ }
+
+ public List<DockerLayerProgress> detail() {
+ return detail;
+ }
+
+ public String error() {
+ return error;
+ }
+
+ public long emitTime() {
+ return emitTime;
+ }
+
+ public double percent() {
+ return percent;
+ }
+
+ public List<DockerLayerProgress> detailAsJava() {
+ return detail;
+ }
+
+ public static DockerPullSnapshot of(
+ List<DockerLayerProgress> detail,
+ String error,
+ long emitTime) {
+ long currentSum = detail.stream().mapToLong(DockerLayerProgress::current).sum();
+ long totalSum = detail.stream().mapToLong(DockerLayerProgress::total).sum();
+ return new DockerPullSnapshot(
+ detail, error, emitTime, Utils.calPercent(currentSum, totalSum));
+ }
+}
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/DockerPushProgress.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/DockerPushProgress.java
new file mode 100644
index 000000000..b5104f631
--- /dev/null
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/DockerPushProgress.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.packer.pipeline;
+
+import org.apache.commons.lang3.StringUtils;
+
+import com.github.dockerjava.api.model.PushResponseItem;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+public class DockerPushProgress {
+
+ private final Map<String, DockerLayerProgress> layers = new LinkedHashMap<>();
+ private String error = "";
+ private long lastTime;
+
+ public DockerPushProgress(long lastTime) {
+ this.lastTime = lastTime;
+ }
+
+ public void update(PushResponseItem pushRsp) {
+ if (pushRsp == null
+ || StringUtils.isBlank(pushRsp.getId())
+ || StringUtils.isBlank(pushRsp.getStatus())) {
+ return;
+ }
+ if (pushRsp.getStatus().contains("complete")) {
+ layers.put(
+ pushRsp.getId(),
+ new DockerLayerProgress(pushRsp.getId(), pushRsp.getStatus(), 1, 1));
+ lastTime = System.currentTimeMillis();
+ } else {
+ long cur =
+ pushRsp.getProgressDetail() == null || pushRsp.getProgressDetail().getCurrent() == null
+ ? 0L
+ : pushRsp.getProgressDetail().getCurrent();
+ long total =
+ pushRsp.getProgressDetail() == null || pushRsp.getProgressDetail().getTotal() == null
+ ? 0L
+ : pushRsp.getProgressDetail().getTotal();
+ layers.put(
+ pushRsp.getId(),
+ new DockerLayerProgress(pushRsp.getId(), pushRsp.getStatus(), cur, total));
+ error =
+ pushRsp.getErrorDetail() == null ? "" : pushRsp.getErrorDetail().getMessage();
+ lastTime = System.currentTimeMillis();
+ }
+ }
+
+ public DockerPushSnapshot snapshot() {
+ return DockerPushSnapshot.of(new ArrayList<>(layers.values()), error, lastTime);
+ }
+
+ public static DockerPushProgress empty() {
+ return new DockerPushProgress(System.currentTimeMillis());
+ }
+}
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/DockerPushSnapshot.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/DockerPushSnapshot.java
new file mode 100644
index 000000000..9d5f04fac
--- /dev/null
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/DockerPushSnapshot.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.packer.pipeline;
+
+import org.apache.streampark.common.util.Utils;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
+import java.util.List;
+
+/** snapshot for pushing docker image progress. */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class DockerPushSnapshot {
+
+ private final List<DockerLayerProgress> detail;
+ private final String error;
+ private final long emitTime;
+ private final double percent;
+
+ public DockerPushSnapshot(
+ List<DockerLayerProgress> detail,
+ String error,
+ long emitTime,
+ double percent) {
+ this.detail = detail;
+ this.error = error;
+ this.emitTime = emitTime;
+ this.percent = percent;
+ }
+
+ public List<DockerLayerProgress> detail() {
+ return detail;
+ }
+
+ public String error() {
+ return error;
+ }
+
+ public long emitTime() {
+ return emitTime;
+ }
+
+ public double percent() {
+ return percent;
+ }
+
+ public List<DockerLayerProgress> detailAsJava() {
+ return detail;
+ }
+
+ public static DockerPushSnapshot of(
+ List<DockerLayerProgress> detail,
+ String error,
+ long emitTime) {
+ long currentSum = detail.stream().mapToLong(DockerLayerProgress::current).sum();
+ long totalSum = detail.stream().mapToLong(DockerLayerProgress::total).sum();
+ return new DockerPushSnapshot(
+ detail, error, emitTime, Utils.calPercent(currentSum, totalSum));
+ }
+}
diff --git a/streampark-flink/streampark-flink-packer/src/test/scala/org/apache/streampark/flink/packer/package.scala b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/DockerResolveProgress.java
similarity index 51%
rename from streampark-flink/streampark-flink-packer/src/test/scala/org/apache/streampark/flink/packer/package.scala
rename to streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/DockerResolveProgress.java
index d7fc18056..9fbc27253 100644
--- a/streampark-flink/streampark-flink-packer/src/test/scala/org/apache/streampark/flink/packer/package.scala
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/DockerResolveProgress.java
@@ -15,31 +15,33 @@
* limitations under the License.
*/
-package org.apache.streampark.flink
+package org.apache.streampark.flink.packer.pipeline;
-import org.codehaus.plexus.util.IOUtil
+/** cache storage for docker resolved progress */
+public class DockerResolveProgress {
-import java.io.{InputStream, IOException}
-import java.util
-import java.util.jar.JarFile
+ private final DockerPullProgress pull;
+ private final DockerBuildProgress build;
+ private final DockerPushProgress push;
-package object packer {
+ public DockerResolveProgress(
+ DockerPullProgress pull,
+ DockerBuildProgress build,
+ DockerPushProgress push) {
+ this.pull = pull;
+ this.build = build;
+ this.push = push;
+ }
- def path: String => String = (path: String) => getClass.getClassLoader.getResource(path).getFile
+ public DockerPullProgress getPull() {
+ return pull;
+ }
- def jarEquals(jar1: JarFile, jar2: JarFile, entry: String): Boolean = {
- var s1: InputStream = null
- var s2: InputStream = null
- try {
- s1 = jar1.getInputStream(jar1.getJarEntry(entry))
- s2 = jar2.getInputStream(jar2.getJarEntry(entry))
- util.Arrays.equals(IOUtil.toByteArray(s1), IOUtil.toByteArray(s2))
- } catch {
- case e: IOException => false
- } finally {
- if (s1 != null) s1.close()
- if (s2 != null) s2.close()
+ public DockerBuildProgress getBuild() {
+ return build;
}
- }
+ public DockerPushProgress getPush() {
+ return push;
+ }
}
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/DockerResolvedSnapshot.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/DockerResolvedSnapshot.java
new file mode 100644
index 000000000..9b19dcda9
--- /dev/null
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/DockerResolvedSnapshot.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.packer.pipeline;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
+/** Snapshot for docker resolved progress */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class DockerResolvedSnapshot {
+
+ private final DockerPullSnapshot pull;
+ private final DockerBuildSnapshot build;
+ private final DockerPushSnapshot push;
+
+ public DockerResolvedSnapshot(
+ DockerPullSnapshot pull,
+ DockerBuildSnapshot build,
+ DockerPushSnapshot push) {
+ this.pull = pull;
+ this.build = build;
+ this.push = push;
+ }
+
+ public DockerPullSnapshot pull() {
+ return pull;
+ }
+
+ public DockerBuildSnapshot build() {
+ return build;
+ }
+
+ public DockerPushSnapshot push() {
+ return push;
+ }
+}
diff --git a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/DockerImageExist.scala b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/ErrorResult.java
similarity index 61%
copy from streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/DockerImageExist.scala
copy to streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/ErrorResult.java
index 11c26e1d0..ddbbb648a 100644
--- a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/DockerImageExist.scala
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/ErrorResult.java
@@ -15,20 +15,26 @@
* limitations under the License.
*/
-package org.apache.streampark.flink.packer.docker
+package org.apache.streampark.flink.packer.pipeline;
-import com.github.dockerjava.api.exception.NotFoundException
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
-class DockerImageExist {
- def doesDockerImageExist(imageName: String): Boolean = {
- usingDockerClient {
- dockerClient =>
- try {
- dockerClient.inspectImageCmd(imageName).exec()
- true
- } catch {
- case _: NotFoundException => false
- }
- }(err => throw new Exception(s"Check docker image failed, imageName=$imageName", err))
- }
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class ErrorResult implements BuildResult {
+
+ private boolean pass = false;
+
+ public ErrorResult() {
+ }
+
+ @Override
+ public boolean pass() {
+ return pass;
+ }
+
+ @JsonProperty("pass")
+ public void setPass(boolean pass) {
+ this.pass = pass;
+ }
}
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/FlinkBuildParam.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/FlinkBuildParam.java
new file mode 100644
index 000000000..b86a61ead
--- /dev/null
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/FlinkBuildParam.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.packer.pipeline;
+
+import org.apache.streampark.common.conf.FlinkVersion;
+import org.apache.streampark.common.conf.Workspace;
+import org.apache.streampark.common.enums.FlinkDeployMode;
+import org.apache.streampark.common.enums.FlinkJobType;
+import org.apache.streampark.flink.packer.maven.DependencyInfo;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+public interface FlinkBuildParam extends BuildParam {
+
+ String workspace();
+
+ FlinkDeployMode deployMode();
+
+ FlinkJobType flinkJobType();
+
+ FlinkVersion flinkVersion();
+
+ DependencyInfo dependencyInfo();
+
+ String customFlinkUserJar();
+
+ default DependencyInfo providedLibs() {
+ Set<String> libs = new HashSet<>(Arrays.asList(
+ Workspace.local().APP_JARS(),
+ customFlinkUserJar()));
+ if (flinkJobType() == FlinkJobType.FLINK_SQL) {
+ libs.add(
+ Workspace.local().APP_SHIMS()
+ + "/flink-"
+ + flinkVersion().majorVersion());
+ }
+ return dependencyInfo().merge(libs);
+ }
+
+ default String getShadedJarPath(String rootWorkspace) {
+ String safeAppName = appName().replaceAll("\\s+", "_");
+ return rootWorkspace + "/streampark-flinkjob_" + safeAppName + ".jar";
+ }
+}
diff --git a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/DockerImageExist.scala b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/FlinkBuildResult.java
similarity index 61%
copy from streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/DockerImageExist.scala
copy to streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/FlinkBuildResult.java
index 11c26e1d0..55afabe05 100644
--- a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/DockerImageExist.scala
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/FlinkBuildResult.java
@@ -15,20 +15,10 @@
* limitations under the License.
*/
-package org.apache.streampark.flink.packer.docker
+package org.apache.streampark.flink.packer.pipeline;
-import com.github.dockerjava.api.exception.NotFoundException
+/** Flink build result with workspace path. */
+public interface FlinkBuildResult extends BuildResult {
-class DockerImageExist {
- def doesDockerImageExist(imageName: String): Boolean = {
- usingDockerClient {
- dockerClient =>
- try {
- dockerClient.inspectImageCmd(imageName).exec()
- true
- } catch {
- case _: NotFoundException => false
- }
- }(err => throw new Exception(s"Check docker image failed, imageName=$imageName", err))
- }
+ String workspacePath();
}
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/FlinkK8sApplicationBuildRequest.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/FlinkK8sApplicationBuildRequest.java
new file mode 100644
index 000000000..b24eb6958
--- /dev/null
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/FlinkK8sApplicationBuildRequest.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.packer.pipeline;
+
+import org.apache.streampark.common.conf.FlinkVersion;
+import org.apache.streampark.common.enums.FlinkDeployMode;
+import org.apache.streampark.common.enums.FlinkJobType;
+import org.apache.streampark.flink.kubernetes.model.K8sPodTemplates;
+import org.apache.streampark.flink.packer.docker.DockerConf;
+import org.apache.streampark.flink.packer.maven.DependencyInfo;
+
+public class FlinkK8sApplicationBuildRequest implements FlinkK8sBuildParam {
+
+ private final String appName;
+ private final String workspace;
+ private final String mainClass;
+ private final String customFlinkUserJar;
+ private final FlinkDeployMode deployMode;
+ private final FlinkJobType flinkJobType;
+ private final FlinkVersion flinkVersion;
+ private final DependencyInfo dependencyInfo;
+ private final String clusterId;
+ private final String k8sNamespace;
+ private final String flinkBaseImage;
+ private final K8sPodTemplates flinkPodTemplate;
+ private final boolean integrateWithHadoop;
+ private final DockerConf dockerConfig;
+ private final String ingressTemplate;
+
+ public FlinkK8sApplicationBuildRequest(
+ String appName,
+ String workspace,
+ String mainClass,
+ String customFlinkUserJar,
+ FlinkDeployMode deployMode,
+ FlinkJobType flinkJobType,
+ FlinkVersion flinkVersion,
+ DependencyInfo dependencyInfo,
+ String clusterId,
+ String k8sNamespace,
+ String flinkBaseImage,
+ K8sPodTemplates flinkPodTemplate,
+ boolean integrateWithHadoop,
+ DockerConf dockerConfig,
+ String ingressTemplate) {
+ this.appName = appName;
+ this.workspace = workspace;
+ this.mainClass = mainClass;
+ this.customFlinkUserJar = customFlinkUserJar;
+ this.deployMode = deployMode;
+ this.flinkJobType = flinkJobType;
+ this.flinkVersion = flinkVersion;
+ this.dependencyInfo = dependencyInfo;
+ this.clusterId = clusterId;
+ this.k8sNamespace = k8sNamespace;
+ this.flinkBaseImage = flinkBaseImage;
+ this.flinkPodTemplate = flinkPodTemplate;
+ this.integrateWithHadoop = integrateWithHadoop;
+ this.dockerConfig = dockerConfig;
+ this.ingressTemplate = ingressTemplate;
+ }
+
+ @Override
+ public String appName() {
+ return appName;
+ }
+
+ @Override
+ public String workspace() {
+ return workspace;
+ }
+
+ @Override
+ public String mainClass() {
+ return mainClass;
+ }
+
+ @Override
+ public String customFlinkUserJar() {
+ return customFlinkUserJar;
+ }
+
+ @Override
+ public FlinkDeployMode deployMode() {
+ return deployMode;
+ }
+
+ @Override
+ public FlinkJobType flinkJobType() {
+ return flinkJobType;
+ }
+
+ @Override
+ public FlinkVersion flinkVersion() {
+ return flinkVersion;
+ }
+
+ @Override
+ public DependencyInfo dependencyInfo() {
+ return dependencyInfo;
+ }
+
+ @Override
+ public String clusterId() {
+ return clusterId;
+ }
+
+ @Override
+ public String k8sNamespace() {
+ return k8sNamespace;
+ }
+
+ public String flinkBaseImage() {
+ return flinkBaseImage;
+ }
+
+ public K8sPodTemplates flinkPodTemplate() {
+ return flinkPodTemplate;
+ }
+
+ public boolean integrateWithHadoop() {
+ return integrateWithHadoop;
+ }
+
+ public DockerConf dockerConfig() {
+ return dockerConfig;
+ }
+
+ public String ingressTemplate() {
+ return ingressTemplate;
+ }
+}
diff --git a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/DockerImageExist.scala b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/FlinkK8sBuildParam.java
similarity index 61%
copy from streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/DockerImageExist.scala
copy to streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/FlinkK8sBuildParam.java
index 11c26e1d0..6f56f2140 100644
--- a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/DockerImageExist.scala
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/FlinkK8sBuildParam.java
@@ -15,20 +15,11 @@
* limitations under the License.
*/
-package org.apache.streampark.flink.packer.docker
+package org.apache.streampark.flink.packer.pipeline;
-import com.github.dockerjava.api.exception.NotFoundException
+public interface FlinkK8sBuildParam extends FlinkBuildParam {
-class DockerImageExist {
- def doesDockerImageExist(imageName: String): Boolean = {
- usingDockerClient {
- dockerClient =>
- try {
- dockerClient.inspectImageCmd(imageName).exec()
- true
- } catch {
- case _: NotFoundException => false
- }
- }(err => throw new Exception(s"Check docker image failed, imageName=$imageName", err))
- }
+ String clusterId();
+
+ String k8sNamespace();
}
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/FlinkK8sSessionBuildRequest.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/FlinkK8sSessionBuildRequest.java
new file mode 100644
index 000000000..fd14dd84f
--- /dev/null
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/FlinkK8sSessionBuildRequest.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.packer.pipeline;
+
+import org.apache.streampark.common.conf.FlinkVersion;
+import org.apache.streampark.common.enums.FlinkDeployMode;
+import org.apache.streampark.common.enums.FlinkJobType;
+import org.apache.streampark.flink.packer.maven.DependencyInfo;
+
+public class FlinkK8sSessionBuildRequest implements FlinkK8sBuildParam {
+
+ private final String appName;
+ private final String workspace;
+ private final String mainClass;
+ private final String customFlinkUserJar;
+ private final FlinkDeployMode deployMode;
+ private final FlinkJobType flinkJobType;
+ private final FlinkVersion flinkVersion;
+ private final DependencyInfo dependencyInfo;
+ private final String clusterId;
+ private final String k8sNamespace;
+
+ public FlinkK8sSessionBuildRequest(
+ String appName,
+ String workspace,
+ String mainClass,
+ String customFlinkUserJar,
+ FlinkDeployMode deployMode,
+ FlinkJobType flinkJobType,
+ FlinkVersion flinkVersion,
+ DependencyInfo dependencyInfo,
+ String clusterId,
+ String k8sNamespace) {
+ this.appName = appName;
+ this.workspace = workspace;
+ this.mainClass = mainClass;
+ this.customFlinkUserJar = customFlinkUserJar;
+ this.deployMode = deployMode;
+ this.flinkJobType = flinkJobType;
+ this.flinkVersion = flinkVersion;
+ this.dependencyInfo = dependencyInfo;
+ this.clusterId = clusterId;
+ this.k8sNamespace = k8sNamespace;
+ }
+
+ @Override
+ public String appName() {
+ return appName;
+ }
+
+ @Override
+ public String workspace() {
+ return workspace;
+ }
+
+ @Override
+ public String mainClass() {
+ return mainClass;
+ }
+
+ @Override
+ public String customFlinkUserJar() {
+ return customFlinkUserJar;
+ }
+
+ @Override
+ public FlinkDeployMode deployMode() {
+ return deployMode;
+ }
+
+ @Override
+ public FlinkJobType flinkJobType() {
+ return flinkJobType;
+ }
+
+ @Override
+ public FlinkVersion flinkVersion() {
+ return flinkVersion;
+ }
+
+ @Override
+ public DependencyInfo dependencyInfo() {
+ return dependencyInfo;
+ }
+
+ @Override
+ public String clusterId() {
+ return clusterId;
+ }
+
+ @Override
+ public String k8sNamespace() {
+ return k8sNamespace;
+ }
+}
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/FlinkRemotePerJobBuildRequest.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/FlinkRemotePerJobBuildRequest.java
new file mode 100644
index 000000000..32922a72b
--- /dev/null
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/FlinkRemotePerJobBuildRequest.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.packer.pipeline;
+
+import org.apache.streampark.common.conf.FlinkVersion;
+import org.apache.streampark.common.enums.FlinkDeployMode;
+import org.apache.streampark.common.enums.FlinkJobType;
+import org.apache.streampark.flink.packer.maven.DependencyInfo;
+
+public class FlinkRemotePerJobBuildRequest implements FlinkBuildParam {
+
+ private final String appName;
+ private final String workspace;
+ private final String mainClass;
+ private final String customFlinkUserJar;
+ private final boolean skipBuild;
+ private final FlinkDeployMode deployMode;
+ private final FlinkJobType flinkJobType;
+ private final FlinkVersion flinkVersion;
+ private final DependencyInfo dependencyInfo;
+
+ public FlinkRemotePerJobBuildRequest(
+ String appName,
+ String workspace,
+ String mainClass,
+ String customFlinkUserJar,
+ boolean skipBuild,
+ FlinkDeployMode deployMode,
+ FlinkJobType flinkJobType,
+ FlinkVersion flinkVersion,
+ DependencyInfo dependencyInfo) {
+ this.appName = appName;
+ this.workspace = workspace;
+ this.mainClass = mainClass;
+ this.customFlinkUserJar = customFlinkUserJar;
+ this.skipBuild = skipBuild;
+ this.deployMode = deployMode;
+ this.flinkJobType = flinkJobType;
+ this.flinkVersion = flinkVersion;
+ this.dependencyInfo = dependencyInfo;
+ }
+
+ @Override
+ public String appName() {
+ return appName;
+ }
+
+ @Override
+ public String workspace() {
+ return workspace;
+ }
+
+ @Override
+ public String mainClass() {
+ return mainClass;
+ }
+
+ @Override
+ public String customFlinkUserJar() {
+ return customFlinkUserJar;
+ }
+
+ public boolean skipBuild() {
+ return skipBuild;
+ }
+
+ @Override
+ public FlinkDeployMode deployMode() {
+ return deployMode;
+ }
+
+ @Override
+ public FlinkJobType flinkJobType() {
+ return flinkJobType;
+ }
+
+ @Override
+ public FlinkVersion flinkVersion() {
+ return flinkVersion;
+ }
+
+ @Override
+ public DependencyInfo dependencyInfo() {
+ return dependencyInfo;
+ }
+}
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/FlinkYarnApplicationBuildRequest.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/FlinkYarnApplicationBuildRequest.java
new file mode 100644
index 000000000..b61f39ea8
--- /dev/null
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/FlinkYarnApplicationBuildRequest.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.packer.pipeline;
+
+import org.apache.streampark.common.enums.FlinkJobType;
+import org.apache.streampark.flink.packer.maven.DependencyInfo;
+
+public class FlinkYarnApplicationBuildRequest implements BuildParam {
+
+ private final String appName;
+ private final String mainClass;
+ private final String localWorkspace;
+ private final String yarnProvidedPath;
+ private final FlinkJobType flinkJobType;
+ private final DependencyInfo dependencyInfo;
+
+ public FlinkYarnApplicationBuildRequest(
+ String appName,
+ String mainClass,
+ String localWorkspace,
+ String yarnProvidedPath,
+ FlinkJobType flinkJobType,
+ DependencyInfo dependencyInfo) {
+ this.appName = appName;
+ this.mainClass = mainClass;
+ this.localWorkspace = localWorkspace;
+ this.yarnProvidedPath = yarnProvidedPath;
+ this.flinkJobType = flinkJobType;
+ this.dependencyInfo = dependencyInfo;
+ }
+
+ @Override
+ public String appName() {
+ return appName;
+ }
+
+ @Override
+ public String mainClass() {
+ return mainClass;
+ }
+
+ public String localWorkspace() {
+ return localWorkspace;
+ }
+
+ public String yarnProvidedPath() {
+ return yarnProvidedPath;
+ }
+
+ public FlinkJobType flinkJobType() {
+ return flinkJobType;
+ }
+
+ public DependencyInfo dependencyInfo() {
+ return dependencyInfo;
+ }
+}
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/K8sAppModeBuildResponse.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/K8sAppModeBuildResponse.java
new file mode 100644
index 000000000..f8fac6c43
--- /dev/null
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/K8sAppModeBuildResponse.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.packer.pipeline;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.Set;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class K8sAppModeBuildResponse implements FlinkBuildResult {
+
+ private String workspacePath;
+ private String flinkBaseImage;
+ private String mainJarPath;
+ private Set<String> extraLibJarPaths;
+ private boolean pass = false;
+
+ public K8sAppModeBuildResponse() {
+ }
+
+ public K8sAppModeBuildResponse(
+ String workspacePath,
+ String flinkBaseImage,
+ String mainJarPath,
+ Set<String> extraLibJarPaths,
+ boolean pass) {
+ this.workspacePath = workspacePath;
+ this.flinkBaseImage = flinkBaseImage;
+ this.mainJarPath = mainJarPath;
+ this.extraLibJarPaths = extraLibJarPaths;
+ this.pass = pass;
+ }
+
+ @Override
+ public String workspacePath() {
+ return workspacePath;
+ }
+
+ public String flinkBaseImage() {
+ return flinkBaseImage;
+ }
+
+ public String mainJarPath() {
+ return mainJarPath;
+ }
+
+ public Set<String> extraLibJarPaths() {
+ return extraLibJarPaths;
+ }
+
+ @Override
+ public boolean pass() {
+ return pass;
+ }
+
+ @JsonProperty("workspacePath")
+ public void setWorkspacePath(String workspacePath) {
+ this.workspacePath = workspacePath;
+ }
+
+ @JsonProperty("flinkBaseImage")
+ public void setFlinkBaseImage(String flinkBaseImage) {
+ this.flinkBaseImage = flinkBaseImage;
+ }
+
+ @JsonProperty("mainJarPath")
+ public void setMainJarPath(String mainJarPath) {
+ this.mainJarPath = mainJarPath;
+ }
+
+ @JsonProperty("extraLibJarPaths")
+ public void setExtraLibJarPaths(Set<String> extraLibJarPaths) {
+ this.extraLibJarPaths = extraLibJarPaths;
+ }
+
+ @JsonProperty("pass")
+ public void setPass(boolean pass) {
+ this.pass = pass;
+ }
+
+ @Override
+ public String toString() {
+ return "{ workspacePath: "
+ + workspacePath
+ + ", flinkBaseImage: "
+ + flinkBaseImage
+ + ", mainJarPath: "
+ + mainJarPath
+ + ", extraLibJarPaths: "
+ + extraLibJarPaths
+ + ", pass: "
+ + pass
+ + " }";
+ }
+}
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/PipeError.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/PipeError.java
new file mode 100644
index 000000000..0a05f8d18
--- /dev/null
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/PipeError.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.packer.pipeline;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import javax.annotation.Nullable;
+
+/** Error details of building pipeline. */
+@JsonIgnoreProperties(ignoreUnknown = true, value = "exception")
+public class PipeError {
+
+ private String summary;
+ @JsonIgnore
+ @Nullable
+ private transient Throwable exception;
+ @Nullable
+ private String exceptionStack;
+
+ public PipeError() {
+ }
+
+ public PipeError(String summary, @Nullable Throwable exception, @Nullable String exceptionStack) {
+ this.summary = summary;
+ this.exception = exception;
+ this.exceptionStack = exceptionStack;
+ }
+
+ public String summary() {
+ return summary;
+ }
+
+ @Nullable
+ public Throwable exception() {
+ return exception;
+ }
+
+ @Nullable
+ public String exceptionStack() {
+ return exceptionStack;
+ }
+
+ public boolean nonEmpty() {
+ return (summary != null && !summary.isEmpty()) || exception != null;
+ }
+
+ public boolean isEmpty() {
+ return !nonEmpty();
+ }
+
+ public PipeError copy() {
+ return new PipeError(summary, exception, exceptionStack);
+ }
+
+ public static PipeError empty() {
+ return of("", null);
+ }
+
+ public static PipeError of(String summary, @Nullable Throwable exception) {
+ String stack = exception == null ? "" : stackTraceToString(exception);
+ return new PipeError(summary, exception, stack);
+ }
+
+ private static String stackTraceToString(Throwable exception) {
+ StringBuilder sb = new StringBuilder();
+ for (StackTraceElement element : exception.getStackTrace()) {
+ if (sb.length() > 0) {
+ sb.append('\n');
+ }
+ sb.append(element.toString());
+ }
+ return sb.toString();
+ }
+
+ @JsonProperty("summary")
+ public void setSummary(String summary) {
+ this.summary = summary;
+ }
+
+ @JsonProperty("exceptionStack")
+ public void setExceptionStack(String exceptionStack) {
+ this.exceptionStack = exceptionStack;
+ }
+}
diff --git a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/PipelineWatcher.scala b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/PipeWatcher.java
similarity index 50%
rename from streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/PipelineWatcher.scala
rename to streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/PipeWatcher.java
index 14ea5d927..2d1560d87 100644
--- a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/PipelineWatcher.scala
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/PipeWatcher.java
@@ -15,32 +15,20 @@
* limitations under the License.
*/
-package org.apache.streampark.flink.packer.pipeline
+package org.apache.streampark.flink.packer.pipeline;
/** Trait for watching a BuildPipeline instance */
-trait PipeWatcher {
+public interface PipeWatcher {
- /** called when the pipeline is launched. */
- @throws[Exception]
- def onStart(snapshot: PipelineSnapshot): Unit
+ /** called when the pipeline is launched. */
+ void onStart(PipelineSnapshot snapshot) throws Exception;
- /** called when the any status of building step is changed. */
- @throws[Exception]
- def onStepStateChange(snapshot: PipelineSnapshot): Unit
+ /** called when the any status of building step is changed. */
+ void onStepStateChange(PipelineSnapshot snapshot) throws Exception;
- /**
- * called when the pipeline is finished, or you can get the results directly from the
- * BuildPipeline.launch() synchronously.
- */
- @throws[Exception]
- def onFinish(snapshot: PipelineSnapshot, result: BuildResult): Unit
-}
-
-class SilentPipeWatcher extends PipeWatcher {
-
- override def onStart(snapshot: PipelineSnapshot): Unit = {}
-
- override def onStepStateChange(snapshot: PipelineSnapshot): Unit = {}
-
- override def onFinish(snapshot: PipelineSnapshot, result: BuildResult): Unit = {}
+ /**
+ * called when the pipeline is finished, or you can get the results directly from the
+ * BuildPipeline.launch() synchronously.
+ */
+ void onFinish(PipelineSnapshot snapshot, BuildResult result) throws Exception;
}
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/PipelineSnapshot.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/PipelineSnapshot.java
new file mode 100644
index 000000000..5d42cb229
--- /dev/null
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/PipelineSnapshot.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.packer.pipeline;
+
+import org.apache.streampark.common.util.Utils;
+
+import java.util.AbstractMap;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * Snapshot for a BuildPipeline instance.
+ *
+ * @param emitTime snapshot interception time
+ * @param stepStatus StepSeq -> (PipeStepStatus -> status update timestamp)
+ */
+public class PipelineSnapshot {
+
+ private final String appName;
+ private final PipelineTypeEnum pipeType;
+ private final PipelineStatusEnum pipeStatus;
+ private final int curStep;
+ private final int allSteps;
+ private final Map<Integer, Map.Entry<PipelineStepStatusEnum, Long>> stepStatus;
+ private final PipeError error;
+ private final long emitTime;
+
+ public PipelineSnapshot(
+ String appName,
+ PipelineTypeEnum pipeType,
+ PipelineStatusEnum pipeStatus,
+ int curStep,
+ int allSteps,
+ Map<Integer, Map.Entry<PipelineStepStatusEnum, Long>> stepStatus,
+ PipeError error,
+ long emitTime) {
+ this.appName = appName;
+ this.pipeType = pipeType;
+ this.pipeStatus = pipeStatus;
+ this.curStep = curStep;
+ this.allSteps = allSteps;
+ this.stepStatus = stepStatus;
+ this.error = error;
+ this.emitTime = emitTime;
+ }
+
+ public String appName() {
+ return appName;
+ }
+
+ public PipelineTypeEnum pipeType() {
+ return pipeType;
+ }
+
+ public PipelineStatusEnum pipeStatus() {
+ return pipeStatus;
+ }
+
+ public int curStep() {
+ return curStep;
+ }
+
+ public int allSteps() {
+ return allSteps;
+ }
+
+ public Map<Integer, Map.Entry<PipelineStepStatusEnum, Long>> stepStatus() {
+ return stepStatus;
+ }
+
+ public PipeError error() {
+ return error;
+ }
+
+ public long emitTime() {
+ return emitTime;
+ }
+
+ public double percent() {
+ return Utils.calPercent(curStep, allSteps);
+ }
+
+ public Map<Integer, Map.Entry<PipelineStepStatusEnum, Long>> stepStatusAsJava() {
+ Map<Integer, Map.Entry<PipelineStepStatusEnum, Long>> result = new HashMap<>();
+ stepStatus.forEach(
+ (key, value) -> result.put(
+ key,
+ new AbstractMap.SimpleEntry<>(value.getKey(), value.getValue())));
+ return result;
+ }
+
+ public Map<Integer, PipelineStepStatusEnum> pureStepStatusAsJava() {
+ return stepStatus.entrySet().stream()
+ .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().getKey()));
+ }
+
+ public Map<Integer, Long> stepStatusTimestampAsJava() {
+ return stepStatus.entrySet().stream()
+ .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().getValue()));
+ }
+}
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/ShadedBuildResponse.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/ShadedBuildResponse.java
new file mode 100644
index 000000000..9e383c5cb
--- /dev/null
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/ShadedBuildResponse.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.packer.pipeline;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class ShadedBuildResponse implements FlinkBuildResult {
+
+ private String workspacePath;
+ private String shadedJarPath;
+ private boolean pass = true;
+
+ public ShadedBuildResponse(String workspacePath, String shadedJarPath) {
+ this(workspacePath, shadedJarPath, true);
+ }
+
+ public ShadedBuildResponse(String workspacePath, String shadedJarPath, boolean pass) {
+ this.workspacePath = workspacePath;
+ this.shadedJarPath = shadedJarPath;
+ this.pass = pass;
+ }
+
+ public ShadedBuildResponse() {
+ }
+
+ @Override
+ public String workspacePath() {
+ return workspacePath;
+ }
+
+ public String shadedJarPath() {
+ return shadedJarPath;
+ }
+
+ @Override
+ public boolean pass() {
+ return pass;
+ }
+
+ @JsonProperty("workspacePath")
+ public void setWorkspacePath(String workspacePath) {
+ this.workspacePath = workspacePath;
+ }
+
+ @JsonProperty("shadedJarPath")
+ public void setShadedJarPath(String shadedJarPath) {
+ this.shadedJarPath = shadedJarPath;
+ }
+
+ @JsonProperty("pass")
+ public void setPass(boolean pass) {
+ this.pass = pass;
+ }
+
+ @Override
+ public String toString() {
+ return "{ workspacePath: "
+ + workspacePath
+ + ", shadedJarPath: "
+ + shadedJarPath
+ + ", pass: "
+ + pass
+ + " }";
+ }
+}
diff --git a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/DockerImageExist.scala b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/SilentDockerProgressWatcher.java
similarity index 57%
copy from streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/DockerImageExist.scala
copy to streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/SilentDockerProgressWatcher.java
index 11c26e1d0..5e6c5e522 100644
--- a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/DockerImageExist.scala
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/SilentDockerProgressWatcher.java
@@ -15,20 +15,23 @@
* limitations under the License.
*/
-package org.apache.streampark.flink.packer.docker
+package org.apache.streampark.flink.packer.pipeline;
-import com.github.dockerjava.api.exception.NotFoundException
+/** silent watcher */
+public class SilentDockerProgressWatcher implements DockerProgressWatcher {
-class DockerImageExist {
- def doesDockerImageExist(imageName: String): Boolean = {
- usingDockerClient {
- dockerClient =>
- try {
- dockerClient.inspectImageCmd(imageName).exec()
- true
- } catch {
- case _: NotFoundException => false
- }
- }(err => throw new Exception(s"Check docker image failed, imageName=$imageName", err))
- }
+ @Override
+ public void onDockerPullProgressChange(DockerPullSnapshot snapshot) {
+ // intentionally empty — silent watcher
+ }
+
+ @Override
+ public void onDockerBuildProgressChange(DockerBuildSnapshot snapshot) {
+ // intentionally empty — silent watcher
+ }
+
+ @Override
+ public void onDockerPushProgressChange(DockerPushSnapshot snapshot) {
+ // intentionally empty — silent watcher
+ }
}
diff --git a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/DockerImageExist.scala b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/SilentPipeWatcher.java
similarity index 59%
rename from streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/DockerImageExist.scala
rename to streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/SilentPipeWatcher.java
index 11c26e1d0..3a433fa66 100644
--- a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/DockerImageExist.scala
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/SilentPipeWatcher.java
@@ -15,20 +15,23 @@
* limitations under the License.
*/
-package org.apache.streampark.flink.packer.docker
+package org.apache.streampark.flink.packer.pipeline;
-import com.github.dockerjava.api.exception.NotFoundException
+/** Silent pipeline watcher. */
+public class SilentPipeWatcher implements PipeWatcher {
-class DockerImageExist {
- def doesDockerImageExist(imageName: String): Boolean = {
- usingDockerClient {
- dockerClient =>
- try {
- dockerClient.inspectImageCmd(imageName).exec()
- true
- } catch {
- case _: NotFoundException => false
- }
- }(err => throw new Exception(s"Check docker image failed, imageName=$imageName", err))
- }
+ @Override
+ public void onStart(PipelineSnapshot snapshot) {
+ // intentionally empty — silent watcher
+ }
+
+ @Override
+ public void onStepStateChange(PipelineSnapshot snapshot) {
+ // intentionally empty — silent watcher
+ }
+
+ @Override
+ public void onFinish(PipelineSnapshot snapshot, BuildResult result) {
+ // intentionally empty — silent watcher
+ }
}
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/SimpleBuildResponse.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/SimpleBuildResponse.java
new file mode 100644
index 000000000..cee1412da
--- /dev/null
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/SimpleBuildResponse.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.packer.pipeline;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class SimpleBuildResponse implements FlinkBuildResult {
+
+ private String workspacePath;
+ private boolean pass = true;
+
+ public SimpleBuildResponse() {
+ }
+
+ public SimpleBuildResponse(String workspacePath, boolean pass) {
+ this.workspacePath = workspacePath;
+ this.pass = pass;
+ }
+
+ @Override
+ public String workspacePath() {
+ return workspacePath;
+ }
+
+ @Override
+ public boolean pass() {
+ return pass;
+ }
+
+ @JsonProperty("workspacePath")
+ public void setWorkspacePath(String workspacePath) {
+ this.workspacePath = workspacePath;
+ }
+
+ @JsonProperty("pass")
+ public void setPass(boolean pass) {
+ this.pass = pass;
+ }
+
+ @Override
+ public String toString() {
+ return "{ workspacePath: " + workspacePath + ", pass: " + pass + " }";
+ }
+}
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/SparkK8sApplicationBuildRequest.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/SparkK8sApplicationBuildRequest.java
new file mode 100644
index 000000000..494026be9
--- /dev/null
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/SparkK8sApplicationBuildRequest.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.packer.pipeline;
+
+import org.apache.streampark.common.conf.SparkVersion;
+import org.apache.streampark.common.enums.SparkDeployMode;
+import org.apache.streampark.common.enums.SparkJobType;
+import org.apache.streampark.flink.packer.docker.DockerConf;
+import org.apache.streampark.flink.packer.maven.DependencyInfo;
+import org.apache.streampark.spark.kubernetes.model.SparkK8sPodTemplates;
+
+public class SparkK8sApplicationBuildRequest implements BuildParam {
+
+ private final String appName;
+ private final String workspace;
+ private final String mainClass;
+ private final String mainJar;
+ private final SparkDeployMode deployMode;
+ private final SparkJobType jobType;
+ private final SparkVersion sparkVersion;
+ private final DependencyInfo dependencyInfo;
+ private final String k8sNamespace;
+ private final String sparkBaseImage;
+ private final SparkK8sPodTemplates sparkPodTemplate;
+ private final boolean integrateWithHadoop;
+ private final DockerConf dockerConfig;
+
+ public SparkK8sApplicationBuildRequest(
+ String appName,
+ String workspace,
+ String mainClass,
+ String mainJar,
+ SparkDeployMode deployMode,
+ SparkJobType jobType,
+ SparkVersion sparkVersion,
+ DependencyInfo dependencyInfo,
+ String k8sNamespace,
+ String sparkBaseImage,
+ SparkK8sPodTemplates sparkPodTemplate,
+ boolean integrateWithHadoop,
+ DockerConf dockerConfig) {
+ this.appName = appName;
+ this.workspace = workspace;
+ this.mainClass = mainClass;
+ this.mainJar = mainJar;
+ this.deployMode = deployMode;
+ this.jobType = jobType;
+ this.sparkVersion = sparkVersion;
+ this.dependencyInfo = dependencyInfo;
+ this.k8sNamespace = k8sNamespace;
+ this.sparkBaseImage = sparkBaseImage;
+ this.sparkPodTemplate = sparkPodTemplate;
+ this.integrateWithHadoop = integrateWithHadoop;
+ this.dockerConfig = dockerConfig;
+ }
+
+ @Override
+ public String appName() {
+ return appName;
+ }
+
+ @Override
+ public String mainClass() {
+ return mainClass;
+ }
+
+ public String workspace() {
+ return workspace;
+ }
+
+ public String mainJar() {
+ return mainJar;
+ }
+
+ public SparkDeployMode deployMode() {
+ return deployMode;
+ }
+
+ public SparkJobType jobType() {
+ return jobType;
+ }
+
+ public SparkVersion sparkVersion() {
+ return sparkVersion;
+ }
+
+ public DependencyInfo dependencyInfo() {
+ return dependencyInfo;
+ }
+
+ public String k8sNamespace() {
+ return k8sNamespace;
+ }
+
+ public String sparkBaseImage() {
+ return sparkBaseImage;
+ }
+
+ public SparkK8sPodTemplates sparkPodTemplate() {
+ return sparkPodTemplate;
+ }
+
+ public boolean integrateWithHadoop() {
+ return integrateWithHadoop;
+ }
+
+ public DockerConf dockerConfig() {
+ return dockerConfig;
+ }
+}
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/SparkYarnBuildRequest.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/SparkYarnBuildRequest.java
new file mode 100644
index 000000000..4b035a351
--- /dev/null
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/SparkYarnBuildRequest.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.packer.pipeline;
+
+import org.apache.streampark.common.enums.SparkDeployMode;
+import org.apache.streampark.common.enums.SparkJobType;
+import org.apache.streampark.flink.packer.maven.DependencyInfo;
+
+public class SparkYarnBuildRequest implements BuildParam {
+
+ private final String appName;
+ private final String mainClass;
+ private final String localWorkspace;
+ private final String yarnProvidedPath;
+ private final SparkJobType jobType;
+ private final SparkDeployMode deployMode;
+ private final DependencyInfo dependencyInfo;
+
+ public SparkYarnBuildRequest(
+ String appName,
+ String mainClass,
+ String localWorkspace,
+ String yarnProvidedPath,
+ SparkJobType jobType,
+ SparkDeployMode deployMode,
+ DependencyInfo dependencyInfo) {
+ this.appName = appName;
+ this.mainClass = mainClass;
+ this.localWorkspace = localWorkspace;
+ this.yarnProvidedPath = yarnProvidedPath;
+ this.jobType = jobType;
+ this.deployMode = deployMode;
+ this.dependencyInfo = dependencyInfo;
+ }
+
+ @Override
+ public String appName() {
+ return appName;
+ }
+
+ @Override
+ public String mainClass() {
+ return mainClass;
+ }
+
+ public String localWorkspace() {
+ return localWorkspace;
+ }
+
+ public String yarnProvidedPath() {
+ return yarnProvidedPath;
+ }
+
+ public SparkJobType jobType() {
+ return jobType;
+ }
+
+ public SparkDeployMode deployMode() {
+ return deployMode;
+ }
+
+ public DependencyInfo dependencyInfo() {
+ return dependencyInfo;
+ }
+}
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/impl/FlinkK8sApplicationBuildPipeline.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/impl/FlinkK8sApplicationBuildPipeline.java
new file mode 100644
index 000000000..250ffa18b
--- /dev/null
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/impl/FlinkK8sApplicationBuildPipeline.java
@@ -0,0 +1,385 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.packer.pipeline.impl;
+
+import org.apache.streampark.common.fs.LfsOperator;
+import org.apache.streampark.common.util.ThreadUtils;
+import org.apache.streampark.flink.kubernetes.PodTemplateTool;
+import org.apache.streampark.flink.kubernetes.ingress.IngressController;
+import org.apache.streampark.flink.kubernetes.model.K8sPodTemplates;
+import org.apache.streampark.flink.packer.docker.DockerConf;
+import org.apache.streampark.flink.packer.docker.DockerUtils;
+import org.apache.streampark.flink.packer.docker.FlinkDockerfileTemplate;
+import org.apache.streampark.flink.packer.docker.FlinkDockerfileTemplateTrait;
+import org.apache.streampark.flink.packer.docker.FlinkHadoopDockerfileTemplate;
+import org.apache.streampark.flink.packer.maven.MavenTool;
+import org.apache.streampark.flink.packer.pipeline.BuildPipeline;
+import org.apache.streampark.flink.packer.pipeline.DockerBuildProgress;
+import org.apache.streampark.flink.packer.pipeline.DockerImageBuildResponse;
+import org.apache.streampark.flink.packer.pipeline.DockerProgressWatcher;
+import org.apache.streampark.flink.packer.pipeline.DockerPullProgress;
+import org.apache.streampark.flink.packer.pipeline.DockerPushProgress;
+import org.apache.streampark.flink.packer.pipeline.DockerResolveProgress;
+import org.apache.streampark.flink.packer.pipeline.FlinkK8sApplicationBuildRequest;
+import org.apache.streampark.flink.packer.pipeline.PipelineTypeEnum;
+import org.apache.streampark.flink.packer.pipeline.SilentDockerProgressWatcher;
+
+import org.apache.commons.lang3.StringUtils;
+
+import com.github.dockerjava.api.command.PushImageCmd;
+import com.github.dockerjava.core.command.HackBuildImageCmd;
+import com.github.dockerjava.core.command.HackPullImageCmd;
+import com.github.dockerjava.core.command.HackPushImageCmd;
+import com.google.common.collect.Sets;
+
+import java.io.File;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import scala.collection.JavaConverters;
+
+/** Building pipeline for flink kubernetes-native application mode */
+public class FlinkK8sApplicationBuildPipeline extends BuildPipeline {
+
+ private static final ExecutorService DOCKER_EXECUTOR =
+ new ThreadPoolExecutor(
+ Runtime.getRuntime().availableProcessors() * 5,
+ Runtime.getRuntime().availableProcessors() * 10,
+ 60L,
+ TimeUnit.SECONDS,
+ new LinkedBlockingQueue<>(2048),
+ ThreadUtils.threadFactory("streampark-docker-progress-watcher-executor"),
+ new ThreadPoolExecutor.DiscardOldestPolicy());
+
+ private final FlinkK8sApplicationBuildRequest request;
+
+ private DockerProgressWatcher dockerProcessWatcher = new SilentDockerProgressWatcher();
+
+ private final DockerResolveProgress dockerProcess =
+ new DockerResolveProgress(
+ DockerPullProgress.empty(),
+ DockerBuildProgress.empty(),
+ DockerPushProgress.empty());
+
+ public FlinkK8sApplicationBuildPipeline(FlinkK8sApplicationBuildRequest request) {
+ this.request = request;
+ }
+
+ @Override
+ public PipelineTypeEnum pipeType() {
+ return PipelineTypeEnum.FLINK_NATIVE_K8S_APPLICATION;
+ }
+
+ @Override
+ public FlinkK8sApplicationBuildRequest offerBuildParam() {
+ return request;
+ }
+
+ public void registerDockerProgressWatcher(DockerProgressWatcher watcher) {
+ this.dockerProcessWatcher = watcher;
+ }
+
+ @Override
+ public DockerImageBuildResponse buildProcess() {
+ String buildWorkspace =
+ execStep(
+ 1,
+ () -> {
+ String workspace =
+ request.workspace()
+ + "/"
+ + request.clusterId()
+ + "@"
+ + request.k8sNamespace();
+ LfsOperator.mkCleanDirs(workspace);
+ logInfo("Recreate building workspace: " + workspace);
+ return workspace;
+ })
+ .orElseThrow(() -> {
+ throw pipelineException();
+ });
+
+ Map<String, String> podTemplatePaths;
+ K8sPodTemplates podTemplate = request.flinkPodTemplate();
+ if (podTemplate.isEmpty()) {
+ skipStep(2);
+ podTemplatePaths = Collections.emptyMap();
+ } else {
+ podTemplatePaths =
+ execStep(
+ 2,
+ () -> {
+ Map<String, String> podTemplateFiles =
+ JavaConverters.mapAsJavaMap(
+ PodTemplateTool.preparePodTemplateFiles(
+ buildWorkspace, podTemplate)
+ .tmplFiles());
+ logInfo(
+ "Export flink podTemplates: "
+ + String.join(",", podTemplateFiles.values()));
+ return podTemplateFiles;
+ })
+ .orElseThrow(() -> {
+ throw pipelineException();
+ });
+ }
+
+ final File shadedJar =
+ execStep(
+ 3,
+ () -> {
+ String shadedJarOutputPath = request.getShadedJarPath(buildWorkspace);
+ File jar =
+ MavenTool.buildFatJar(
+ request.mainClass(),
+ request.providedLibs(),
+ shadedJarOutputPath);
+ logInfo("Output shaded flink job jar: " + jar.getAbsolutePath());
+ return jar;
+ })
+ .orElseThrow(
+ () -> {
+ throw pipelineException();
+ });
+ final Set<String> extJarLibs = request.dependencyInfo().extJarLibs();
+
+ File dockerfile;
+ FlinkDockerfileTemplateTrait dockerFileTemplate;
+ Object[] dockerResult =
+ execStep(
+ 4,
+ () -> {
+ FlinkDockerfileTemplateTrait template;
+ if (request.integrateWithHadoop()) {
+ template =
+ FlinkHadoopDockerfileTemplate.fromSystemHadoopConf(
+ buildWorkspace,
+ request.flinkBaseImage(),
+ shadedJar.getAbsolutePath(),
+ extJarLibs);
+ } else {
+ template =
+ new FlinkDockerfileTemplate(
+ buildWorkspace,
+ request.flinkBaseImage(),
+ shadedJar.getAbsolutePath(),
+ extJarLibs);
+ }
+ File dockerFile = template.writeDockerfile();
+ logInfo(
+ "Output flink dockerfile: "
+ + dockerFile.getAbsolutePath()
+ + ", content: \n"
+ + template.offerDockerfileContent());
+ return new Object[]{dockerFile, template};
+ })
+ .orElseThrow(() -> {
+ throw pipelineException();
+ });
+ dockerfile = (File) dockerResult[0];
+ dockerFileTemplate = (FlinkDockerfileTemplateTrait) dockerResult[1];
+
+ DockerConf dockerConf = request.dockerConfig();
+ String baseImageTag = request.flinkBaseImage().trim();
+ if (request.k8sNamespace().isEmpty() || request.clusterId().isEmpty()) {
+ throw new IllegalArgumentException("k8sNamespace or clusterId cannot be empty");
+ }
+ String expectedImageTag =
+ "streampark-flinkjob-" + request.k8sNamespace() + "-" + request.clusterId();
+ String pushImageTag =
+ compileTag(
+ expectedImageTag, dockerConf.registerAddress(), dockerConf.imageNamespace());
+
+ execStep(
+ 5,
+ () -> {
+ DockerUtils.usingDockerClient(
+ dockerClient -> {
+ boolean imgExists =
+ dockerClient.listImagesCmd().exec().stream()
+ .anyMatch(
+ image -> image.getRepoTags() != null
+ && java.util.Arrays.stream(image.getRepoTags())
+ .anyMatch(tag -> tag.contains(baseImageTag)));
+ if (imgExists) {
+ logInfo(
+ "found local docker image "
+ + baseImageTag
+ + ", no need to pull from remote.");
+ } else {
+ HackPullImageCmd pullImageCmd;
+ if (dockerConf.registerAddress() != null
+ && !baseImageTag.startsWith(dockerConf.registerAddress())) {
+ pullImageCmd =
+ (HackPullImageCmd) dockerClient.pullImageCmd(baseImageTag);
+ } else {
+ pullImageCmd =
+ (HackPullImageCmd) dockerClient
+ .pullImageCmd(baseImageTag)
+ .withAuthConfig(dockerConf.toAuthConf());
+ }
+ try {
+ pullImageCmd
+ .start(
+ DockerUtils.watchDockerPullProcess(
+ pullRsp -> {
+ dockerProcess.getPull().update(pullRsp);
+ DOCKER_EXECUTOR.submit(
+ () -> dockerProcessWatcher
+ .onDockerPullProgressChange(
+ dockerProcess.getPull().snapshot()));
+ }))
+ .awaitCompletion();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException(e);
+ }
+ logInfo(
+ "Already pulled docker image from remote register, imageTag="
+ + baseImageTag);
+ }
+ return null;
+ },
+ err -> {
+ throw new RuntimeException(
+ "Pull docker image failed, imageTag=" + baseImageTag, err);
+ });
+ return null;
+ })
+ .orElseThrow(() -> {
+ throw pipelineException();
+ });
+
+ execStep(
+ 6,
+ () -> {
+ DockerUtils.usingDockerClient(
+ dockerClient -> {
+ HackBuildImageCmd buildImageCmd =
+ (HackBuildImageCmd) dockerClient
+ .buildImageCmd()
+ .withBaseDirectory(new File(buildWorkspace))
+ .withDockerfile(dockerfile)
+ .withTags(Sets.newHashSet(pushImageTag));
+ String imageId =
+ buildImageCmd
+ .start(
+ DockerUtils.watchDockerBuildStep(
+ buildStep -> {
+ dockerProcess.getBuild().update(buildStep);
+ DOCKER_EXECUTOR.submit(
+ () -> dockerProcessWatcher
+ .onDockerBuildProgressChange(
+ dockerProcess.getBuild().snapshot()));
+ }))
+ .awaitImageId();
+ logInfo(
+ "Built docker image, imageId="
+ + imageId
+ + ", imageTag="
+ + pushImageTag);
+ return null;
+ },
+ err -> {
+ throw new RuntimeException(
+ "Build docker image failed. tag=" + pushImageTag, err);
+ });
+ return null;
+ })
+ .orElseThrow(() -> {
+ throw pipelineException();
+ });
+
+ execStep(
+ 7,
+ () -> {
+ DockerUtils.usingDockerClient(
+ dockerClient -> {
+ PushImageCmd pushCmd =
+ dockerClient
+ .pushImageCmd(pushImageTag)
+ .withAuthConfig(dockerConf.toAuthConf());
+ try {
+ ((HackPushImageCmd) pushCmd)
+ .start(
+ DockerUtils.watchDockerPushProcess(
+ pushRsp -> {
+ dockerProcess.getPush().update(pushRsp);
+ DOCKER_EXECUTOR.submit(
+ () -> dockerProcessWatcher.onDockerPushProgressChange(
+ dockerProcess.getPush().snapshot()));
+ }))
+ .awaitCompletion();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException(e);
+ }
+ logInfo("Already pushed docker image, imageTag=" + pushImageTag);
+ return null;
+ },
+ err -> {
+ throw new RuntimeException(
+ "Push docker image failed. tag=" + pushImageTag, err);
+ });
+ return null;
+ })
+ .orElseThrow(() -> {
+ throw pipelineException();
+ });
+
+ if (StringUtils.isBlank(request.ingressTemplate())) {
+ skipStep(8);
+ } else {
+ execStep(
+ 8,
+ () -> {
+ String ingressOutputPath =
+ IngressController.prepareIngressTemplateFiles(
+ buildWorkspace, request.ingressTemplate());
+ logInfo("Export flink ingress: " + ingressOutputPath);
+ return ingressOutputPath;
+ })
+ .orElseThrow(() -> {
+ throw pipelineException();
+ });
+ }
+
+ return new DockerImageBuildResponse(
+ buildWorkspace,
+ pushImageTag,
+ podTemplatePaths,
+ dockerFileTemplate.innerMainJarPath());
+ }
+
+ private String compileTag(String tag, String registerAddress, String imageNamespace) {
+ String tagName = tag.contains("/") ? tag : imageNamespace + "/" + tag;
+ if (StringUtils.isNotBlank(registerAddress) && !tagName.startsWith(registerAddress)) {
+ tagName = registerAddress + "/" + tagName;
+ }
+ return tagName.toLowerCase();
+ }
+
+ public static FlinkK8sApplicationBuildPipeline of(FlinkK8sApplicationBuildRequest request) {
+ return new FlinkK8sApplicationBuildPipeline(request);
+ }
+}
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/impl/FlinkK8sSessionBuildPipeline.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/impl/FlinkK8sSessionBuildPipeline.java
new file mode 100644
index 000000000..33296075c
--- /dev/null
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/impl/FlinkK8sSessionBuildPipeline.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.packer.pipeline.impl;
+
+import org.apache.streampark.common.fs.LfsOperator;
+import org.apache.streampark.flink.packer.maven.MavenTool;
+import org.apache.streampark.flink.packer.pipeline.BuildPipeline;
+import org.apache.streampark.flink.packer.pipeline.FlinkK8sSessionBuildRequest;
+import org.apache.streampark.flink.packer.pipeline.PipelineTypeEnum;
+import org.apache.streampark.flink.packer.pipeline.ShadedBuildResponse;
+
+import java.io.File;
+
+/** Building pipeline for flink kubernetes-native session mode */
+public class FlinkK8sSessionBuildPipeline extends BuildPipeline {
+
+ private final FlinkK8sSessionBuildRequest request;
+
+ public FlinkK8sSessionBuildPipeline(FlinkK8sSessionBuildRequest request) {
+ this.request = request;
+ }
+
+ @Override
+ public PipelineTypeEnum pipeType() {
+ return PipelineTypeEnum.FLINK_NATIVE_K8S_SESSION;
+ }
+
+ @Override
+ public FlinkK8sSessionBuildRequest offerBuildParam() {
+ return request;
+ }
+
+ @Override
+ public ShadedBuildResponse buildProcess() {
+ String buildWorkspace =
+ execStep(
+ 1,
+ () -> {
+ String workspace =
+ request.workspace()
+ + "/"
+ + request.clusterId()
+ + "@"
+ + request.k8sNamespace();
+ LfsOperator.mkCleanDirs(workspace);
+ logInfo("Recreate building workspace: " + workspace);
+ return workspace;
+ })
+ .orElseThrow(() -> {
+ throw pipelineException();
+ });
+
+ File shadedJar =
+ execStep(
+ 2,
+ () -> {
+ File output =
+ MavenTool.buildFatJar(
+ request.mainClass(),
+ request.providedLibs(),
+ request.getShadedJarPath(buildWorkspace));
+ logInfo("Output shaded flink job jar: " + output.getAbsolutePath());
+ return output;
+ })
+ .orElseThrow(() -> {
+ throw pipelineException();
+ });
+
+ return new ShadedBuildResponse(buildWorkspace, shadedJar.getAbsolutePath());
+ }
+
+ public static FlinkK8sSessionBuildPipeline of(FlinkK8sSessionBuildRequest request) {
+ return new FlinkK8sSessionBuildPipeline(request);
+ }
+}
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/impl/FlinkRemoteBuildPipeline.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/impl/FlinkRemoteBuildPipeline.java
new file mode 100644
index 000000000..a901826f5
--- /dev/null
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/impl/FlinkRemoteBuildPipeline.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.packer.pipeline.impl;
+
+import org.apache.streampark.common.enums.FlinkJobType;
+import org.apache.streampark.common.fs.FsOperator;
+import org.apache.streampark.common.fs.LfsOperator;
+import org.apache.streampark.flink.packer.maven.MavenTool;
+import org.apache.streampark.flink.packer.pipeline.BuildPipeline;
+import org.apache.streampark.flink.packer.pipeline.FlinkRemotePerJobBuildRequest;
+import org.apache.streampark.flink.packer.pipeline.PipelineTypeEnum;
+import org.apache.streampark.flink.packer.pipeline.ShadedBuildResponse;
+
+import java.io.File;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/** Building pipeline for flink standalone session mode */
+public class FlinkRemoteBuildPipeline extends BuildPipeline {
+
+ private final FlinkRemotePerJobBuildRequest request;
+
+ public FlinkRemoteBuildPipeline(FlinkRemotePerJobBuildRequest request) {
+ this.request = request;
+ }
+
+ @Override
+ public PipelineTypeEnum pipeType() {
+ return PipelineTypeEnum.FLINK_STANDALONE;
+ }
+
+ @Override
+ public FlinkRemotePerJobBuildRequest offerBuildParam() {
+ return request;
+ }
+
+ @Override
+ public ShadedBuildResponse buildProcess() {
+ if (request.skipBuild()) {
+ return new ShadedBuildResponse(request.workspace(), request.customFlinkUserJar());
+ }
+
+ execStep(1, () -> {
+ LfsOperator.mkCleanDirs(request.workspace());
+ logInfo("Recreate building workspace: " + request.workspace());
+ return null;
+ }).orElseThrow(() -> {
+ throw pipelineException();
+ });
+
+ File shadedJar =
+ execStep(
+ 2,
+ () -> {
+ if (request.flinkJobType() == FlinkJobType.FLINK_SQL) {
+ File output =
+ MavenTool.buildFatJar(
+ request.mainClass(),
+ request.providedLibs(),
+ request.getShadedJarPath(request.workspace()));
+ logInfo("output shaded flink job jar: " + output.getAbsolutePath());
+ return output;
+ }
+ return new File(request.customFlinkUserJar());
+ })
+ .orElseThrow(() -> {
+ throw pipelineException();
+ });
+
+ List<String> mavenJars =
+ execStep(
+ 3,
+ () -> {
+ if (request.flinkJobType() == FlinkJobType.PYFLINK) {
+ List<File> mavenArts =
+ MavenTool.resolveArtifacts(request.dependencyInfo().mavenArts());
+ List<String> paths =
+ mavenArts.stream()
+ .map(File::getAbsolutePath)
+ .collect(Collectors.toList());
+ paths.addAll(request.dependencyInfo().extJarLibs());
+ return paths;
+ }
+ return Collections.<String>emptyList();
+ })
+ .orElseThrow(() -> {
+ throw pipelineException();
+ });
+
+ execStep(
+ 4,
+ () -> {
+ if (request.flinkJobType() == FlinkJobType.PYFLINK) {
+ FsOperator lfs = FsOperator.lfs();
+ String lib = request.workspace().concat("/lib");
+ lfs.mkdirsIfNotExists(lib);
+ for (String jar : mavenJars) {
+ File originFile = new File(jar);
+ if (originFile.isFile()) {
+ lfs.copy(originFile.getAbsolutePath(), lib);
+ }
+ }
+ }
+ return null;
+ })
+ .orElseThrow(() -> {
+ throw pipelineException();
+ });
+
+ return new ShadedBuildResponse(request.workspace(), shadedJar.getAbsolutePath());
+ }
+
+ public static FlinkRemoteBuildPipeline of(FlinkRemotePerJobBuildRequest request) {
+ return new FlinkRemoteBuildPipeline(request);
+ }
+}
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/impl/FlinkYarnApplicationBuildPipeline.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/impl/FlinkYarnApplicationBuildPipeline.java
new file mode 100644
index 000000000..0bf759194
--- /dev/null
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/impl/FlinkYarnApplicationBuildPipeline.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.packer.pipeline.impl;
+
+import org.apache.streampark.common.conf.Workspace;
+import org.apache.streampark.common.enums.FlinkJobType;
+import org.apache.streampark.common.fs.FsOperator;
+import org.apache.streampark.common.fs.HdfsOperator;
+import org.apache.streampark.common.fs.LfsOperator;
+import org.apache.streampark.common.util.AutoCloseUtils;
+import org.apache.streampark.flink.packer.maven.MavenTool;
+import org.apache.streampark.flink.packer.pipeline.BuildPipeline;
+import org.apache.streampark.flink.packer.pipeline.FlinkYarnApplicationBuildRequest;
+import org.apache.streampark.flink.packer.pipeline.PipelineTypeEnum;
+import org.apache.streampark.flink.packer.pipeline.SimpleBuildResponse;
+
+import org.apache.commons.codec.digest.DigestUtils;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/** Building pipeline for flink yarn application mode */
+public class FlinkYarnApplicationBuildPipeline extends BuildPipeline {
+
+ private final FlinkYarnApplicationBuildRequest request;
+
+ public FlinkYarnApplicationBuildPipeline(FlinkYarnApplicationBuildRequest request) {
+ this.request = request;
+ }
+
+ @Override
+ public PipelineTypeEnum pipeType() {
+ return PipelineTypeEnum.FLINK_YARN_APPLICATION;
+ }
+
+ @Override
+ public FlinkYarnApplicationBuildRequest offerBuildParam() {
+ return request;
+ }
+
+ @Override
+ public SimpleBuildResponse buildProcess() {
+ execStep(
+ 1,
+ () -> {
+ if (request.flinkJobType() == FlinkJobType.FLINK_SQL
+ || request.flinkJobType() == FlinkJobType.PYFLINK) {
+ LfsOperator.mkCleanDirs(request.localWorkspace());
+ HdfsOperator.mkCleanDirs(request.yarnProvidedPath());
+ }
+ logInfo("Recreate building workspace: " + request.yarnProvidedPath());
+ return null;
+ })
+ .orElseThrow(() -> {
+ throw pipelineException();
+ });
+
+ List<String> mavenJars =
+ execStep(
+ 2,
+ () -> {
+ if (request.flinkJobType() == FlinkJobType.FLINK_SQL
+ || request.flinkJobType() == FlinkJobType.PYFLINK) {
+ List<File> mavenArts =
+ MavenTool.resolveArtifacts(request.dependencyInfo().mavenArts());
+ List<String> paths =
+ mavenArts.stream()
+ .map(File::getAbsolutePath)
+ .collect(Collectors.toList());
+ paths.addAll(request.dependencyInfo().extJarLibs());
+ return paths;
+ }
+ return Collections.<String>emptyList();
+ })
+ .orElseThrow(() -> {
+ throw pipelineException();
+ });
+
+ execStep(
+ 3,
+ () -> {
+ for (String jar : mavenJars) {
+ uploadJarToHdfsOrLfs(FsOperator.lfs(), jar, request.localWorkspace());
+ uploadJarToHdfsOrLfs(FsOperator.hdfs(), jar, request.yarnProvidedPath());
+ }
+ return null;
+ })
+ .orElseThrow(() -> {
+ throw pipelineException();
+ });
+
+ return new SimpleBuildResponse();
+ }
+
+ private void uploadJarToHdfsOrLfs(FsOperator fsOperator, String origin, String target) throws IOException {
+ File originFile = new File(origin);
+ if (!fsOperator.exists(target)) {
+ fsOperator.mkdirs(target);
+ }
+ if (originFile.isFile()) {
+ if (fsOperator == FsOperator.lfs()) {
+ fsOperator.copy(originFile.getAbsolutePath(), target);
+ } else {
+ String uploadFile = Workspace.remote().APP_UPLOADS() + "/" + originFile.getName();
+ if (fsOperator.exists(uploadFile)) {
+ AutoCloseUtils.using(
+ new FileInputStream(originFile),
+ inputStream -> {
+ try {
+ if (!DigestUtils.md5Hex(inputStream)
+ .equals(fsOperator.fileMd5(uploadFile))) {
+ fsOperator.upload(originFile.getAbsolutePath(), uploadFile);
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return null;
+ });
+ } else {
+ fsOperator.upload(originFile.getAbsolutePath(), uploadFile);
+ }
+ fsOperator.copy(uploadFile, target);
+ }
+ } else if (fsOperator == FsOperator.hdfs()) {
+ fsOperator.upload(originFile.getAbsolutePath(), target);
+ }
+ }
+
+ public static FlinkYarnApplicationBuildPipeline of(FlinkYarnApplicationBuildRequest request) {
+ return new FlinkYarnApplicationBuildPipeline(request);
+ }
+}
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/impl/SparkK8sApplicationBuildPipeline.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/impl/SparkK8sApplicationBuildPipeline.java
new file mode 100644
index 000000000..8b1b8d125
--- /dev/null
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/impl/SparkK8sApplicationBuildPipeline.java
@@ -0,0 +1,347 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.packer.pipeline.impl;
+
+import org.apache.streampark.common.fs.LfsOperator;
+import org.apache.streampark.common.util.ThreadUtils;
+import org.apache.streampark.flink.kubernetes.PodTemplateTool;
+import org.apache.streampark.flink.packer.docker.DockerConf;
+import org.apache.streampark.flink.packer.docker.DockerUtils;
+import org.apache.streampark.flink.packer.docker.SparkDockerfileTemplate;
+import org.apache.streampark.flink.packer.docker.SparkDockerfileTemplateTrait;
+import org.apache.streampark.flink.packer.docker.SparkHadoopDockerfileTemplate;
+import org.apache.streampark.flink.packer.pipeline.BuildPipeline;
+import org.apache.streampark.flink.packer.pipeline.DockerBuildProgress;
+import org.apache.streampark.flink.packer.pipeline.DockerImageBuildResponse;
+import org.apache.streampark.flink.packer.pipeline.DockerProgressWatcher;
+import org.apache.streampark.flink.packer.pipeline.DockerPullProgress;
+import org.apache.streampark.flink.packer.pipeline.DockerPushProgress;
+import org.apache.streampark.flink.packer.pipeline.DockerResolveProgress;
+import org.apache.streampark.flink.packer.pipeline.PipelineTypeEnum;
+import org.apache.streampark.flink.packer.pipeline.SilentDockerProgressWatcher;
+import org.apache.streampark.flink.packer.pipeline.SparkK8sApplicationBuildRequest;
+import org.apache.streampark.spark.kubernetes.model.SparkK8sPodTemplates;
+
+import org.apache.commons.lang3.StringUtils;
+
+import com.github.dockerjava.api.command.PushImageCmd;
+import com.github.dockerjava.core.command.HackBuildImageCmd;
+import com.github.dockerjava.core.command.HackPullImageCmd;
+import com.github.dockerjava.core.command.HackPushImageCmd;
+import com.google.common.collect.Sets;
+
+import java.io.File;
+import java.nio.file.Paths;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import scala.collection.JavaConverters;
+
+/** Building pipeline for Spark kubernetes-native application mode */
+public class SparkK8sApplicationBuildPipeline extends BuildPipeline {
+
+ private static final ExecutorService DOCKER_EXECUTOR =
+ new ThreadPoolExecutor(
+ Runtime.getRuntime().availableProcessors() * 5,
+ Runtime.getRuntime().availableProcessors() * 10,
+ 60L,
+ TimeUnit.SECONDS,
+ new LinkedBlockingQueue<>(2048),
+ ThreadUtils.threadFactory("streampark-docker-progress-watcher-executor"),
+ new ThreadPoolExecutor.DiscardOldestPolicy());
+
+ private final SparkK8sApplicationBuildRequest request;
+
+ private DockerProgressWatcher dockerProcessWatcher = new SilentDockerProgressWatcher();
+
+ private final DockerResolveProgress dockerProcess =
+ new DockerResolveProgress(
+ DockerPullProgress.empty(),
+ DockerBuildProgress.empty(),
+ DockerPushProgress.empty());
+
+ public SparkK8sApplicationBuildPipeline(SparkK8sApplicationBuildRequest request) {
+ this.request = request;
+ }
+
+ @Override
+ public PipelineTypeEnum pipeType() {
+ return PipelineTypeEnum.SPARK_NATIVE_K8S_APPLICATION;
+ }
+
+ @Override
+ public SparkK8sApplicationBuildRequest offerBuildParam() {
+ return request;
+ }
+
+ public void registerDockerProgressWatcher(DockerProgressWatcher watcher) {
+ this.dockerProcessWatcher = watcher;
+ }
+
+ @Override
+ public DockerImageBuildResponse buildProcess() {
+ String buildWorkspace =
+ execStep(
+ 1,
+ () -> {
+ String workspace = request.workspace() + "/" + request.k8sNamespace();
+ LfsOperator.mkCleanDirs(workspace);
+ logInfo("Recreate building workspace: " + workspace);
+ return workspace;
+ })
+ .orElseThrow(() -> {
+ throw pipelineException();
+ });
+
+ Map<String, String> podTemplatePaths;
+ SparkK8sPodTemplates podTemplate = request.sparkPodTemplate();
+ if (podTemplate.isEmpty()) {
+ skipStep(2);
+ podTemplatePaths = Collections.emptyMap();
+ } else {
+ podTemplatePaths =
+ execStep(
+ 2,
+ () -> {
+ Map<String, String> podTemplateFiles =
+ JavaConverters.mapAsJavaMap(
+ PodTemplateTool.preparePodTemplateFiles(
+ buildWorkspace, podTemplate)
+ .tmplFiles());
+ logInfo(
+ "Export spark podTemplates: "
+ + String.join(",", podTemplateFiles.values()));
+ return podTemplateFiles;
+ })
+ .orElseThrow(() -> {
+ throw pipelineException();
+ });
+ }
+
+ final String mainJarPath =
+ execStep(
+ 3,
+ () -> {
+ String mainJarName =
+ Paths.get(request.mainJar()).getFileName().toString();
+ String path = buildWorkspace + "/" + mainJarName;
+ LfsOperator.copy(request.mainJar(), path);
+ logInfo("Prepared spark job jar: " + path);
+ return path;
+ })
+ .orElseThrow(
+ () -> {
+ throw pipelineException();
+ });
+ final Set<String> extJarLibs = new HashSet<>();
+
+ File dockerfile;
+ SparkDockerfileTemplateTrait dockerFileTemplate;
+ Object[] dockerResult =
+ execStep(
+ 4,
+ () -> {
+ SparkDockerfileTemplateTrait template;
+ if (request.integrateWithHadoop()) {
+ template =
+ SparkHadoopDockerfileTemplate.fromSystemHadoopConf(
+ buildWorkspace,
+ request.sparkBaseImage(),
+ mainJarPath,
+ extJarLibs);
+ } else {
+ template =
+ new SparkDockerfileTemplate(
+ buildWorkspace,
+ request.sparkBaseImage(),
+ mainJarPath,
+ extJarLibs);
+ }
+ File dockerFile = template.writeDockerfile();
+ logInfo(
+ "Output spark dockerfile: "
+ + dockerFile.getAbsolutePath()
+ + ", content: \n"
+ + template.offerDockerfileContent());
+ return new Object[]{dockerFile, template};
+ })
+ .orElseThrow(() -> {
+ throw pipelineException();
+ });
+ dockerfile = (File) dockerResult[0];
+ dockerFileTemplate = (SparkDockerfileTemplateTrait) dockerResult[1];
+
+ DockerConf dockerConf = request.dockerConfig();
+ String baseImageTag = request.sparkBaseImage().trim();
+ if (request.k8sNamespace().isEmpty() || request.appName().isEmpty()) {
+ throw new IllegalArgumentException("k8sNamespace or appName cannot be empty");
+ }
+ String expectedImageTag =
+ "streampark-sparkjob-" + request.k8sNamespace() + "-" + request.appName();
+ String pushImageTag =
+ compileTag(
+ expectedImageTag, dockerConf.registerAddress(), dockerConf.imageNamespace());
+
+ execStep(
+ 5,
+ () -> {
+ DockerUtils.usingDockerClient(
+ dockerClient -> {
+ HackPullImageCmd pullImageCmd;
+ if (dockerConf.registerAddress() != null
+ && !baseImageTag.startsWith(dockerConf.registerAddress())) {
+ pullImageCmd =
+ (HackPullImageCmd) dockerClient.pullImageCmd(baseImageTag);
+ } else {
+ pullImageCmd =
+ (HackPullImageCmd) dockerClient
+ .pullImageCmd(baseImageTag)
+ .withAuthConfig(dockerConf.toAuthConf());
+ }
+ try {
+ pullImageCmd
+ .start(
+ DockerUtils.watchDockerPullProcess(
+ pullRsp -> {
+ dockerProcess.getPull().update(pullRsp);
+ DOCKER_EXECUTOR.submit(
+ () -> dockerProcessWatcher.onDockerPullProgressChange(
+ dockerProcess.getPull().snapshot()));
+ }))
+ .awaitCompletion();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException(e);
+ }
+ logInfo(
+ "Already pulled docker image from remote register, imageTag="
+ + baseImageTag);
+ return null;
+ },
+ err -> {
+ throw new RuntimeException(
+ "Pull docker image failed, imageTag=" + baseImageTag, err);
+ });
+ return null;
+ })
+ .orElseThrow(() -> {
+ throw pipelineException();
+ });
+
+ execStep(
+ 6,
+ () -> {
+ DockerUtils.usingDockerClient(
+ dockerClient -> {
+ HackBuildImageCmd buildImageCmd =
+ (HackBuildImageCmd) dockerClient
+ .buildImageCmd()
+ .withBaseDirectory(new File(buildWorkspace))
+ .withDockerfile(dockerfile)
+ .withTags(Sets.newHashSet(pushImageTag));
+ String imageId =
+ buildImageCmd
+ .start(
+ DockerUtils.watchDockerBuildStep(
+ buildStep -> {
+ dockerProcess.getBuild().update(buildStep);
+ DOCKER_EXECUTOR.submit(
+ () -> dockerProcessWatcher
+ .onDockerBuildProgressChange(
+ dockerProcess.getBuild().snapshot()));
+ }))
+ .awaitImageId();
+ logInfo(
+ "Built docker image, imageId="
+ + imageId
+ + ", imageTag="
+ + pushImageTag);
+ return null;
+ },
+ err -> {
+ throw new RuntimeException(
+ "Build docker image failed. tag=" + pushImageTag, err);
+ });
+ return null;
+ })
+ .orElseThrow(() -> {
+ throw pipelineException();
+ });
+
+ execStep(
+ 7,
+ () -> {
+ DockerUtils.usingDockerClient(
+ dockerClient -> {
+ PushImageCmd pushCmd =
+ dockerClient
+ .pushImageCmd(pushImageTag)
+ .withAuthConfig(dockerConf.toAuthConf());
+ try {
+ ((HackPushImageCmd) pushCmd)
+ .start(
+ DockerUtils.watchDockerPushProcess(
+ pushRsp -> {
+ dockerProcess.getPush().update(pushRsp);
+ DOCKER_EXECUTOR.submit(
+ () -> dockerProcessWatcher.onDockerPushProgressChange(
+ dockerProcess.getPush().snapshot()));
+ }))
+ .awaitCompletion();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException(e);
+ }
+ logInfo("Already pushed docker image, imageTag=" + pushImageTag);
+ return null;
+ },
+ err -> {
+ throw new RuntimeException(
+ "Push docker image failed. tag=" + pushImageTag, err);
+ });
+ return null;
+ })
+ .orElseThrow(() -> {
+ throw pipelineException();
+ });
+
+ return new DockerImageBuildResponse(
+ buildWorkspace,
+ pushImageTag,
+ podTemplatePaths,
+ dockerFileTemplate.innerMainJarPath());
+ }
+
+ private String compileTag(String tag, String registerAddress, String imageNamespace) {
+ String tagName = tag.contains("/") ? tag : imageNamespace + "/" + tag;
+ if (StringUtils.isNotBlank(registerAddress) && !tagName.startsWith(registerAddress)) {
+ tagName = registerAddress + "/" + tagName;
+ }
+ return tagName.toLowerCase();
+ }
+
+ public static SparkK8sApplicationBuildPipeline of(SparkK8sApplicationBuildRequest request) {
+ return new SparkK8sApplicationBuildPipeline(request);
+ }
+}
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/impl/SparkYarnBuildPipeline.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/impl/SparkYarnBuildPipeline.java
new file mode 100644
index 000000000..52cc4f590
--- /dev/null
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/impl/SparkYarnBuildPipeline.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.packer.pipeline.impl;
+
+import org.apache.streampark.common.conf.Workspace;
+import org.apache.streampark.common.enums.SparkJobType;
+import org.apache.streampark.common.fs.FsOperator;
+import org.apache.streampark.common.fs.HdfsOperator;
+import org.apache.streampark.common.fs.LfsOperator;
+import org.apache.streampark.common.util.AutoCloseUtils;
+import org.apache.streampark.flink.packer.maven.MavenTool;
+import org.apache.streampark.flink.packer.pipeline.BuildPipeline;
+import org.apache.streampark.flink.packer.pipeline.PipelineTypeEnum;
+import org.apache.streampark.flink.packer.pipeline.SimpleBuildResponse;
+import org.apache.streampark.flink.packer.pipeline.SparkYarnBuildRequest;
+
+import org.apache.commons.codec.digest.DigestUtils;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/** Building pipeline for spark yarn application mode */
+public class SparkYarnBuildPipeline extends BuildPipeline {
+
+ private final SparkYarnBuildRequest request;
+
+ public SparkYarnBuildPipeline(SparkYarnBuildRequest request) {
+ this.request = request;
+ }
+
+ @Override
+ public PipelineTypeEnum pipeType() {
+ return PipelineTypeEnum.SPARK_CLUSTER;
+ }
+
+ @Override
+ public SparkYarnBuildRequest offerBuildParam() {
+ return request;
+ }
+
+ @Override
+ public SimpleBuildResponse buildProcess() {
+ execStep(
+ 1,
+ () -> {
+ if (request.jobType() == SparkJobType.SPARK_SQL) {
+ LfsOperator.mkCleanDirs(request.localWorkspace());
+ HdfsOperator.mkCleanDirs(request.yarnProvidedPath());
+ }
+ logInfo("Recreate building workspace: " + request.yarnProvidedPath());
+ return null;
+ })
+ .orElseThrow(() -> {
+ throw pipelineException();
+ });
+
+ List<String> mavenJars =
+ execStep(
+ 2,
+ () -> {
+ if (request.jobType() == SparkJobType.SPARK_SQL) {
+ List<File> mavenArts =
+ MavenTool.resolveArtifacts(request.dependencyInfo().mavenArts());
+ List<String> paths =
+ mavenArts.stream()
+ .map(File::getAbsolutePath)
+ .collect(Collectors.toList());
+ paths.addAll(request.dependencyInfo().extJarLibs());
+ return paths;
+ }
+ return Collections.<String>emptyList();
+ })
+ .orElseThrow(() -> {
+ throw pipelineException();
+ });
+
+ execStep(
+ 3,
+ () -> {
+ for (String jar : mavenJars) {
+ uploadJarToHdfsOrLfs(FsOperator.lfs(), jar, request.localWorkspace());
+ uploadJarToHdfsOrLfs(FsOperator.hdfs(), jar, request.yarnProvidedPath());
+ }
+ return null;
+ })
+ .orElseThrow(() -> {
+ throw pipelineException();
+ });
+
+ return new SimpleBuildResponse();
+ }
+
+ private void uploadJarToHdfsOrLfs(FsOperator fsOperator, String origin, String target) throws IOException {
+ File originFile = new File(origin);
+ if (!fsOperator.exists(target)) {
+ fsOperator.mkdirs(target);
+ }
+ if (originFile.isFile()) {
+ if (fsOperator == FsOperator.lfs()) {
+ fsOperator.copy(originFile.getAbsolutePath(), target);
+ } else {
+ String uploadFile = Workspace.remote().APP_UPLOADS() + "/" + originFile.getName();
+ if (fsOperator.exists(uploadFile)) {
+ AutoCloseUtils.using(
+ new FileInputStream(originFile),
+ inputStream -> {
+ try {
+ if (!DigestUtils.md5Hex(inputStream)
+ .equals(fsOperator.fileMd5(uploadFile))) {
+ fsOperator.upload(originFile.getAbsolutePath(), uploadFile);
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return null;
+ });
+ } else {
+ fsOperator.upload(originFile.getAbsolutePath(), uploadFile);
+ }
+ fsOperator.copy(uploadFile, target);
+ }
+ } else if (fsOperator == FsOperator.hdfs()) {
+ fsOperator.upload(originFile.getAbsolutePath(), target);
+ }
+ }
+
+ public static SparkYarnBuildPipeline of(SparkYarnBuildRequest request) {
+ return new SparkYarnBuildPipeline(request);
+ }
+}
diff --git a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/PackerResourceGC.scala b/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/PackerResourceGC.scala
deleted file mode 100644
index e37aac507..000000000
--- a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/PackerResourceGC.scala
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.packer
-
-import org.apache.streampark.common.conf.Workspace
-import org.apache.streampark.common.constants.Constants
-import org.apache.streampark.common.util.Logger
-
-import org.apache.commons.io.FileUtils
-
-import java.io.File
-
-import scala.util.Try
-
-/** Garbage resource collector during packing. */
-object PackerResourceGC extends Logger {
-
- private val appWorkspacePath: String = Workspace.local.APP_WORKSPACE
-
- /**
- * Start a building legacy resources collection process.
- *
- * @param expiredHours
- * Expected expiration time of building resources.
- */
- def startGc(expiredHours: Integer): Unit = {
- val appWorkspace = new File(appWorkspacePath)
- if (!appWorkspace.exists()) return
- val evictedBarrier = System.currentTimeMillis - expiredHours * 3600 * 1000
-
- // find flink building path that should be evicted, which are based on pattern
- // matching of filename.
- val evictedFiles = appWorkspace.listFiles
- .filter(_.isDirectory)
- .filter(_.getName.contains("@"))
- .flatMap(findLastModifiedOfSubFile)
- .filter(_._2 < evictedBarrier)
- .map(_._1)
-
- if (evictedFiles.isEmpty) return
- logInfo(s"Delete expired building resources, ${evictedFiles.mkString(", ")}")
- evictedFiles.foreach(path => Try(FileUtils.deleteDirectory(path)))
- }
-
- private def findLastModifiedOfSubFile(file: File): Array[(File, Long)] = {
- val isApplicationMode =
- file.listFiles.map(_.getName).exists(_.contains(Constants.JAR_SUFFIX))
- if (isApplicationMode) {
- Array(file -> file.listFiles.map(_.lastModified).max)
- } else {
- file.listFiles
- .filter(_.isDirectory)
- .map(subFile => subFile -> subFile.listFiles.map(_.lastModified).max)
- }
- }
-
-}
diff --git a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/DockerConf.scala b/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/DockerConf.scala
deleted file mode 100644
index c53a117a2..000000000
--- a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/DockerConf.scala
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.packer.docker
-
-import com.github.dockerjava.api.model.AuthConfig
-
-import javax.annotation.Nullable
-
-/**
- * Authentication Configuration of Remote Docker Register
- *
- * @param registerAddress
- * docker image regoster address. when this configuration item is empty, it means that the
- * dockerhub public repository is used.
- * @param registerUsername
- * login username of docker image regoster.
- * @param registerPassword
- * login password of docker image regoster.
- */
-case class DockerConf(
- @Nullable registerAddress: String,
- imageNamespace: String,
- registerUsername: String,
- registerPassword: String) {
-
- /** covert to com.github.docker.java.api.model.AuthConfig */
- def toAuthConf: AuthConfig = new AuthConfig()
- .withRegistryAddress(registerAddress)
- .withUsername(registerUsername)
- .withPassword(registerPassword)
-
-}
-
-object DockerConf {
-
- def of(
- @Nullable registerAddress: String,
- imageNameSpace: String,
- registerUsername: String,
- registerPassword: String): DockerConf =
- DockerConf(registerAddress, imageNameSpace, registerUsername, registerPassword)
-}
diff --git a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/DockerRetriever.scala b/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/DockerRetriever.scala
deleted file mode 100644
index d966a751e..000000000
--- a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/DockerRetriever.scala
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.packer.docker
-
-import org.apache.streampark.common.conf.{CommonConfig, InternalConfigHolder}
-import org.apache.streampark.common.util.Utils
-
-import com.github.dockerjava.api.DockerClient
-import com.github.dockerjava.core.{DefaultDockerClientConfig, DockerClientConfig, HackDockerClient}
-import com.github.dockerjava.httpclient5.ApacheDockerHttpClient
-
-import java.net.URI
-import java.time.Duration
-
-object DockerRetriever {
-
- /**
- * docker config param from system properties, see in
- * https://github.com/docker-java/docker-java/blob/master/docs/getting_started.md#properties-docker-javaproperties
- * todo support custom docker configuration parameters in unified configurations in the future
- */
- lazy val dockerClientConf: DockerClientConfig = {
- DefaultDockerClientConfig.createDefaultConfigBuilder().build()
- }
-
- /**
- * docker http client builder, use ApacheDockerHttpClient by default todo support custom http
- * client configuration parameters in unified configurations in the future
- */
- private lazy val dockerHttpClientBuilder: ApacheDockerHttpClient.Builder =
- new ApacheDockerHttpClient.Builder()
- .dockerHost(dockerClientConf.getDockerHost)
- .sslConfig(dockerClientConf.getSSLConfig)
- .maxConnections(InternalConfigHolder.get(CommonConfig.DOCKER_MAX_CONNECTIONS))
- .connectionTimeout(
- Duration.ofSeconds(InternalConfigHolder.get(CommonConfig.DOCKER_CONNECTION_TIMEOUT_SEC)))
- .responseTimeout(Duration.ofSeconds(
- InternalConfigHolder.get(CommonConfig.DOCKER_RESPONSE_TIMEOUT_SEC)))
-
- /** get new DockerClient instance */
- def newDockerClient(): DockerClient = {
- setDockerHost()
- HackDockerClient.getInstance(dockerClientConf, dockerHttpClientBuilder.build())
- }
-
- /** set docker-host for kata */
- private def setDockerHost(): Unit = {
- val dockerHost: String = InternalConfigHolder.get(CommonConfig.DOCKER_HOST)
- if (Utils.isNotEmpty(dockerHost)) {
- val dockerHostUri: URI = new URI(dockerHost)
- dockerHttpClientBuilder.dockerHost(dockerHostUri)
- }
- }
-
-}
diff --git a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/FlinkDockerfileTemplate.scala b/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/FlinkDockerfileTemplate.scala
deleted file mode 100644
index b2a11c59e..000000000
--- a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/FlinkDockerfileTemplate.scala
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.packer.docker
-
-/**
- * Base flink docker file image template.
- *
- * @param workspacePath
- * Path of dockerfile workspace, it should be a directory.
- * @param flinkBaseImage
- * Flink base docker image name, see https://hub.docker.com/_/flink.
- * @param flinkMainJarPath
- * Path of flink job main jar which would copy to $FLINK_HOME/usrlib/
- * @param flinkExtraLibPaths
- * Path of additional flink lib path which would copy to $FLINK_HOME/lib/
- */
-case class FlinkDockerfileTemplate(
- workspacePath: String,
- flinkBaseImage: String,
- flinkMainJarPath: String,
- flinkExtraLibPaths: Set[String])
- extends FlinkDockerfileTemplateTrait {
-
- /** offer content of DockerFile */
- override def offerDockerfileContent: String = {
- s"""FROM $flinkBaseImage
- |RUN mkdir -p $FLINK_HOME/usrlib
- |COPY $extraLibName $FLINK_HOME/lib/
- |COPY $mainJarName $FLINK_HOME/usrlib/$mainJarName
- |""".stripMargin
- }
-
-}
diff --git a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/FlinkDockerfileTemplateTrait.scala b/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/FlinkDockerfileTemplateTrait.scala
deleted file mode 100644
index 95208388b..000000000
--- a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/FlinkDockerfileTemplateTrait.scala
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.packer.docker
-
-import org.apache.streampark.common.constants.Constants
-import org.apache.streampark.common.fs.LfsOperator
-
-import org.apache.commons.io.FileUtils
-
-import java.io.File
-import java.nio.file.{Path, Paths}
-
-/** Flink image dockerfile template. */
-trait FlinkDockerfileTemplateTrait {
-
- /** Path of dockerfile workspace, it should be a directory. */
- def workspacePath: String
-
- /** Flink base docker image name, see https://hub.docker.com/_/flink. */
- def flinkBaseImage: String
-
- /** Path of flink job main jar which would copy to $FLINK_HOME/usrlib/ */
- def flinkMainJarPath: String
-
- /** Path of additional flink lib path which would copy to $FLINK_HOME/lib/ */
- def flinkExtraLibPaths: Set[String]
-
- /** Offer content of DockerFile. */
- def offerDockerfileContent: String
-
- /** Startup flink main jar path inner Docker */
- def innerMainJarPath: String = s"local:///opt/flink/usrlib/$mainJarName"
-
- /** output dockerfile name */
- protected val DEFAULT_DOCKER_FILE_NAME = "Dockerfile"
- protected val FLINK_LIB_PATH = "lib"
- protected val FLINK_HOME: String = "$FLINK_HOME"
-
- /** Dockerfile building workspace. */
- lazy val workspace: Path = {
- val path = Paths.get(workspacePath).toAbsolutePath
- if (!LfsOperator.exists(workspacePath)) LfsOperator.mkdirs(workspacePath)
- path
- }
-
- /**
- * flink main jar name, the main jar would copy from `flinkMainjarPath` to
- * `workspacePath/mainJarName.jar`.
- */
- lazy val mainJarName: String = {
- val mainJarPath = Paths.get(flinkMainJarPath).toAbsolutePath
- if (mainJarPath.getParent != workspace) {
- LfsOperator.copy(
- mainJarPath.toString,
- s"${workspace.toString}/${mainJarPath.getFileName.toString}")
- }
- mainJarPath.getFileName.toString
- }
-
- /**
- * flink extra jar lib, the jar file in `flinkExtraLibPaths` would be copyed into
- * `FLINK_LIB_PATH`.
- */
- lazy val extraLibName: String = {
- LfsOperator.mkCleanDirs(s"${workspace.toString}/$FLINK_LIB_PATH")
- flinkExtraLibPaths
- .map(new File(_))
- .filter(_.exists())
- .filter(_.getName.endsWith(Constants.JAR_SUFFIX))
- .flatMap {
- case f if f.isDirectory =>
- f.listFiles
- .filter(_.isFile)
- .filter(_.getName.endsWith(Constants.JAR_SUFFIX))
- .map(_.getAbsolutePath)
- case f if f.isFile => Array(f.getAbsolutePath)
- }
- .foreach(LfsOperator.copy(_, s"${workspace.toString}/$FLINK_LIB_PATH"))
- FLINK_LIB_PATH
- }
-
- /**
- * write content of DockerFile to outputPath, the output dockerfile name is "dockerfile".
- *
- * @return
- * File Object for actual output Dockerfile
- */
- def writeDockerfile: File = {
- val output = new File(s"$workspacePath/$DEFAULT_DOCKER_FILE_NAME")
- FileUtils.write(output, offerDockerfileContent, "UTF-8")
- output
- }
-
- /**
- * write content of DockerFile to outputPath using specified output dockerfile name.
- *
- * @return
- * File Object for actual output Dockerfile
- */
- def writeDockerfile(dockerfileName: String): File = {
- val output = new File(s"$workspacePath/$dockerfileName")
- FileUtils.write(output, offerDockerfileContent, "UTF-8")
- output
- }
-
-}
diff --git a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/FlinkHadoopDockerfileTemplate.scala b/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/FlinkHadoopDockerfileTemplate.scala
deleted file mode 100644
index c7c53d671..000000000
--- a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/FlinkHadoopDockerfileTemplate.scala
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.packer.docker
-
-import org.apache.streampark.common.fs.LfsOperator
-import org.apache.streampark.common.util.HadoopConfigUtils
-
-import javax.annotation.Nullable
-
-import java.nio.file.Paths
-
-/**
- * flink-hadoop integration docker image template.
- *
- * @param workspacePath
- * Path of dockerfile workspace, it should be a directory.
- * @param flinkBaseImage
- * Flink base docker image name, see https://hub.docker.com/_/flink.
- * @param flinkMainJarPath
- * Path of flink job main jar which would copy to $FLINK_HOME/usrlib/
- * @param flinkExtraLibPaths
- * Path of additional flink lib path which would copy to $FLINK_HOME/lib/
- * @param hadoopConfDirPath
- * Path of hadoop conf directory.
- * @param hiveConfDirPath
- * Path of hive conf directory.
- */
-case class FlinkHadoopDockerfileTemplate(
- workspacePath: String,
- flinkBaseImage: String,
- flinkMainJarPath: String,
- flinkExtraLibPaths: Set[String],
- @Nullable hadoopConfDirPath: String,
- @Nullable hiveConfDirPath: String)
- extends FlinkDockerfileTemplateTrait {
-
- val hadoopConfDir: String =
- workspace
- .relativize(Paths.get(Option(hadoopConfDirPath).getOrElse("")))
- .toString
-
- val hiveConfDir: String =
- workspace
- .relativize(Paths.get(Option(hiveConfDirPath).getOrElse("")))
- .toString
-
- /** offer content of DockerFile */
- override def offerDockerfileContent: String = {
- var dockerfile =
- s"""FROM $flinkBaseImage
- |RUN mkdir -p $FLINK_HOME/usrlib
- |""".stripMargin
- if (hadoopConfDir.nonEmpty) {
- dockerfile +=
- s"""
- |COPY $hadoopConfDir /opt/hadoop-conf
- |ENV HADOOP_CONF_DIR /opt/hadoop-conf
- |""".stripMargin
- }
- if (hiveConfDir.nonEmpty) {
- dockerfile +=
- s"""
- |COPY $hiveConfDir /opt/hive-conf
- |ENV HIVE_CONF_DIR /opt/hive-conf
- |""".stripMargin
- }
- dockerfile +=
- s"""
- |COPY $extraLibName $FLINK_HOME/lib/
- |COPY $mainJarName $FLINK_HOME/usrlib/$mainJarName
- |""".stripMargin
- dockerfile
- }
-
-}
-
-object FlinkHadoopDockerfileTemplate {
-
- /** Use relevant system variables as the value of hadoopConfDirPath, hiveConfDirPath. */
- def fromSystemHadoopConf(
- workspacePath: String,
- flinkBaseImage: String,
- flinkMainJarPath: String,
- flinkExtraLibPaths: Set[String]): FlinkHadoopDockerfileTemplate = {
- // get hadoop and hive config directory from system and copy to workspacePath
- val hadoopConfDir = HadoopConfigUtils.getSystemHadoopConfDir match {
- case None => ""
- case hadoopConf if !LfsOperator.exists(hadoopConf.get) => ""
- case hadoopConf =>
- val dstDir = s"$workspacePath/hadoop-conf"
- LfsOperator.mkCleanDirs(dstDir)
- LfsOperator.copyDir(hadoopConf.get, dstDir)
- dstDir
- }
- val hiveConfDir = HadoopConfigUtils.getSystemHiveConfDir match {
- case None => ""
- case hiveConf if !LfsOperator.exists(hiveConf.get) => ""
- case hiveConf =>
- val dstDir = s"$workspacePath/hive-conf"
- LfsOperator.mkCleanDirs(dstDir)
- LfsOperator.copyDir(hiveConf.get, dstDir)
- dstDir
- }
- FlinkHadoopDockerfileTemplate(
- workspacePath,
- flinkBaseImage,
- flinkMainJarPath,
- flinkExtraLibPaths,
- hadoopConfDir,
- hiveConfDir)
- }
-
-}
diff --git a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/SparkDockerfileTemplate.scala b/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/SparkDockerfileTemplate.scala
deleted file mode 100644
index c4f9b14a6..000000000
--- a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/SparkDockerfileTemplate.scala
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.packer.docker
-
-/**
- * Base spark docker file image template.
- *
- * @param workspacePath
- * Path of dockerfile workspace, it should be a directory.
- * @param sparkBaseImage
- * Spark base docker image name, see https://hub.docker.com/r/apache/spark.
- * @param sparkMainJarPath
- * Path of spark job main jar which would copy to $SPARK_HOME/usrlib/
- * @param sparkExtraLibPaths
- * Path of additional spark lib path which would copy to $SPARK_HOME/lib/
- */
-case class SparkDockerfileTemplate(
- workspacePath: String,
- sparkBaseImage: String,
- sparkMainJarPath: String,
- sparkExtraLibPaths: Set[String])
- extends SparkDockerfileTemplateTrait {
-
- /** offer content of DockerFile */
- override def offerDockerfileContent: String = {
- s"""FROM $sparkBaseImage
- |USER root
- |RUN mkdir -p $SPARK_HOME/usrlib
- |COPY $mainJarName $SPARK_HOME/usrlib/$mainJarName
- |COPY $extraLibName $SPARK_HOME/lib/
- |""".stripMargin
- }
-}
diff --git a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/SparkDockerfileTemplateTrait.scala b/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/SparkDockerfileTemplateTrait.scala
deleted file mode 100644
index 729aca65b..000000000
--- a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/SparkDockerfileTemplateTrait.scala
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.packer.docker
-
-import org.apache.streampark.common.constants.Constants
-import org.apache.streampark.common.fs.LfsOperator
-
-import org.apache.commons.io.FileUtils
-
-import java.io.File
-import java.nio.file.{Path, Paths}
-
-/** Spark image dockerfile template. */
-trait SparkDockerfileTemplateTrait {
-
- /** Path of dockerfile workspace, it should be a directory. */
- def workspacePath: String
-
- /** Spark base docker image name, see https://hub.docker.com/r/apache/spark */
- def sparkBaseImage: String
-
- /** Path of spark job main jar which would copy to $SPARK_HOME/usrlib/ */
- def sparkMainJarPath: String
-
- /** Path of additional spark lib path which would copy to $SPARK_HOME/lib/ */
- def sparkExtraLibPaths: Set[String]
-
- /** Offer content of DockerFile. */
- def offerDockerfileContent: String
-
- /** Startup spark main jar path inner Docker */
- def innerMainJarPath: String = s"local:///opt/spark/usrlib/$mainJarName"
-
- /** output dockerfile name */
- protected val DEFAULT_DOCKER_FILE_NAME = "Dockerfile"
- protected val SPARK_LIB_PATH = "lib"
- protected val SPARK_HOME: String = "$SPARK_HOME"
-
- /** Dockerfile building workspace. */
- lazy val workspace: Path = {
- val path = Paths.get(workspacePath).toAbsolutePath
- if (!LfsOperator.exists(workspacePath)) LfsOperator.mkdirs(workspacePath)
- path
- }
-
- /**
- * spark main jar name, the main jar would copy from `sparkMainjarPath` to
- * `workspacePath/mainJarName.jar`.
- */
- lazy val mainJarName: String = {
- val mainJarPath = Paths.get(sparkMainJarPath).toAbsolutePath
- if (mainJarPath.getParent != workspace) {
- LfsOperator.copy(
- mainJarPath.toString,
- s"${workspace.toString}/${mainJarPath.getFileName.toString}")
- }
- mainJarPath.getFileName.toString
- }
-
- /**
- * spark extra jar lib, the jar file in `sparkExtraLibPaths` would be copyed into
- * `SPARK_LIB_PATH`.
- */
- lazy val extraLibName: String = {
- LfsOperator.mkCleanDirs(s"${workspace.toString}/$SPARK_LIB_PATH")
- sparkExtraLibPaths
- .map(new File(_))
- .filter(_.exists())
- .filter(_.getName.endsWith(Constants.JAR_SUFFIX))
- .flatMap {
- case f if f.isDirectory =>
- f.listFiles
- .filter(_.isFile)
- .filter(_.getName.endsWith(Constants.JAR_SUFFIX))
- .map(_.getAbsolutePath)
- case f if f.isFile => Array(f.getAbsolutePath)
- }
- .foreach(LfsOperator.copy(_, s"${workspace.toString}/$SPARK_LIB_PATH"))
- SPARK_LIB_PATH
- }
-
- /**
- * write content of DockerFile to outputPath, the output dockerfile name is "dockerfile".
- *
- * @return
- * File Object for actual output Dockerfile
- */
- def writeDockerfile: File = {
- val output = new File(s"$workspacePath/$DEFAULT_DOCKER_FILE_NAME")
- FileUtils.write(output, offerDockerfileContent, "UTF-8")
- output
- }
-
- /**
- * write content of DockerFile to outputPath using specified output dockerfile name.
- *
- * @return
- * File Object for actual output Dockerfile
- */
- def writeDockerfile(dockerfileName: String): File = {
- val output = new File(s"$workspacePath/$dockerfileName")
- FileUtils.write(output, offerDockerfileContent, "UTF-8")
- output
- }
-
-}
diff --git a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/SparkHadoopDockerfileTemplate.scala b/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/SparkHadoopDockerfileTemplate.scala
deleted file mode 100644
index 9bc3d0f84..000000000
--- a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/SparkHadoopDockerfileTemplate.scala
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.packer.docker
-
-import org.apache.streampark.common.fs.LfsOperator
-import org.apache.streampark.common.util.HadoopConfigUtils
-
-import javax.annotation.Nullable
-
-import java.nio.file.Paths
-
-/**
- * flink-hadoop integration docker image template.
- *
- * @param workspacePath
- * Path of dockerfile workspace, it should be a directory.
- * @param sparkBaseImage
- * Flink base docker image name, see https://hub.docker.com/_/flink.
- * @param sparkMainJarPath
- * Path of spark job main jar which would copy to $FLINK_HOME/usrlib/
- * @param sparkExtraLibPaths
- * Path of additional flink lib path which would copy to $FLINK_HOME/lib/
- * @param hadoopConfDirPath
- * Path of hadoop conf directory.
- * @param hiveConfDirPath
- * Path of hive conf directory.
- */
-case class SparkHadoopDockerfileTemplate(
- workspacePath: String,
- sparkBaseImage: String,
- sparkMainJarPath: String,
- sparkExtraLibPaths: Set[String],
- @Nullable hadoopConfDirPath: String,
- @Nullable hiveConfDirPath: String)
- extends SparkDockerfileTemplateTrait {
-
- val hadoopConfDir: String =
- workspace
- .relativize(Paths.get(Option(hadoopConfDirPath).getOrElse("")))
- .toString
-
- val hiveConfDir: String =
- workspace
- .relativize(Paths.get(Option(hiveConfDirPath).getOrElse("")))
- .toString
-
- /** offer content of DockerFile */
- override def offerDockerfileContent: String = {
- var dockerfile =
- s"""FROM $sparkBaseImage
- |RUN mkdir -p $SPARK_HOME/usrlib
- |""".stripMargin
- if (hadoopConfDir.nonEmpty) {
- dockerfile +=
- s"""
- |COPY $hadoopConfDir /opt/hadoop-conf
- |ENV HADOOP_CONF_DIR /opt/hadoop-conf
- |""".stripMargin
- }
- if (hiveConfDir.nonEmpty) {
- dockerfile +=
- s"""
- |COPY $hiveConfDir /opt/hive-conf
- |ENV HIVE_CONF_DIR /opt/hive-conf
- |""".stripMargin
- }
- dockerfile +=
- s"""
- |COPY $extraLibName $SPARK_HOME/lib/
- |COPY $mainJarName $SPARK_HOME/usrlib/$mainJarName
- |""".stripMargin
- dockerfile
- }
-
-}
-
-object SparkHadoopDockerfileTemplate {
-
- /** Use relevant system variables as the value of hadoopConfDirPath, hiveConfDirPath. */
- def fromSystemHadoopConf(
- workspacePath: String,
- sparkBaseImage: String,
- sparkMainJarPath: String,
- sparkExtraLibPaths: Set[String]): SparkHadoopDockerfileTemplate = {
- // get hadoop and hive config directory from system and copy to workspacePath
- val hadoopConfDir = HadoopConfigUtils.getSystemHadoopConfDir match {
- case None => ""
- case hadoopConf if !LfsOperator.exists(hadoopConf.get) => ""
- case hadoopConf =>
- val dstDir = s"$workspacePath/hadoop-conf"
- LfsOperator.mkCleanDirs(dstDir)
- LfsOperator.copyDir(hadoopConf.get, dstDir)
- dstDir
- }
- val hiveConfDir = HadoopConfigUtils.getSystemHiveConfDir match {
- case None => ""
- case hiveConf if !LfsOperator.exists(hiveConf.get) => ""
- case hiveConf =>
- val dstDir = s"$workspacePath/hive-conf"
- LfsOperator.mkCleanDirs(dstDir)
- LfsOperator.copyDir(hiveConf.get, dstDir)
- dstDir
- }
- SparkHadoopDockerfileTemplate(
- workspacePath,
- sparkBaseImage,
- sparkMainJarPath,
- sparkExtraLibPaths,
- hadoopConfDir,
- hiveConfDir)
- }
-
-}
diff --git a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/package.scala b/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/package.scala
deleted file mode 100644
index 790fce723..000000000
--- a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/docker/package.scala
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.packer
-
-import com.github.dockerjava.api.DockerClient
-import com.github.dockerjava.api.listener.{BuildImageCallbackListener, PullImageCallbackListener, PushImageCallbackListener}
-import com.github.dockerjava.api.model.{PullResponseItem, PushResponseItem}
-
-import scala.util.{Failure, Success, Try}
-
-package object docker {
-
- def watchDockerBuildStep(func: String => Unit): BuildImageCallbackListener =
- new BuildImageCallbackListener() {
- def watchBuildStep(buildStepMsg: String): Unit = func(buildStepMsg)
- }
-
- def watchDockerPullProcess(func: PullResponseItem => Unit): PullImageCallbackListener =
- new PullImageCallbackListener {
- override def watchPullProcess(processDetail: PullResponseItem): Unit =
- func(processDetail)
- }
-
- def watchDockerPushProcess(func: PushResponseItem => Unit): PushImageCallbackListener =
- new PushImageCallbackListener {
- override def watchPushProcess(processDetail: PushResponseItem): Unit =
- func(processDetail)
- }
-
- def usingDockerClient[R](process: DockerClient => R)(handleException: Throwable => R): R = {
- Try(DockerRetriever.newDockerClient()) match {
- case Success(client) =>
- val r = process(client)
- client.close()
- r
- case Failure(e) => handleException(e)
- }
- }
-
-}
diff --git a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/maven/Artifact.scala b/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/maven/Artifact.scala
deleted file mode 100644
index cff3c0398..000000000
--- a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/maven/Artifact.scala
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.packer.maven
-
-import org.eclipse.aether.artifact.{Artifact => AetherArtifact}
-
-import java.util.regex.Pattern
-
-case class Artifact(
- groupId: String,
- artifactId: String,
- version: String,
- classifier: String = null) {
-
- def eq(artifact: AetherArtifact): Boolean = {
- artifact.getGroupId match {
- case g if g == groupId =>
- artifact.getArtifactId match {
- case "*" => true
- case a => a == artifactId
- }
- case _ => false
- }
- }
-
-}
-
-object Artifact {
-
- private lazy val PATTERN = Pattern.compile("([^: ]+):([^: ]+):([^: ]+)")
-
- /** build from coords */
- def of(coords: String): Artifact = {
- PATTERN.matcher(coords) match {
- case m if m.matches() =>
- val g = m.group(1)
- val a = m.group(2)
- val v = m.group(3)
- Artifact(g, a, v)
- case _ =>
- throw new IllegalArgumentException(
- s"Bad artifact coordinates $coords, expected format is <groupId>:<artifactId>[:<extension>[:<classifier>]]:<version>")
- }
- }
-}
diff --git a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/maven/DependencyInfo.scala b/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/maven/DependencyInfo.scala
deleted file mode 100644
index bc035bdd8..000000000
--- a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/maven/DependencyInfo.scala
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.packer.maven
-
-import org.apache.streampark.common.util.Implicits._
-
-/**
- * @param mavenArts
- * collection of maven artifacts
- * @param extJarLibs
- * collection of jar lib paths, which elements can be a directory or file path.
- */
-case class DependencyInfo(mavenArts: Set[Artifact] = Set(), extJarLibs: Set[String] = Set()) {
-
- def this(mavenArts: JavaList[Artifact], extJarLibs: JavaList[String]) {
- this(mavenArts.toSet, extJarLibs.toSet)
- }
-
- def merge(jarLibs: Set[String]): DependencyInfo =
- if (jarLibs != null) {
- DependencyInfo(mavenArts, extJarLibs ++ jarLibs)
- } else {
- this.copy()
- }
-
- def merge(mvnPoms: JavaList[Artifact], jarLibs: JavaList[String]): DependencyInfo =
- if (mvnPoms != null && jarLibs != null) {
- DependencyInfo(mavenArts ++ mvnPoms.toSet, extJarLibs ++ jarLibs.toSet)
- } else if (mvnPoms != null) {
- DependencyInfo(mavenArts ++ mvnPoms.toSet, extJarLibs)
- } else if (jarLibs != null) {
- DependencyInfo(mavenArts, extJarLibs ++ jarLibs.toSet)
- } else {
- this.copy()
- }
-}
-
-object DependencyInfo {
- def empty: DependencyInfo = new DependencyInfo()
-}
diff --git a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/maven/MavenTool.scala b/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/maven/MavenTool.scala
deleted file mode 100644
index 9855eae8d..000000000
--- a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/maven/MavenTool.scala
+++ /dev/null
@@ -1,268 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.packer.maven
-
-import org.apache.streampark.common.conf.{InternalConfigHolder, Workspace}
-import org.apache.streampark.common.conf.CommonConfig.{MAVEN_AUTH_PASSWORD, MAVEN_AUTH_USER, MAVEN_REMOTE_URL}
-import org.apache.streampark.common.constants.Constants
-import org.apache.streampark.common.util.{AssertUtils, Logger, Utils}
-import org.apache.streampark.common.util.Implicits._
-
-import com.google.common.collect.Lists
-import org.apache.maven.plugins.shade.{DefaultShader, ShadeRequest}
-import org.apache.maven.plugins.shade.filter.Filter
-import org.apache.maven.plugins.shade.resource.{ManifestResourceTransformer, ResourceTransformer, ServicesResourceTransformer}
-import org.apache.maven.repository.internal.MavenRepositorySystemUtils
-import org.codehaus.plexus.logging.{Logger => PlexusLog}
-import org.codehaus.plexus.logging.console.ConsoleLogger
-import org.eclipse.aether.{RepositorySystem, RepositorySystemSession}
-import org.eclipse.aether.artifact.DefaultArtifact
-import org.eclipse.aether.connector.basic.BasicRepositoryConnectorFactory
-import org.eclipse.aether.repository.{LocalRepository, RemoteRepository}
-import org.eclipse.aether.resolution.{ArtifactDescriptorRequest, ArtifactRequest}
-import org.eclipse.aether.spi.connector.RepositoryConnectorFactory
-import org.eclipse.aether.spi.connector.transport.TransporterFactory
-import org.eclipse.aether.transport.file.FileTransporterFactory
-import org.eclipse.aether.transport.http.HttpTransporterFactory
-import org.eclipse.aether.util.repository.AuthenticationBuilder
-
-import javax.annotation.{Nonnull, Nullable}
-
-import java.io.File
-import java.util
-
-import scala.collection.mutable.ArrayBuffer
-import scala.util.Try
-
-object MavenTool extends Logger {
-
- private[this] lazy val plexusLog =
- new ConsoleLogger(PlexusLog.LEVEL_INFO, "streampark-maven")
-
- private[this] val excludeArtifact = List(
- Artifact.of("org.apache.flink:force-shading:*"),
- Artifact.of("com.google.code.findbugs:jsr305:*"),
- Artifact.of("org.apache.logging.log4j:*:*"))
-
- private[this] def getRemoteRepos(): List[RemoteRepository] = {
- val builder =
- new RemoteRepository.Builder(
- "central",
- Constants.DEFAULT,
- InternalConfigHolder.get(MAVEN_REMOTE_URL))
- val remoteRepository = {
- val buildState =
- InternalConfigHolder.get(MAVEN_AUTH_USER) == null || InternalConfigHolder.get(
- MAVEN_AUTH_PASSWORD) == null
- if (buildState) {
- builder.build()
- } else {
- val authentication = new AuthenticationBuilder()
- .addUsername(InternalConfigHolder.get[String](MAVEN_AUTH_USER))
- .addPassword(InternalConfigHolder.get[String](MAVEN_AUTH_PASSWORD))
- .build()
- builder.setAuthentication(authentication).build()
- }
- }
- List(remoteRepository)
- }
-
- private val isJarFile = (file: File) =>
- file.isFile && Try(Utils.requireCheckJarFile(file.toURI.toURL)).isSuccess
-
- /**
- * Build a fat-jar with custom jar libraries.
- *
- * @param jarLibs
- * list of jar lib paths for building fat-jar
- * @param outFatJarPath
- * output paths of fat-jar, like "/streampark/workspace/233/my-fat.jar"
- * @return
- * File Object of output fat-jar
- */
- @throws[Exception]
- def buildFatJar(
- @Nullable mainClass: String,
- @Nonnull jarLibs: Set[String],
- @Nonnull outFatJarPath: String): File = {
- // check userJarPath
- val uberJar = new File(outFatJarPath)
- require(
- outFatJarPath.endsWith(Constants.JAR_SUFFIX) && !uberJar.isDirectory,
- s"[StreamPark] streampark-packer: outFatJarPath($outFatJarPath) should be a JAR file.")
- uberJar.delete()
- // resolve all jarLibs
- val jarSet = new util.HashSet[File]
- jarLibs
- .map(lib => new File(lib))
- .filter(_.exists)
- .foreach {
- case libFile if isJarFile(libFile) => jarSet.add(libFile)
- case libFile if libFile.isDirectory =>
- libFile.listFiles.filter(isJarFile).foreach(jarSet.add)
- case _ =>
- }
- logInfo(s"start shaded fat-jar: ${jarLibs.mkString(",")}")
- // shade jars
- val shadeRequest = {
- val req = new ShadeRequest
- req.setJars(jarSet)
- req.setUberJar(uberJar)
-
- val transformer = ArrayBuffer[ResourceTransformer]()
- // ref https://ci.apache.org/projects/flink/flink-docs-master/docs/connectors/table/overview/#transform-table-connectorformat-resources
- transformer += new ServicesResourceTransformer()
- if (mainClass != null) {
- val manifest = new ManifestResourceTransformer()
- manifest.setMainClass(mainClass)
- transformer += manifest
- }
- req.setResourceTransformers(transformer.toList)
- req.setFilters(List(new ShadeFilter))
- req.setRelocators(Lists.newArrayList())
- req
- }
- val shader = new DefaultShader()
- shader.enableLogging(plexusLog)
- shader.shade(shadeRequest)
- logInfo(s"finish build fat-jar: ${uberJar.getAbsolutePath}")
- uberJar
- }
-
- /**
- * Build a fat-jar with custom jar librarties and maven artifacts.
- *
- * @param dependencyInfo
- * maven artifacts and jar libraries for building a fat-jar
- * @param outFatJarPath
- * output paths of fat-jar, like "/streampark/workspace/233/my-fat.jar"
- */
- @throws[Exception]
- def buildFatJar(
- @Nullable mainClass: String,
- @Nonnull dependencyInfo: DependencyInfo,
- @Nonnull outFatJarPath: String): File = {
- val jarLibs = dependencyInfo.extJarLibs
- val arts = dependencyInfo.mavenArts
- AssertUtils.required(
- !(jarLibs.isEmpty && arts.isEmpty),
- s"[StreamPark] streampark-packer: empty artifacts.")
-
- val artFilePaths = resolveArtifacts(arts).map(_.getAbsolutePath)
- buildFatJar(mainClass, jarLibs ++ artFilePaths, outFatJarPath)
- }
-
- @throws[Exception]
- def resolveArtifacts(mavenArtifact: Artifact): JavaList[File] =
- resolveArtifacts(Set(mavenArtifact))
-
- /**
- * Resolve the collectoin of artifacts, Artifacts will be download to ConfigConst.MAVEN_LOCAL_DIR
- * if necessary. notes: Only compile scope dependencies will be resolved.
- *
- * @param mavenArtifacts
- * collection of maven artifacts
- * @return
- * jar File Object of resolved artifacts
- */
- @throws[Exception]
- def resolveArtifacts(mavenArtifacts: JavaSet[Artifact]): JavaList[File] = {
- if (mavenArtifacts == null) List.empty[File]
- else {
- val (repoSystem, session) = getMavenEndpoint()
- val artifacts = mavenArtifacts.map(e => {
- val artifact =
- new DefaultArtifact(e.groupId, e.artifactId, e.classifier, "jar", e.version)
- artifact.getProperties
- artifact
- })
- logInfo(s"start resolving dependencies: ${artifacts.mkString}")
-
- val remoteRepos = getRemoteRepos()
- // read relevant artifact descriptor info
- // plz don't simplify the following lambda syntax to maintain the readability of the code.
- val resolvedArtifacts = artifacts
- .map(artifact => new ArtifactDescriptorRequest(artifact, remoteRepos, null))
- .map(artDescReq => repoSystem.readArtifactDescriptor(session, artDescReq))
- .flatMap(_.getDependencies)
- .filter(_.getScope == "compile")
- .filter(x => !excludeArtifact.exists(_.eq(x.getArtifact)))
- .map(_.getArtifact)
-
- val mergedArtifacts = artifacts ++ resolvedArtifacts
- logInfo(s"resolved dependencies: ${mergedArtifacts.mkString}")
-
- // download artifacts
- val artReqs =
- mergedArtifacts.map(artifact => new ArtifactRequest(artifact, remoteRepos, null))
- repoSystem
- .resolveArtifacts(session, artReqs)
- .map(_.getArtifact.getFile)
- .toList
- }
- }
-
- /** create composite maven endpoint */
- private[this] def getMavenEndpoint(): (RepositorySystem, RepositorySystemSession) = {
-
- /** create maven repository endpoint */
-
- lazy val locator = MavenRepositorySystemUtils.newServiceLocator
-
- /** default maven local repository */
- lazy val localRepo = new LocalRepository(Workspace.MAVEN_LOCAL_PATH)
-
- def newRepoSystem(): RepositorySystem = {
- locator.addService(
- classOf[RepositoryConnectorFactory],
- classOf[BasicRepositoryConnectorFactory])
- locator.addService(classOf[TransporterFactory], classOf[FileTransporterFactory])
- locator.addService(classOf[TransporterFactory], classOf[HttpTransporterFactory])
- locator.getService(classOf[RepositorySystem])
- }
-
- /** create maven repository session endpoint */
- def newSession(system: RepositorySystem): RepositorySystemSession = {
- val session = MavenRepositorySystemUtils.newSession
- session.setLocalRepositoryManager(system.newLocalRepositoryManager(session, localRepo))
- session
- }
-
- val repoSystem = newRepoSystem()
- val session = newSession(repoSystem)
- (repoSystem, session)
- }
-
- private[this] class ShadeFilter extends Filter {
- override def canFilter(jar: File): Boolean = true
-
- override def isFiltered(name: String): Boolean = {
- val isFilteredState =
- name.startsWith("META-INF/") && name.endsWith(".SF") || name.endsWith(".DSA") || name
- .endsWith(".RSA")
- if (isFilteredState) {
- logInfo(s"shade ignore file: $name")
- return true
- }
- false
- }
-
- override def finished(): Unit = {}
- }
-
-}
diff --git a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/BuildPipeline.scala b/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/BuildPipeline.scala
deleted file mode 100644
index 25062f94b..000000000
--- a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/BuildPipeline.scala
+++ /dev/null
@@ -1,202 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.packer.pipeline
-
-import org.apache.streampark.common.util.{Logger, ThreadUtils}
-import org.apache.streampark.common.util.Implicits._
-import org.apache.streampark.flink.packer.pipeline.BuildPipeline.executor
-
-import java.util.concurrent.{Callable, LinkedBlockingQueue, ThreadPoolExecutor, TimeUnit}
-
-import scala.collection.mutable
-import scala.concurrent.{ExecutionContext, ExecutionContextExecutorService}
-import scala.util.{Failure, Success, Try}
-
-/** Behavior that BuildPipeline subclasses must inherit to implement. */
-trait BuildPipelineProcess {
-
- /** the type of pipeline */
- def pipeType: PipelineTypeEnum
-
- /**
- * the actual build process. the effective steps progress should be implemented in multiple
- * BuildPipeline.execStep() functions.
- */
- @throws[Throwable]
- protected def buildProcess(): BuildResult
-
- /** the build params of build process */
- protected def offerBuildParam: BuildParam
-}
-
-/** Callable methods exposed by BuildPipeline to the outside. */
-trait BuildPipelineExpose {
-
- /** get current state of the pipeline instance */
- def getPipeStatus: PipelineStatusEnum
-
- /** get error of pipeline instance */
- def getError: PipeError
-
- /** get all of the steps status StepSeq -> (PipeStepStatus -> status update timestamp) */
- def getStepsStatus: Map[Int, (PipelineStepStatusEnum, Long)]
-
- /** get current build step index */
- def getCurStep: Int
-
- /** get count of all build steps */
- def allSteps: Int
-
- /** launch the pipeline instance */
- def launch(): BuildResult
-
- def as[T <: BuildPipeline](implicit clz: Class[T]): T = this.asInstanceOf[T]
-}
-
-/** Building pipeline trait. */
-trait BuildPipeline extends BuildPipelineProcess with BuildPipelineExpose with Logger {
-
- protected var pipeStatus: PipelineStatusEnum = PipelineStatusEnum.pending
-
- protected var error: PipeError = PipeError.empty()
-
- protected var curStep: Int = 0
-
- protected val stepsStatus: mutable.Map[Int, (PipelineStepStatusEnum, Long)] =
- mutable.Map(
- pipeType.getSteps
- .map(e => e._1.toInt -> (PipelineStepStatusEnum.waiting -> System.currentTimeMillis))
- .toSeq: _*)
-
- /** use to identify the log record that belongs to which pipeline instance */
- private val logSuffix: String = s"appName=${offerBuildParam.appName}"
-
- protected var watcher: PipeWatcher = new SilentPipeWatcher
-
- def registerWatcher(watcher: PipeWatcher): BuildPipeline = {
- this.watcher = watcher
- this
- }
-
- protected def execStep[R](seq: Int)(process: => R): Option[R] = {
- Try {
- curStep = seq
- stepsStatus(seq) = PipelineStepStatusEnum.running -> System.currentTimeMillis
- logInfo(s"Building pipeline step[$seq/$allSteps] running => ${pipeType.getSteps.get(seq)}")
- watcher.onStepStateChange(snapshot)
- process
- } match {
- case Success(result) =>
- stepsStatus(seq) = PipelineStepStatusEnum.success -> System.currentTimeMillis
- logInfo(s"Building pipeline step[$seq/$allSteps] success")
- watcher.onStepStateChange(snapshot)
- Some(result)
- case Failure(cause) =>
- stepsStatus(seq) = PipelineStepStatusEnum.failure -> System.currentTimeMillis
- pipeStatus = PipelineStatusEnum.failure
- error = PipeError.of(cause.getMessage, cause)
- logInfo(s"Building pipeline step[$seq/$allSteps] failure => ${pipeType.getSteps
- .get(seq)}")
- watcher.onStepStateChange(snapshot)
- None
- }
- }
-
- protected def skipStep(step: Int): Unit = {
- curStep = step
- stepsStatus(step) = PipelineStepStatusEnum.skipped -> System.currentTimeMillis
- logInfo(s"Building pipeline step[$step/$allSteps] skipped => ${pipeType.getSteps.get(step)}")
- watcher.onStepStateChange(snapshot)
- }
-
- /** Launch the building pipeline. */
- override def launch(): BuildResult = {
- pipeStatus = PipelineStatusEnum.running
- Try {
- watcher.onStart(snapshot)
- logInfo(s"Building pipeline is launching, params=${offerBuildParam.toString}")
- executor
- .submit(new Callable[BuildResult] {
- override def call(): BuildResult = buildProcess()
- })
- .get(20, TimeUnit.MINUTES)
- } match {
- case Success(result) =>
- pipeStatus = PipelineStatusEnum.success
- logInfo(s"Building pipeline has finished successfully.")
- watcher.onFinish(snapshot, result)
- result
- case Failure(cause) =>
- pipeStatus = PipelineStatusEnum.failure
- error = PipeError.of(cause.getMessage, cause)
- // log and print error trace stack
- logError(s"Building pipeline has failed.", cause)
- val result = ErrorResult()
- watcher.onFinish(snapshot, result)
- result
- }
- }
-
- override def getPipeStatus: PipelineStatusEnum = pipeStatus
-
- override def getError: PipeError = error.copy()
-
- override def getStepsStatus: Map[Int, (PipelineStepStatusEnum, Long)] =
- stepsStatus.toMap
-
- override def getCurStep: Int = curStep
-
- override def allSteps: Int = pipeType.getSteps.size
-
- override def logInfo(msg: => String): Unit =
- super.logInfo(s"[streampark-packer] $msg | $logSuffix")
-
- override def logError(msg: => String): Unit =
- super.logError(s"[streampark-packer] $msg | $logSuffix")
-
- override def logError(msg: => String, throwable: Throwable): Unit =
- super.logError(s"[streampark-packer] $msg | $logSuffix", throwable)
-
- /** intercept snapshot */
- def snapshot: PipelineSnapshot = PipelineSnapshot(
- offerBuildParam.appName,
- pipeType,
- getPipeStatus,
- getCurStep,
- allSteps,
- getStepsStatus,
- getError,
- System.currentTimeMillis)
-
-}
-
-object BuildPipeline {
-
- val execPool = new ThreadPoolExecutor(
- Runtime.getRuntime.availableProcessors * 2,
- 300,
- 60L,
- TimeUnit.SECONDS,
- new LinkedBlockingQueue[Runnable](2048),
- ThreadUtils.threadFactory("streampark-pipeline-watcher-executor"),
- new ThreadPoolExecutor.AbortPolicy)
-
- implicit val executor: ExecutionContextExecutorService =
- ExecutionContext.fromExecutorService(execPool)
-
-}
diff --git a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/BuildRequest.scala b/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/BuildRequest.scala
deleted file mode 100644
index ed3393889..000000000
--- a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/BuildRequest.scala
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.packer.pipeline
-
-import org.apache.streampark.common.conf.{FlinkVersion, SparkVersion, Workspace}
-import org.apache.streampark.common.enums.{FlinkDeployMode, FlinkJobType, SparkDeployMode, SparkJobType}
-import org.apache.streampark.flink.kubernetes.model.{K8sPodTemplates => FlinkK8sPodTemplates}
-import org.apache.streampark.flink.packer.docker.DockerConf
-import org.apache.streampark.flink.packer.maven.DependencyInfo
-import org.apache.streampark.spark.kubernetes.model.SparkK8sPodTemplates
-
-import scala.collection.mutable.ArrayBuffer
-
-/** Params of a BuildPipeline instance. */
-sealed trait BuildParam {
- def appName: String
-
- def mainClass: String
-}
-
-sealed trait FlinkBuildParam extends BuildParam {
-
- private[this] val localWorkspace = Workspace.local
-
- def workspace: String
-
- def deployMode: FlinkDeployMode
-
- def flinkJobType: FlinkJobType
-
- def flinkVersion: FlinkVersion
-
- def dependencyInfo: DependencyInfo
-
- def customFlinkUserJar: String
-
- lazy val providedLibs: DependencyInfo = {
- val providedLibs =
- ArrayBuffer(localWorkspace.APP_JARS, customFlinkUserJar)
- if (flinkJobType == FlinkJobType.FLINK_SQL) {
- providedLibs += s"${localWorkspace.APP_SHIMS}/flink-${flinkVersion.majorVersion}"
- }
- dependencyInfo.merge(providedLibs.toSet)
- }
-
- def getShadedJarPath(rootWorkspace: String): String = {
- val safeAppName: String = appName.replaceAll("\\s+", "_")
- s"$rootWorkspace/streampark-flinkjob_$safeAppName.jar"
- }
-
-}
-
-sealed trait FlinkK8sBuildParam extends FlinkBuildParam {
-
- def clusterId: String
-
- def k8sNamespace: String
-}
-
-case class FlinkK8sSessionBuildRequest(
- appName: String,
- workspace: String,
- mainClass: String,
- customFlinkUserJar: String,
- deployMode: FlinkDeployMode,
- flinkJobType: FlinkJobType,
- flinkVersion: FlinkVersion,
- dependencyInfo: DependencyInfo,
- clusterId: String,
- k8sNamespace: String)
- extends FlinkK8sBuildParam
-
-case class FlinkK8sApplicationBuildRequest(
- appName: String,
- workspace: String,
- mainClass: String,
- customFlinkUserJar: String,
- deployMode: FlinkDeployMode,
- flinkJobType: FlinkJobType,
- flinkVersion: FlinkVersion,
- dependencyInfo: DependencyInfo,
- clusterId: String,
- k8sNamespace: String,
- flinkBaseImage: String,
- flinkPodTemplate: FlinkK8sPodTemplates,
- integrateWithHadoop: Boolean = false,
- dockerConfig: DockerConf,
- ingressTemplate: String)
- extends FlinkK8sBuildParam
-
-case class FlinkRemotePerJobBuildRequest(
- appName: String,
- workspace: String,
- mainClass: String,
- customFlinkUserJar: String,
- skipBuild: Boolean,
- deployMode: FlinkDeployMode,
- flinkJobType: FlinkJobType,
- flinkVersion: FlinkVersion,
- dependencyInfo: DependencyInfo)
- extends FlinkBuildParam
-
-case class FlinkYarnApplicationBuildRequest(
- appName: String,
- mainClass: String,
- localWorkspace: String,
- yarnProvidedPath: String,
- flinkJobType: FlinkJobType,
- dependencyInfo: DependencyInfo)
- extends BuildParam
-
-case class SparkYarnBuildRequest(
- appName: String,
- mainClass: String,
- localWorkspace: String,
- yarnProvidedPath: String,
- jobType: SparkJobType,
- deployMode: SparkDeployMode,
- dependencyInfo: DependencyInfo) extends BuildParam
-
-case class SparkK8sApplicationBuildRequest(
- appName: String,
- workspace: String,
- mainClass: String,
- mainJar: String,
- deployMode: SparkDeployMode,
- jobType: SparkJobType,
- sparkVersion: SparkVersion,
- dependencyInfo: DependencyInfo,
- k8sNamespace: String,
- sparkBaseImage: String,
- sparkPodTemplate: SparkK8sPodTemplates,
- integrateWithHadoop: Boolean = false,
- dockerConfig: DockerConf) extends BuildParam
diff --git a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/BuildResponse.scala b/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/BuildResponse.scala
deleted file mode 100644
index a368a9ade..000000000
--- a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/BuildResponse.scala
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.packer.pipeline
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties
-
-/** Result of a BuildPipeline instance. */
-sealed trait BuildResult {
-
- /** is pass aka is successfully */
- def pass: Boolean
-
- def as[T <: BuildResult](implicit clz: Class[T]): T = this.asInstanceOf[T]
-}
-
-sealed trait FlinkBuildResult extends BuildResult {
- def workspacePath: String
-}
-
-@JsonIgnoreProperties(ignoreUnknown = true)
-case class ErrorResult(pass: Boolean = false) extends BuildResult {}
-
-@JsonIgnoreProperties(ignoreUnknown = true)
-case class SimpleBuildResponse(workspacePath: String = null, pass: Boolean = true)
- extends FlinkBuildResult {
- override def toString: String =
- s"{ workspacePath: $workspacePath, pass: $pass }"
-}
-
-@JsonIgnoreProperties(ignoreUnknown = true)
-case class ShadedBuildResponse(workspacePath: String, shadedJarPath: String, pass: Boolean = true)
- extends FlinkBuildResult {
- override def toString: String =
- s"{ workspacePath: $workspacePath, " +
- s"shadedJarPath: $shadedJarPath, " +
- s"pass: $pass }"
-
-}
-
-@JsonIgnoreProperties(ignoreUnknown = true)
-case class DockerImageBuildResponse(
- workspacePath: String,
- flinkImageTag: String,
- podTemplatePaths: Map[String, String],
- dockerInnerMainJarPath: String,
- pass: Boolean = true)
- extends FlinkBuildResult {
- override def toString: String =
- s"{ workspacePath: $workspacePath, " +
- s"flinkImageTag: $flinkImageTag, " +
- s"podTemplatePaths: ${podTemplatePaths.mkString(",")}, " +
- s"dockerInnerMainJarPath: $dockerInnerMainJarPath, " +
- s"pass: $pass }"
-}
-
-@JsonIgnoreProperties(ignoreUnknown = true)
-case class K8sAppModeBuildResponse(
- workspacePath: String,
- flinkBaseImage: String,
- mainJarPath: String,
- extraLibJarPaths: Set[String],
- pass: Boolean = false) extends FlinkBuildResult {
- override def toString: String =
- s"{ workspacePath: $workspacePath, " +
- s"flinkBaseImage: $flinkBaseImage, " +
- s"mainJarPath: $mainJarPath, " +
- s"extraLibJarPaths: ${extraLibJarPaths.mkString(",")}, " +
- s"pass: $pass }"
-}
diff --git a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/DockerResolveProgress.scala b/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/DockerResolveProgress.scala
deleted file mode 100644
index a565f9d6c..000000000
--- a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/DockerResolveProgress.scala
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.packer.pipeline
-
-import org.apache.streampark.common.util.Utils
-
-import com.github.dockerjava.api.model.{PullResponseItem, PushResponseItem}
-import org.apache.commons.lang3.StringUtils
-
-import scala.collection.mutable
-import scala.collection.mutable.ArrayBuffer
-
-/** cache storage for docker resolved progress */
-class DockerResolveProgress(
- val pull: DockerPullProgress,
- val build: DockerBuildProgress,
- val push: DockerPushProgress)
-
-class DockerPullProgress(
- val layers: mutable.Map[String, DockerLayerProgress],
- var error: String,
- var lastTime: Long) {
- // noinspection DuplicatedCode
- def update(pullRsp: PullResponseItem): Unit = {
- val nonPullUpdateState =
- pullRsp == null || StringUtils.isBlank(pullRsp.getId) || StringUtils
- .isBlank(pullRsp.getStatus)
- if (nonPullUpdateState) {
- return
- }
- if (pullRsp.getStatus.contains("complete")) {
- layers += pullRsp.getId -> DockerLayerProgress(pullRsp.getId, pullRsp.getStatus, 1, 1)
- lastTime = System.currentTimeMillis
- } else {
- val cur =
- if (pullRsp.getProgressDetail == null || pullRsp.getProgressDetail.getCurrent == null) 0
- else pullRsp.getProgressDetail.getCurrent.toLong
- val total =
- if (pullRsp.getProgressDetail == null || pullRsp.getProgressDetail.getTotal == null) 0
- else pullRsp.getProgressDetail.getTotal.toLong
- layers += pullRsp.getId -> DockerLayerProgress(pullRsp.getId, pullRsp.getStatus, cur, total)
- error = Option(pullRsp.getErrorDetail).map(_.getMessage).getOrElse("")
- lastTime = System.currentTimeMillis
- }
- }
-
- def snapshot: DockerPullSnapshot =
- DockerPullSnapshot.of(layers.values.toSeq, error, lastTime)
-}
-
-class DockerBuildProgress(val steps: ArrayBuffer[String], var lastTime: Long) {
- def update(buildStep: String): Unit = {
- if (StringUtils.isNotBlank(buildStep)) {
- steps += buildStep
- lastTime = System.currentTimeMillis
- }
- }
-
- def snapshot: DockerBuildSnapshot = DockerBuildSnapshot(steps, lastTime)
-}
-
-class DockerPushProgress(
- val layers: mutable.Map[String, DockerLayerProgress],
- var error: String,
- var lastTime: Long) {
- // noinspection DuplicatedCode
- def update(pushRsp: PushResponseItem): Unit = {
- val nonPushUpdateState =
- pushRsp == null || StringUtils.isBlank(pushRsp.getId) || StringUtils
- .isBlank(pushRsp.getStatus)
- if (nonPushUpdateState) {
- return
- }
- if (pushRsp.getStatus.contains("complete")) {
- layers += pushRsp.getId -> DockerLayerProgress(pushRsp.getId, pushRsp.getStatus, 1, 1)
- lastTime = System.currentTimeMillis
- } else {
- val cur =
- if (pushRsp.getProgressDetail == null || pushRsp.getProgressDetail.getCurrent == null) 0L
- else pushRsp.getProgressDetail.getCurrent.toLong
- val total =
- if (pushRsp.getProgressDetail == null || pushRsp.getProgressDetail.getTotal == null) 0L
- else pushRsp.getProgressDetail.getTotal.toLong
- layers += pushRsp.getId -> DockerLayerProgress(pushRsp.getId, pushRsp.getStatus, cur, total)
- error = Option(pushRsp.getErrorDetail).map(_.getMessage).getOrElse("")
- lastTime = System.currentTimeMillis
- }
- }
-
- def snapshot: DockerPushSnapshot =
- DockerPushSnapshot.of(layers.values.toSeq, error, lastTime)
-}
-
-object DockerPullProgress {
- def empty(): DockerPullProgress =
- new DockerPullProgress(mutable.Map(), "", System.currentTimeMillis)
-}
-
-object DockerBuildProgress {
- def empty(): DockerBuildProgress =
- new DockerBuildProgress(ArrayBuffer(), System.currentTimeMillis)
-}
-
-object DockerPushProgress {
- def empty(): DockerPushProgress =
- new DockerPushProgress(mutable.Map(), "", System.currentTimeMillis)
-}
-
-/**
- * push/pull progress of per docker layer.
- *
- * @param current
- * already download size (byte)
- * @param total
- * layer size (byte)
- */
-case class DockerLayerProgress(layerId: String, status: String, current: Long, total: Long) {
-
- def percent: Double = Utils.calPercent(current, total)
-
- def currentMb: Double =
- if (current == 0) 0
- else "%.2f".format(current.toDouble / (1024 * 1024)).toDouble
-
- def totalMb: Double =
- if (total == 0) 0
- else "%.2f".format(total.toDouble / (1024 * 1024)).toDouble
-}
diff --git a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/DockerResolveSnapshot.scala b/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/DockerResolveSnapshot.scala
deleted file mode 100644
index 61fa36bc5..000000000
--- a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/DockerResolveSnapshot.scala
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.packer.pipeline
-
-import org.apache.streampark.common.util.Implicits._
-import org.apache.streampark.common.util.Utils
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties
-
-/** Snapshot for docker resolved progress */
-@JsonIgnoreProperties(ignoreUnknown = true)
-case class DockerResolvedSnapshot(
- pull: DockerPullSnapshot,
- build: DockerBuildSnapshot,
- push: DockerPushSnapshot)
-
-/** snapshot for pulling docker image progress. */
-@JsonIgnoreProperties(ignoreUnknown = true)
-case class DockerPullSnapshot(
- detail: Seq[DockerLayerProgress],
- error: String,
- emitTime: Long,
- percent: Double) {
- def detailAsJava: JavaList[DockerLayerProgress] = detail
-}
-
-/** snapshot for building docker image progress. */
-@JsonIgnoreProperties(ignoreUnknown = true)
-case class DockerBuildSnapshot(detail: Seq[String], emitTime: Long) {
- def detailAsJava: JavaList[String] = detail
-}
-
-/** snapshot for pushing docker image progress. */
-@JsonIgnoreProperties(ignoreUnknown = true)
-case class DockerPushSnapshot(
- detail: Seq[DockerLayerProgress],
- error: String,
- emitTime: Long,
- percent: Double) {
- def detailAsJava: JavaList[DockerLayerProgress] = detail
-}
-
-object DockerPullSnapshot {
- def of(detail: Seq[DockerLayerProgress], error: String, emitTime: Long): DockerPullSnapshot =
- DockerPullSnapshot(
- detail,
- error,
- emitTime,
- Utils.calPercent(detail.map(_.current).sum, detail.map(_.total).sum))
-}
-
-object DockerPushSnapshot {
- def of(detail: Seq[DockerLayerProgress], error: String, emitTime: Long): DockerPushSnapshot =
- DockerPushSnapshot(
- detail,
- error,
- emitTime,
- Utils.calPercent(detail.map(_.current).sum, detail.map(_.total).sum))
-}
diff --git a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/PipelineError.scala b/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/PipelineError.scala
deleted file mode 100644
index ebd2141bd..000000000
--- a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/PipelineError.scala
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.packer.pipeline
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties
-
-import javax.annotation.Nullable
-
-/**
- * Error details of building pipeline.
- *
- * @param summary
- * summary of error
- * @param exception
- * exception stack
- */
-@JsonIgnoreProperties(ignoreUnknown = true, value = Array("exception"))
-case class PipeError(
- summary: String,
- @Nullable exception: Throwable,
- @Nullable exceptionStack: String) {
-
- def nonEmpty: Boolean =
- Option(summary).exists(_.nonEmpty) || exception != null
-
- def isEmpty: Boolean = !nonEmpty
-}
-
-object PipeError {
-
- def empty(): PipeError = of("", null)
-
- def of(summary: String, @Nullable exception: Throwable): PipeError =
- PipeError(
- summary,
- exception,
- if (exception == null) "" else exception.getStackTrace.mkString("\n"))
-}
diff --git a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/PipelineSnapshot.scala b/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/PipelineSnapshot.scala
deleted file mode 100644
index 2bf492034..000000000
--- a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/PipelineSnapshot.scala
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.packer.pipeline
-
-import org.apache.streampark.common.util.Implicits._
-import org.apache.streampark.common.util.Utils
-
-import java.lang.{Long => JavaLong}
-
-/**
- * Snapshot for a BuildPipeline instance. see
- * org.apache.streampark.flink.packer.pipeline.BuildPipeline
- *
- * @param emitTime
- * snapshot interception time
- * @param stepStatus
- * StepSeq -> (PipeStepStatus -> status update timestamp)
- */
-
-case class PipelineSnapshot(
- appName: String,
- pipeType: PipelineTypeEnum,
- pipeStatus: PipelineStatusEnum,
- curStep: Int,
- allSteps: Int,
- stepStatus: Map[Int, (PipelineStepStatusEnum, Long)],
- error: PipeError,
- emitTime: Long) {
-
- def percent(): Double = Utils.calPercent(curStep, allSteps)
-
- def stepStatusAsJava: JavaMap[Integer, (PipelineStepStatusEnum, JavaLong)] = {
- stepStatus.toSeq
- .map(e => Integer.valueOf(e._1) -> (e._2._1 -> JavaLong.valueOf(e._2._2)))
- .toMap
- .asJava
- }
-
- def pureStepStatusAsJava: JavaMap[Integer, PipelineStepStatusEnum] = {
- stepStatus.toSeq.map(e => Integer.valueOf(e._1) -> e._2._1).toMap.asJava
- }
-
- def stepStatusTimestampAsJava: JavaMap[Integer, JavaLong] = {
- stepStatus.toSeq
- .map(e => Integer.valueOf(e._1) -> JavaLong.valueOf(e._2._2))
- .toMap
- .asJava
- }
-
-}
diff --git a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/impl/FlinkK8sApplicationBuildPipeline.scala b/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/impl/FlinkK8sApplicationBuildPipeline.scala
deleted file mode 100644
index 484dc5bed..000000000
--- a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/impl/FlinkK8sApplicationBuildPipeline.scala
+++ /dev/null
@@ -1,271 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.packer.pipeline.impl
-
-import org.apache.streampark.common.fs.LfsOperator
-import org.apache.streampark.common.util.Implicits._
-import org.apache.streampark.common.util.ThreadUtils
-import org.apache.streampark.flink.kubernetes.PodTemplateTool
-import org.apache.streampark.flink.kubernetes.ingress.IngressController
-import org.apache.streampark.flink.packer.docker._
-import org.apache.streampark.flink.packer.maven.MavenTool
-import org.apache.streampark.flink.packer.pipeline._
-import org.apache.streampark.flink.packer.pipeline.BuildPipeline.executor
-
-import com.github.dockerjava.api.command.PushImageCmd
-import com.github.dockerjava.core.command.{HackBuildImageCmd, HackPullImageCmd, HackPushImageCmd}
-import com.google.common.collect.Sets
-import org.apache.commons.lang3.StringUtils
-
-import java.io.File
-import java.util.concurrent.{LinkedBlockingQueue, ThreadPoolExecutor, TimeUnit}
-
-import scala.concurrent.{ExecutionContext, Future}
-
-/** Building pipeline for flink kubernetes-native application mode */
-class FlinkK8sApplicationBuildPipeline(request: FlinkK8sApplicationBuildRequest)
- extends BuildPipeline {
-
- override def pipeType: PipelineTypeEnum =
- PipelineTypeEnum.FLINK_NATIVE_K8S_APPLICATION
-
- private var dockerProcessWatcher: DockerProgressWatcher =
- new SilentDockerProgressWatcher
-
- // non-thread-safe
- private val dockerProcess = new DockerResolveProgress(
- DockerPullProgress.empty(),
- DockerBuildProgress.empty(),
- DockerPushProgress.empty())
-
- override protected def offerBuildParam: FlinkK8sApplicationBuildRequest =
- request
-
- def registerDockerProgressWatcher(watcher: DockerProgressWatcher): Unit = {
- dockerProcessWatcher = watcher
- }
-
- @throws[Throwable]
- override protected def buildProcess(): DockerImageBuildResponse = {
-
- // Step-1: init build workspace of flink job
- // the sub workspace dir like: APP_WORKSPACE/k8s-clusterId@k8s-namespace/
- val buildWorkspace =
- execStep(1) {
- val buildWorkspace =
- s"${request.workspace}/${request.clusterId}@${request.k8sNamespace}"
- LfsOperator.mkCleanDirs(buildWorkspace)
- logInfo(s"Recreate building workspace: $buildWorkspace")
- buildWorkspace
- }.getOrElse(throw getError.exception)
-
- // Step-2: export k8s pod template files
- val podTemplatePaths = request.flinkPodTemplate match {
- case podTemplate if podTemplate.isEmpty =>
- skipStep(2)
- Map[String, String]()
- case podTemplate =>
- execStep(2) {
- val podTemplateFiles =
- PodTemplateTool
- .preparePodTemplateFiles(buildWorkspace, podTemplate)
- .tmplFiles
- logInfo(s"Export flink podTemplates: ${podTemplateFiles.values.mkString(",")}")
- podTemplateFiles
- }.getOrElse(throw getError.exception)
- }
-
- // Step-3: build shaded flink job jar and handle extra jars
- // the output shaded jar file name like: streampark-flinkjob_myjob-test.jar
- val (shadedJar, extJarLibs) =
- execStep(3) {
- val shadedJarOutputPath = request.getShadedJarPath(buildWorkspace)
- val shadedJar =
- MavenTool.buildFatJar(request.mainClass, request.providedLibs, shadedJarOutputPath)
- logInfo(s"Output shaded flink job jar: ${shadedJar.getAbsolutePath}")
- shadedJar -> request.dependencyInfo.extJarLibs
- }.getOrElse(throw getError.exception)
-
- // Step-4: generate and Export flink image dockerfiles
- val (dockerfile, dockerFileTemplate) =
- execStep(4) {
- val dockerFileTemplate = {
- if (request.integrateWithHadoop) {
- FlinkHadoopDockerfileTemplate.fromSystemHadoopConf(
- buildWorkspace,
- request.flinkBaseImage,
- shadedJar.getAbsolutePath,
- extJarLibs)
- } else {
- FlinkDockerfileTemplate(
- buildWorkspace,
- request.flinkBaseImage,
- shadedJar.getAbsolutePath,
- extJarLibs)
- }
- }
- val dockerFile = dockerFileTemplate.writeDockerfile
- logInfo(
- s"Output flink dockerfile: ${dockerFile.getAbsolutePath}, content: \n${dockerFileTemplate.offerDockerfileContent}")
- dockerFile -> dockerFileTemplate
- }.getOrElse(throw getError.exception)
-
- val dockerConf = request.dockerConfig
- val baseImageTag = request.flinkBaseImage.trim
- val pushImageTag = {
- if (request.k8sNamespace.isEmpty || request.clusterId.isEmpty) {
- throw new IllegalArgumentException("k8sNamespace or clusterId cannot be empty")
- }
- val expectedImageTag =
- s"streampark-flinkjob-${request.k8sNamespace}-${request.clusterId}"
- compileTag(expectedImageTag, dockerConf.registerAddress, dockerConf.imageNamespace)
- }
-
- // Step-5: pull flink base image
- execStep(5) {
- usingDockerClient {
- dockerClient =>
- val imgExists = dockerClient.listImagesCmd().exec().exists(_.getRepoTags.exists(_.contains(baseImageTag)))
- if (imgExists) {
- logInfo(s"found local docker image $baseImageTag, no need to pull from remote.")
- } else {
- val pullImageCmd = {
- // when the register address prefix is explicitly identified on base image tag,
- // the user's pre-saved docker register auth info would be used.
- val pullImageCmdState =
- dockerConf.registerAddress != null && !baseImageTag.startsWith(
- dockerConf.registerAddress)
- if (pullImageCmdState) {
- dockerClient.pullImageCmd(baseImageTag)
- } else {
- dockerClient
- .pullImageCmd(baseImageTag)
- .withAuthConfig(dockerConf.toAuthConf)
- }
- }
- val pullCmdCallback = pullImageCmd
- .asInstanceOf[HackPullImageCmd]
- .start(watchDockerPullProcess {
- pullRsp =>
- dockerProcess.pull.update(pullRsp)
- Future(dockerProcessWatcher.onDockerPullProgressChange(dockerProcess.pull.snapshot))
- })
- pullCmdCallback.awaitCompletion
- logInfo(s"Already pulled docker image from remote register, imageTag=$baseImageTag")
- }
- }(err => throw new Exception(s"Pull docker image failed, imageTag=$baseImageTag", err))
- }.getOrElse(throw getError.exception)
-
- // Step-6: build flink image
- execStep(6) {
- usingDockerClient {
- dockerClient =>
- val buildImageCmd = dockerClient
- .buildImageCmd()
- .withBaseDirectory(new File(buildWorkspace))
- .withDockerfile(dockerfile)
- .withTags(Sets.newHashSet(pushImageTag))
-
- val buildCmdCallback = buildImageCmd
- .asInstanceOf[HackBuildImageCmd]
- .start(watchDockerBuildStep {
- buildStep =>
- dockerProcess.build.update(buildStep)
- Future(
- dockerProcessWatcher.onDockerBuildProgressChange(dockerProcess.build.snapshot))
- })
- val imageId = buildCmdCallback.awaitImageId
- logInfo(s"Built docker image, imageId=$imageId, imageTag=$pushImageTag")
- }(err => throw new Exception(s"Build docker image failed. tag=$pushImageTag", err))
- }.getOrElse(throw getError.exception)
-
- // Step-7: push flink image
- execStep(7) {
- usingDockerClient {
- dockerClient =>
- val pushCmd: PushImageCmd = dockerClient
- .pushImageCmd(pushImageTag)
- .withAuthConfig(dockerConf.toAuthConf)
-
- val pushCmdCallback = pushCmd
- .asInstanceOf[HackPushImageCmd]
- .start(watchDockerPushProcess {
- pushRsp =>
- dockerProcess.push.update(pushRsp)
- Future(dockerProcessWatcher.onDockerPushProgressChange(dockerProcess.push.snapshot))
- })
- pushCmdCallback.awaitCompletion
- logInfo(s"Already pushed docker image, imageTag=$pushImageTag")
- }(err => throw new Exception(s"Push docker image failed. tag=$pushImageTag", err))
- }.getOrElse(throw getError.exception)
-
- // Step-8: init build workspace of ingress
- val ingressOutputPath = request.ingressTemplate match {
- case ingress if StringUtils.isBlank(ingress) =>
- skipStep(8)
- ""
- case _ =>
- execStep(8) {
- val ingressOutputPath =
- IngressController.prepareIngressTemplateFiles(buildWorkspace, request.ingressTemplate)
- logInfo(s"Export flink ingress: $ingressOutputPath")
- ingressOutputPath
- }.getOrElse(throw getError.exception)
- }
-
- DockerImageBuildResponse(
- buildWorkspace,
- pushImageTag,
- podTemplatePaths,
- dockerFileTemplate.innerMainJarPath)
- }
-
- /** compile image tag with namespace and remote address. */
- private[this] def compileTag(
- tag: String,
- registerAddress: String,
- imageNamespace: String): String = {
- var tagName = if (tag.contains("/")) tag else s"$imageNamespace/$tag"
- val addRegisterAddressState =
- StringUtils.isNotBlank(registerAddress) && !tagName.startsWith(registerAddress)
- if (addRegisterAddressState) {
- tagName = s"$registerAddress/$tagName"
- }
- tagName.toLowerCase
- }
-
-}
-
-object FlinkK8sApplicationBuildPipeline {
-
- val execPool = new ThreadPoolExecutor(
- Runtime.getRuntime.availableProcessors * 5,
- Runtime.getRuntime.availableProcessors() * 10,
- 60L,
- TimeUnit.SECONDS,
- new LinkedBlockingQueue[Runnable](2048),
- ThreadUtils.threadFactory("streampark-docker-progress-watcher-executor"),
- new ThreadPoolExecutor.DiscardOldestPolicy)
-
- implicit val executor: ExecutionContext =
- ExecutionContext.fromExecutorService(execPool)
-
- def of(request: FlinkK8sApplicationBuildRequest): FlinkK8sApplicationBuildPipeline =
- new FlinkK8sApplicationBuildPipeline(request)
-
-}
diff --git a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/impl/FlinkK8sSessionBuildPipeline.scala b/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/impl/FlinkK8sSessionBuildPipeline.scala
deleted file mode 100644
index 0c0c865fa..000000000
--- a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/impl/FlinkK8sSessionBuildPipeline.scala
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.packer.pipeline.impl
-
-import org.apache.streampark.common.fs.LfsOperator
-import org.apache.streampark.flink.packer.maven.MavenTool
-import org.apache.streampark.flink.packer.pipeline._
-
-/** Building pipeline for flink kubernetes-native session mode */
-class FlinkK8sSessionBuildPipeline(request: FlinkK8sSessionBuildRequest) extends BuildPipeline {
-
- override def pipeType: PipelineTypeEnum =
- PipelineTypeEnum.FLINK_NATIVE_K8S_SESSION
-
- override def offerBuildParam: FlinkK8sSessionBuildRequest = request
-
- /** The construction logic needs to be implemented by subclasses */
- @throws[Throwable]
- override protected def buildProcess(): ShadedBuildResponse = {
-
- // create workspace.
- // the sub workspace path like: APP_WORKSPACE/k8s-clusterId@k8s-namespace/
- val buildWorkspace =
- execStep(1) {
- val buildWorkspace =
- s"${request.workspace}/${request.clusterId}@${request.k8sNamespace}"
- LfsOperator.mkCleanDirs(buildWorkspace)
- logInfo(s"Recreate building workspace: $buildWorkspace")
- buildWorkspace
- }.getOrElse(throw getError.exception)
-
- // build flink job shaded jar.
- // the output shaded file name like: streampark-flinkjob_myjob_20211024134822
- val shadedJar =
- execStep(2) {
- val output = MavenTool.buildFatJar(
- request.mainClass,
- request.providedLibs,
- request.getShadedJarPath(buildWorkspace))
- logInfo(s"Output shaded flink job jar: ${output.getAbsolutePath}")
- output
- }.getOrElse(throw getError.exception)
-
- ShadedBuildResponse(buildWorkspace, shadedJar.getAbsolutePath)
- }
-}
-
-object FlinkK8sSessionBuildPipeline {
- def of(request: FlinkK8sSessionBuildRequest): FlinkK8sSessionBuildPipeline =
- new FlinkK8sSessionBuildPipeline(request)
-}
diff --git a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/impl/FlinkRemoteBuildPipeline.scala b/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/impl/FlinkRemoteBuildPipeline.scala
deleted file mode 100644
index 93a965068..000000000
--- a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/impl/FlinkRemoteBuildPipeline.scala
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.packer.pipeline.impl
-
-import org.apache.streampark.common.enums.FlinkJobType
-import org.apache.streampark.common.fs.{FsOperator, LfsOperator}
-import org.apache.streampark.common.util.Implicits._
-import org.apache.streampark.flink.packer.maven.MavenTool
-import org.apache.streampark.flink.packer.pipeline._
-
-import java.io.File
-
-/** Building pipeline for flink standalone session mode */
-class FlinkRemoteBuildPipeline(request: FlinkRemotePerJobBuildRequest) extends BuildPipeline {
-
- override def pipeType: PipelineTypeEnum = PipelineTypeEnum.FLINK_STANDALONE
-
- override def offerBuildParam: FlinkRemotePerJobBuildRequest = request
-
- /** The construction logic needs to be implemented by subclasses */
- @throws[Throwable]
- override protected def buildProcess(): ShadedBuildResponse = {
- // create workspace.
- // the sub workspace path like: APP_WORKSPACE/jobName
- if (request.skipBuild) {
- ShadedBuildResponse(request.workspace, request.customFlinkUserJar)
- } else {
- execStep(1) {
- LfsOperator.mkCleanDirs(request.workspace)
- logInfo(s"Recreate building workspace: ${request.workspace}")
- }.getOrElse(throw getError.exception)
- // build flink job shaded jar
- val shadedJar =
- execStep(2) {
- request.flinkJobType match {
- case FlinkJobType.FLINK_SQL =>
- val output = MavenTool.buildFatJar(
- request.mainClass,
- request.providedLibs,
- request.getShadedJarPath(request.workspace))
- logInfo(s"output shaded flink job jar: ${output.getAbsolutePath}")
- output
- case _ => new File(request.customFlinkUserJar);
- }
- }.getOrElse(throw getError.exception)
-
- val mavenJars =
- execStep(3) {
- request.flinkJobType match {
- case FlinkJobType.PYFLINK =>
- val mavenArts =
- MavenTool.resolveArtifacts(request.dependencyInfo.mavenArts)
- mavenArts.map(_.getAbsolutePath) ++ request.dependencyInfo.extJarLibs
- case _ => List[String]()
- }
- }.getOrElse(throw getError.exception)
-
- execStep(4) {
- request.flinkJobType match {
- case FlinkJobType.PYFLINK =>
- mavenJars.foreach(jar => {
- val lfs: FsOperator = FsOperator.lfs
- val lib = request.workspace.concat("/lib")
- lfs.mkdirsIfNotExists(lib)
- val originFile = new File(jar)
- if (originFile.isFile) {
- lfs.copy(originFile.getAbsolutePath, lib)
- }
-
- })
- case _ =>
- }
- }.getOrElse(throw getError.exception)
- ShadedBuildResponse(request.workspace, shadedJar.getAbsolutePath)
- }
-
- }
-
-}
-
-object FlinkRemoteBuildPipeline {
- def of(request: FlinkRemotePerJobBuildRequest): FlinkRemoteBuildPipeline =
- new FlinkRemoteBuildPipeline(request)
-}
diff --git a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/impl/FlinkYarnApplicationBuildPipeline.scala b/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/impl/FlinkYarnApplicationBuildPipeline.scala
deleted file mode 100644
index 4fdd0766e..000000000
--- a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/impl/FlinkYarnApplicationBuildPipeline.scala
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.packer.pipeline.impl
-
-import org.apache.streampark.common.conf.Workspace
-import org.apache.streampark.common.enums.FlinkJobType
-import org.apache.streampark.common.fs.{FsOperator, HdfsOperator, LfsOperator}
-import org.apache.streampark.common.util.Implicits._
-import org.apache.streampark.flink.packer.maven.MavenTool
-import org.apache.streampark.flink.packer.pipeline._
-
-import org.apache.commons.codec.digest.DigestUtils
-
-import java.io.{File, FileInputStream, IOException}
-
-/** Building pipeline for flink yarn application mode */
-class FlinkYarnApplicationBuildPipeline(request: FlinkYarnApplicationBuildRequest)
- extends BuildPipeline {
-
- /** the type of pipeline */
- override def pipeType: PipelineTypeEnum =
- PipelineTypeEnum.FLINK_YARN_APPLICATION
-
- override def offerBuildParam: FlinkYarnApplicationBuildRequest = request
-
- /**
- * the actual build process. the effective steps progress should be implemented in multiple
- * BuildPipeline.execStep() functions.
- */
- @throws[Throwable]
- override protected def buildProcess(): SimpleBuildResponse = {
- execStep(1) {
- request.flinkJobType match {
- case FlinkJobType.FLINK_SQL | FlinkJobType.PYFLINK =>
- LfsOperator.mkCleanDirs(request.localWorkspace)
- HdfsOperator.mkCleanDirs(request.yarnProvidedPath)
- case _ =>
- }
- logInfo(s"Recreate building workspace: ${request.yarnProvidedPath}")
- }.getOrElse(throw getError.exception)
-
- val mavenJars =
- execStep(2) {
- request.flinkJobType match {
- case FlinkJobType.FLINK_SQL | FlinkJobType.FLINK_SQL | FlinkJobType.PYFLINK =>
- val mavenArts =
- MavenTool.resolveArtifacts(request.dependencyInfo.mavenArts)
- mavenArts.map(_.getAbsolutePath) ++ request.dependencyInfo.extJarLibs
- case _ => List[String]()
- }
- }.getOrElse(throw getError.exception)
-
- execStep(3) {
- mavenJars.foreach(jar => {
- uploadJarToHdfsOrLfs(FsOperator.lfs, jar, request.localWorkspace)
- uploadJarToHdfsOrLfs(FsOperator.hdfs, jar, request.yarnProvidedPath)
- })
- }.getOrElse(throw getError.exception)
-
- SimpleBuildResponse()
- }
-
- @throws[IOException]
- private[this] def uploadJarToHdfsOrLfs(
- fsOperator: FsOperator,
- origin: String,
- target: String): Unit = {
- val originFile = new File(origin)
- if (!fsOperator.exists(target)) {
- fsOperator.mkdirs(target)
- }
- if (originFile.isFile) {
- // check file in upload dir
- fsOperator match {
- case FsOperator.lfs =>
- fsOperator.copy(originFile.getAbsolutePath, target)
- case FsOperator.hdfs =>
- val uploadFile =
- s"${Workspace.remote.APP_UPLOADS}/${originFile.getName}"
- if (fsOperator.exists(uploadFile)) {
- new FileInputStream(originFile).using(inputStream => {
- if (DigestUtils.md5Hex(inputStream) != fsOperator.fileMd5(uploadFile)) {
- fsOperator.upload(originFile.getAbsolutePath, uploadFile)
- }
- })
- } else {
- fsOperator.upload(originFile.getAbsolutePath, uploadFile)
- }
- // copy jar from upload dir to target dir
- fsOperator.copy(uploadFile, target)
- }
- } else {
- fsOperator match {
- case FsOperator.hdfs =>
- fsOperator.upload(originFile.getAbsolutePath, target)
- case _ =>
- }
- }
- }
-
-}
-
-object FlinkYarnApplicationBuildPipeline {
- def of(request: FlinkYarnApplicationBuildRequest): FlinkYarnApplicationBuildPipeline =
- new FlinkYarnApplicationBuildPipeline(request)
-}
diff --git a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/impl/SparkK8sApplicationBuildPipeline.scala b/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/impl/SparkK8sApplicationBuildPipeline.scala
deleted file mode 100644
index d90e8d758..000000000
--- a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/impl/SparkK8sApplicationBuildPipeline.scala
+++ /dev/null
@@ -1,249 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.packer.pipeline.impl
-
-import org.apache.streampark.common.fs.LfsOperator
-import org.apache.streampark.common.util.ThreadUtils
-import org.apache.streampark.flink.kubernetes.PodTemplateTool
-import org.apache.streampark.flink.packer.docker._
-import org.apache.streampark.flink.packer.pipeline._
-import org.apache.streampark.flink.packer.pipeline.BuildPipeline.executor
-
-import com.github.dockerjava.api.command.PushImageCmd
-import com.github.dockerjava.core.command.{HackBuildImageCmd, HackPullImageCmd, HackPushImageCmd}
-import com.google.common.collect.Sets
-import org.apache.commons.lang3.StringUtils
-
-import java.io.File
-import java.nio.file.Paths
-import java.util.concurrent.{LinkedBlockingQueue, ThreadPoolExecutor, TimeUnit}
-
-import scala.concurrent.{ExecutionContext, Future}
-
-/** Building pipeline for Spark kubernetes-native application mode */
-class SparkK8sApplicationBuildPipeline(request: SparkK8sApplicationBuildRequest)
- extends BuildPipeline {
-
- override def pipeType: PipelineTypeEnum =
- PipelineTypeEnum.SPARK_NATIVE_K8S_APPLICATION
-
- private var dockerProcessWatcher: DockerProgressWatcher =
- new SilentDockerProgressWatcher
-
- // non-thread-safe
- private val dockerProcess = new DockerResolveProgress(
- DockerPullProgress.empty(),
- DockerBuildProgress.empty(),
- DockerPushProgress.empty())
-
- override protected def offerBuildParam: SparkK8sApplicationBuildRequest =
- request
-
- def registerDockerProgressWatcher(watcher: DockerProgressWatcher): Unit = {
- dockerProcessWatcher = watcher
- }
-
- @throws[Throwable]
- override protected def buildProcess(): DockerImageBuildResponse = {
-
- // Step-1: init build workspace of spark job
- // the sub workspace dir like: APP_WORKSPACE/k8s-clusterId@k8s-namespace/
- val buildWorkspace =
- execStep(1) {
- val buildWorkspace =
- s"${request.workspace}/${request.k8sNamespace}"
- LfsOperator.mkCleanDirs(buildWorkspace)
- logInfo(s"Recreate building workspace: $buildWorkspace")
- buildWorkspace
- }.getOrElse(throw getError.exception)
-
- // Step-2: export k8s pod template files
- val podTemplatePaths = request.sparkPodTemplate match {
- case podTemplate if podTemplate.isEmpty =>
- skipStep(2)
- Map[String, String]()
- case podTemplate =>
- execStep(2) {
- val podTemplateFiles =
- PodTemplateTool
- .preparePodTemplateFiles(buildWorkspace, podTemplate)
- .tmplFiles
- logInfo(s"Export spark podTemplates: ${podTemplateFiles.values.mkString(",")}")
- podTemplateFiles
- }.getOrElse(throw getError.exception)
- }
-
- // Step-3: prepare spark job jar
- val (mainJarPath, extJarLibs) =
- execStep(3) {
- val mainJarName = Paths.get(request.mainJar).getFileName
- val mainJarPath = s"$buildWorkspace/$mainJarName"
- LfsOperator.copy(request.mainJar, mainJarPath)
- logInfo(s"Prepared spark job jar: $mainJarPath")
- mainJarPath -> Set[String]()
- }.getOrElse(throw getError.exception)
-
- // Step-4: generate and Export spark image dockerfiles
- val (dockerfile, dockerFileTemplate) =
- execStep(4) {
- val dockerFileTemplate = {
- if (request.integrateWithHadoop) {
- SparkHadoopDockerfileTemplate.fromSystemHadoopConf(
- buildWorkspace,
- request.sparkBaseImage,
- mainJarPath,
- extJarLibs)
- } else {
- SparkDockerfileTemplate(
- buildWorkspace,
- request.sparkBaseImage,
- mainJarPath,
- extJarLibs)
- }
- }
- val dockerFile = dockerFileTemplate.writeDockerfile
- logInfo(
- s"Output spark dockerfile: ${dockerFile.getAbsolutePath}, content: \n${dockerFileTemplate.offerDockerfileContent}")
- dockerFile -> dockerFileTemplate
- }.getOrElse(throw getError.exception)
-
- val dockerConf = request.dockerConfig
- val baseImageTag = request.sparkBaseImage.trim
- val pushImageTag = {
- if (request.k8sNamespace.isEmpty || request.appName.isEmpty) {
- throw new IllegalArgumentException("k8sNamespace or appName cannot be empty")
- }
- val expectedImageTag =
- s"streampark-sparkjob-${request.k8sNamespace}-${request.appName}"
- compileTag(expectedImageTag, dockerConf.registerAddress, dockerConf.imageNamespace)
- }
-
- // Step-5: pull spark base image
- execStep(5) {
- usingDockerClient {
- dockerClient =>
- val pullImageCmd = {
- // when the register address prefix is explicitly identified on base image tag,
- // the user's pre-saved docker register auth info would be used.
- val pullImageCmdState =
- dockerConf.registerAddress != null && !baseImageTag.startsWith(
- dockerConf.registerAddress)
- if (pullImageCmdState) {
- dockerClient.pullImageCmd(baseImageTag)
- } else {
- dockerClient
- .pullImageCmd(baseImageTag)
- .withAuthConfig(dockerConf.toAuthConf)
- }
- }
- val pullCmdCallback = pullImageCmd
- .asInstanceOf[HackPullImageCmd]
- .start(watchDockerPullProcess {
- pullRsp =>
- dockerProcess.pull.update(pullRsp)
- Future(dockerProcessWatcher.onDockerPullProgressChange(dockerProcess.pull.snapshot))
- })
- pullCmdCallback.awaitCompletion
- logInfo(s"Already pulled docker image from remote register, imageTag=$baseImageTag")
- }(err => throw new Exception(s"Pull docker image failed, imageTag=$baseImageTag", err))
- }.getOrElse(throw getError.exception)
-
- // Step-6: build spark image
- execStep(6) {
- usingDockerClient {
- dockerClient =>
- val buildImageCmd = dockerClient
- .buildImageCmd()
- .withBaseDirectory(new File(buildWorkspace))
- .withDockerfile(dockerfile)
- .withTags(Sets.newHashSet(pushImageTag))
-
- val buildCmdCallback = buildImageCmd
- .asInstanceOf[HackBuildImageCmd]
- .start(watchDockerBuildStep {
- buildStep =>
- dockerProcess.build.update(buildStep)
- Future(
- dockerProcessWatcher.onDockerBuildProgressChange(dockerProcess.build.snapshot))
- })
- val imageId = buildCmdCallback.awaitImageId
- logInfo(s"Built docker image, imageId=$imageId, imageTag=$pushImageTag")
- }(err => throw new Exception(s"Build docker image failed. tag=$pushImageTag", err))
- }.getOrElse(throw getError.exception)
-
- // Step-7: push spark image
- execStep(7) {
- usingDockerClient {
- dockerClient =>
- val pushCmd: PushImageCmd = dockerClient
- .pushImageCmd(pushImageTag)
- .withAuthConfig(dockerConf.toAuthConf)
-
- val pushCmdCallback = pushCmd
- .asInstanceOf[HackPushImageCmd]
- .start(watchDockerPushProcess {
- pushRsp =>
- dockerProcess.push.update(pushRsp)
- Future(dockerProcessWatcher.onDockerPushProgressChange(dockerProcess.push.snapshot))
- })
- pushCmdCallback.awaitCompletion
- logInfo(s"Already pushed docker image, imageTag=$pushImageTag")
- }(err => throw new Exception(s"Push docker image failed. tag=$pushImageTag", err))
- }.getOrElse(throw getError.exception)
-
- DockerImageBuildResponse(
- buildWorkspace,
- pushImageTag,
- podTemplatePaths,
- dockerFileTemplate.innerMainJarPath)
- }
-
- /** compile image tag with namespace and remote address. */
- private[this] def compileTag(
- tag: String,
- registerAddress: String,
- imageNamespace: String): String = {
- var tagName = if (tag.contains("/")) tag else s"$imageNamespace/$tag"
- val addRegisterAddressState =
- StringUtils.isNotBlank(registerAddress) && !tagName.startsWith(registerAddress)
- if (addRegisterAddressState) {
- tagName = s"$registerAddress/$tagName"
- }
- tagName.toLowerCase
- }
-
-}
-
-object SparkK8sApplicationBuildPipeline {
-
- val execPool = new ThreadPoolExecutor(
- Runtime.getRuntime.availableProcessors * 5,
- Runtime.getRuntime.availableProcessors() * 10,
- 60L,
- TimeUnit.SECONDS,
- new LinkedBlockingQueue[Runnable](2048),
- ThreadUtils.threadFactory("streampark-docker-progress-watcher-executor"),
- new ThreadPoolExecutor.DiscardOldestPolicy)
-
- implicit val executor: ExecutionContext =
- ExecutionContext.fromExecutorService(execPool)
-
- def of(request: SparkK8sApplicationBuildRequest): SparkK8sApplicationBuildPipeline =
- new SparkK8sApplicationBuildPipeline(request)
-
-}
diff --git a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/impl/SparkYarnBuildPipeline.scala b/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/impl/SparkYarnBuildPipeline.scala
deleted file mode 100644
index 2c92067b6..000000000
--- a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/impl/SparkYarnBuildPipeline.scala
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.packer.pipeline.impl
-
-import org.apache.streampark.common.conf.Workspace
-import org.apache.streampark.common.enums.SparkJobType
-import org.apache.streampark.common.fs.{FsOperator, HdfsOperator, LfsOperator}
-import org.apache.streampark.common.util.Implicits._
-import org.apache.streampark.flink.packer.maven.MavenTool
-import org.apache.streampark.flink.packer.pipeline._
-
-import org.apache.commons.codec.digest.DigestUtils
-
-import java.io.{File, FileInputStream, IOException}
-
-/** Building pipeline for spark yarn application mode */
-class SparkYarnBuildPipeline(request: SparkYarnBuildRequest)
- extends BuildPipeline {
-
- /** the type of pipeline */
- override def pipeType: PipelineTypeEnum =
- PipelineTypeEnum.SPARK_CLUSTER
-
- override def offerBuildParam: SparkYarnBuildRequest = request
-
- /**
- * the actual build process. the effective steps progress should be implemented in multiple
- * BuildPipeline.execStep() functions.
- */
- /**
- * the actual build process. the effective steps progress should be implemented in multiple
- * BuildPipeline.execStep() functions.
- */
- @throws[Throwable]
- override protected def buildProcess(): SimpleBuildResponse = {
- execStep(1) {
- request.jobType match {
- case SparkJobType.SPARK_SQL =>
- LfsOperator.mkCleanDirs(request.localWorkspace)
- HdfsOperator.mkCleanDirs(request.yarnProvidedPath)
- case _ =>
- }
- logInfo(s"Recreate building workspace: ${request.yarnProvidedPath}")
- }.getOrElse(throw getError.exception)
-
- val mavenJars =
- execStep(2) {
- request.jobType match {
- case SparkJobType.SPARK_SQL =>
- val mavenArts =
- MavenTool.resolveArtifacts(request.dependencyInfo.mavenArts)
- mavenArts.map(_.getAbsolutePath) ++ request.dependencyInfo.extJarLibs
- case _ => List[String]()
- }
- }.getOrElse(throw getError.exception)
-
- execStep(3) {
- mavenJars.foreach(jar => {
- uploadJarToHdfsOrLfs(FsOperator.lfs, jar, request.localWorkspace)
- uploadJarToHdfsOrLfs(FsOperator.hdfs, jar, request.yarnProvidedPath)
- })
- }.getOrElse(throw getError.exception)
- SimpleBuildResponse()
- }
-
- @throws[IOException]
- private[this] def uploadJarToHdfsOrLfs(
- fsOperator: FsOperator,
- origin: String,
- target: String): Unit = {
- val originFile = new File(origin)
- if (!fsOperator.exists(target)) {
- fsOperator.mkdirs(target)
- }
- if (originFile.isFile) {
- // check file in upload dir
- fsOperator match {
- case FsOperator.lfs =>
- fsOperator.copy(originFile.getAbsolutePath, target)
- case FsOperator.hdfs =>
- val uploadFile =
- s"${Workspace.remote.APP_UPLOADS}/${originFile.getName}"
- if (fsOperator.exists(uploadFile)) {
- new FileInputStream(originFile).using(inputStream => {
- if (DigestUtils.md5Hex(inputStream) != fsOperator.fileMd5(uploadFile)) {
- fsOperator.upload(originFile.getAbsolutePath, uploadFile)
- }
- })
- } else {
- fsOperator.upload(originFile.getAbsolutePath, uploadFile)
- }
- // copy jar from upload dir to target dir
- fsOperator.copy(uploadFile, target)
- }
- } else {
- fsOperator match {
- case FsOperator.hdfs =>
- fsOperator.upload(originFile.getAbsolutePath, target)
- case _ =>
- }
- }
- }
-
-}
-
-object SparkYarnBuildPipeline {
- def of(request: SparkYarnBuildRequest): SparkYarnBuildPipeline =
- new SparkYarnBuildPipeline(request)
-}
diff --git a/streampark-flink/streampark-flink-packer/src/test/java/org/apache/streampark/flink/packer/DockerClientTest.java b/streampark-flink/streampark-flink-packer/src/test/java/org/apache/streampark/flink/packer/DockerClientTest.java
new file mode 100644
index 000000000..3e4d85ba8
--- /dev/null
+++ b/streampark-flink/streampark-flink-packer/src/test/java/org/apache/streampark/flink/packer/DockerClientTest.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.packer;
+
+import org.apache.streampark.common.conf.CommonConfig;
+import org.apache.streampark.common.conf.InternalConfigHolder;
+import org.apache.streampark.flink.packer.docker.DockerImageExist;
+import org.apache.streampark.flink.packer.docker.DockerRetriever;
+import org.apache.streampark.flink.packer.docker.DockerUtils;
+
+import com.github.dockerjava.api.command.PullImageResultCallback;
+import com.github.dockerjava.api.exception.NotFoundException;
+import com.github.dockerjava.httpclient5.ApacheDockerHttpClient;
+import org.junit.jupiter.api.Assumptions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.time.Duration;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class DockerClientTest {
+
+ private static final String TEST_IMAGE = "flink:1.18.1-scala_2.12-java8";
+
+ private static boolean dockerAvailable;
+
+ @BeforeAll
+ static void prepareDocker() {
+ String dockerHost = System.getenv("DOCKER_HOST");
+ if (dockerHost == null) {
+ dockerHost = "/var/run/docker.sock";
+ }
+ dockerAvailable = Files.exists(Paths.get(dockerHost.replace("unix://", "")));
+ if (!dockerAvailable) {
+ return;
+ }
+ DockerUtils.usingDockerClient(
+ client -> {
+ try {
+ client.inspectImageCmd(TEST_IMAGE).exec();
+ } catch (NotFoundException e) {
+ try {
+ client.pullImageCmd("hello-world:latest")
+ .exec(new PullImageResultCallback())
+ .awaitCompletion(2, TimeUnit.MINUTES);
+ } catch (InterruptedException interrupted) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException(
+ "Interrupted while pulling docker test image: hello-world:latest",
+ interrupted);
+ }
+ client.tagImageCmd("hello-world:latest", "flink", "1.18.1-scala_2.12-java8")
+ .exec();
+ }
+ return null;
+ },
+ err -> {
+ throw new RuntimeException("Failed to prepare docker test image: " + TEST_IMAGE, err);
+ });
+ }
+
+ @Test
+ void dockerClientConfigBuilder() {
+ ApacheDockerHttpClient.Builder dockerHttpClientBuilder =
+ new ApacheDockerHttpClient.Builder()
+ .dockerHost(DockerRetriever.dockerClientConf.getDockerHost())
+ .sslConfig(DockerRetriever.dockerClientConf.getSSLConfig())
+ .maxConnections(InternalConfigHolder.get(CommonConfig.DOCKER_MAX_CONNECTIONS()))
+ .connectionTimeout(
+ Duration.ofSeconds(
+ InternalConfigHolder.get(CommonConfig.DOCKER_CONNECTION_TIMEOUT_SEC())))
+ .responseTimeout(
+ Duration.ofSeconds(
+ InternalConfigHolder.get(CommonConfig.DOCKER_RESPONSE_TIMEOUT_SEC())));
+ assertNotNull(dockerHttpClientBuilder);
+ }
+
+ @Test
+ void returnTrueIfImageExists() {
+ Assumptions.assumeTrue(dockerAvailable, "Docker daemon is not available");
+ DockerImageExist dockerImageExist = new DockerImageExist();
+ assertTrue(dockerImageExist.doesDockerImageExist(TEST_IMAGE));
+ }
+
+ @Test
+ void returnFalseIfImageDoesNotExist() {
+ Assumptions.assumeTrue(dockerAvailable, "Docker daemon is not available");
+ DockerImageExist dockerImageExist = new DockerImageExist();
+ String imageName = "flink:1.18.1-scala_2.12-java8-fail";
+ assertFalse(dockerImageExist.doesDockerImageExist(imageName));
+ }
+}
diff --git a/streampark-flink/streampark-flink-packer/src/test/java/org/apache/streampark/flink/packer/FlinkDockerfileTemplateTest.java b/streampark-flink/streampark-flink-packer/src/test/java/org/apache/streampark/flink/packer/FlinkDockerfileTemplateTest.java
new file mode 100644
index 000000000..5d97e14aa
--- /dev/null
+++ b/streampark-flink/streampark-flink-packer/src/test/java/org/apache/streampark/flink/packer/FlinkDockerfileTemplateTest.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.packer;
+
+import org.apache.streampark.flink.packer.docker.FlinkDockerfileTemplate;
+
+import org.apache.commons.io.FileUtils;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.io.File;
+import java.nio.file.Paths;
+import java.util.Collections;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+class FlinkDockerfileTemplateTest {
+
+ private final File outputDir = new File("FlinkDockerfileTemplateSpec-output/");
+ private String mainJarPath;
+
+ @BeforeEach
+ void setUp() throws Exception {
+ outputDir.mkdir();
+ mainJarPath =
+ PackerTestUtils
+ .createJar(
+ Paths.get(outputDir.getAbsolutePath(), "WordCountSQL.jar"),
+ "org/apache/flink/WordCountSQL.class",
+ new byte[]{1})
+ .toString();
+ }
+
+ @AfterEach
+ void tearDown() throws Exception {
+ FileUtils.forceDelete(outputDir);
+ }
+
+ @Test
+ void buildDockerfileContent() {
+ FlinkDockerfileTemplate template =
+ new FlinkDockerfileTemplate(
+ outputDir.getAbsolutePath(), "1.13-scala_2.11", mainJarPath, Collections.emptySet());
+ String expected =
+ "FROM 1.13-scala_2.11\n"
+ + "RUN mkdir -p $FLINK_HOME/usrlib\n"
+ + "COPY lib $FLINK_HOME/lib/\n"
+ + "COPY WordCountSQL.jar $FLINK_HOME/usrlib/WordCountSQL.jar\n";
+ assertEquals(expected, template.offerDockerfileContent());
+ }
+
+ @Test
+ void writeDockerfileToFile() throws Exception {
+ FlinkDockerfileTemplate template =
+ new FlinkDockerfileTemplate(
+ outputDir.getAbsolutePath(), "1.13-scala_2.11", mainJarPath, Collections.emptySet());
+ String expected =
+ "FROM 1.13-scala_2.11\n"
+ + "RUN mkdir -p $FLINK_HOME/usrlib\n"
+ + "COPY lib $FLINK_HOME/lib/\n"
+ + "COPY WordCountSQL.jar $FLINK_HOME/usrlib/WordCountSQL.jar\n";
+ File outFile = template.writeDockerfile();
+ assertEquals("Dockerfile", outFile.getName());
+ assertEquals(expected, FileUtils.readFileToString(outFile, "UTF-8"));
+ }
+
+ @Test
+ void writeDockerfileWithSpecialName() throws Exception {
+ FlinkDockerfileTemplate template =
+ new FlinkDockerfileTemplate(
+ outputDir.getAbsolutePath(), "1.13-scala_2.11", mainJarPath, Collections.emptySet());
+ String expected =
+ "FROM 1.13-scala_2.11\n"
+ + "RUN mkdir -p $FLINK_HOME/usrlib\n"
+ + "COPY lib $FLINK_HOME/lib/\n"
+ + "COPY WordCountSQL.jar $FLINK_HOME/usrlib/WordCountSQL.jar\n";
+ File outFile = template.writeDockerfile("Dockerfile");
+ assertEquals("Dockerfile", outFile.getName());
+ assertEquals(expected, FileUtils.readFileToString(outFile, "UTF-8"));
+ }
+}
diff --git a/streampark-flink/streampark-flink-packer/src/test/java/org/apache/streampark/flink/packer/MavenArtifactTest.java b/streampark-flink/streampark-flink-packer/src/test/java/org/apache/streampark/flink/packer/MavenArtifactTest.java
new file mode 100644
index 000000000..5f6376232
--- /dev/null
+++ b/streampark-flink/streampark-flink-packer/src/test/java/org/apache/streampark/flink/packer/MavenArtifactTest.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.packer;
+
+import org.apache.streampark.flink.packer.maven.Artifact;
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+class MavenArtifactTest {
+
+ static final String[] ILLEGAL_ARTIFACT_COORDS_CASES = {
+ ":::",
+ "org.apache.flink:flink-table:",
+ ":flink-table:1.13.0",
+ "org.apache.flink::1.13.0",
+ "org.apache.flink:flink-table:",
+ "org.apache.flink:"
+ };
+
+ @Test
+ void createWithLegalCoords() {
+ Artifact art = Artifact.of("org.apache.flink:flink-table:1.13.0");
+ assertEquals("org.apache.flink", art.groupId());
+ assertEquals("flink-table", art.artifactId());
+ assertEquals("1.13.0", art.version());
+ }
+
+ @Test
+ void createWithIllegalCoords() {
+ for (String coord : ILLEGAL_ARTIFACT_COORDS_CASES) {
+ assertThrows(IllegalArgumentException.class, () -> Artifact.of(coord));
+ }
+ }
+}
diff --git a/streampark-flink/streampark-flink-packer/src/test/java/org/apache/streampark/flink/packer/MavenToolTest.java b/streampark-flink/streampark-flink-packer/src/test/java/org/apache/streampark/flink/packer/MavenToolTest.java
new file mode 100644
index 000000000..a1922c226
--- /dev/null
+++ b/streampark-flink/streampark-flink-packer/src/test/java/org/apache/streampark/flink/packer/MavenToolTest.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.packer;
+
+import org.apache.streampark.common.conf.Workspace;
+import org.apache.streampark.flink.packer.maven.Artifact;
+import org.apache.streampark.flink.packer.maven.DependencyInfo;
+import org.apache.streampark.flink.packer.maven.MavenTool;
+
+import org.apache.commons.io.FileUtils;
+
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import java.io.File;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.jar.JarFile;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class MavenToolTest {
+
+ private static final String OUTPUT_DIR = "MavenToolSpec-output/";
+ private static final Path JAR_DIR = Paths.get(OUTPUT_DIR, "jars");
+ private static String jar1Path;
+ private static String jar2Path;
+ private static String jar3Path;
+
+ @BeforeAll
+ static void beforeAll() throws Exception {
+ FileUtils.forceMkdir(new File(Workspace.MAVEN_LOCAL_PATH()));
+ File output = new File(OUTPUT_DIR);
+ FileUtils.deleteDirectory(output);
+ FileUtils.forceMkdir(output);
+ FileUtils.forceMkdir(JAR_DIR.toFile());
+
+ jar1Path =
+ PackerTestUtils
+ .createJar(
+ JAR_DIR.resolve("commons-cli-1.4.jar"),
+ "org/apache/commons/cli/DefaultParser.class",
+ new byte[]{1, 2, 3})
+ .toString();
+ jar2Path =
+ PackerTestUtils
+ .createJar(
+ JAR_DIR.resolve("commons-dbutils-1.7.jar"),
+ "org/apache/commons/dbutils/DbUtils.class",
+ new byte[]{4, 5, 6})
+ .toString();
+ jar3Path =
+ PackerTestUtils
+ .createJar(
+ JAR_DIR.resolve("commons-logging-1.2.jar"),
+ "org/apache/commons/logging/Log.class",
+ new byte[]{7, 8, 9})
+ .toString();
+ }
+
+ @AfterAll
+ static void afterAll() throws Exception {
+ FileUtils.deleteDirectory(new File(OUTPUT_DIR));
+ }
+
+ @Test
+ void buildFatJarWithJarLibs() throws Exception {
+ String fatJarPath = OUTPUT_DIR.concat("fat-1.jar");
+ File fatJar =
+ MavenTool.buildFatJar(null, new HashSet<>(Arrays.asList(jar1Path, jar2Path)), fatJarPath);
+ assertTrue(fatJar.exists());
+ assertTrue(
+ PackerTestUtils.jarEquals(
+ new JarFile(fatJarPath), new JarFile(jar1Path), "org/apache/commons/cli/DefaultParser.class"));
+ assertTrue(
+ PackerTestUtils.jarEquals(
+ new JarFile(fatJarPath), new JarFile(jar2Path), "org/apache/commons/dbutils/DbUtils.class"));
+ }
+
+ @Test
+ void buildFatJarWithJarLibsUnderDirectory() throws Exception {
+ String fatJarPath = OUTPUT_DIR.concat("fat-2.jar");
+ File fatJar = MavenTool.buildFatJar(null, Collections.singleton(JAR_DIR.toString()), fatJarPath);
+ assertTrue(fatJar.exists());
+ assertTrue(
+ PackerTestUtils.jarEquals(
+ new JarFile(fatJarPath), new JarFile(jar1Path), "org/apache/commons/cli/DefaultParser.class"));
+ assertTrue(
+ PackerTestUtils.jarEquals(
+ new JarFile(fatJarPath), new JarFile(jar2Path), "org/apache/commons/dbutils/DbUtils.class"));
+ assertTrue(
+ PackerTestUtils.jarEquals(
+ new JarFile(fatJarPath), new JarFile(jar3Path), "org/apache/commons/logging/Log.class"));
+ }
+
+ @Test
+ void buildFatJarWithJarLibsAndMavenArtifacts() throws Exception {
+ String fatJarPath = OUTPUT_DIR.concat("fat-3.jar");
+ File fatJar =
+ MavenTool.buildFatJar(
+ null,
+ new DependencyInfo(
+ Collections.singleton(
+ Artifact.of("org.apache.flink:flink-connector-kafka_2.11:1.13.0")),
+ Collections.singleton(jar2Path)),
+ fatJarPath);
+ assertTrue(fatJar.exists());
+ assertTrue(
+ PackerTestUtils.jarEquals(
+ new JarFile(fatJarPath), new JarFile(jar2Path), "org/apache/commons/dbutils/DbUtils.class"));
+ assertNotNull(new JarFile(fatJarPath).getJarEntry("org/apache/kafka/clients/ClientUtils.class"));
+ assertNotNull(
+ new JarFile(fatJarPath)
+ .getJarEntry("org/apache/flink/connector/base/source/reader/SourceReaderBase.class"));
+ }
+
+ @Test
+ void resolveSingleArtifact() throws Exception {
+ List<File> jars =
+ MavenTool.resolveArtifacts(
+ Collections.singleton(
+ Artifact.of("org.apache.flink:flink-connector-kafka_2.11:1.13.0")));
+ String[] expectJars = {
+ "flink-connector-kafka_2.11-1.13.0.jar",
+ "flink-connector-base-1.13.0.jar",
+ "kafka-clients-2.4.1.jar"
+ };
+ assertNotEquals(0, jars.size());
+ assertTrue(jars.stream().allMatch(File::exists));
+ assertEquals(
+ new HashSet<>(Arrays.asList(expectJars)),
+ jars.stream().map(File::getName).collect(java.util.stream.Collectors.toSet()));
+ }
+
+ @Test
+ void resolveMultipleArtifacts() throws Exception {
+ List<File> jars =
+ MavenTool.resolveArtifacts(
+ new HashSet<>(
+ Arrays.asList(
+ Artifact.of("org.apache.flink:flink-connector-kafka_2.11:1.13.0"),
+ Artifact.of("org.apache.flink:flink-connector-base:1.13.0"))));
+ String[] expectJars = {
+ "flink-connector-kafka_2.11-1.13.0.jar",
+ "flink-core-1.13.0.jar",
+ "flink-connector-base-1.13.0.jar",
+ "kafka-clients-2.4.1.jar"
+ };
+ assertTrue(jars.stream().allMatch(File::exists));
+ assertEquals(
+ new HashSet<>(Arrays.asList(expectJars)),
+ jars.stream().map(File::getName).collect(java.util.stream.Collectors.toSet()));
+ }
+}
diff --git a/streampark-flink/streampark-flink-packer/src/test/java/org/apache/streampark/flink/packer/PackerTestUtils.java b/streampark-flink/streampark-flink-packer/src/test/java/org/apache/streampark/flink/packer/PackerTestUtils.java
new file mode 100644
index 000000000..d66499851
--- /dev/null
+++ b/streampark-flink/streampark-flink-packer/src/test/java/org/apache/streampark/flink/packer/PackerTestUtils.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.packer;
+
+import org.codehaus.plexus.util.IOUtil;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
+import java.util.jar.JarOutputStream;
+
+/** Test utilities for packer module. */
+public final class PackerTestUtils {
+
+ private PackerTestUtils() {
+ }
+
+ public static String path(String resourcePath) {
+ java.net.URL resource = PackerTestUtils.class.getClassLoader().getResource(resourcePath);
+ if (resource == null) {
+ throw new IllegalStateException("Missing test resource: " + resourcePath);
+ }
+ return resource.getFile();
+ }
+
+ public static Path createJar(Path jarPath, String entryName, byte[] content) throws IOException {
+ Files.createDirectories(jarPath.getParent());
+ try (
+ OutputStream out = Files.newOutputStream(jarPath);
+ JarOutputStream jarOut = new JarOutputStream(out)) {
+ jarOut.putNextEntry(new JarEntry(entryName));
+ jarOut.write(content);
+ jarOut.closeEntry();
+ }
+ return jarPath;
+ }
+
+ public static boolean jarEquals(JarFile jar1, JarFile jar2, String entry) {
+ InputStream s1 = null;
+ InputStream s2 = null;
+ try {
+ s1 = jar1.getInputStream(jar1.getJarEntry(entry));
+ s2 = jar2.getInputStream(jar2.getJarEntry(entry));
+ return Arrays.equals(IOUtil.toByteArray(s1), IOUtil.toByteArray(s2));
+ } catch (IOException e) {
+ return false;
+ } finally {
+ if (s1 != null) {
+ try {
+ s1.close();
+ } catch (IOException ignored) {
+ }
+ }
+ if (s2 != null) {
+ try {
+ s2.close();
+ } catch (IOException ignored) {
+ }
+ }
+ }
+ }
+}
diff --git a/streampark-flink/streampark-flink-packer/src/test/scala/org/apache/streampark/flink/packer/DockerClientTest.scala b/streampark-flink/streampark-flink-packer/src/test/scala/org/apache/streampark/flink/packer/DockerClientTest.scala
deleted file mode 100644
index ddee08137..000000000
--- a/streampark-flink/streampark-flink-packer/src/test/scala/org/apache/streampark/flink/packer/DockerClientTest.scala
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.packer
-
-import org.apache.streampark.common.conf.{CommonConfig, InternalConfigHolder}
-import org.apache.streampark.flink.packer.docker.DockerImageExist
-import org.apache.streampark.flink.packer.docker.DockerRetriever.dockerClientConf
-
-import com.github.dockerjava.httpclient5.ApacheDockerHttpClient
-import org.scalatest.BeforeAndAfterAll
-import org.scalatest.matchers.must.Matchers
-import org.scalatest.wordspec.AnyWordSpec
-
-import java.time.Duration
-
-class DockerClientTest extends AnyWordSpec with BeforeAndAfterAll with Matchers {
-
- "Docker Client Config" when {
- val dockerHttpClientBuilder: ApacheDockerHttpClient.Builder =
- new ApacheDockerHttpClient.Builder()
- .dockerHost(dockerClientConf.getDockerHost)
- .sslConfig(dockerClientConf.getSSLConfig)
- .maxConnections(InternalConfigHolder.get(CommonConfig.DOCKER_MAX_CONNECTIONS))
- .connectionTimeout(
- Duration.ofSeconds(InternalConfigHolder.get(CommonConfig.DOCKER_CONNECTION_TIMEOUT_SEC)))
- .responseTimeout(Duration.ofSeconds(
- InternalConfigHolder.get(CommonConfig.DOCKER_RESPONSE_TIMEOUT_SEC)))
- }
-
- "Docker Image Exist" should {
- "return true if the image exists" in {
- val dockerImageExist = new DockerImageExist()
- val imageName = "flink:1.18.1-scala_2.12-java8"
- val result = dockerImageExist.doesDockerImageExist(imageName)
- assert(result)
- }
-
- "return false if the image does not exist" in {
- val dockerImageExist = new DockerImageExist()
- val imageName = "flink:1.18.1-scala_2.12-java8-fail"
- val result = dockerImageExist.doesDockerImageExist(imageName)
- assert(!result)
- }
- }
-
-}
diff --git a/streampark-flink/streampark-flink-packer/src/test/scala/org/apache/streampark/flink/packer/FlinkDockerfileTemplateSpec.scala b/streampark-flink/streampark-flink-packer/src/test/scala/org/apache/streampark/flink/packer/FlinkDockerfileTemplateSpec.scala
deleted file mode 100644
index e3225a9bd..000000000
--- a/streampark-flink/streampark-flink-packer/src/test/scala/org/apache/streampark/flink/packer/FlinkDockerfileTemplateSpec.scala
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.packer
-
-import org.apache.streampark.flink.packer.docker.FlinkDockerfileTemplate
-
-import org.apache.commons.io.FileUtils
-import org.scalatest.BeforeAndAfter
-import org.scalatest.matchers.must.Matchers
-import org.scalatest.wordspec.AnyWordSpec
-
-import java.io.File
-
-class FlinkDockerfileTemplateSpec extends AnyWordSpec with BeforeAndAfter with Matchers {
-
- val outputDir = new File("FlinkDockerfileTemplateSpec-output/")
-
- val assertDockerFileContent: String =
- """FROM 1.13-scala_2.11
- |RUN mkdir -p $FLINK_HOME/usrlib
- |COPY /WordCountSQL.jar $FLINK_HOME/usrlib/WordCountSQL.jar
- |""".stripMargin
-
- before {
- outputDir.mkdir()
- }
- after {
- FileUtils.forceDelete(outputDir)
- }
-
- "FlinkDockerfileTemplate" when {
-
- "create dockerfile" should {
- val template = FlinkDockerfileTemplate(
- outputDir.getAbsolutePath,
- "1.13-scala_2.11",
- path("flink/WordCountSQL.jar"),
- Set())
- val assertDockerFileContent =
- """FROM 1.13-scala_2.11
- |RUN mkdir -p $FLINK_HOME/usrlib
- |COPY /WordCountSQL.jar $FLINK_HOME/usrlib/WordCountSQL.jar
- |COPY lib $FLINK_HOME/lib/
- |""".stripMargin
- "build Dockerfile content" in {
- template.offerDockerfileContent mustBe assertDockerFileContent
- }
- "write Dockerfile to file" in {
- val outFile = template.writeDockerfile
- outFile.getName mustBe "dockerfile"
- FileUtils.readFileToString(outFile, "UTF-8") mustBe assertDockerFileContent
- }
- "write Dockerfile with special name" in {
- val outFile = template.writeDockerfile("Dockerfile")
- outFile.getName mustBe "my-dockerfile"
- FileUtils.readFileToString(outFile, "UTF-8") mustBe assertDockerFileContent
- }
- }
-
- }
-
-}
diff --git a/streampark-flink/streampark-flink-packer/src/test/scala/org/apache/streampark/flink/packer/MavenArtifactSpec.scala b/streampark-flink/streampark-flink-packer/src/test/scala/org/apache/streampark/flink/packer/MavenArtifactSpec.scala
deleted file mode 100644
index 1109d67fa..000000000
--- a/streampark-flink/streampark-flink-packer/src/test/scala/org/apache/streampark/flink/packer/MavenArtifactSpec.scala
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.packer
-
-import org.apache.streampark.flink.packer.MavenArtifactSpec.illegalArtifactCoordsCases
-import org.apache.streampark.flink.packer.maven.Artifact
-
-import org.scalatest.matchers.must.Matchers
-import org.scalatest.wordspec.AnyWordSpec
-
-object MavenArtifactSpec {
-
- private[packer] lazy val illegalArtifactCoordsCases = Array(
- ":::",
- "org.apache.flink:flink-table:",
- ":flink-table:1.13.0",
- "org.apache.flink::1.13.0",
- "org.apache.flink:flink-table:",
- "org.apache.flink:")
-}
-
-class MavenArtifactSpec extends AnyWordSpec with Matchers {
-
- "Artifact" when {
- "created" should {
- "with legal coords" in {
- val art = Artifact.of("org.apache.flink:flink-table:1.13.0")
- art.groupId mustBe "org.apache.flink"
- art.artifactId mustBe "flink-table"
- art.version mustBe "1.13.0"
- }
- "with illegal coords" in {
- illegalArtifactCoordsCases.foreach(coord => {
- assertThrows[IllegalArgumentException] {
- Artifact.of(coord)
- }
- })
- }
- }
- }
-
-}
diff --git a/streampark-flink/streampark-flink-packer/src/test/scala/org/apache/streampark/flink/packer/MavenToolSpec.scala b/streampark-flink/streampark-flink-packer/src/test/scala/org/apache/streampark/flink/packer/MavenToolSpec.scala
deleted file mode 100644
index 67735d9cf..000000000
--- a/streampark-flink/streampark-flink-packer/src/test/scala/org/apache/streampark/flink/packer/MavenToolSpec.scala
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.packer
-
-import org.apache.streampark.common.util.Implicits._
-import org.apache.streampark.flink.packer.maven.{Artifact, DependencyInfo, MavenTool}
-
-import org.apache.commons.io.FileUtils
-import org.scalatest.BeforeAndAfterAll
-import org.scalatest.matchers.must.Matchers
-import org.scalatest.wordspec.AnyWordSpec
-
-import java.io.File
-import java.util.jar.JarFile
-
-class MavenToolSpec extends AnyWordSpec with BeforeAndAfterAll with Matchers {
-
- val outputDir = "MavenToolSpec-output/"
- val preWorkSpaceVal: String = "/streampark"
-
- override protected def beforeAll(): Unit = {
- val output = new File(outputDir)
- FileUtils.deleteDirectory(output)
- FileUtils.forceMkdir(output)
- }
-
- override protected def afterAll(): Unit = {
- FileUtils.deleteDirectory(new File(outputDir))
- }
-
- "MavenTools" when {
- "build fat-jar" should {
- "with jarlibs" in {
- val fatJarPath = outputDir.concat("fat-1.jar")
- val fatJar = MavenTool.buildFatJar(
- null,
- Set(path("jars/commons-cli-1.4.jar"), path("jars/commons-dbutils-1.7.jar")),
- fatJarPath)
- fatJar.exists() mustBe true
- assert(
- jarEquals(
- new JarFile(fatJarPath),
- new JarFile(path("jars/commons-cli-1.4.jar")),
- "org/apache/commons/cli/DefaultParser.class"))
- assert(
- jarEquals(
- new JarFile(fatJarPath),
- new JarFile(path("jars/commons-dbutils-1.7.jar")),
- "org/apache/commons/dbutils/DbUtils.class"))
- }
- "with jarlibs under directory" in {
- val fatJarPath = outputDir.concat("fat-2.jar")
- val fatJar = MavenTool.buildFatJar(null, Set(path("jars/")), fatJarPath)
- fatJar.exists() mustBe true
- assert(
- jarEquals(
- new JarFile(fatJarPath),
- new JarFile(path("jars/commons-cli-1.4.jar")),
- "org/apache/commons/cli/DefaultParser.class"))
- assert(
- jarEquals(
- new JarFile(fatJarPath),
- new JarFile(path("jars/commons-dbutils-1.7.jar")),
- "org/apache/commons/dbutils/DbUtils.class"))
- assert(
- jarEquals(
- new JarFile(fatJarPath),
- new JarFile(path("jars/commons-logging-1.2.jar")),
- "org/apache/commons/logging/Log.class"))
- }
- "with jarlibs and maven artifacts" in {
- val fatJarPath = outputDir.concat("fat-3.jar")
- val fatJar = MavenTool.buildFatJar(
- null,
- DependencyInfo(
- Set(Artifact.of("org.apache.flink:flink-connector-kafka_2.11:1.13.0")),
- Set(path("jars/commons-dbutils-1.7.jar"))),
- fatJarPath)
- fatJar.exists() mustBe true
- assert(
- jarEquals(
- new JarFile(fatJarPath),
- new JarFile(path("jars/commons-dbutils-1.7.jar")),
- "org/apache/commons/dbutils/DbUtils.class"))
- new JarFile(fatJarPath).getJarEntry(
- "org/apache/kafka/clients/ClientUtils.class") mustNot be(null)
- new JarFile(fatJarPath).getJarEntry(
- "org/apache/flink/connector/base/source/reader/SourceReaderBase.class") mustNot be(null)
- }
- }
-
- "resolve artifacts" should {
- "with single artifact" in {
- val jars = MavenTool.resolveArtifacts(
- Set(Artifact.of("org.apache.flink:flink-connector-kafka_2.11:1.13.0")))
- val expectJars = Array(
- "force-shading-1.13.0.jar",
- "flink-connector-base-1.13.0.jar",
- "kafka-clients-2.4.1.jar")
- jars.size mustNot be(0)
- jars.forall(jar => jar.exists()) mustBe true
- jars.map(jar => jar.getName).sameElements(expectJars) mustBe true
- }
- "with mutiply artifact" in {
- val jars = MavenTool.resolveArtifacts(
- Set(
- Artifact.of("org.apache.flink:flink-connector-kafka_2.11:1.13.0"),
- Artifact.of("org.apache.flink:flink-connector-base:1.13.0")))
- val expectJars = Array(
- "flink-core-1.13.0.jar",
- "force-shading-1.13.0.jar",
- "flink-connector-base-1.13.0.jar",
- "kafka-clients-2.4.1.jar")
- // scalastyle:off println
- jars.foreach(jar => println(jar.getName))
- // scalastyle:on println
- jars.forall(_.exists) mustBe true
- jars.map(_.getName).sameElements(expectJars) mustBe true
- FileUtils.deleteDirectory(new File(outputDir))
- }
-
- }
- }
-
-}