You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampark.apache.org by be...@apache.org on 2026/08/03 01:23:54 UTC
(streampark) branch dev updated: [Flink] Migrate streampark-flink-client from Scala to Java (#4467)
This is an automated email from the ASF dual-hosted git repository.
wolfboys pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/streampark.git
The following commit(s) were added to refs/heads/dev by this push:
new 940ca347b [Flink] Migrate streampark-flink-client from Scala to Java (#4467)
940ca347b is described below
commit 940ca347b0d1a41a93299c23df79ad1b06c24027
Author: Assert <42...@users.noreply.github.com>
AuthorDate: Mon Aug 3 09:23:49 2026 +0800
[Flink] Migrate streampark-flink-client from Scala to Java (#4467)
* [Migrate] #4452 Migrate streampark-flink-client from Scala to Java
Convert all 27 Scala sources in client-api and client-core to Java,
replace json4s with Jackson in session REST submit, and preserve
FlinkShimsProxy reflection contracts for console compatibility.
Closes #4452
* [CI] Fix Sonar issues for #4467 flink-client migration
Address 31 Sonar findings: serializable request fields (S1948), refactor
complex methods (S3776/S1141), replace instanceof checks (S1193), remove
printStackTrace (S4507), and improve exception/boolean handling (S112/S5411).
* [CI] Replace Sonar suppressions with proper fixes for #4467
Use parameter objects (JobClientTarget, SubmitApplicationSpec, etc.) for
S107, make BuildResult Serializable for S1948, and standardize client
API on FlinkException instead of @SuppressWarnings for S112.
* [Flink] Fix remaining Sonar S112/S1141 in flink-client entrypoint and Yarn trait
Replace throws Exception with FlinkException in FlinkClientEntrypoint and
refactor YarnClientTrait to eliminate nested try blocks without suppressions.
* [Flink] Reduce duplicated code in flink-client migration for Sonar
Extract callAsFlinkException helpers, AbstractSavepointClientRequest base
class, and generic entrypoint routing to lower new-code duplication.
* [Flink] Further reduce Sonar duplicated lines in flink-client migration
Extract shared logging, job-graph submit, and packer build response base
classes to bring new-code duplication below the quality gate threshold.
* [Flink] Remove unused import in SimpleBuildResponse
* [Flink] Fix Sonar duplication in client-api bean classes
Extract shared deploy request base, hdfs workspace helper, and compose
SubmitRequest from SubmitApplicationSpec to eliminate 221 duplicated lines.
* [Flink] Fix Spotless formatting in flink-client-api bean classes
Align constructor parameter indentation with the project formatter
so CI spotless:check passes.
* [Flink] Fix Spotless formatting in flink-client-core Java classes
Apply project formatter to client-core implementations and traits so
CI spotless:check passes during backend build.
---
.../impl/FlinkApplicationActionServiceImpl.java | 50 +-
.../service/impl/FlinkSavepointServiceImpl.java | 9 +-
.../streampark-flink-client-api/pom.xml | 6 +
.../flink/client/ExitSecurityManager.java} | 27 +-
.../streampark/flink/client/FlinkClient.java | 116 +++
.../client/bean/AbstractDeployClientRequest.java | 88 ++
.../bean/AbstractSavepointClientRequest.java | 87 ++
.../flink/client/bean/CancelRequest.java | 62 ++
.../flink/client/bean/CancelResponse.java} | 19 +-
.../flink/client/bean/ClientBeanUtils.java | 87 ++
.../flink/client/bean/DeployRequest.java | 49 ++
.../flink/client/bean/DeployRequestTrait.java} | 25 +-
.../flink/client/bean/DeployResponse.java | 66 ++
.../flink/client/bean/HdfsWorkspace.java | 71 ++
.../flink/client/bean/JobClientTarget.java | 51 ++
.../flink/client/bean/KubernetesDeployParam.java | 90 +++
.../flink/client/bean/SavepointCancelOptions.java | 58 ++
.../flink/client/bean/SavepointRequestTrait.java} | 37 +-
.../flink/client/bean/SavepointResponse.java} | 19 +-
.../client/bean/SavepointTriggerOptions.java} | 39 +-
.../flink/client/bean/ShutDownRequest.java} | 26 +-
.../flink/client/bean/ShutDownResponse.java} | 23 +-
.../flink/client/bean/SubmitApplicationSpec.java | 166 ++++
.../flink/client/bean/SubmitClusterSpec.java | 70 ++
.../flink/client/bean/SubmitRequest.java | 460 +++++++++++
.../flink/client/bean/SubmitResponse.java | 75 ++
.../flink/client/bean/TriggerSavepointRequest.java | 54 ++
.../streampark/flink/client/FlinkClient.scala | 108 ---
.../flink/client/bean/CancelRequest.scala | 39 -
.../flink/client/bean/CancelResponse.scala | 20 -
.../flink/client/bean/DeployRequest.scala | 72 --
.../flink/client/bean/SubmitRequest.scala | 236 ------
.../client/bean/TriggerSavepointRequest.scala | 38 -
.../streampark-flink-client-core/pom.xml | 42 +-
.../flink/client/FlinkClientEntrypoint.java | 121 +++
.../flink/client/impl/FlinkConfigurationOps.java} | 28 +-
.../impl/KubernetesNativeApplicationClient.java | 118 +++
.../client/impl/KubernetesNativeSessionClient.java | 267 +++++++
.../streampark/flink/client/impl/LocalClient.java | 120 +++
.../streampark/flink/client/impl/RemoteClient.java | 182 +++++
.../flink/client/impl/YarnApplicationClient.java | 167 ++++
.../flink/client/impl/YarnPerJobClient.java | 126 +++
.../flink/client/impl/YarnSessionClient.java | 281 +++++++
.../client/tool/FlinkSessionSubmitHelper.java | 188 +++++
.../flink/client/trait/FlinkClientTrait.java | 884 +++++++++++++++++++++
.../client/trait/KubernetesNativeClientTrait.java | 250 ++++++
.../flink/client/trait/YarnClientTrait.java | 271 +++++++
.../flink/client/FlinkClientEntrypoint.scala | 87 --
.../impl/KubernetesNativeApplicationClient.scala | 105 ---
.../impl/KubernetesNativeSessionClient.scala | 216 -----
.../streampark/flink/client/impl/LocalClient.scala | 108 ---
.../flink/client/impl/RemoteClient.scala | 171 ----
.../flink/client/impl/YarnApplicationClient.scala | 155 ----
.../flink/client/impl/YarnPerJobClient.scala | 114 ---
.../flink/client/impl/YarnSessionClient.scala | 209 -----
.../client/tool/FlinkSessionClientHelper.scala | 137 ----
.../flink/client/trait/FlinkClientTrait.scala | 617 --------------
.../client/trait/KubernetesNativeClientTrait.scala | 190 -----
.../flink/client/trait/YarnClientTrait.scala | 199 -----
.../flink/client/test/ParameterTestCase.java | 45 ++
.../flink/client/test/ParameterTestCase.scala | 44 -
.../flink/client/test/YarnPerJobTestCase.scala | 208 -----
...sponse.java => AbstractFlinkBuildResponse.java} | 16 +-
.../flink/packer/pipeline/BuildResult.java | 4 +-
.../packer/pipeline/DockerImageBuildResponse.java | 31 +-
.../packer/pipeline/K8sAppModeBuildResponse.java | 31 +-
.../flink/packer/pipeline/ShadedBuildResponse.java | 31 +-
.../flink/packer/pipeline/SimpleBuildResponse.java | 31 +-
68 files changed, 4903 insertions(+), 3334 deletions(-)
diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationActionServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationActionServiceImpl.java
index d79911d4c..299f68ab4 100644
--- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationActionServiceImpl.java
+++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationActionServiceImpl.java
@@ -75,6 +75,10 @@ import org.apache.streampark.console.core.watcher.FlinkK8sWatcherWrapper;
import org.apache.streampark.flink.client.FlinkClient;
import org.apache.streampark.flink.client.bean.CancelRequest;
import org.apache.streampark.flink.client.bean.CancelResponse;
+import org.apache.streampark.flink.client.bean.JobClientTarget;
+import org.apache.streampark.flink.client.bean.SavepointCancelOptions;
+import org.apache.streampark.flink.client.bean.SubmitApplicationSpec;
+import org.apache.streampark.flink.client.bean.SubmitClusterSpec;
import org.apache.streampark.flink.client.bean.SubmitRequest;
import org.apache.streampark.flink.client.bean.SubmitResponse;
import org.apache.streampark.flink.kubernetes.FlinkK8sWatcher;
@@ -309,13 +313,12 @@ public class FlinkApplicationActionServiceImpl
flinkEnv.getFlinkVersion(),
FlinkDeployMode.of(application.getDeployMode()),
properties,
- clusterId,
- application.getJobId(),
- appParam.getRestoreOrTriggerSavepoint(),
- appParam.getDrain(),
- customSavepoint,
- appParam.getNativeFormat(),
- namespace);
+ new JobClientTarget(clusterId, application.getJobId(), namespace),
+ new SavepointCancelOptions(
+ appParam.getRestoreOrTriggerSavepoint(),
+ appParam.getDrain(),
+ customSavepoint,
+ appParam.getNativeFormat()));
final Date triggerTime = new Date();
CompletableFuture<CancelResponse> cancelFuture =
@@ -459,22 +462,25 @@ public class FlinkApplicationActionServiceImpl
flinkEnv.getFlinkVersion(),
FlinkDeployMode.of(application.getDeployMode()),
getProperties(application, dynamicProperties),
- flinkEnv.getFlinkConf(),
- FlinkJobType.of(application.getJobType()),
- application.getId(),
- new JobID().toHexString(),
- application.getJobName(),
- appConf,
- application.getApplicationType(),
- getSavepointPath(appParam),
- FlinkRestoreMode.of(appParam.getRestoreMode()),
- applicationArgs,
- k8sClusterId,
- application.getHadoopUser(),
+ SubmitApplicationSpec.builder()
+ .flinkYaml(flinkEnv.getFlinkConf())
+ .jobType(FlinkJobType.of(application.getJobType()))
+ .id(application.getId())
+ .jobId(new JobID().toHexString())
+ .appName(application.getJobName())
+ .appConf(appConf)
+ .applicationType(application.getApplicationType())
+ .savePoint(getSavepointPath(appParam))
+ .restoreMode(FlinkRestoreMode.of(appParam.getRestoreMode()))
+ .args(applicationArgs)
+ .build(),
+ new SubmitClusterSpec(
+ k8sClusterId,
+ application.getHadoopUser(),
+ k8sNamespace,
+ exposedType),
buildResult,
- extraParameter,
- k8sNamespace,
- exposedType);
+ extraParameter);
CompletableFuture<SubmitResponse> future =
CompletableFuture.supplyAsync(() -> FlinkClient.submit(submitRequest), executorService);
diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/FlinkSavepointServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/FlinkSavepointServiceImpl.java
index 743cbe615..78b1a552b 100644
--- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/FlinkSavepointServiceImpl.java
+++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/FlinkSavepointServiceImpl.java
@@ -46,7 +46,9 @@ import org.apache.streampark.console.core.service.application.FlinkApplicationMa
import org.apache.streampark.console.core.util.ServiceHelper;
import org.apache.streampark.console.core.watcher.FlinkAppHttpWatcher;
import org.apache.streampark.flink.client.FlinkClient;
+import org.apache.streampark.flink.client.bean.JobClientTarget;
import org.apache.streampark.flink.client.bean.SavepointResponse;
+import org.apache.streampark.flink.client.bean.SavepointTriggerOptions;
import org.apache.streampark.flink.client.bean.TriggerSavepointRequest;
import org.apache.streampark.flink.util.FlinkUtils;
@@ -503,10 +505,7 @@ public class FlinkSavepointServiceImpl extends ServiceImpl<FlinkSavepointMapper,
flinkEnv.getFlinkVersion(),
application.getDeployModeEnum(),
properties,
- clusterId,
- application.getJobId(),
- customSavepoint,
- nativeFormat,
- application.getK8sNamespace());
+ new JobClientTarget(clusterId, application.getJobId(), application.getK8sNamespace()),
+ new SavepointTriggerOptions(customSavepoint, nativeFormat));
}
}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/pom.xml b/streampark-flink/streampark-flink-client/streampark-flink-client-api/pom.xml
index 2aba03fab..be2413c38 100644
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/pom.xml
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/pom.xml
@@ -72,6 +72,12 @@
<artifactId>hadoop-client-runtime</artifactId>
<scope>provided</scope>
</dependency>
+
+ <dependency>
+ <groupId>org.projectlombok</groupId>
+ <artifactId>lombok</artifactId>
+ <scope>provided</scope>
+ </dependency>
</dependencies>
<profiles>
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/DeployRequestTrait.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/ExitSecurityManager.java
similarity index 59%
rename from streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/DeployRequestTrait.scala
rename to streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/ExitSecurityManager.java
index d72918e57..e1784690b 100644
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/DeployRequestTrait.scala
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/ExitSecurityManager.java
@@ -15,20 +15,23 @@
* limitations under the License.
*/
-package org.apache.streampark.flink.client.bean
+package org.apache.streampark.flink.client;
-import org.apache.streampark.common.conf.FlinkVersion
-import org.apache.streampark.common.enums.FlinkDeployMode
-import org.apache.streampark.common.util.Implicits.JavaMap
+import java.security.Permission;
-import javax.annotation.Nullable
+/** Used to mask JVM requests for external operations. */
+public class ExitSecurityManager extends SecurityManager {
-trait DeployRequestTrait {
+ @Override
+ public void checkExit(int status) {
+ throw new SecurityException(
+ "System.exit("
+ + status
+ + ") was called in your flink job, The job has been stopped, please check your program...");
+ }
- val flinkVersion: FlinkVersion
- val deployMode: FlinkDeployMode
- val properties: JavaMap[String, Any]
- val clusterId: String
- val id: Long
- @Nullable val k8sParam: KubernetesDeployParam
+ @Override
+ public void checkPermission(Permission perm) {
+ // no-op
+ }
}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/FlinkClient.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/FlinkClient.java
new file mode 100644
index 000000000..73c0e89e2
--- /dev/null
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/FlinkClient.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.client;
+
+import org.apache.streampark.common.conf.FlinkVersion;
+import org.apache.streampark.common.util.LoggerSupport;
+import org.apache.streampark.flink.client.bean.CancelRequest;
+import org.apache.streampark.flink.client.bean.CancelResponse;
+import org.apache.streampark.flink.client.bean.DeployRequest;
+import org.apache.streampark.flink.client.bean.DeployResponse;
+import org.apache.streampark.flink.client.bean.SavepointResponse;
+import org.apache.streampark.flink.client.bean.ShutDownRequest;
+import org.apache.streampark.flink.client.bean.ShutDownResponse;
+import org.apache.streampark.flink.client.bean.SubmitRequest;
+import org.apache.streampark.flink.client.bean.SubmitResponse;
+import org.apache.streampark.flink.client.bean.TriggerSavepointRequest;
+import org.apache.streampark.flink.proxy.FlinkShimsProxy;
+
+import java.util.function.Function;
+
+public final class FlinkClient extends LoggerSupport {
+
+ private static final String FLINK_CLIENT_ENTRYPOINT_CLASS =
+ "org.apache.streampark.flink.client.FlinkClientEntrypoint";
+
+ private static final String SUBMIT_REQUEST =
+ "org.apache.streampark.flink.client.bean.SubmitRequest";
+
+ private static final String DEPLOY_REQUEST =
+ "org.apache.streampark.flink.client.bean.DeployRequest";
+
+ private static final String CANCEL_REQUEST =
+ "org.apache.streampark.flink.client.bean.CancelRequest";
+
+ private static final String SHUTDOWN_REQUEST =
+ "org.apache.streampark.flink.client.bean.ShutDownRequest";
+
+ private static final String SAVEPOINT_REQUEST =
+ "org.apache.streampark.flink.client.bean.TriggerSavepointRequest";
+
+ private FlinkClient() {
+ }
+
+ public static SubmitResponse submit(SubmitRequest submitRequest) {
+ SecurityManager securityManager = System.getSecurityManager();
+ try {
+ System.setSecurityManager(new ExitSecurityManager());
+ return proxy(submitRequest, submitRequest.flinkVersion(), SUBMIT_REQUEST, "submit");
+ } finally {
+ System.setSecurityManager(securityManager);
+ }
+ }
+
+ public static CancelResponse cancel(CancelRequest stopRequest) {
+ return proxy(stopRequest, stopRequest.flinkVersion(), CANCEL_REQUEST, "cancel");
+ }
+
+ public static DeployResponse deploy(DeployRequest deployRequest) {
+ return proxy(deployRequest, deployRequest.flinkVersion(), DEPLOY_REQUEST, "deploy");
+ }
+
+ public static ShutDownResponse shutdown(ShutDownRequest shutDownRequest) {
+ return proxy(shutDownRequest, shutDownRequest.flinkVersion(), SHUTDOWN_REQUEST, "shutdown");
+ }
+
+ public static SavepointResponse triggerSavepoint(TriggerSavepointRequest savepointRequest) {
+ return proxy(
+ savepointRequest, savepointRequest.flinkVersion(), SAVEPOINT_REQUEST, "triggerSavepoint");
+ }
+
+ @SuppressWarnings("unchecked")
+ private static <T> T proxy(
+ Object request,
+ FlinkVersion flinkVersion,
+ String requestClassName,
+ String methodName) {
+ flinkVersion.checkVersion();
+ return FlinkShimsProxy.proxy(
+ flinkVersion,
+ (Function<ClassLoader, T>) classLoader -> {
+ try {
+ Class<?> submitClass = classLoader.loadClass(FLINK_CLIENT_ENTRYPOINT_CLASS);
+ Class<?> requestClass = classLoader.loadClass(requestClassName);
+ java.lang.reflect.Method method =
+ submitClass.getDeclaredMethod(methodName, requestClass);
+ method.setAccessible(true);
+ Object obj =
+ method.invoke(
+ null, FlinkShimsProxy.getObject(classLoader, request));
+ if (obj == null) {
+ return null;
+ }
+ return FlinkShimsProxy.getObject(FlinkClient.class.getClassLoader(), obj);
+ } catch (RuntimeException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ });
+ }
+}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/AbstractDeployClientRequest.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/AbstractDeployClientRequest.java
new file mode 100644
index 000000000..53b2b901e
--- /dev/null
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/AbstractDeployClientRequest.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.client.bean;
+
+import org.apache.streampark.common.conf.FlinkVersion;
+import org.apache.streampark.common.enums.FlinkDeployMode;
+
+import javax.annotation.Nullable;
+
+import java.io.Serializable;
+import java.util.Map;
+
+/** Shared fields for deploy and shutdown client requests. */
+abstract class AbstractDeployClientRequest implements DeployRequestTrait, Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private final FlinkVersion flinkVersion;
+ private final FlinkDeployMode deployMode;
+ @Nullable
+ private final Map<String, Serializable> properties;
+ private final String clusterId;
+ private final long id;
+ @Nullable
+ private final KubernetesDeployParam k8sParam;
+
+ AbstractDeployClientRequest(
+ FlinkVersion flinkVersion,
+ FlinkDeployMode deployMode,
+ @Nullable Map<String, Object> properties,
+ String clusterId,
+ long id,
+ @Nullable KubernetesDeployParam k8sParam) {
+ this.flinkVersion = flinkVersion;
+ this.deployMode = deployMode;
+ this.properties = ClientBeanUtils.toSerializableMap(properties);
+ this.clusterId = clusterId;
+ this.id = id;
+ this.k8sParam = k8sParam;
+ }
+
+ @Override
+ public FlinkVersion flinkVersion() {
+ return flinkVersion;
+ }
+
+ @Override
+ public FlinkDeployMode deployMode() {
+ return deployMode;
+ }
+
+ @Override
+ @Nullable
+ public Map<String, Object> properties() {
+ return ClientBeanUtils.copyPropertiesMap(properties);
+ }
+
+ @Override
+ public String clusterId() {
+ return clusterId;
+ }
+
+ @Override
+ public long id() {
+ return id;
+ }
+
+ @Override
+ @Nullable
+ public KubernetesDeployParam k8sParam() {
+ return k8sParam;
+ }
+}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/AbstractSavepointClientRequest.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/AbstractSavepointClientRequest.java
new file mode 100644
index 000000000..ffe029b45
--- /dev/null
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/AbstractSavepointClientRequest.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.client.bean;
+
+import org.apache.streampark.common.conf.FlinkVersion;
+import org.apache.streampark.common.enums.FlinkDeployMode;
+
+import javax.annotation.Nullable;
+
+import java.io.Serializable;
+import java.util.Map;
+
+/** Shared fields for cancel and trigger-savepoint client requests. */
+abstract class AbstractSavepointClientRequest implements SavepointRequestTrait, Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private final long id;
+ private final FlinkVersion flinkVersion;
+ private final FlinkDeployMode deployMode;
+ @Nullable
+ private final Map<String, Serializable> properties;
+ private final JobClientTarget target;
+
+ AbstractSavepointClientRequest(
+ long id,
+ FlinkVersion flinkVersion,
+ FlinkDeployMode deployMode,
+ @Nullable Map<String, Object> properties,
+ JobClientTarget target) {
+ this.id = id;
+ this.flinkVersion = flinkVersion;
+ this.deployMode = deployMode;
+ this.properties = ClientBeanUtils.toSerializableMap(properties);
+ this.target = target;
+ }
+
+ public long id() {
+ return id;
+ }
+
+ @Override
+ public FlinkVersion flinkVersion() {
+ return flinkVersion;
+ }
+
+ @Override
+ public FlinkDeployMode deployMode() {
+ return deployMode;
+ }
+
+ @Override
+ @Nullable
+ public Map<String, Object> properties() {
+ return ClientBeanUtils.copyPropertiesMap(properties);
+ }
+
+ @Override
+ public String clusterId() {
+ return target.clusterId();
+ }
+
+ @Override
+ public String jobId() {
+ return target.jobId();
+ }
+
+ @Override
+ public String kubernetesNamespace() {
+ return target.kubernetesNamespace();
+ }
+}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/CancelRequest.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/CancelRequest.java
new file mode 100644
index 000000000..02d47158c
--- /dev/null
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/CancelRequest.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.client.bean;
+
+import org.apache.streampark.common.conf.FlinkVersion;
+import org.apache.streampark.common.enums.FlinkDeployMode;
+
+import javax.annotation.Nullable;
+
+import java.util.Map;
+
+public class CancelRequest extends AbstractSavepointClientRequest {
+
+ private static final long serialVersionUID = 1L;
+
+ private final SavepointCancelOptions savepointOptions;
+
+ public CancelRequest(
+ long id,
+ FlinkVersion flinkVersion,
+ FlinkDeployMode deployMode,
+ @Nullable Map<String, Object> properties,
+ JobClientTarget target,
+ SavepointCancelOptions savepointOptions) {
+ super(id, flinkVersion, deployMode, properties, target);
+ this.savepointOptions = savepointOptions;
+ }
+
+ @Override
+ public boolean withSavepoint() {
+ return savepointOptions.withSavepoint();
+ }
+
+ public boolean withDrain() {
+ return savepointOptions.withDrain();
+ }
+
+ @Override
+ public String savepointPath() {
+ return savepointOptions.savepointPath();
+ }
+
+ @Override
+ public boolean nativeFormat() {
+ return savepointOptions.nativeFormat();
+ }
+}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/ShutDownResponse.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/CancelResponse.java
similarity index 66%
rename from streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/ShutDownResponse.scala
rename to streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/CancelResponse.java
index de96a55aa..1960d1e5e 100644
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/ShutDownResponse.scala
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/CancelResponse.java
@@ -15,6 +15,21 @@
* limitations under the License.
*/
-package org.apache.streampark.flink.client.bean
+package org.apache.streampark.flink.client.bean;
-case class ShutDownResponse(clusterId: String)
+import java.io.Serializable;
+
+public class CancelResponse implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private final String savepointDir;
+
+ public CancelResponse(String savepointDir) {
+ this.savepointDir = savepointDir;
+ }
+
+ public String savepointDir() {
+ return savepointDir;
+ }
+}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/ClientBeanUtils.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/ClientBeanUtils.java
new file mode 100644
index 000000000..625a917da
--- /dev/null
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/ClientBeanUtils.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.client.bean;
+
+import org.apache.streampark.common.conf.FlinkVersion;
+import org.apache.streampark.common.conf.Workspace;
+import org.apache.streampark.flink.util.FlinkUtils;
+
+import org.apache.commons.io.FileUtils;
+
+import javax.annotation.Nullable;
+
+import java.io.File;
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+
+final class ClientBeanUtils {
+
+ private ClientBeanUtils() {
+ }
+
+ static HdfsWorkspace createHdfsWorkspace(FlinkVersion flinkVersion) {
+ Workspace workspace = Workspace.remote();
+ String flinkHome = flinkVersion.flinkHome;
+ File flinkHomeDir = new File(flinkHome);
+ String flinkName;
+ try {
+ flinkName =
+ FileUtils.isSymlink(flinkHomeDir)
+ ? flinkHomeDir.getCanonicalFile().getName()
+ : flinkHomeDir.getName();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ String flinkHdfsHome = workspace.APP_FLINK() + "/" + flinkName;
+ return new HdfsWorkspace(
+ flinkName,
+ flinkHome,
+ FlinkUtils.getFlinkDistJar(flinkHome),
+ flinkHdfsHome + "/lib",
+ flinkHdfsHome + "/plugins",
+ workspace.APP_JARS());
+ }
+
+ @Nullable
+ static Map<String, Object> copyPropertiesMap(@Nullable Map<String, Serializable> properties) {
+ if (properties == null) {
+ return null;
+ }
+ Map<String, Object> result = new HashMap<>();
+ result.putAll(properties);
+ return result;
+ }
+
+ @Nullable
+ static Map<String, Serializable> toSerializableMap(@Nullable Map<String, Object> properties) {
+ if (properties == null) {
+ return null;
+ }
+ Map<String, Serializable> result = new HashMap<>();
+ for (Map.Entry<String, Object> entry : properties.entrySet()) {
+ Object value = entry.getValue();
+ if (value instanceof Serializable) {
+ result.put(entry.getKey(), (Serializable) value);
+ } else if (value != null) {
+ result.put(entry.getKey(), value.toString());
+ }
+ }
+ return result;
+ }
+}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/DeployRequest.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/DeployRequest.java
new file mode 100644
index 000000000..d4a4505a5
--- /dev/null
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/DeployRequest.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.client.bean;
+
+import org.apache.streampark.common.conf.FlinkVersion;
+import org.apache.streampark.common.enums.FlinkDeployMode;
+
+import javax.annotation.Nullable;
+
+import java.util.Map;
+
+public class DeployRequest extends AbstractDeployClientRequest {
+
+ private static final long serialVersionUID = 1L;
+
+ private transient HdfsWorkspace hdfsWorkspace;
+
+ public DeployRequest(
+ FlinkVersion flinkVersion,
+ FlinkDeployMode deployMode,
+ Map<String, Object> properties,
+ String clusterId,
+ long id,
+ @Nullable KubernetesDeployParam k8sParam) {
+ super(flinkVersion, deployMode, properties, clusterId, id, k8sParam);
+ }
+
+ public HdfsWorkspace hdfsWorkspace() {
+ if (hdfsWorkspace == null) {
+ hdfsWorkspace = ClientBeanUtils.createHdfsWorkspace(flinkVersion());
+ }
+ return hdfsWorkspace;
+ }
+}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/DeployResponse.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/DeployRequestTrait.java
similarity index 63%
rename from streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/DeployResponse.scala
rename to streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/DeployRequestTrait.java
index 6e1edbaff..e48d51a7d 100644
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/DeployResponse.scala
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/DeployRequestTrait.java
@@ -15,6 +15,27 @@
* limitations under the License.
*/
-package org.apache.streampark.flink.client.bean
+package org.apache.streampark.flink.client.bean;
-case class DeployResponse(address: String = null, clusterId: String = null, error: Throwable = null)
+import org.apache.streampark.common.conf.FlinkVersion;
+import org.apache.streampark.common.enums.FlinkDeployMode;
+
+import javax.annotation.Nullable;
+
+import java.util.Map;
+
+public interface DeployRequestTrait {
+
+ FlinkVersion flinkVersion();
+
+ FlinkDeployMode deployMode();
+
+ Map<String, Object> properties();
+
+ String clusterId();
+
+ long id();
+
+ @Nullable
+ KubernetesDeployParam k8sParam();
+}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/DeployResponse.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/DeployResponse.java
new file mode 100644
index 000000000..559b52180
--- /dev/null
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/DeployResponse.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.client.bean;
+
+import javax.annotation.Nullable;
+
+import java.io.Serializable;
+
+public class DeployResponse implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ @Nullable
+ private final String address;
+ @Nullable
+ private final String clusterId;
+ @Nullable
+ private final Throwable error;
+
+ public DeployResponse(@Nullable String address, @Nullable String clusterId) {
+ this(address, clusterId, null);
+ }
+
+ public DeployResponse(@Nullable Throwable error) {
+ this(null, null, error);
+ }
+
+ public DeployResponse(
+ @Nullable String address,
+ @Nullable String clusterId,
+ @Nullable Throwable error) {
+ this.address = address;
+ this.clusterId = clusterId;
+ this.error = error;
+ }
+
+ @Nullable
+ public String address() {
+ return address;
+ }
+
+ @Nullable
+ public String clusterId() {
+ return clusterId;
+ }
+
+ @Nullable
+ public Throwable error() {
+ return error;
+ }
+}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/HdfsWorkspace.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/HdfsWorkspace.java
new file mode 100644
index 000000000..bbe328b5a
--- /dev/null
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/HdfsWorkspace.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.client.bean;
+
+import java.io.Serializable;
+
+public class HdfsWorkspace implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private final String flinkName;
+ private final String flinkHome;
+ private final String flinkDistJar;
+ private final String flinkLib;
+ private final String flinkPlugins;
+ private final String appJars;
+
+ public HdfsWorkspace(
+ String flinkName,
+ String flinkHome,
+ String flinkDistJar,
+ String flinkLib,
+ String flinkPlugins,
+ String appJars) {
+ this.flinkName = flinkName;
+ this.flinkHome = flinkHome;
+ this.flinkDistJar = flinkDistJar;
+ this.flinkLib = flinkLib;
+ this.flinkPlugins = flinkPlugins;
+ this.appJars = appJars;
+ }
+
+ public String flinkName() {
+ return flinkName;
+ }
+
+ public String flinkHome() {
+ return flinkHome;
+ }
+
+ public String flinkDistJar() {
+ return flinkDistJar;
+ }
+
+ public String flinkLib() {
+ return flinkLib;
+ }
+
+ public String flinkPlugins() {
+ return flinkPlugins;
+ }
+
+ public String appJars() {
+ return appJars;
+ }
+}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/JobClientTarget.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/JobClientTarget.java
new file mode 100644
index 000000000..7bca4278a
--- /dev/null
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/JobClientTarget.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.client.bean;
+
+import org.apache.streampark.common.constants.Constants;
+
+import java.io.Serializable;
+
+/** Cluster and job identity for client cancel/savepoint requests. */
+public final class JobClientTarget implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private final String clusterId;
+ private final String jobId;
+ private final String kubernetesNamespace;
+
+ public JobClientTarget(String clusterId, String jobId, String kubernetesNamespace) {
+ this.clusterId = clusterId;
+ this.jobId = jobId;
+ this.kubernetesNamespace =
+ kubernetesNamespace != null ? kubernetesNamespace : Constants.DEFAULT;
+ }
+
+ public String clusterId() {
+ return clusterId;
+ }
+
+ public String jobId() {
+ return jobId;
+ }
+
+ public String kubernetesNamespace() {
+ return kubernetesNamespace;
+ }
+}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/KubernetesDeployParam.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/KubernetesDeployParam.java
new file mode 100644
index 000000000..26c11cf16
--- /dev/null
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/KubernetesDeployParam.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.client.bean;
+
+import org.apache.streampark.common.enums.FlinkK8sRestExposedType;
+
+import org.apache.flink.kubernetes.configuration.KubernetesConfigOptions;
+
+import javax.annotation.Nullable;
+
+import java.io.Serializable;
+
+public class KubernetesDeployParam implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private final String clusterId;
+ private final String kubernetesNamespace;
+ private final String kubeConf;
+ private final String serviceAccount;
+ private final String flinkImage;
+ @Nullable
+ private final FlinkK8sRestExposedType flinkRestExposedType;
+
+ public KubernetesDeployParam(String clusterId) {
+ this(
+ clusterId,
+ KubernetesConfigOptions.NAMESPACE.defaultValue(),
+ "~/.kube/config",
+ KubernetesConfigOptions.KUBERNETES_SERVICE_ACCOUNT.defaultValue(),
+ KubernetesConfigOptions.CONTAINER_IMAGE.defaultValue(),
+ FlinkK8sRestExposedType.CLUSTER_IP);
+ }
+
+ public KubernetesDeployParam(
+ String clusterId,
+ String kubernetesNamespace,
+ String kubeConf,
+ String serviceAccount,
+ String flinkImage,
+ @Nullable FlinkK8sRestExposedType flinkRestExposedType) {
+ this.clusterId = clusterId;
+ this.kubernetesNamespace = kubernetesNamespace;
+ this.kubeConf = kubeConf;
+ this.serviceAccount = serviceAccount;
+ this.flinkImage = flinkImage;
+ this.flinkRestExposedType =
+ flinkRestExposedType != null ? flinkRestExposedType : FlinkK8sRestExposedType.CLUSTER_IP;
+ }
+
+ public String clusterId() {
+ return clusterId;
+ }
+
+ public String kubernetesNamespace() {
+ return kubernetesNamespace;
+ }
+
+ public String kubeConf() {
+ return kubeConf;
+ }
+
+ public String serviceAccount() {
+ return serviceAccount;
+ }
+
+ public String flinkImage() {
+ return flinkImage;
+ }
+
+ @Nullable
+ public FlinkK8sRestExposedType flinkRestExposedType() {
+ return flinkRestExposedType;
+ }
+}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SavepointCancelOptions.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SavepointCancelOptions.java
new file mode 100644
index 000000000..baae013f4
--- /dev/null
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SavepointCancelOptions.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.client.bean;
+
+import java.io.Serializable;
+
+/** Savepoint-related options for job cancel requests. */
+public final class SavepointCancelOptions implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private final boolean withSavepoint;
+ private final boolean withDrain;
+ private final String savepointPath;
+ private final boolean nativeFormat;
+
+ public SavepointCancelOptions(
+ boolean withSavepoint,
+ boolean withDrain,
+ String savepointPath,
+ boolean nativeFormat) {
+ this.withSavepoint = withSavepoint;
+ this.withDrain = withDrain;
+ this.savepointPath = savepointPath;
+ this.nativeFormat = nativeFormat;
+ }
+
+ public boolean withSavepoint() {
+ return withSavepoint;
+ }
+
+ public boolean withDrain() {
+ return withDrain;
+ }
+
+ public String savepointPath() {
+ return savepointPath;
+ }
+
+ public boolean nativeFormat() {
+ return nativeFormat;
+ }
+}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/SavepointRequestTrait.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SavepointRequestTrait.java
similarity index 54%
copy from streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/SavepointRequestTrait.scala
copy to streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SavepointRequestTrait.java
index 739845151..219a48581 100644
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/SavepointRequestTrait.scala
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SavepointRequestTrait.java
@@ -15,33 +15,38 @@
* limitations under the License.
*/
-package org.apache.streampark.flink.client.bean
+package org.apache.streampark.flink.client.bean;
-import org.apache.streampark.common.conf.FlinkVersion
-import org.apache.streampark.common.constants.Constants
-import org.apache.streampark.common.enums.FlinkDeployMode
-import org.apache.streampark.common.util.Implicits._
+import org.apache.streampark.common.conf.FlinkVersion;
+import org.apache.streampark.common.constants.Constants;
+import org.apache.streampark.common.enums.FlinkDeployMode;
-import javax.annotation.Nullable
+import javax.annotation.Nullable;
-trait SavepointRequestTrait {
+import java.util.Map;
- val flinkVersion: FlinkVersion
+public interface SavepointRequestTrait {
- val deployMode: FlinkDeployMode
+ FlinkVersion flinkVersion();
- val clusterId: String
+ FlinkDeployMode deployMode();
- val jobId: String
+ String clusterId();
- val withSavepoint: Boolean = true
+ String jobId();
- val savepointPath: String
+ default boolean withSavepoint() {
+ return true;
+ }
- val nativeFormat: Boolean
+ String savepointPath();
- val kubernetesNamespace: String = Constants.DEFAULT
+ boolean nativeFormat();
- @Nullable val properties: JavaMap[String, Any]
+ default String kubernetesNamespace() {
+ return Constants.DEFAULT;
+ }
+ @Nullable
+ Map<String, Object> properties();
}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/SavepointResponse.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SavepointResponse.java
similarity index 68%
rename from streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/SavepointResponse.scala
rename to streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SavepointResponse.java
index 6127f11e2..76d341da6 100644
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/SavepointResponse.scala
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SavepointResponse.java
@@ -15,7 +15,22 @@
* limitations under the License.
*/
-package org.apache.streampark.flink.client.bean
+package org.apache.streampark.flink.client.bean;
+
+import java.io.Serializable;
/** Result class of trigger savepoint presents savepoint path. */
-case class SavepointResponse(savepointDir: String)
+public class SavepointResponse implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private final String savepointDir;
+
+ public SavepointResponse(String savepointDir) {
+ this.savepointDir = savepointDir;
+ }
+
+ public String savepointDir() {
+ return savepointDir;
+ }
+}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/SavepointRequestTrait.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SavepointTriggerOptions.java
similarity index 55%
rename from streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/SavepointRequestTrait.scala
rename to streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SavepointTriggerOptions.java
index 739845151..e22c99f1a 100644
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/SavepointRequestTrait.scala
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SavepointTriggerOptions.java
@@ -15,33 +15,28 @@
* limitations under the License.
*/
-package org.apache.streampark.flink.client.bean
+package org.apache.streampark.flink.client.bean;
-import org.apache.streampark.common.conf.FlinkVersion
-import org.apache.streampark.common.constants.Constants
-import org.apache.streampark.common.enums.FlinkDeployMode
-import org.apache.streampark.common.util.Implicits._
+import java.io.Serializable;
-import javax.annotation.Nullable
+/** Savepoint trigger options for savepoint requests. */
+public final class SavepointTriggerOptions implements Serializable {
-trait SavepointRequestTrait {
+ private static final long serialVersionUID = 1L;
- val flinkVersion: FlinkVersion
+ private final String savepointPath;
+ private final boolean nativeFormat;
- val deployMode: FlinkDeployMode
+ public SavepointTriggerOptions(String savepointPath, boolean nativeFormat) {
+ this.savepointPath = savepointPath;
+ this.nativeFormat = nativeFormat;
+ }
- val clusterId: String
-
- val jobId: String
-
- val withSavepoint: Boolean = true
-
- val savepointPath: String
-
- val nativeFormat: Boolean
-
- val kubernetesNamespace: String = Constants.DEFAULT
-
- @Nullable val properties: JavaMap[String, Any]
+ public String savepointPath() {
+ return savepointPath;
+ }
+ public boolean nativeFormat() {
+ return nativeFormat;
+ }
}
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/BuildResult.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/ShutDownRequest.java
similarity index 51%
copy from streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/BuildResult.java
copy to streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/ShutDownRequest.java
index 56d9969fc..4fc78d223 100644
--- a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/BuildResult.java
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/ShutDownRequest.java
@@ -15,16 +15,26 @@
* limitations under the License.
*/
-package org.apache.streampark.flink.packer.pipeline;
+package org.apache.streampark.flink.client.bean;
-/** Result of a BuildPipeline instance. */
-public interface BuildResult {
+import org.apache.streampark.common.conf.FlinkVersion;
+import org.apache.streampark.common.enums.FlinkDeployMode;
- /** is pass aka is successfully */
- boolean pass();
+import javax.annotation.Nullable;
- @SuppressWarnings("unchecked")
- default <T extends BuildResult> T as(Class<T> clz) {
- return (T) this;
+import java.util.Map;
+
+public class ShutDownRequest extends AbstractDeployClientRequest {
+
+ private static final long serialVersionUID = 1L;
+
+ public ShutDownRequest(
+ FlinkVersion flinkVersion,
+ FlinkDeployMode deployMode,
+ @Nullable Map<String, Object> properties,
+ String clusterId,
+ long id,
+ @Nullable KubernetesDeployParam k8sParam) {
+ super(flinkVersion, deployMode, properties, clusterId, id, k8sParam);
}
}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/SubmitResponse.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/ShutDownResponse.java
similarity index 67%
rename from streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/SubmitResponse.scala
rename to streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/ShutDownResponse.java
index e2466a204..736b4bbe9 100644
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/SubmitResponse.scala
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/ShutDownResponse.java
@@ -15,14 +15,21 @@
* limitations under the License.
*/
-package org.apache.streampark.flink.client.bean
+package org.apache.streampark.flink.client.bean;
-import org.apache.streampark.common.util.Implicits.JavaMap
+import java.io.Serializable;
-import javax.annotation.Nullable
+public class ShutDownResponse implements Serializable {
-case class SubmitResponse(
- clusterId: String,
- flinkConfig: JavaMap[String, String],
- @Nullable jobId: String = "",
- @Nullable jobManagerUrl: String = "")
+ private static final long serialVersionUID = 1L;
+
+ private final String clusterId;
+
+ public ShutDownResponse(String clusterId) {
+ this.clusterId = clusterId;
+ }
+
+ public String clusterId() {
+ return clusterId;
+ }
+}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SubmitApplicationSpec.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SubmitApplicationSpec.java
new file mode 100644
index 000000000..3729d9921
--- /dev/null
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SubmitApplicationSpec.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.client.bean;
+
+import org.apache.streampark.common.enums.ApplicationType;
+import org.apache.streampark.common.enums.FlinkJobType;
+import org.apache.streampark.common.enums.FlinkRestoreMode;
+
+import java.io.Serializable;
+
+/** Application metadata for Flink submit requests. */
+public final class SubmitApplicationSpec implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private final String flinkYaml;
+ private final FlinkJobType jobType;
+ private final long id;
+ private final String jobId;
+ private final String appName;
+ private final String appConf;
+ private final ApplicationType applicationType;
+ private final String savePoint;
+ private final FlinkRestoreMode restoreMode;
+ private final String args;
+
+ private SubmitApplicationSpec(Builder builder) {
+ this.flinkYaml = builder.flinkYaml;
+ this.jobType = builder.jobType;
+ this.id = builder.id;
+ this.jobId = builder.jobId;
+ this.appName = builder.appName;
+ this.appConf = builder.appConf;
+ this.applicationType = builder.applicationType;
+ this.savePoint = builder.savePoint;
+ this.restoreMode = builder.restoreMode;
+ this.args = builder.args;
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public String flinkYaml() {
+ return flinkYaml;
+ }
+
+ public FlinkJobType jobType() {
+ return jobType;
+ }
+
+ public long id() {
+ return id;
+ }
+
+ public String jobId() {
+ return jobId;
+ }
+
+ public String appName() {
+ return appName;
+ }
+
+ public String appConf() {
+ return appConf;
+ }
+
+ public ApplicationType applicationType() {
+ return applicationType;
+ }
+
+ public String savePoint() {
+ return savePoint;
+ }
+
+ public FlinkRestoreMode restoreMode() {
+ return restoreMode;
+ }
+
+ public String args() {
+ return args;
+ }
+
+ public static final class Builder {
+
+ private String flinkYaml;
+ private FlinkJobType jobType;
+ private long id;
+ private String jobId;
+ private String appName;
+ private String appConf;
+ private ApplicationType applicationType;
+ private String savePoint;
+ private FlinkRestoreMode restoreMode;
+ private String args;
+
+ public Builder flinkYaml(String flinkYaml) {
+ this.flinkYaml = flinkYaml;
+ return this;
+ }
+
+ public Builder jobType(FlinkJobType jobType) {
+ this.jobType = jobType;
+ return this;
+ }
+
+ public Builder id(long id) {
+ this.id = id;
+ return this;
+ }
+
+ public Builder jobId(String jobId) {
+ this.jobId = jobId;
+ return this;
+ }
+
+ public Builder appName(String appName) {
+ this.appName = appName;
+ return this;
+ }
+
+ public Builder appConf(String appConf) {
+ this.appConf = appConf;
+ return this;
+ }
+
+ public Builder applicationType(ApplicationType applicationType) {
+ this.applicationType = applicationType;
+ return this;
+ }
+
+ public Builder savePoint(String savePoint) {
+ this.savePoint = savePoint;
+ return this;
+ }
+
+ public Builder restoreMode(FlinkRestoreMode restoreMode) {
+ this.restoreMode = restoreMode;
+ return this;
+ }
+
+ public Builder args(String args) {
+ this.args = args;
+ return this;
+ }
+
+ public SubmitApplicationSpec build() {
+ return new SubmitApplicationSpec(this);
+ }
+ }
+}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SubmitClusterSpec.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SubmitClusterSpec.java
new file mode 100644
index 000000000..a6ffd8f07
--- /dev/null
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SubmitClusterSpec.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.client.bean;
+
+import org.apache.streampark.common.enums.FlinkK8sRestExposedType;
+
+import javax.annotation.Nullable;
+
+import java.io.Serializable;
+
+/** Cluster deployment context for Flink submit requests. */
+public final class SubmitClusterSpec implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ @Nullable
+ private final String clusterId;
+ @Nullable
+ private final String hadoopUser;
+ @Nullable
+ private final String kubernetesNamespace;
+ @Nullable
+ private final FlinkK8sRestExposedType flinkRestExposedType;
+
+ public SubmitClusterSpec(
+ @Nullable String clusterId,
+ @Nullable String hadoopUser,
+ @Nullable String kubernetesNamespace,
+ @Nullable FlinkK8sRestExposedType flinkRestExposedType) {
+ this.clusterId = clusterId;
+ this.hadoopUser = hadoopUser;
+ this.kubernetesNamespace = kubernetesNamespace;
+ this.flinkRestExposedType = flinkRestExposedType;
+ }
+
+ @Nullable
+ public String clusterId() {
+ return clusterId;
+ }
+
+ @Nullable
+ public String hadoopUser() {
+ return hadoopUser;
+ }
+
+ @Nullable
+ public String kubernetesNamespace() {
+ return kubernetesNamespace;
+ }
+
+ @Nullable
+ public FlinkK8sRestExposedType flinkRestExposedType() {
+ return flinkRestExposedType;
+ }
+}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SubmitRequest.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SubmitRequest.java
new file mode 100644
index 000000000..3bca741e9
--- /dev/null
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SubmitRequest.java
@@ -0,0 +1,460 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.client.bean;
+
+import org.apache.streampark.common.conf.ConfigKeys;
+import org.apache.streampark.common.conf.FlinkVersion;
+import org.apache.streampark.common.conf.Workspace;
+import org.apache.streampark.common.constants.Constants;
+import org.apache.streampark.common.enums.ApplicationType;
+import org.apache.streampark.common.enums.FlinkDeployMode;
+import org.apache.streampark.common.enums.FlinkJobType;
+import org.apache.streampark.common.enums.FlinkK8sRestExposedType;
+import org.apache.streampark.common.enums.FlinkRestoreMode;
+import org.apache.streampark.common.util.AssertUtils;
+import org.apache.streampark.common.util.DeflaterUtils;
+import org.apache.streampark.common.util.HdfsUtils;
+import org.apache.streampark.common.util.PropertiesUtils;
+import org.apache.streampark.flink.packer.pipeline.BuildResult;
+import org.apache.streampark.flink.packer.pipeline.ShadedBuildResponse;
+
+import org.apache.streampark.shaded.com.fasterxml.jackson.core.type.TypeReference;
+import org.apache.streampark.shaded.com.fasterxml.jackson.databind.ObjectMapper;
+
+import org.apache.commons.collections.MapUtils;
+import org.apache.flink.runtime.jobgraph.SavepointConfigOptions;
+import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings;
+
+import javax.annotation.Nullable;
+
+import java.io.File;
+import java.io.Serializable;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+public class SubmitRequest implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private final FlinkVersion flinkVersion;
+ private final FlinkDeployMode deployMode;
+ private final Map<String, Serializable> properties;
+ private final SubmitApplicationSpec application;
+ @Nullable
+ private final SubmitClusterSpec cluster;
+ @Nullable
+ private final BuildResult buildResult;
+ @Nullable
+ private final Map<String, Serializable> extraParameter;
+
+ private transient Map<String, String> appProperties;
+ private transient Map<String, String> appOption;
+ private transient String appMain;
+ private transient String effectiveAppName;
+ private transient List<URL> libs;
+ private transient List<URL> classPaths;
+ private transient String flinkSQL;
+ private transient Boolean allowNonRestoredState;
+ private transient SavepointRestoreSettings savepointRestoreSettings;
+ private transient File userJarFile;
+ private transient Boolean safePackageProgram;
+ private transient HdfsWorkspace hdfsWorkspace;
+
+ public SubmitRequest(
+ FlinkVersion flinkVersion,
+ FlinkDeployMode deployMode,
+ Map<String, Object> properties,
+ SubmitApplicationSpec application,
+ @Nullable SubmitClusterSpec cluster,
+ @Nullable BuildResult buildResult,
+ @Nullable Map<String, Object> extraParameter) {
+ this.flinkVersion = flinkVersion;
+ this.deployMode = deployMode;
+ this.properties = ClientBeanUtils.toSerializableMap(properties);
+ this.application = application;
+ this.cluster = cluster;
+ this.buildResult = buildResult;
+ this.extraParameter = ClientBeanUtils.toSerializableMap(extraParameter);
+ }
+
+ public FlinkVersion flinkVersion() {
+ return flinkVersion;
+ }
+
+ public FlinkDeployMode deployMode() {
+ return deployMode;
+ }
+
+ public Map<String, Object> properties() {
+ return ClientBeanUtils.copyPropertiesMap(properties);
+ }
+
+ public String flinkYaml() {
+ return application.flinkYaml();
+ }
+
+ public FlinkJobType jobType() {
+ return application.jobType();
+ }
+
+ public long id() {
+ return application.id();
+ }
+
+ public String jobId() {
+ return application.jobId();
+ }
+
+ public String appName() {
+ return application.appName();
+ }
+
+ public String appConf() {
+ return application.appConf();
+ }
+
+ public ApplicationType applicationType() {
+ return application.applicationType();
+ }
+
+ public String savePoint() {
+ return application.savePoint();
+ }
+
+ public FlinkRestoreMode restoreMode() {
+ return application.restoreMode();
+ }
+
+ public String args() {
+ return application.args();
+ }
+
+ @Nullable
+ public String clusterId() {
+ return cluster != null ? cluster.clusterId() : null;
+ }
+
+ @Nullable
+ public String hadoopUser() {
+ return cluster != null ? cluster.hadoopUser() : null;
+ }
+
+ @Nullable
+ public BuildResult buildResult() {
+ return buildResult;
+ }
+
+ @Nullable
+ public Map<String, Object> extraParameter() {
+ return ClientBeanUtils.copyPropertiesMap(extraParameter);
+ }
+
+ @Nullable
+ public String kubernetesNamespace() {
+ return cluster != null ? cluster.kubernetesNamespace() : null;
+ }
+
+ @Nullable
+ public FlinkK8sRestExposedType flinkRestExposedType() {
+ return cluster != null ? cluster.flinkRestExposedType() : null;
+ }
+
+ public Map<String, String> appProperties() {
+ if (appProperties == null) {
+ appProperties = getParameterMap(ConfigKeys.KEY_FLINK_PROPERTY_PREFIX());
+ }
+ return appProperties;
+ }
+
+ public Map<String, String> appOption() {
+ if (appOption == null) {
+ appOption = getParameterMap(ConfigKeys.KEY_FLINK_OPTION_PREFIX());
+ }
+ return appOption;
+ }
+
+ public String appMain() {
+ if (appMain == null) {
+ if (jobType() == FlinkJobType.FLINK_SQL) {
+ appMain = Constants.STREAMPARK_FLINKSQL_CLIENT_CLASS;
+ } else if (jobType() == FlinkJobType.PYFLINK) {
+ appMain = Constants.PYTHON_FLINK_DRIVER_CLASS_NAME;
+ } else {
+ appMain = appProperties().get(ConfigKeys.KEY_FLINK_APPLICATION_MAIN_CLASS());
+ }
+ }
+ return appMain;
+ }
+
+ public String effectiveAppName() {
+ if (effectiveAppName == null) {
+ effectiveAppName =
+ appName() == null ? appProperties().get(ConfigKeys.KEY_FLINK_APP_NAME()) : appName();
+ }
+ return effectiveAppName;
+ }
+
+ public List<URL> libs() {
+ if (libs == null) {
+ File libDir = new File(new File(Workspace.local().APP_WORKSPACE(), String.valueOf(id())), "lib");
+ File[] files = libDir.listFiles();
+ if (files == null) {
+ libs = Collections.emptyList();
+ } else {
+ List<URL> urls = new ArrayList<>();
+ for (File file : files) {
+ try {
+ urls.add(file.toURI().toURL());
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ libs = urls;
+ }
+ }
+ return libs;
+ }
+
+ public List<URL> classPaths() {
+ if (classPaths == null) {
+ try {
+ List<URL> combined = new ArrayList<>(flinkVersion.getFlinkLibs());
+ combined.addAll(libs());
+ classPaths = combined;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ return classPaths;
+ }
+
+ public String flinkSQL() {
+ if (flinkSQL == null) {
+ flinkSQL = extraParameter.get(ConfigKeys.KEY_FLINK_SQL()).toString();
+ }
+ return flinkSQL;
+ }
+
+ public boolean allowNonRestoredState() {
+ if (allowNonRestoredState == null) {
+ Object value =
+ properties.get(SavepointConfigOptions.SAVEPOINT_IGNORE_UNCLAIMED_STATE.key());
+ if (value == null) {
+ allowNonRestoredState = false;
+ } else {
+ try {
+ allowNonRestoredState = Boolean.parseBoolean(value.toString());
+ } catch (Exception e) {
+ allowNonRestoredState = false;
+ }
+ }
+ }
+ return allowNonRestoredState;
+ }
+
+ public SavepointRestoreSettings savepointRestoreSettings() {
+ if (savepointRestoreSettings == null) {
+ if (savePoint() == null || savePoint().isEmpty()) {
+ savepointRestoreSettings = SavepointRestoreSettings.none();
+ } else {
+ savepointRestoreSettings =
+ SavepointRestoreSettings.forPath(savePoint(), allowNonRestoredState());
+ }
+ }
+ return savepointRestoreSettings;
+ }
+
+ public File userJarFile() {
+ if (userJarFile == null) {
+ if (deployMode() == FlinkDeployMode.KUBERNETES_NATIVE_APPLICATION) {
+ userJarFile = null;
+ } else {
+ checkBuildResult();
+ ShadedBuildResponse shadedBuildResult = buildResult.as(ShadedBuildResponse.class);
+ userJarFile = new File(shadedBuildResult.shadedJarPath());
+ }
+ }
+ return userJarFile;
+ }
+
+ public boolean safePackageProgram() {
+ if (safePackageProgram == null) {
+ String version = flinkVersion.version();
+ String[] parts = version.split("\\.");
+ if (parts.length >= 3) {
+ try {
+ int a = Integer.parseInt(parts[0].trim());
+ int b = Integer.parseInt(parts[1].trim());
+ int c = Integer.parseInt(parts[2].trim());
+ safePackageProgram = a >= 1 && (b > 12 || (b == 12 && c >= 2));
+ } catch (NumberFormatException e) {
+ safePackageProgram = false;
+ }
+ } else {
+ safePackageProgram = false;
+ }
+ }
+ return safePackageProgram;
+ }
+
+ public boolean hasProp(String key) {
+ return MapUtils.isNotEmpty(properties) && properties.containsKey(key);
+ }
+
+ public Object getProp(String key) {
+ return properties.get(key);
+ }
+
+ public void putProperty(String key, Serializable value) {
+ if (properties != null) {
+ properties.put(key, value);
+ }
+ }
+
+ public boolean hasExtra(String key) {
+ return MapUtils.isNotEmpty(extraParameter) && extraParameter.containsKey(key);
+ }
+
+ public Object getExtra(String key) {
+ return extraParameter.get(key);
+ }
+
+ public HdfsWorkspace hdfsWorkspace() {
+ if (hdfsWorkspace == null) {
+ hdfsWorkspace = ClientBeanUtils.createHdfsWorkspace(flinkVersion);
+ }
+ return hdfsWorkspace;
+ }
+
+ public void checkBuildResult() {
+ if (deployMode() == FlinkDeployMode.KUBERNETES_NATIVE_SESSION) {
+ AssertUtils.required(
+ buildResult != null,
+ "[flink-submit] current job: "
+ + effectiveAppName()
+ + " was not yet built, buildResult is empty"
+ + ",clusterId="
+ + clusterId()
+ + ","
+ + ",namespace="
+ + kubernetesNamespace());
+ AssertUtils.required(
+ buildResult.pass(),
+ "[flink-submit] current job "
+ + effectiveAppName()
+ + " build failed, clusterId"
+ + ",clusterId="
+ + clusterId()
+ + ","
+ + ",namespace="
+ + kubernetesNamespace());
+ } else {
+ AssertUtils.required(
+ buildResult != null,
+ "[flink-submit] current job: "
+ + effectiveAppName()
+ + " was not yet built, buildResult is empty");
+ AssertUtils.required(
+ buildResult.pass(),
+ "[flink-submit] current job " + effectiveAppName() + " build failed, please check");
+ }
+ }
+
+ private Map<String, String> getParameterMap(String prefix) {
+ if (appConf() == null) {
+ return Collections.emptyMap();
+ }
+ String format = appConf().substring(0, Math.min(appConf().length(), 7));
+ Map<String, String> map = loadAppConfMap(format);
+ return filterByPrefix(map, prefix);
+ }
+
+ private Map<String, String> loadAppConfMap(String format) {
+ if ("json://".equals(format)) {
+ return parseJsonAppConf();
+ }
+ String content = DeflaterUtils.unzipString(appConf().trim().substring(7));
+ switch (format) {
+ case "yaml://":
+ return PropertiesUtils.fromYamlTextAsJava(content);
+ case "conf://":
+ return PropertiesUtils.fromHoconTextAsJava(content);
+ case "prop://":
+ return PropertiesUtils.fromPropertiesTextAsJava(content);
+ case "hdfs://":
+ return parseHdfsAppConf();
+ default:
+ throw new IllegalArgumentException("[StreamPark] application config format error.");
+ }
+ }
+
+ private Map<String, String> parseJsonAppConf() {
+ String json = appConf().substring(7);
+ try {
+ Map<String, String> map =
+ new ObjectMapper()
+ .readValue(json, new TypeReference<Map<String, String>>() {
+ });
+ return map.entrySet().stream()
+ .filter(e -> e.getValue() != null)
+ .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private Map<String, String> parseHdfsAppConf() {
+ try {
+ String text = HdfsUtils.read(appConf());
+ String extension = appConf().split("\\.")[appConf().split("\\.").length - 1].toLowerCase();
+ switch (extension) {
+ case "yml":
+ case "yaml":
+ return PropertiesUtils.fromYamlTextAsJava(text);
+ case "conf":
+ return PropertiesUtils.fromHoconTextAsJava(text);
+ case "properties":
+ return PropertiesUtils.fromPropertiesTextAsJava(text);
+ default:
+ throw new IllegalArgumentException(
+ "[StreamPark] Usage: application config format error,must be [yaml|conf|properties]");
+ }
+ } catch (RuntimeException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static Map<String, String> filterByPrefix(Map<String, String> map, String prefix) {
+ Map<String, String> result = new HashMap<>();
+ for (Map.Entry<String, String> entry : map.entrySet()) {
+ String key = entry.getKey();
+ String value = entry.getValue();
+ if (key.startsWith(prefix) && value != null && !value.isEmpty()) {
+ result.put(key.substring(prefix.length()), value);
+ }
+ }
+ return result;
+ }
+}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SubmitResponse.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SubmitResponse.java
new file mode 100644
index 000000000..985ebc67c
--- /dev/null
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SubmitResponse.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.client.bean;
+
+import javax.annotation.Nullable;
+
+import java.io.Serializable;
+import java.util.Map;
+
+public class SubmitResponse implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private final String clusterId;
+ private final Map<String, String> flinkConfig;
+ @Nullable
+ private final String jobId;
+ @Nullable
+ private final String jobManagerUrl;
+
+ public SubmitResponse(String clusterId, Map<String, String> flinkConfig) {
+ this(clusterId, flinkConfig, "", "");
+ }
+
+ public SubmitResponse(
+ String clusterId,
+ Map<String, String> flinkConfig,
+ @Nullable String jobManagerUrl) {
+ this(clusterId, flinkConfig, "", jobManagerUrl);
+ }
+
+ public SubmitResponse(
+ String clusterId,
+ Map<String, String> flinkConfig,
+ @Nullable String jobId,
+ @Nullable String jobManagerUrl) {
+ this.clusterId = clusterId;
+ this.flinkConfig = flinkConfig;
+ this.jobId = jobId != null ? jobId : "";
+ this.jobManagerUrl = jobManagerUrl != null ? jobManagerUrl : "";
+ }
+
+ public String clusterId() {
+ return clusterId;
+ }
+
+ public Map<String, String> flinkConfig() {
+ return flinkConfig;
+ }
+
+ @Nullable
+ public String jobId() {
+ return jobId;
+ }
+
+ @Nullable
+ public String jobManagerUrl() {
+ return jobManagerUrl;
+ }
+}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/TriggerSavepointRequest.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/TriggerSavepointRequest.java
new file mode 100644
index 000000000..4b972d9ce
--- /dev/null
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/TriggerSavepointRequest.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.client.bean;
+
+import org.apache.streampark.common.conf.FlinkVersion;
+import org.apache.streampark.common.enums.FlinkDeployMode;
+
+import javax.annotation.Nullable;
+
+import java.util.Map;
+
+/** Trigger savepoint request. */
+public class TriggerSavepointRequest extends AbstractSavepointClientRequest {
+
+ private static final long serialVersionUID = 1L;
+
+ private final SavepointTriggerOptions savepointOptions;
+
+ public TriggerSavepointRequest(
+ long id,
+ FlinkVersion flinkVersion,
+ FlinkDeployMode deployMode,
+ @Nullable Map<String, Object> properties,
+ JobClientTarget target,
+ SavepointTriggerOptions savepointOptions) {
+ super(id, flinkVersion, deployMode, properties, target);
+ this.savepointOptions = savepointOptions;
+ }
+
+ @Override
+ public String savepointPath() {
+ return savepointOptions.savepointPath();
+ }
+
+ @Override
+ public boolean nativeFormat() {
+ return savepointOptions.nativeFormat();
+ }
+}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/FlinkClient.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/FlinkClient.scala
deleted file mode 100644
index 2e7fdd0d5..000000000
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/FlinkClient.scala
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.client
-
-import org.apache.streampark.common.conf.FlinkVersion
-import org.apache.streampark.common.util.Logger
-import org.apache.streampark.flink.client.bean._
-import org.apache.streampark.flink.proxy.FlinkShimsProxy
-
-import java.security.Permission
-
-import scala.reflect.ClassTag
-
-object FlinkClient extends Logger {
-
- private[this] val FLINK_CLIENT_ENTRYPOINT_CLASS =
- "org.apache.streampark.flink.client.FlinkClientEntrypoint"
-
- private[this] val SUBMIT_REQUEST =
- "org.apache.streampark.flink.client.bean.SubmitRequest" -> "submit"
-
- private[this] val DEPLOY_REQUEST =
- "org.apache.streampark.flink.client.bean.DeployRequest" -> "deploy"
-
- private[this] val CANCEL_REQUEST =
- "org.apache.streampark.flink.client.bean.CancelRequest" -> "cancel"
-
- private[this] val SHUTDOWN_REQUEST =
- "org.apache.streampark.flink.client.bean.ShutDownRequest" -> "shutdown"
-
- private[this] val SAVEPOINT_REQUEST =
- "org.apache.streampark.flink.client.bean.TriggerSavepointRequest" -> "triggerSavepoint"
-
- def submit(submitRequest: SubmitRequest): SubmitResponse = {
- val securityManager = System.getSecurityManager
- try {
- System.setSecurityManager(new ExitSecurityManager())
- proxy[SubmitResponse](submitRequest, submitRequest.flinkVersion, SUBMIT_REQUEST)
- } catch {
- case e: Exception => throw e
- } finally {
- System.setSecurityManager(securityManager)
- }
- }
-
- def cancel(stopRequest: CancelRequest): CancelResponse = {
- proxy[CancelResponse](stopRequest, stopRequest.flinkVersion, CANCEL_REQUEST)
- }
-
- def deploy(deployRequest: DeployRequest): DeployResponse = {
- proxy[DeployResponse](deployRequest, deployRequest.flinkVersion, DEPLOY_REQUEST)
- }
-
- def shutdown(shutDownRequest: ShutDownRequest): ShutDownResponse = {
- proxy[ShutDownResponse](shutDownRequest, shutDownRequest.flinkVersion, SHUTDOWN_REQUEST)
- }
-
- def triggerSavepoint(savepointRequest: TriggerSavepointRequest): SavepointResponse = {
- proxy[SavepointResponse](savepointRequest, savepointRequest.flinkVersion, SAVEPOINT_REQUEST)
- }
-
- private[this] def proxy[T: ClassTag](
- request: Object,
- flinkVersion: FlinkVersion,
- requestBody: (String, String)): T = {
- flinkVersion.checkVersion()
- FlinkShimsProxy.proxy(
- flinkVersion,
- (classLoader: ClassLoader) => {
- val submitClass = classLoader.loadClass(FLINK_CLIENT_ENTRYPOINT_CLASS)
- val requestClass = classLoader.loadClass(requestBody._1)
- val method = submitClass.getDeclaredMethod(requestBody._2, requestClass)
- method.setAccessible(true)
- val obj = method.invoke(null, FlinkShimsProxy.getObject(classLoader, request))
- if (obj == null) {
- null.asInstanceOf[T]
- } else {
- FlinkShimsProxy.getObject[T](this.getClass.getClassLoader, obj)
- }
- })
- }
-
-}
-
-/** Used to mask JVM requests for external operations */
-class ExitSecurityManager extends SecurityManager {
- override def checkExit(status: Int): Unit = {
- throw new SecurityException(
- s"System.exit($status) was called in your flink job, The job has been stopped, please check your program...")
- }
-
- override def checkPermission(perm: Permission): Unit = {}
-}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/CancelRequest.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/CancelRequest.scala
deleted file mode 100644
index 0d5c9f7fd..000000000
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/CancelRequest.scala
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.client.bean
-
-import org.apache.streampark.common.conf.FlinkVersion
-import org.apache.streampark.common.constants.Constants
-import org.apache.streampark.common.enums.FlinkDeployMode
-import org.apache.streampark.common.util.Implicits.JavaMap
-
-import javax.annotation.Nullable
-
-case class CancelRequest(
- id: Long,
- flinkVersion: FlinkVersion,
- deployMode: FlinkDeployMode,
- @Nullable properties: JavaMap[String, Any],
- clusterId: String,
- jobId: String,
- override val withSavepoint: Boolean,
- withDrain: Boolean,
- savepointPath: String,
- nativeFormat: Boolean,
- override val kubernetesNamespace: String = Constants.DEFAULT)
- extends SavepointRequestTrait
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/CancelResponse.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/CancelResponse.scala
deleted file mode 100644
index 668a3af89..000000000
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/CancelResponse.scala
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.client.bean
-
-case class CancelResponse(savepointDir: String)
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/DeployRequest.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/DeployRequest.scala
deleted file mode 100644
index d567c640f..000000000
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/DeployRequest.scala
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.client.bean
-
-import org.apache.streampark.common.conf.{FlinkVersion, Workspace}
-import org.apache.streampark.common.enums.{FlinkDeployMode, FlinkK8sRestExposedType}
-import org.apache.streampark.common.util.Implicits._
-import org.apache.streampark.flink.util.FlinkUtils
-
-import org.apache.commons.io.FileUtils
-import org.apache.flink.kubernetes.configuration.KubernetesConfigOptions
-
-import javax.annotation.Nullable
-
-import java.io.File
-
-case class DeployRequest(
- flinkVersion: FlinkVersion,
- deployMode: FlinkDeployMode,
- properties: JavaMap[String, Any],
- clusterId: String,
- id: Long,
- @Nullable k8sParam: KubernetesDeployParam)
- extends DeployRequestTrait {
-
- private[client] lazy val hdfsWorkspace = {
-
- /**
- * You must keep the flink version and configuration in the native flink and hdfs exactly the
- * same.
- */
- val workspace = Workspace.remote
- val flinkHome = flinkVersion.flinkHome
- val flinkHomeDir = new File(flinkHome)
- val flinkName = if (FileUtils.isSymlink(flinkHomeDir)) {
- flinkHomeDir.getCanonicalFile.getName
- } else {
- flinkHomeDir.getName
- }
- val flinkHdfsHome = s"${workspace.APP_FLINK}/$flinkName"
- HdfsWorkspace(
- flinkName,
- flinkHome,
- flinkLib = s"$flinkHdfsHome/lib",
- flinkPlugins = s"$flinkHdfsHome/plugins",
- flinkDistJar = FlinkUtils.getFlinkDistJar(flinkHome),
- appJars = workspace.APP_JARS)
- }
-}
-
-case class KubernetesDeployParam(
- clusterId: String,
- kubernetesNamespace: String = KubernetesConfigOptions.NAMESPACE.defaultValue(),
- kubeConf: String = "~/.kube/config",
- serviceAccount: String = KubernetesConfigOptions.KUBERNETES_SERVICE_ACCOUNT.defaultValue(),
- flinkImage: String = KubernetesConfigOptions.CONTAINER_IMAGE.defaultValue(),
- @Nullable flinkRestExposedType: FlinkK8sRestExposedType = FlinkK8sRestExposedType.CLUSTER_IP)
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/SubmitRequest.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/SubmitRequest.scala
deleted file mode 100644
index 007aa217b..000000000
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/SubmitRequest.scala
+++ /dev/null
@@ -1,236 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.client.bean
-
-import org.apache.streampark.common.conf.{FlinkVersion, Workspace}
-import org.apache.streampark.common.conf.ConfigKeys._
-import org.apache.streampark.common.constants.Constants
-import org.apache.streampark.common.enums._
-import org.apache.streampark.common.util._
-import org.apache.streampark.common.util.Implicits._
-import org.apache.streampark.flink.packer.pipeline.{BuildResult, ShadedBuildResponse}
-import org.apache.streampark.flink.util.FlinkUtils
-import org.apache.streampark.shaded.com.fasterxml.jackson.databind.ObjectMapper
-
-import org.apache.commons.collections.MapUtils
-import org.apache.commons.io.FileUtils
-import org.apache.flink.runtime.jobgraph.{SavepointConfigOptions, SavepointRestoreSettings}
-
-import javax.annotation.Nullable
-
-import java.io.File
-import java.net.URL
-
-import scala.util.Try
-
-case class SubmitRequest(
- flinkVersion: FlinkVersion,
- deployMode: FlinkDeployMode,
- properties: JavaMap[String, Any],
- flinkYaml: String,
- jobType: FlinkJobType,
- id: Long,
- jobId: String,
- appName: String,
- appConf: String,
- applicationType: ApplicationType,
- savePoint: String,
- restoreMode: FlinkRestoreMode,
- args: String,
- @Nullable clusterId: String,
- @Nullable hadoopUser: String,
- @Nullable buildResult: BuildResult,
- @Nullable extraParameter: JavaMap[String, Any],
- @Nullable kubernetesNamespace: String,
- @Nullable flinkRestExposedType: FlinkK8sRestExposedType) {
-
- private[this] lazy val appProperties: Map[String, String] = getParameterMap(
- KEY_FLINK_PROPERTY_PREFIX)
-
- lazy val appOption: Map[String, String] = getParameterMap(KEY_FLINK_OPTION_PREFIX)
-
- lazy val appMain: String = this.jobType match {
- case FlinkJobType.FLINK_SQL =>
- Constants.STREAMPARK_FLINKSQL_CLIENT_CLASS
- case FlinkJobType.PYFLINK => Constants.PYTHON_FLINK_DRIVER_CLASS_NAME
- case _ => appProperties(KEY_FLINK_APPLICATION_MAIN_CLASS)
- }
-
- lazy val effectiveAppName: String =
- if (this.appName == null) appProperties(KEY_FLINK_APP_NAME)
- else this.appName
-
- lazy val libs: List[URL] = {
- val path = s"${Workspace.local.APP_WORKSPACE}/$id/lib"
- Try(new File(path).listFiles().map(_.toURI.toURL).toList)
- .getOrElse(List.empty[URL])
- }
-
- lazy val classPaths: List[URL] = flinkVersion.flinkLibs ++ libs
-
- lazy val flinkSQL: String = extraParameter.get(KEY_FLINK_SQL()).toString
-
- lazy val allowNonRestoredState: Boolean = Try(
- properties
- .get(SavepointConfigOptions.SAVEPOINT_IGNORE_UNCLAIMED_STATE.key)
- .toString
- .toBoolean)
- .getOrElse(false)
-
- lazy val savepointRestoreSettings: SavepointRestoreSettings = {
- savePoint match {
- case sp if Try(sp.isEmpty).getOrElse(true) =>
- SavepointRestoreSettings.none
- case sp => SavepointRestoreSettings.forPath(sp, allowNonRestoredState)
- }
- }
-
- lazy val userJarFile: File = {
- deployMode match {
- case FlinkDeployMode.KUBERNETES_NATIVE_APPLICATION => null
- case _ =>
- checkBuildResult()
- new File(buildResult.asInstanceOf[ShadedBuildResponse].shadedJarPath)
- }
- }
-
- lazy val safePackageProgram: Boolean = {
- // ref FLINK-21164 FLINK-9844 packageProgram.close()
- // must be flink 1.12.2 and above
- flinkVersion.version.split("\\.").map(_.trim.toInt) match {
- case Array(a, b, c) if a >= 1 => b > 12 || (b == 12 && c >= 2)
- case _ => false
- }
- }
-
- def hasProp(key: String): Boolean = MapUtils.isNotEmpty(properties) && properties.containsKey(key)
-
- def getProp(key: String): Any = properties.get(key)
-
- def hasExtra(key: String): Boolean = MapUtils.isNotEmpty(extraParameter) && extraParameter.containsKey(key)
-
- def getExtra(key: String): Any = extraParameter.get(key)
-
- private[this] def getParameterMap(prefix: String = ""): Map[String, String] = {
- if (this.appConf == null) {
- return Map.empty[String, String]
- }
- val format = this.appConf.substring(0, 7)
- if (format == "json://") {
- val json = this.appConf.drop(7)
- new ObjectMapper()
- .readValue[JavaMap[String, String]](json, classOf[JavaMap[String, String]])
- .toMap
- .filter(_._2 != null)
- } else {
- lazy val content = DeflaterUtils.unzipString(this.appConf.trim.drop(7))
- val map = format match {
- case "yaml://" => PropertiesUtils.fromYamlText(content)
- case "conf://" => PropertiesUtils.fromHoconText(content)
- case "prop://" => PropertiesUtils.fromPropertiesText(content)
- case "hdfs://" =>
- /**
- * If the configuration file is HDFS mode, you need to copy the HDFS related configuration
- * file to resources.
- */
- val text = HdfsUtils.read(this.appConf)
- val extension = this.appConf.split("\\.").last.toLowerCase
- extension match {
- case "yml" | "yaml" => PropertiesUtils.fromYamlText(text)
- case "conf" => PropertiesUtils.fromHoconText(text)
- case "properties" => PropertiesUtils.fromPropertiesText(text)
- case _ =>
- throw new IllegalArgumentException(
- "[StreamPark] Usage: application config format error,must be [yaml|conf|properties]")
- }
- case _ =>
- throw new IllegalArgumentException("[StreamPark] application config format error.")
- }
- map
- .filter(_._1.startsWith(prefix))
- .filter(_._2.nonEmpty)
- .map(x => x._1.drop(prefix.length) -> x._2)
- }
- }
-
- private[client] lazy val hdfsWorkspace = {
-
- /**
- * The flink version and configuration in the native flink and hdfs must be kept exactly the
- * same.
- */
- val workspace = Workspace.remote
- val flinkHome = flinkVersion.flinkHome
- val flinkHomeDir = new File(flinkHome)
- val flinkName = if (FileUtils.isSymlink(flinkHomeDir)) {
- flinkHomeDir.getCanonicalFile.getName
- } else {
- flinkHomeDir.getName
- }
- val flinkHdfsHome = s"${workspace.APP_FLINK}/$flinkName"
- HdfsWorkspace(
- flinkName,
- flinkHome,
- flinkLib = s"$flinkHdfsHome/lib",
- flinkPlugins = s"$flinkHdfsHome/plugins",
- flinkDistJar = FlinkUtils.getFlinkDistJar(flinkHome),
- appJars = workspace.APP_JARS)
- }
-
- @throws[Exception]
- def checkBuildResult(): Unit = {
- deployMode match {
- case FlinkDeployMode.KUBERNETES_NATIVE_SESSION =>
- AssertUtils.required(
- buildResult != null,
- s"[flink-submit] current job: ${this.effectiveAppName} was not yet built, buildResult is empty" +
- s",clusterId=$clusterId," +
- s",namespace=$kubernetesNamespace")
- AssertUtils.required(
- buildResult.pass,
- s"[flink-submit] current job ${this.effectiveAppName} build failed, clusterId" +
- s",clusterId=$clusterId," +
- s",namespace=$kubernetesNamespace")
- case _ =>
- AssertUtils.required(
- this.buildResult != null,
- s"[flink-submit] current job: ${this.effectiveAppName} was not yet built, buildResult is empty")
- AssertUtils.required(
- this.buildResult.pass,
- s"[flink-submit] current job ${this.effectiveAppName} build failed, please check")
- }
- }
-
-}
-
-/**
- * @param flinkName
- * @param flinkHome
- * @param flinkDistJar
- * @param flinkLib
- * @param flinkPlugins
- * @param appJars
- * @param appPlugins
- */
-case class HdfsWorkspace(
- flinkName: String,
- flinkHome: String,
- flinkDistJar: String,
- flinkLib: String,
- flinkPlugins: String,
- appJars: String)
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/TriggerSavepointRequest.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/TriggerSavepointRequest.scala
deleted file mode 100644
index 482d7b181..000000000
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/TriggerSavepointRequest.scala
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.client.bean
-
-import org.apache.streampark.common.conf.FlinkVersion
-import org.apache.streampark.common.constants.Constants
-import org.apache.streampark.common.enums.FlinkDeployMode
-import org.apache.streampark.common.util.Implicits.JavaMap
-
-import javax.annotation.Nullable
-
-/** Trigger savepoint request. */
-case class TriggerSavepointRequest(
- id: Long,
- flinkVersion: FlinkVersion,
- deployMode: FlinkDeployMode,
- @Nullable properties: JavaMap[String, Any],
- clusterId: String,
- jobId: String,
- savepointPath: String,
- nativeFormat: Boolean,
- override val kubernetesNamespace: String = Constants.DEFAULT)
- extends SavepointRequestTrait
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/pom.xml b/streampark-flink/streampark-flink-client/streampark-flink-client-core/pom.xml
index d9fe1ff05..6d5edbb87 100644
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/pom.xml
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/pom.xml
@@ -91,43 +91,23 @@
</dependency>
<dependency>
- <groupId>org.json4s</groupId>
- <artifactId>json4s-jackson_${scala.binary.version}</artifactId>
+ <groupId>org.projectlombok</groupId>
+ <artifactId>lombok</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-engine</artifactId>
+ <scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>shade</goal>
- </goals>
- <phase>package</phase>
- <configuration>
- <createDependencyReducedPom>true</createDependencyReducedPom>
- <dependencyReducedPomLocation>${project.basedir}/target/dependency-reduced-pom.xml</dependencyReducedPomLocation>
- <artifactSet>
- <includes>
- <include>org.json4s:*</include>
- </includes>
- </artifactSet>
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- </excludes>
- </filter>
- </filters>
- </configuration>
- </execution>
- </executions>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/FlinkClientEntrypoint.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/FlinkClientEntrypoint.java
new file mode 100644
index 000000000..9a075434f
--- /dev/null
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/FlinkClientEntrypoint.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.client;
+
+import org.apache.streampark.common.enums.FlinkDeployMode;
+import org.apache.streampark.flink.client.bean.CancelRequest;
+import org.apache.streampark.flink.client.bean.CancelResponse;
+import org.apache.streampark.flink.client.bean.DeployRequest;
+import org.apache.streampark.flink.client.bean.DeployResponse;
+import org.apache.streampark.flink.client.bean.SavepointResponse;
+import org.apache.streampark.flink.client.bean.ShutDownRequest;
+import org.apache.streampark.flink.client.bean.ShutDownResponse;
+import org.apache.streampark.flink.client.bean.SubmitRequest;
+import org.apache.streampark.flink.client.bean.SubmitResponse;
+import org.apache.streampark.flink.client.bean.TriggerSavepointRequest;
+import org.apache.streampark.flink.client.impl.KubernetesNativeApplicationClient;
+import org.apache.streampark.flink.client.impl.KubernetesNativeSessionClient;
+import org.apache.streampark.flink.client.impl.LocalClient;
+import org.apache.streampark.flink.client.impl.RemoteClient;
+import org.apache.streampark.flink.client.impl.YarnApplicationClient;
+import org.apache.streampark.flink.client.impl.YarnPerJobClient;
+import org.apache.streampark.flink.client.impl.YarnSessionClient;
+import org.apache.streampark.flink.client.trait.FlinkClientTrait;
+
+import org.apache.flink.util.FlinkException;
+
+import java.util.EnumMap;
+import java.util.Map;
+
+/** Entry point for Flink client operations routed by deploy mode. */
+public final class FlinkClientEntrypoint {
+
+ private static final Map<FlinkDeployMode, FlinkClientTrait> CLIENTS = new EnumMap<>(FlinkDeployMode.class);
+
+ static {
+ CLIENTS.put(FlinkDeployMode.LOCAL, LocalClient.INSTANCE);
+ CLIENTS.put(FlinkDeployMode.REMOTE, RemoteClient.INSTANCE);
+ CLIENTS.put(FlinkDeployMode.YARN_APPLICATION, YarnApplicationClient.INSTANCE);
+ CLIENTS.put(FlinkDeployMode.YARN_SESSION, YarnSessionClient.INSTANCE);
+ CLIENTS.put(FlinkDeployMode.YARN_PER_JOB, YarnPerJobClient.INSTANCE);
+ CLIENTS.put(
+ FlinkDeployMode.KUBERNETES_NATIVE_SESSION, KubernetesNativeSessionClient.INSTANCE);
+ CLIENTS.put(
+ FlinkDeployMode.KUBERNETES_NATIVE_APPLICATION,
+ KubernetesNativeApplicationClient.INSTANCE);
+ }
+
+ private FlinkClientEntrypoint() {
+ }
+
+ @FunctionalInterface
+ private interface ClientInvoker<R> {
+
+ R invoke(FlinkClientTrait client) throws FlinkException;
+ }
+
+ private static <R> R invokeClient(
+ FlinkDeployMode deployMode,
+ ClientInvoker<R> invoker,
+ String action) throws FlinkException {
+ FlinkClientTrait client = CLIENTS.get(deployMode);
+ if (client != null) {
+ return invoker.invoke(client);
+ }
+ throw new UnsupportedOperationException("Unsupported " + deployMode + " " + action);
+ }
+
+ public static SubmitResponse submit(SubmitRequest submitRequest) throws FlinkException {
+ return invokeClient(
+ submitRequest.deployMode(), client -> client.submit(submitRequest), "submit");
+ }
+
+ public static CancelResponse cancel(CancelRequest cancelRequest) throws FlinkException {
+ return invokeClient(
+ cancelRequest.deployMode(), client -> client.cancel(cancelRequest), "cancel");
+ }
+
+ public static SavepointResponse triggerSavepoint(TriggerSavepointRequest savepointRequest) throws FlinkException {
+ return invokeClient(
+ savepointRequest.deployMode(),
+ client -> client.triggerSavepoint(savepointRequest),
+ "triggerSavepoint");
+ }
+
+ public static DeployResponse deploy(DeployRequest deployRequest) throws Exception {
+ if (deployRequest.deployMode() == FlinkDeployMode.YARN_SESSION) {
+ return YarnSessionClient.INSTANCE.deploy(deployRequest);
+ }
+ if (deployRequest.deployMode() == FlinkDeployMode.KUBERNETES_NATIVE_SESSION) {
+ return KubernetesNativeSessionClient.INSTANCE.deploy(deployRequest);
+ }
+ throw new UnsupportedOperationException(
+ "Unsupported " + deployRequest.deployMode() + " deploy cluster ");
+ }
+
+ public static ShutDownResponse shutdown(ShutDownRequest shutDownRequest) throws Exception {
+ if (shutDownRequest.deployMode() == FlinkDeployMode.YARN_SESSION) {
+ return YarnSessionClient.INSTANCE.shutdown(shutDownRequest);
+ }
+ if (shutDownRequest.deployMode() == FlinkDeployMode.KUBERNETES_NATIVE_SESSION) {
+ return KubernetesNativeSessionClient.INSTANCE.shutdown(shutDownRequest);
+ }
+ throw new UnsupportedOperationException(
+ "Unsupported " + shutDownRequest.deployMode() + " shutdown cluster ");
+ }
+}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/ShutDownRequest.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/FlinkConfigurationOps.java
similarity index 57%
rename from streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/ShutDownRequest.scala
rename to streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/FlinkConfigurationOps.java
index 52881d534..58a99bb38 100644
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/ShutDownRequest.scala
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/FlinkConfigurationOps.java
@@ -15,19 +15,21 @@
* limitations under the License.
*/
-package org.apache.streampark.flink.client.bean
+package org.apache.streampark.flink.client.impl;
-import org.apache.streampark.common.conf.FlinkVersion
-import org.apache.streampark.common.enums.FlinkDeployMode
-import org.apache.streampark.common.util.Implicits.JavaMap
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.Configuration;
-import javax.annotation.Nullable
+/** Configuration helpers mirroring {@code FlinkClientTrait.EnhanceFlinkConfiguration#safeSet}. */
+final class FlinkConfigurationOps {
-case class ShutDownRequest(
- flinkVersion: FlinkVersion,
- deployMode: FlinkDeployMode,
- @Nullable properties: JavaMap[String, Any],
- clusterId: String,
- id: Long,
- @Nullable k8sParam: KubernetesDeployParam)
- extends DeployRequestTrait
+ private FlinkConfigurationOps() {
+ }
+
+ static <T> Configuration safeSet(Configuration flinkConfig, ConfigOption<T> option, T value) {
+ if (value != null && !value.toString().isEmpty()) {
+ flinkConfig.set(option, value);
+ }
+ return flinkConfig;
+ }
+}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/KubernetesNativeApplicationClient.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/KubernetesNativeApplicationClient.java
new file mode 100644
index 000000000..b2e8f4e24
--- /dev/null
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/KubernetesNativeApplicationClient.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.client.impl;
+
+import org.apache.streampark.common.enums.FlinkDeployMode;
+import org.apache.streampark.flink.client.bean.CancelRequest;
+import org.apache.streampark.flink.client.bean.CancelResponse;
+import org.apache.streampark.flink.client.bean.SavepointResponse;
+import org.apache.streampark.flink.client.bean.SubmitRequest;
+import org.apache.streampark.flink.client.bean.SubmitResponse;
+import org.apache.streampark.flink.client.bean.TriggerSavepointRequest;
+import org.apache.streampark.flink.client.trait.KubernetesNativeClientTrait;
+import org.apache.streampark.flink.packer.pipeline.DockerImageBuildResponse;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.flink.client.deployment.ClusterSpecification;
+import org.apache.flink.client.deployment.application.ApplicationConfiguration;
+import org.apache.flink.client.program.ClusterClient;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.DeploymentOptions;
+import org.apache.flink.configuration.PipelineOptions;
+import org.apache.flink.kubernetes.KubernetesClusterDescriptor;
+import org.apache.flink.kubernetes.configuration.KubernetesConfigOptions;
+import org.apache.flink.util.FlinkException;
+
+import com.google.common.collect.Lists;
+
+import scala.Tuple2;
+
+/** Kubernetes native application mode submit. */
+public final class KubernetesNativeApplicationClient extends KubernetesNativeClientTrait {
+
+ public static final KubernetesNativeApplicationClient INSTANCE =
+ new KubernetesNativeApplicationClient();
+
+ private KubernetesNativeApplicationClient() {
+ }
+
+ @Override
+ public SubmitResponse doSubmit(SubmitRequest submitRequest, Configuration flinkConfig) throws FlinkException {
+ if (StringUtils.isBlank(submitRequest.clusterId())) {
+ throw new IllegalArgumentException(
+ String.format(
+ "[flink-submit] submit flink job failed, clusterId is null, mode=%s",
+ flinkConfig.get(DeploymentOptions.TARGET)));
+ }
+
+ return callAsFlinkException(
+ () -> {
+ submitRequest.checkBuildResult();
+
+ DockerImageBuildResponse buildResult =
+ (DockerImageBuildResponse) submitRequest.buildResult();
+
+ FlinkConfigurationOps.safeSet(
+ flinkConfig,
+ PipelineOptions.JARS,
+ Lists.newArrayList(buildResult.dockerInnerMainJarPath()));
+ FlinkConfigurationOps.safeSet(
+ flinkConfig, KubernetesConfigOptions.CONTAINER_IMAGE, buildResult.flinkImageTag());
+
+ Tuple2<KubernetesClusterDescriptor, ClusterSpecification> descriptorAndSpec =
+ getK8sClusterDescriptorAndSpecification(flinkConfig);
+ KubernetesClusterDescriptor clusterDescriptor = descriptorAndSpec._1();
+ ClusterSpecification clusterSpecification = descriptorAndSpec._2();
+
+ ApplicationConfiguration applicationConfig =
+ ApplicationConfiguration.fromConfiguration(flinkConfig);
+ ClusterClient<String> clusterClient =
+ clusterDescriptor
+ .deployApplicationCluster(clusterSpecification, applicationConfig)
+ .getClusterClient();
+
+ String clusterId = clusterClient.getClusterId();
+ SubmitResponse result =
+ new SubmitResponse(
+ clusterId,
+ flinkConfig.toMap(),
+ submitRequest.jobId(),
+ clusterClient.getWebInterfaceURL());
+ logInfo(
+ "[flink-submit] flink job has been submitted. "
+ + flinkConfIdentifierInfo(flinkConfig));
+
+ closeSubmit(submitRequest, clusterDescriptor, clusterClient);
+ return result;
+ });
+ }
+
+ @Override
+ public CancelResponse doCancel(CancelRequest cancelRequest, Configuration flinkConf) throws FlinkException {
+ setK8sDeployTarget(flinkConf, FlinkDeployMode.KUBERNETES_NATIVE_APPLICATION);
+ return super.doCancel(cancelRequest, flinkConf);
+ }
+
+ @Override
+ public SavepointResponse doTriggerSavepoint(
+ TriggerSavepointRequest request,
+ Configuration flinkConf) throws FlinkException {
+ setK8sDeployTarget(flinkConf, FlinkDeployMode.KUBERNETES_NATIVE_APPLICATION);
+ return super.doTriggerSavepoint(request, flinkConf);
+ }
+}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/KubernetesNativeSessionClient.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/KubernetesNativeSessionClient.java
new file mode 100644
index 000000000..e91ef15dc
--- /dev/null
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/KubernetesNativeSessionClient.java
@@ -0,0 +1,267 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.client.impl;
+
+import org.apache.streampark.common.enums.FlinkDeployMode;
+import org.apache.streampark.common.util.Utils;
+import org.apache.streampark.flink.client.bean.CancelRequest;
+import org.apache.streampark.flink.client.bean.CancelResponse;
+import org.apache.streampark.flink.client.bean.DeployRequest;
+import org.apache.streampark.flink.client.bean.DeployRequestTrait;
+import org.apache.streampark.flink.client.bean.DeployResponse;
+import org.apache.streampark.flink.client.bean.SavepointResponse;
+import org.apache.streampark.flink.client.bean.ShutDownRequest;
+import org.apache.streampark.flink.client.bean.ShutDownResponse;
+import org.apache.streampark.flink.client.bean.SubmitRequest;
+import org.apache.streampark.flink.client.bean.SubmitResponse;
+import org.apache.streampark.flink.client.bean.TriggerSavepointRequest;
+import org.apache.streampark.flink.client.tool.FlinkSessionSubmitHelper;
+import org.apache.streampark.flink.client.trait.KubernetesNativeClientTrait;
+import org.apache.streampark.flink.core.FlinkKubernetesClient;
+import org.apache.streampark.flink.kubernetes.KubernetesRetriever;
+import org.apache.streampark.flink.kubernetes.enums.FlinkK8sDeployMode;
+import org.apache.streampark.flink.kubernetes.model.ClusterKey;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.flink.client.deployment.ClusterSpecification;
+import org.apache.flink.client.program.ClusterClient;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.DeploymentOptions;
+import org.apache.flink.configuration.DeploymentOptionsInternal;
+import org.apache.flink.kubernetes.KubernetesClusterDescriptor;
+import org.apache.flink.kubernetes.configuration.KubernetesConfigOptions;
+import org.apache.flink.kubernetes.configuration.KubernetesConfigOptions.ServiceExposedType;
+import org.apache.flink.kubernetes.configuration.KubernetesDeploymentTarget;
+import org.apache.flink.kubernetes.kubeclient.FlinkKubeClient;
+import org.apache.flink.kubernetes.kubeclient.FlinkKubeClientFactory;
+import org.apache.flink.util.FlinkException;
+
+import java.io.File;
+
+import scala.Tuple2;
+
+/** Kubernetes native session mode submit. */
+public final class KubernetesNativeSessionClient extends KubernetesNativeClientTrait {
+
+ public static final KubernetesNativeSessionClient INSTANCE =
+ new KubernetesNativeSessionClient();
+
+ private KubernetesNativeSessionClient() {
+ }
+
+ @Override
+ public SubmitResponse doSubmit(SubmitRequest submitRequest, Configuration flinkConfig) throws FlinkException {
+ if (StringUtils.isBlank(submitRequest.clusterId())) {
+ throw new IllegalArgumentException(
+ String.format(
+ "[flink-submit] submit flink job failed, clusterId is null, mode=%s",
+ flinkConfig.get(DeploymentOptions.TARGET)));
+ }
+ return trySubmit(
+ submitRequest,
+ flinkConfig,
+ submitRequest.userJarFile(),
+ this::jobGraphSubmit,
+ this::restApiSubmit);
+ }
+
+ /** Submit flink session job via rest api. */
+ public SubmitResponse restApiSubmit(
+ SubmitRequest submitRequest,
+ Configuration flinkConfig,
+ File fatJar) throws FlinkException {
+ return callAsFlinkException(
+ () -> {
+ ClusterKey clusterKey =
+ ClusterKey.builder()
+ .executeMode(FlinkK8sDeployMode.SESSION)
+ .namespace(submitRequest.kubernetesNamespace())
+ .clusterId(submitRequest.clusterId())
+ .build();
+ String jmRestUrl =
+ KubernetesRetriever.retrieveFlinkRestUrl(clusterKey)
+ .orElseThrow(
+ () -> new FlinkException(
+ "[flink-submit] retrieve flink session rest url failed, clusterKey="
+ + clusterKey));
+ String jobId =
+ FlinkSessionSubmitHelper.submitViaRestApi(jmRestUrl, fatJar, flinkConfig);
+ return new SubmitResponse(
+ clusterKey.clusterId(), flinkConfig.toMap(), jobId, jmRestUrl);
+ });
+ }
+
+ /** Submit flink session job with building JobGraph via ClusterClient api. */
+ public SubmitResponse jobGraphSubmit(
+ SubmitRequest submitRequest,
+ Configuration flinkConfig,
+ File jarFile) throws FlinkException {
+ return callAsFlinkException(
+ () -> {
+ SubmitResponse result =
+ submitJobGraphToCluster(
+ submitRequest,
+ flinkConfig,
+ jarFile,
+ () -> getK8sClusterDescriptor(flinkConfig)
+ .retrieve(flinkConfig.getString(KubernetesConfigOptions.CLUSTER_ID))
+ .getClusterClient(),
+ () -> flinkConfig.getString(KubernetesConfigOptions.CLUSTER_ID));
+ logInfo(
+ "[flink-submit] flink job has been submitted. "
+ + flinkConfIdentifierInfo(flinkConfig)
+ + ", jobId: "
+ + result.jobId());
+ return result;
+ });
+ }
+
+ @Override
+ public CancelResponse doCancel(CancelRequest cancelRequest, Configuration flinkConfig) throws FlinkException {
+ setK8sDeployTarget(flinkConfig, FlinkDeployMode.KUBERNETES_NATIVE_SESSION);
+ return super.doCancel(cancelRequest, flinkConfig);
+ }
+
+ public DeployResponse deploy(DeployRequest deployRequest) throws Exception {
+ logInfo(
+ String.format(
+ "%n--------------------------------------- kubernetes cluster start "
+ + "---------------------------------------%n"
+ + " userFlinkHome : %s%n"
+ + " flinkVersion : %s%n"
+ + " deployMode : %s%n"
+ + " clusterId : %s%n"
+ + " namespace : %s%n"
+ + " exposedType : %s%n"
+ + " serviceAccount : %s%n"
+ + " flinkImage : %s%n"
+ + " properties : %s%n"
+ + "--------------------------------------------------------------------------------------------------------%n",
+ deployRequest.flinkVersion().getFlinkHome(),
+ deployRequest.flinkVersion().version(),
+ deployRequest.deployMode().name(),
+ deployRequest.clusterId(),
+ deployRequest.k8sParam().kubernetesNamespace(),
+ deployRequest.k8sParam().flinkRestExposedType(),
+ deployRequest.k8sParam().serviceAccount(),
+ deployRequest.k8sParam().flinkImage(),
+ deployRequest.properties()));
+
+ Configuration flinkConfig = getFlinkK8sConfig(deployRequest);
+ FlinkKubeClient kubeClient =
+ FlinkKubeClientFactory.getInstance().fromConfiguration(flinkConfig, "client");
+
+ KubernetesClusterDescriptor clusterDescriptor = null;
+ ClusterClient<String> client = null;
+
+ try {
+ Tuple2<KubernetesClusterDescriptor, ClusterSpecification> kubernetesClusterDescriptor =
+ getK8sClusterDescriptorAndSpecification(flinkConfig);
+ clusterDescriptor = kubernetesClusterDescriptor._1();
+
+ FlinkKubernetesClient kubeClientWrapper = new FlinkKubernetesClient(kubeClient);
+ if (kubeClientWrapper.getService(deployRequest.clusterId()).isPresent()) {
+ client =
+ clusterDescriptor.retrieve(deployRequest.clusterId()).getClusterClient();
+ } else {
+ client =
+ clusterDescriptor
+ .deploySessionCluster(kubernetesClusterDescriptor._2())
+ .getClusterClient();
+ }
+ return new DeployResponse(
+ client.getWebInterfaceURL(), client.getClusterId(), null);
+ } catch (Exception e) {
+ return new DeployResponse(null, null, e);
+ } finally {
+ Utils.close(client, clusterDescriptor, kubeClient);
+ }
+ }
+
+ public ShutDownResponse shutdown(ShutDownRequest shutDownRequest) throws Exception {
+ FlinkKubeClient kubeClient = null;
+ try {
+ Configuration flinkConfig = getFlinkK8sConfig(shutDownRequest);
+ kubeClient =
+ FlinkKubeClientFactory.getInstance().fromConfiguration(flinkConfig, "client");
+ FlinkKubernetesClient kubeClientWrapper = new FlinkKubernetesClient(kubeClient);
+
+ boolean stopAndCleanupState =
+ shutDownRequest.clusterId() != null
+ && kubeClientWrapper.getService(shutDownRequest.clusterId()).isPresent();
+ if (stopAndCleanupState) {
+ kubeClient.stopAndCleanupCluster(shutDownRequest.clusterId());
+ return new ShutDownResponse(shutDownRequest.clusterId());
+ }
+ return null;
+ } catch (Exception e) {
+ logError(
+ "shutdown flink session fail in " + shutDownRequest.deployMode() + " mode", e);
+ throw e;
+ } finally {
+ Utils.close(kubeClient);
+ }
+ }
+
+ @Override
+ public SavepointResponse doTriggerSavepoint(
+ TriggerSavepointRequest triggerSavepointRequest,
+ Configuration flinkConfig) throws FlinkException {
+ FlinkConfigurationOps.safeSet(
+ flinkConfig,
+ DeploymentOptions.TARGET,
+ FlinkDeployMode.KUBERNETES_NATIVE_SESSION.getName());
+ return super.doTriggerSavepoint(triggerSavepointRequest, flinkConfig);
+ }
+
+ private Configuration getFlinkK8sConfig(DeployRequestTrait deployRequest) throws Exception {
+ Configuration flinkConfig =
+ extractConfiguration(
+ deployRequest.flinkVersion().getFlinkHome(), deployRequest.properties());
+ FlinkConfigurationOps.safeSet(
+ flinkConfig, DeploymentOptions.TARGET, KubernetesDeploymentTarget.SESSION.getName());
+ FlinkConfigurationOps.safeSet(
+ flinkConfig,
+ KubernetesConfigOptions.NAMESPACE,
+ deployRequest.k8sParam().kubernetesNamespace());
+ FlinkConfigurationOps.safeSet(
+ flinkConfig,
+ KubernetesConfigOptions.KUBERNETES_SERVICE_ACCOUNT,
+ deployRequest.k8sParam().serviceAccount());
+ FlinkConfigurationOps.safeSet(
+ flinkConfig, KubernetesConfigOptions.CLUSTER_ID, deployRequest.clusterId());
+ FlinkConfigurationOps.safeSet(
+ flinkConfig,
+ KubernetesConfigOptions.CONTAINER_IMAGE,
+ deployRequest.k8sParam().flinkImage());
+ FlinkConfigurationOps.safeSet(
+ flinkConfig,
+ KubernetesConfigOptions.REST_SERVICE_EXPOSED_TYPE,
+ ServiceExposedType.valueOf(
+ deployRequest.k8sParam().flinkRestExposedType().getName()));
+ FlinkConfigurationOps.safeSet(
+ flinkConfig,
+ KubernetesConfigOptions.KUBE_CONFIG_FILE,
+ getDefaultKubernetesConf(deployRequest.k8sParam().kubeConf()));
+ FlinkConfigurationOps.safeSet(
+ flinkConfig,
+ DeploymentOptionsInternal.CONF_DIR,
+ deployRequest.flinkVersion().getFlinkHome() + "/conf");
+ return flinkConfig;
+ }
+}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/LocalClient.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/LocalClient.java
new file mode 100644
index 000000000..9df26a32e
--- /dev/null
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/LocalClient.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.client.impl;
+
+import org.apache.streampark.flink.client.bean.CancelRequest;
+import org.apache.streampark.flink.client.bean.CancelResponse;
+import org.apache.streampark.flink.client.bean.SavepointResponse;
+import org.apache.streampark.flink.client.bean.SubmitRequest;
+import org.apache.streampark.flink.client.bean.SubmitResponse;
+import org.apache.streampark.flink.client.bean.TriggerSavepointRequest;
+import org.apache.streampark.flink.client.trait.FlinkClientTrait;
+
+import org.apache.flink.client.deployment.executors.RemoteExecutor;
+import org.apache.flink.client.program.MiniClusterClient;
+import org.apache.flink.client.program.PackagedProgram;
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.DeploymentOptions;
+import org.apache.flink.configuration.JobManagerOptions;
+import org.apache.flink.configuration.PipelineOptions;
+import org.apache.flink.configuration.RestOptions;
+import org.apache.flink.configuration.TaskManagerOptions;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.minicluster.MiniCluster;
+import org.apache.flink.runtime.minicluster.MiniClusterConfiguration;
+import org.apache.flink.util.FlinkException;
+
+import scala.Tuple2;
+
+/** Submit Flink jobs to a local mini cluster. */
+public final class LocalClient extends FlinkClientTrait {
+
+ public static final LocalClient INSTANCE = new LocalClient();
+
+ private LocalClient() {
+ }
+
+ @Override
+ public void setConfig(SubmitRequest submitRequest, Configuration flinkConfig) {
+ FlinkConfigurationOps.safeSet(flinkConfig, PipelineOptions.NAME, submitRequest.effectiveAppName());
+ logEffectiveSubmitConfiguration(flinkConfig);
+ }
+
+ @Override
+ public SubmitResponse doSubmit(SubmitRequest submitRequest, Configuration flinkConfig) throws FlinkException {
+ return callAsFlinkException(
+ () -> {
+ Tuple2<PackagedProgram, JobGraph> programJobGraph =
+ getJobGraph(flinkConfig, submitRequest, submitRequest.userJarFile());
+ PackagedProgram packageProgram = programJobGraph._1();
+ JobGraph jobGraph = programJobGraph._2();
+ MiniClusterClient client = createLocalCluster(flinkConfig);
+ String jobId = client.submitJob(jobGraph).get().toString();
+ SubmitResponse resp =
+ new SubmitResponse(jobId, flinkConfig.toMap(), jobId, client.getWebInterfaceURL());
+ closeSubmit(submitRequest, packageProgram, client);
+ return resp;
+ });
+ }
+
+ @Override
+ public SavepointResponse doTriggerSavepoint(
+ TriggerSavepointRequest savepointRequest,
+ Configuration flinkConfig) throws FlinkException {
+ return RemoteClient.INSTANCE.doTriggerSavepoint(savepointRequest, flinkConfig);
+ }
+
+ @Override
+ public CancelResponse doCancel(CancelRequest cancelRequest, Configuration flinkConfig) throws FlinkException {
+ return RemoteClient.INSTANCE.doCancel(cancelRequest, flinkConfig);
+ }
+
+ private MiniClusterClient createLocalCluster(Configuration flinkConfig) throws Exception {
+ FlinkConfigurationOps.safeSet(flinkConfig, JobManagerOptions.PORT, 0);
+
+ int numTaskManagers =
+ flinkConfig.getInteger(
+ ConfigConstants.LOCAL_NUMBER_TASK_MANAGER,
+ ConfigConstants.DEFAULT_LOCAL_NUMBER_TASK_MANAGER);
+ int numSlotsPerTaskManager = flinkConfig.getInteger(TaskManagerOptions.NUM_TASK_SLOTS);
+
+ MiniClusterConfiguration miniClusterConfig =
+ new MiniClusterConfiguration.Builder()
+ .setConfiguration(flinkConfig)
+ .setNumSlotsPerTaskManager(numSlotsPerTaskManager)
+ .setNumTaskManagers(numTaskManagers)
+ .build();
+
+ MiniCluster cluster = new MiniCluster(miniClusterConfig);
+ cluster.start();
+
+ String host = "localhost";
+ int port = cluster.getRestAddress().get().getPort();
+
+ FlinkConfigurationOps.safeSet(flinkConfig, JobManagerOptions.ADDRESS, host);
+ FlinkConfigurationOps.safeSet(flinkConfig, JobManagerOptions.PORT, port);
+ FlinkConfigurationOps.safeSet(flinkConfig, RestOptions.ADDRESS, host);
+ FlinkConfigurationOps.safeSet(flinkConfig, RestOptions.PORT, port);
+ FlinkConfigurationOps.safeSet(flinkConfig, DeploymentOptions.TARGET, RemoteExecutor.NAME);
+
+ logInfo(String.format("%nStarting local Flink cluster (host: localhost, port: %d).%n", port));
+
+ return new MiniClusterClient(flinkConfig, cluster);
+ }
+}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/RemoteClient.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/RemoteClient.java
new file mode 100644
index 000000000..58e9024f9
--- /dev/null
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/RemoteClient.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.client.impl;
+
+import org.apache.streampark.flink.client.bean.CancelRequest;
+import org.apache.streampark.flink.client.bean.CancelResponse;
+import org.apache.streampark.flink.client.bean.SavepointRequestTrait;
+import org.apache.streampark.flink.client.bean.SavepointResponse;
+import org.apache.streampark.flink.client.bean.SubmitRequest;
+import org.apache.streampark.flink.client.bean.SubmitResponse;
+import org.apache.streampark.flink.client.bean.TriggerSavepointRequest;
+import org.apache.streampark.flink.client.tool.FlinkSessionSubmitHelper;
+import org.apache.streampark.flink.client.trait.FlinkClientTrait;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.client.deployment.ClusterClientFactory;
+import org.apache.flink.client.deployment.DefaultClusterClientServiceLoader;
+import org.apache.flink.client.deployment.StandaloneClusterDescriptor;
+import org.apache.flink.client.deployment.StandaloneClusterId;
+import org.apache.flink.client.program.ClusterClient;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.DeploymentOptions;
+import org.apache.flink.configuration.RestOptions;
+import org.apache.flink.util.FlinkException;
+
+import java.io.File;
+
+import scala.Tuple2;
+
+/** Submit Flink jobs to a remote standalone cluster. */
+public final class RemoteClient extends FlinkClientTrait {
+
+ public static final RemoteClient INSTANCE = new RemoteClient();
+
+ private RemoteClient() {
+ }
+
+ @Override
+ public void setConfig(SubmitRequest submitRequest, Configuration flinkConfig) {
+ // no extra configuration
+ }
+
+ @Override
+ public SubmitResponse doSubmit(SubmitRequest submitRequest, Configuration flinkConfig) throws FlinkException {
+ return callAsFlinkException(
+ () -> trySubmit(
+ submitRequest,
+ flinkConfig,
+ submitRequest.userJarFile(),
+ this::jobGraphSubmit,
+ this::restApiSubmit));
+ }
+
+ @Override
+ public CancelResponse doCancel(CancelRequest cancelRequest, Configuration flinkConfig) throws FlinkException {
+ return executeClientAction(
+ cancelRequest,
+ flinkConfig,
+ (jobId, clusterClient) -> toCancelResponse(cancelRequest, jobId, clusterClient));
+ }
+
+ @Override
+ public SavepointResponse doTriggerSavepoint(
+ TriggerSavepointRequest savepointRequest,
+ Configuration flinkConfig) throws FlinkException {
+ return executeClientAction(
+ savepointRequest,
+ flinkConfig,
+ (jobId, clusterClient) -> toSavepointResponse(savepointRequest, jobId, clusterClient));
+ }
+
+ /** Submit flink session job via rest api. */
+ public SubmitResponse restApiSubmit(
+ SubmitRequest submitRequest,
+ Configuration flinkConfig,
+ File fatJar) throws FlinkException {
+ return callAsFlinkException(
+ () -> {
+ Tuple2<StandaloneClusterId, StandaloneClusterDescriptor> standAloneDescriptor =
+ getStandAloneClusterDescriptor(flinkConfig);
+ StandaloneClusterId yarnClusterId = standAloneDescriptor._1();
+ StandaloneClusterDescriptor clusterDescriptor = standAloneDescriptor._2();
+
+ ClusterClient<StandaloneClusterId> client =
+ clusterDescriptor.retrieve(yarnClusterId).getClusterClient();
+ String jobId =
+ FlinkSessionSubmitHelper.submitViaRestApi(
+ client.getWebInterfaceURL(), fatJar, flinkConfig);
+ logInfo(
+ String.format(
+ "%s mode submit by restApi, WebInterfaceURL %s, jobId: %s",
+ submitRequest.deployMode(), client.getWebInterfaceURL(), jobId));
+ SubmitResponse resp =
+ new SubmitResponse(null, flinkConfig.toMap(), jobId, client.getWebInterfaceURL());
+ closeSubmit(submitRequest, client, clusterDescriptor);
+ return resp;
+ });
+ }
+
+ /** Submit flink session job with building JobGraph via Standalone ClusterClient api. */
+ public SubmitResponse jobGraphSubmit(
+ SubmitRequest submitRequest,
+ Configuration flinkConfig,
+ File jarFile) throws FlinkException {
+ Tuple2<StandaloneClusterId, StandaloneClusterDescriptor> standAloneDescriptor =
+ getStandAloneClusterDescriptor(flinkConfig);
+ return submitJobGraphToCluster(
+ submitRequest,
+ flinkConfig,
+ jarFile,
+ () -> standAloneDescriptor._2().retrieve(standAloneDescriptor._1()).getClusterClient(),
+ () -> null,
+ standAloneDescriptor._2());
+ }
+
+ private <O, R extends SavepointRequestTrait> O executeClientAction(
+ R request,
+ Configuration flinkConfig,
+ ClientAction<O> actFunc) throws FlinkException {
+ return callAsFlinkException(
+ () -> {
+ FlinkConfigurationOps.safeSet(
+ flinkConfig, DeploymentOptions.TARGET, request.deployMode().getName());
+ FlinkConfigurationOps.safeSet(
+ flinkConfig,
+ RestOptions.ADDRESS,
+ request.properties().get(RestOptions.ADDRESS.key()).toString());
+ FlinkConfigurationOps.safeSet(
+ flinkConfig,
+ RestOptions.PORT,
+ Integer.parseInt(
+ request.properties().get(RestOptions.PORT.key()).toString()));
+ logEffectiveSubmitConfiguration(flinkConfig);
+ Tuple2<StandaloneClusterId, StandaloneClusterDescriptor> descriptor =
+ getStandAloneClusterDescriptor(flinkConfig);
+ ClusterClient<StandaloneClusterId> clusterClient =
+ descriptor._2().retrieve(descriptor._1()).getClusterClient();
+ try {
+ return actFunc.apply(JobID.fromHexString(request.jobId()), clusterClient);
+ } finally {
+ clusterClient.close();
+ descriptor._2().close();
+ }
+ },
+ e -> {
+ logError(
+ "Do " + request.getClass().getSimpleName() + " for flink standalone job fail", e);
+ });
+ }
+
+ private Tuple2<StandaloneClusterId, StandaloneClusterDescriptor> getStandAloneClusterDescriptor(
+ Configuration flinkConfig) {
+ DefaultClusterClientServiceLoader serviceLoader = new DefaultClusterClientServiceLoader();
+ ClusterClientFactory<StandaloneClusterId> clientFactory =
+ serviceLoader.getClusterClientFactory(flinkConfig);
+ StandaloneClusterId standaloneClusterId = clientFactory.getClusterId(flinkConfig);
+ StandaloneClusterDescriptor standaloneClusterDescriptor =
+ (StandaloneClusterDescriptor) clientFactory.createClusterDescriptor(flinkConfig);
+ return new Tuple2<>(standaloneClusterId, standaloneClusterDescriptor);
+ }
+
+ @FunctionalInterface
+ private interface ClientAction<O> {
+
+ O apply(JobID jobId, ClusterClient<?> clusterClient) throws FlinkException;
+ }
+}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/YarnApplicationClient.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/YarnApplicationClient.java
new file mode 100644
index 000000000..36d12ee6b
--- /dev/null
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/YarnApplicationClient.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.client.impl;
+
+import org.apache.streampark.common.conf.Workspace;
+import org.apache.streampark.common.constants.Constants;
+import org.apache.streampark.common.enums.FlinkJobType;
+import org.apache.streampark.common.fs.FsOperator;
+import org.apache.streampark.common.util.AssertUtils;
+import org.apache.streampark.common.util.FileUtils;
+import org.apache.streampark.common.util.HdfsUtils;
+import org.apache.streampark.flink.client.bean.HdfsWorkspace;
+import org.apache.streampark.flink.client.bean.SubmitRequest;
+import org.apache.streampark.flink.client.bean.SubmitResponse;
+import org.apache.streampark.flink.client.trait.YarnClientTrait;
+import org.apache.streampark.flink.packer.pipeline.ShadedBuildResponse;
+
+import org.apache.flink.client.deployment.ClusterSpecification;
+import org.apache.flink.client.deployment.application.ApplicationConfiguration;
+import org.apache.flink.client.program.ClusterClient;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.PipelineOptions;
+import org.apache.flink.python.PythonOptions;
+import org.apache.flink.util.FlinkException;
+import org.apache.flink.yarn.YarnClusterDescriptor;
+import org.apache.flink.yarn.configuration.YarnConfigOptions;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import scala.Tuple2;
+
+/** Yarn application mode submit. */
+public final class YarnApplicationClient extends YarnClientTrait {
+
+ public static final YarnApplicationClient INSTANCE = new YarnApplicationClient();
+
+ private static final Workspace WORKSPACE = Workspace.remote;
+
+ private YarnApplicationClient() {
+ }
+
+ @Override
+ public void setConfig(SubmitRequest submitRequest, Configuration flinkConfig) {
+ super.setConfig(submitRequest, flinkConfig);
+
+ List<String> providedLibs = new ArrayList<>();
+ HdfsWorkspace hdfsWorkspace = submitRequest.hdfsWorkspace();
+ providedLibs.add(hdfsWorkspace.flinkLib());
+ providedLibs.add(hdfsWorkspace.flinkPlugins());
+ providedLibs.add(hdfsWorkspace.appJars());
+
+ if (submitRequest.jobType() == FlinkJobType.FLINK_SQL) {
+ providedLibs.add(
+ WORKSPACE.APP_SHIMS() + "/flink-" + submitRequest.flinkVersion().majorVersion());
+ String jobLib = WORKSPACE.APP_WORKSPACE() + "/" + submitRequest.id() + "/lib";
+ try {
+ if (HdfsUtils.exists(jobLib)) {
+ providedLibs.add(jobLib);
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ FlinkConfigurationOps.safeSet(flinkConfig, YarnConfigOptions.PROVIDED_LIB_DIRS, providedLibs);
+ FlinkConfigurationOps.safeSet(
+ flinkConfig, YarnConfigOptions.FLINK_DIST_JAR, hdfsWorkspace.flinkDistJar());
+ FlinkConfigurationOps.safeSet(
+ flinkConfig,
+ PipelineOptions.JARS,
+ Collections.singletonList(
+ ((ShadedBuildResponse) submitRequest.buildResult()).shadedJarPath()));
+ FlinkConfigurationOps.safeSet(
+ flinkConfig, YarnConfigOptions.APPLICATION_NAME, submitRequest.effectiveAppName());
+ FlinkConfigurationOps.safeSet(
+ flinkConfig,
+ YarnConfigOptions.APPLICATION_TYPE,
+ submitRequest.applicationType().getName());
+
+ if (submitRequest.jobType() == FlinkJobType.PYFLINK) {
+ String pyVenv = WORKSPACE.APP_PYTHON_VENV();
+ AssertUtils.required(FsOperator.hdfs().exists(pyVenv), pyVenv + " File does not exist");
+
+ String localLib =
+ Workspace.local().APP_WORKSPACE() + "/" + submitRequest.id() + "/lib";
+ if (FileUtils.exists(localLib) && FileUtils.directoryNotBlank(localLib)) {
+ FlinkConfigurationOps.safeSet(flinkConfig, PipelineOptions.JARS, Arrays.asList(localLib));
+ }
+
+ ArrayList<String> shipFiles = new ArrayList<>();
+ shipFiles.add(submitRequest.userJarFile().getParentFile().getAbsolutePath());
+
+ FlinkConfigurationOps.safeSet(flinkConfig, YarnConfigOptions.SHIP_FILES, shipFiles);
+ FlinkConfigurationOps.safeSet(
+ flinkConfig,
+ PythonOptions.PYTHON_FILES,
+ submitRequest.userJarFile().getParentFile().getName());
+ FlinkConfigurationOps.safeSet(flinkConfig, PythonOptions.PYTHON_ARCHIVES, pyVenv);
+ FlinkConfigurationOps.safeSet(
+ flinkConfig, PythonOptions.PYTHON_CLIENT_EXECUTABLE, Constants.PYTHON_EXECUTABLE);
+ FlinkConfigurationOps.safeSet(
+ flinkConfig, PythonOptions.PYTHON_EXECUTABLE, Constants.PYTHON_EXECUTABLE);
+
+ List<String> args = flinkConfig.get(ApplicationConfiguration.APPLICATION_ARGS);
+ ArrayList<String> argsList = new ArrayList<>(args);
+ argsList.add("-pym");
+ argsList.add(
+ submitRequest
+ .userJarFile()
+ .getName()
+ .substring(
+ 0,
+ submitRequest.userJarFile().getName().length()
+ - Constants.PYTHON_SUFFIX.length()));
+ FlinkConfigurationOps.safeSet(
+ flinkConfig, ApplicationConfiguration.APPLICATION_ARGS, argsList);
+ }
+
+ logEffectiveSubmitConfiguration(flinkConfig);
+ }
+
+ @Override
+ public SubmitResponse doSubmit(SubmitRequest submitRequest, Configuration flinkConfig) throws FlinkException {
+ return callAsFlinkException(
+ () -> {
+ Tuple2<ClusterSpecification, YarnClusterDescriptor> deployDescriptor =
+ getYarnClusterDeployDescriptor(flinkConfig, submitRequest.hadoopUser());
+ ClusterSpecification clusterSpecification = deployDescriptor._1();
+ YarnClusterDescriptor clusterDescriptor = deployDescriptor._2();
+ logClusterSpecification(clusterSpecification);
+
+ ApplicationConfiguration applicationConfiguration =
+ ApplicationConfiguration.fromConfiguration(flinkConfig);
+ ClusterClient<ApplicationId> clusterClient =
+ clusterDescriptor
+ .deployApplicationCluster(clusterSpecification, applicationConfiguration)
+ .getClusterClient();
+ ApplicationId applicationId = clusterClient.getClusterId();
+ String jobManagerUrl = clusterClient.getWebInterfaceURL();
+ logYarnJobStarted(applicationId);
+
+ SubmitResponse resp =
+ new SubmitResponse(applicationId.toString(), flinkConfig.toMap(), "", jobManagerUrl);
+ closeSubmit(submitRequest, clusterClient, clusterDescriptor);
+ return resp;
+ });
+ }
+}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/YarnPerJobClient.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/YarnPerJobClient.java
new file mode 100644
index 000000000..ac57e11ea
--- /dev/null
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/YarnPerJobClient.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.client.impl;
+
+import org.apache.streampark.flink.client.bean.CancelRequest;
+import org.apache.streampark.flink.client.bean.CancelResponse;
+import org.apache.streampark.flink.client.bean.SubmitRequest;
+import org.apache.streampark.flink.client.bean.SubmitResponse;
+import org.apache.streampark.flink.client.trait.YarnClientTrait;
+import org.apache.streampark.flink.util.FlinkUtils;
+
+import org.apache.flink.client.deployment.ClusterSpecification;
+import org.apache.flink.client.program.ClusterClient;
+import org.apache.flink.client.program.PackagedProgram;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.DeploymentOptions;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.util.FlinkException;
+import org.apache.flink.yarn.YarnClusterDescriptor;
+import org.apache.flink.yarn.configuration.YarnDeploymentTarget;
+import org.apache.flink.yarn.entrypoint.YarnJobClusterEntrypoint;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+
+import scala.Tuple2;
+
+/** Yarn per-job mode submit. */
+public final class YarnPerJobClient extends YarnClientTrait {
+
+ public static final YarnPerJobClient INSTANCE = new YarnPerJobClient();
+
+ private YarnPerJobClient() {
+ }
+
+ @Override
+ public void setConfig(SubmitRequest submitRequest, Configuration flinkConfig) {
+ super.setConfig(submitRequest, flinkConfig);
+ FlinkConfigurationOps.safeSet(
+ flinkConfig, DeploymentOptions.TARGET, YarnDeploymentTarget.PER_JOB.getName());
+ FlinkConfigurationOps.safeSet(flinkConfig, DeploymentOptions.ATTACHED, true);
+ FlinkConfigurationOps.safeSet(flinkConfig, DeploymentOptions.SHUTDOWN_IF_ATTACHED, true);
+
+ logEffectiveSubmitConfiguration(flinkConfig);
+ }
+
+ @Override
+ public SubmitResponse doSubmit(SubmitRequest submitRequest, Configuration flinkConfig) throws FlinkException {
+ return callAsFlinkException(
+ () -> {
+ String flinkHome = submitRequest.flinkVersion().getFlinkHome();
+
+ Tuple2<ClusterSpecification, YarnClusterDescriptor> deployDescriptor =
+ getYarnClusterDeployDescriptor(flinkConfig, submitRequest.hadoopUser());
+ ClusterSpecification clusterSpecification = deployDescriptor._1();
+ YarnClusterDescriptor clusterDescriptor = deployDescriptor._2();
+
+ String flinkDistJar = FlinkUtils.getFlinkDistJar(flinkHome);
+ clusterDescriptor.setLocalJarPath(new Path(flinkDistJar));
+ clusterDescriptor.addShipFiles(java.util.Collections.singletonList(new Path(flinkHome + "/lib")));
+
+ PackagedProgram packagedProgram = null;
+ ClusterClient<ApplicationId> clusterClient;
+ logClusterSpecification(clusterSpecification);
+
+ Tuple2<PackagedProgram, JobGraph> programJobGraph =
+ getJobGraph(flinkConfig, submitRequest, submitRequest.userJarFile());
+ packagedProgram = programJobGraph._1();
+ JobGraph jobGraph = programJobGraph._2();
+
+ logInfo(
+ String.format(
+ "%n-------------------------<<applicationId>>------------------------%n"
+ + "jobGraph getJobID: %s%n"
+ + "__________________________________________________________________%n",
+ jobGraph.getJobID()));
+
+ clusterClient =
+ deployInternal(
+ clusterDescriptor,
+ clusterSpecification,
+ submitRequest.effectiveAppName(),
+ YarnJobClusterEntrypoint.class.getName(),
+ jobGraph,
+ true)
+ .getClusterClient();
+
+ ApplicationId applicationId = clusterClient.getClusterId();
+ String jobManagerUrl = clusterClient.getWebInterfaceURL();
+ logYarnJobStarted(applicationId);
+
+ SubmitResponse resp =
+ new SubmitResponse(applicationId.toString(), flinkConfig.toMap(), "", jobManagerUrl);
+ closeSubmit(submitRequest, packagedProgram, clusterClient, clusterDescriptor);
+ return resp;
+ });
+ }
+
+ @Override
+ public CancelResponse doCancel(CancelRequest cancelRequest, Configuration flinkConfig) throws FlinkException {
+ return callAsFlinkException(
+ () -> {
+ CancelResponse response = super.doCancel(cancelRequest, flinkConfig);
+ Tuple2<ApplicationId, YarnClusterDescriptor> yarnClusterDescriptor =
+ getYarnClusterDescriptor(flinkConfig);
+ yarnClusterDescriptor
+ ._2()
+ .killCluster(ApplicationId.fromString(cancelRequest.clusterId()));
+ return response;
+ });
+ }
+}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/YarnSessionClient.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/YarnSessionClient.java
new file mode 100644
index 000000000..0d88a992e
--- /dev/null
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/YarnSessionClient.java
@@ -0,0 +1,281 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.client.impl;
+
+import org.apache.streampark.common.util.Utils;
+import org.apache.streampark.flink.client.bean.CancelRequest;
+import org.apache.streampark.flink.client.bean.CancelResponse;
+import org.apache.streampark.flink.client.bean.DeployRequest;
+import org.apache.streampark.flink.client.bean.DeployResponse;
+import org.apache.streampark.flink.client.bean.HdfsWorkspace;
+import org.apache.streampark.flink.client.bean.SavepointResponse;
+import org.apache.streampark.flink.client.bean.ShutDownRequest;
+import org.apache.streampark.flink.client.bean.ShutDownResponse;
+import org.apache.streampark.flink.client.bean.SubmitRequest;
+import org.apache.streampark.flink.client.bean.SubmitResponse;
+import org.apache.streampark.flink.client.bean.TriggerSavepointRequest;
+import org.apache.streampark.flink.client.trait.YarnClientTrait;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.flink.client.deployment.ClusterSpecification;
+import org.apache.flink.client.program.ClusterClient;
+import org.apache.flink.client.program.PackagedProgram;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.DeploymentOptions;
+import org.apache.flink.configuration.DeploymentOptionsInternal;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.util.FlinkException;
+import org.apache.flink.yarn.YarnClusterDescriptor;
+import org.apache.flink.yarn.configuration.YarnConfigOptions;
+import org.apache.flink.yarn.configuration.YarnDeploymentTarget;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+
+import java.util.ArrayList;
+import java.util.Map;
+
+import scala.Tuple2;
+
+/** Submit Flink jobs to a YARN session cluster. */
+public final class YarnSessionClient extends YarnClientTrait {
+
+ public static final YarnSessionClient INSTANCE = new YarnSessionClient();
+
+ private YarnSessionClient() {
+ }
+
+ @Override
+ public void setConfig(SubmitRequest submitRequest, Configuration flinkConfig) {
+ super.setConfig(submitRequest, flinkConfig);
+ FlinkConfigurationOps.safeSet(
+ flinkConfig, DeploymentOptions.TARGET, YarnDeploymentTarget.SESSION.getName());
+ logEffectiveSubmitConfiguration(flinkConfig);
+ }
+
+ public void deployClusterConfig(DeployRequest deployRequest, Configuration flinkConfig) {
+ ArrayList<String> shipFiles = new ArrayList<>();
+ shipFiles.add(deployRequest.flinkVersion().getFlinkHome() + "/lib");
+ shipFiles.add(deployRequest.flinkVersion().getFlinkHome() + "/plugins");
+
+ HdfsWorkspace hdfsWorkspace = deployRequest.hdfsWorkspace();
+ FlinkConfigurationOps.safeSet(
+ flinkConfig, YarnConfigOptions.FLINK_DIST_JAR, hdfsWorkspace.flinkDistJar());
+ FlinkConfigurationOps.safeSet(flinkConfig, YarnConfigOptions.SHIP_FILES, shipFiles);
+ FlinkConfigurationOps.safeSet(
+ flinkConfig, DeploymentOptions.TARGET, YarnDeploymentTarget.SESSION.getName());
+ FlinkConfigurationOps.safeSet(
+ flinkConfig,
+ DeploymentOptionsInternal.CONF_DIR,
+ deployRequest.flinkVersion().getFlinkHome() + "/conf");
+
+ logEffectiveSubmitConfiguration(flinkConfig);
+ }
+
+ @Override
+ public SubmitResponse doSubmit(SubmitRequest submitRequest, Configuration flinkConfig) throws FlinkException {
+ return callAsFlinkException(
+ () -> {
+ Tuple2<ApplicationId, YarnClusterDescriptor> yarnClusterDescriptor =
+ getYarnClusterDescriptor(flinkConfig);
+ ApplicationId yarnClusterId = yarnClusterDescriptor._1();
+ YarnClusterDescriptor clusterDescriptor = yarnClusterDescriptor._2();
+
+ Tuple2<PackagedProgram, JobGraph> programJobGraph =
+ getJobGraph(flinkConfig, submitRequest, submitRequest.userJarFile());
+ PackagedProgram packageProgram = programJobGraph._1();
+ JobGraph jobGraph = programJobGraph._2();
+
+ ClusterClient<ApplicationId> client =
+ clusterDescriptor.retrieve(yarnClusterId).getClusterClient();
+ String jobId = client.submitJob(jobGraph).get().toString();
+ String jobManagerUrl = client.getWebInterfaceURL();
+
+ logInfo(
+ String.format(
+ "%n-------------------------<<applicationId>>------------------------%n"
+ + "Flink Job Started: jobId: %s , applicationId: %s%n"
+ + "__________________________________________________________________%n",
+ jobId, yarnClusterId));
+
+ SubmitResponse resp =
+ new SubmitResponse(
+ yarnClusterId.toString(), flinkConfig.toMap(), jobId, jobManagerUrl);
+ closeSubmit(submitRequest, packageProgram, client, clusterDescriptor);
+ return resp;
+ });
+ }
+
+ @Override
+ public CancelResponse doCancel(CancelRequest cancelRequest, Configuration flinkConfig) throws FlinkException {
+ FlinkConfigurationOps.safeSet(
+ flinkConfig, DeploymentOptions.TARGET, YarnDeploymentTarget.SESSION.getName());
+ return super.doCancel(cancelRequest, flinkConfig);
+ }
+
+ @Override
+ public SavepointResponse doTriggerSavepoint(
+ TriggerSavepointRequest request,
+ Configuration flinkConfig) throws FlinkException {
+ FlinkConfigurationOps.safeSet(
+ flinkConfig, DeploymentOptions.TARGET, YarnDeploymentTarget.SESSION.getName());
+ return super.doTriggerSavepoint(request, flinkConfig);
+ }
+
+ public DeployResponse deploy(DeployRequest deployRequest) throws Exception {
+ logInfo(
+ String.format(
+ "%n--------------------------------------- flink yarn sesion start "
+ + "---------------------------------------%n"
+ + " userFlinkHome : %s%n"
+ + " flinkVersion : %s%n"
+ + " deployMode : %s%n"
+ + " clusterId : %s%n"
+ + " properties : %s%n"
+ + "-------------------------------------------------------------------------------------------------------%n",
+ deployRequest.flinkVersion().getFlinkHome(),
+ deployRequest.flinkVersion().version(),
+ deployRequest.deployMode().name(),
+ deployRequest.clusterId(),
+ deployRequest.properties()));
+
+ YarnClusterDescriptor clusterDescriptor = null;
+ ClusterClient<ApplicationId> client = null;
+ try {
+ Configuration flinkConfig =
+ extractConfiguration(
+ deployRequest.flinkVersion().getFlinkHome(), deployRequest.properties());
+ deployClusterConfig(deployRequest, flinkConfig);
+ Tuple2<ClusterSpecification, YarnClusterDescriptor> yarnClusterDescriptor =
+ getYarnClusterDeployDescriptor(flinkConfig);
+ clusterDescriptor = yarnClusterDescriptor._2();
+
+ if (StringUtils.isNotBlank(deployRequest.clusterId())) {
+ DeployResponse existingSession =
+ tryReuseExistingYarnSession(deployRequest, clusterDescriptor);
+ if (existingSession != null) {
+ return existingSession;
+ }
+ }
+
+ client =
+ clusterDescriptor
+ .deploySessionCluster(yarnClusterDescriptor._1())
+ .getClusterClient();
+ if (client.getWebInterfaceURL() != null) {
+ return new DeployResponse(
+ client.getWebInterfaceURL(), client.getClusterId().toString(), null);
+ }
+ return new DeployResponse(
+ null,
+ null,
+ new RuntimeException("get the cluster getWebInterfaceURL failed."));
+ } catch (Exception e) {
+ logError("start flink session fail in " + deployRequest.deployMode() + " mode");
+ throw e;
+ } finally {
+ Utils.close(client, clusterDescriptor);
+ }
+ }
+
+ private DeployResponse tryReuseExistingYarnSession(
+ DeployRequest deployRequest,
+ YarnClusterDescriptor clusterDescriptor) throws Exception {
+ try {
+ FinalApplicationStatus applicationStatus =
+ clusterDescriptor
+ .getYarnClient()
+ .getApplicationReport(
+ ApplicationId.fromString(deployRequest.clusterId()))
+ .getFinalApplicationStatus();
+ if (FinalApplicationStatus.UNDEFINED != applicationStatus) {
+ return null;
+ }
+ ClusterClient<ApplicationId> yarnClient =
+ clusterDescriptor
+ .retrieve(ApplicationId.fromString(deployRequest.clusterId()))
+ .getClusterClient();
+ if (yarnClient.getWebInterfaceURL() == null) {
+ return null;
+ }
+ return new DeployResponse(
+ yarnClient.getWebInterfaceURL(),
+ yarnClient.getClusterId().toString(),
+ null);
+ } catch (ApplicationNotFoundException e) {
+ logInfo("this applicationId have not managed by yarn ,need deploy ...");
+ return null;
+ }
+ }
+
+ public ShutDownResponse shutdown(ShutDownRequest shutDownRequest) throws Exception {
+ YarnClusterDescriptor clusterDescriptor = null;
+ ClusterClient<ApplicationId> client = null;
+ try {
+ Configuration flinkConfig =
+ getFlinkDefaultConfiguration(shutDownRequest.flinkVersion().getFlinkHome());
+ if (shutDownRequest.properties() != null) {
+ for (Map.Entry<String, Object> entry : shutDownRequest.properties().entrySet()) {
+ if (entry.getValue() != null) {
+ flinkConfig.setString(entry.getKey(), entry.getValue().toString());
+ }
+ }
+ }
+ FlinkConfigurationOps.safeSet(
+ flinkConfig, YarnConfigOptions.APPLICATION_ID, shutDownRequest.clusterId());
+ FlinkConfigurationOps.safeSet(
+ flinkConfig, DeploymentOptions.TARGET, YarnDeploymentTarget.SESSION.getName());
+
+ Tuple2<ApplicationId, YarnClusterDescriptor> yarnClusterDescriptor =
+ getYarnClusterDescriptor(flinkConfig);
+ clusterDescriptor = yarnClusterDescriptor._2();
+
+ boolean shutDownState =
+ FinalApplicationStatus.UNDEFINED.equals(
+ clusterDescriptor
+ .getYarnClient()
+ .getApplicationReport(
+ ApplicationId.fromString(shutDownRequest.clusterId()))
+ .getFinalApplicationStatus());
+ if (shutDownState) {
+ client =
+ clusterDescriptor
+ .retrieve(yarnClusterDescriptor._1())
+ .getClusterClient();
+ client.shutDownCluster();
+ }
+
+ logInfo(
+ "the "
+ + shutDownRequest.clusterId()
+ + "'s final status is "
+ + clusterDescriptor
+ .getYarnClient()
+ .getApplicationReport(
+ ConverterUtils.toApplicationId(shutDownRequest.clusterId()))
+ .getFinalApplicationStatus());
+ return new ShutDownResponse(shutDownRequest.clusterId());
+ } catch (Exception e) {
+ logError("shutdown flink session fail in " + shutDownRequest.deployMode() + " mode");
+ throw e;
+ } finally {
+ Utils.close(client, clusterDescriptor);
+ }
+ }
+}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/tool/FlinkSessionSubmitHelper.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/tool/FlinkSessionSubmitHelper.java
new file mode 100644
index 000000000..364f9e6a5
--- /dev/null
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/tool/FlinkSessionSubmitHelper.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.client.tool;
+
+import org.apache.streampark.common.util.AssertUtils;
+import org.apache.streampark.common.util.JsonUtils;
+import org.apache.streampark.common.util.LoggerSupport;
+import org.apache.streampark.flink.kubernetes.KubernetesRetriever;
+
+import org.apache.streampark.shaded.com.fasterxml.jackson.databind.JsonNode;
+
+import org.apache.flink.client.deployment.application.ApplicationConfiguration;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.CoreOptions;
+import org.apache.flink.runtime.jobgraph.SavepointConfigOptions;
+import org.apache.hc.client5.http.entity.mime.MultipartEntityBuilder;
+import org.apache.hc.client5.http.fluent.Request;
+import org.apache.hc.core5.http.ContentType;
+import org.apache.hc.core5.http.io.entity.StringEntity;
+
+import java.io.File;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+
+/** Submit Flink jobs to session clusters via the REST API. */
+public final class FlinkSessionSubmitHelper extends LoggerSupport {
+
+ private static final FlinkSessionSubmitHelper INSTANCE = new FlinkSessionSubmitHelper();
+
+ private FlinkSessionSubmitHelper() {
+ }
+
+ /**
+ * Submit Flink Job via Rest API.
+ *
+ * @param jmRestUrl jobmanager rest url of target flink cluster
+ * @param flinkJobJar flink job jar file
+ * @param flinkConfig flink configuration
+ * @return jobID of submitted flink job
+ */
+ public static String submitViaRestApi(
+ String jmRestUrl, File flinkJobJar,
+ Configuration flinkConfig) throws Exception {
+ return INSTANCE.doSubmitViaRestApi(jmRestUrl, flinkJobJar, flinkConfig);
+ }
+
+ private String doSubmitViaRestApi(String jmRestUrl, File flinkJobJar, Configuration flinkConfig) throws Exception {
+ String uploadResult =
+ Request.post(jmRestUrl + "/jars/upload")
+ .connectTimeout(KubernetesRetriever.FLINK_REST_AWAIT_TIMEOUT_SEC)
+ .responseTimeout(KubernetesRetriever.FLINK_CLIENT_TIMEOUT_SEC)
+ .body(
+ MultipartEntityBuilder.create()
+ .addBinaryBody(
+ "jarfile",
+ flinkJobJar,
+ ContentType.create("application/java-archive"),
+ flinkJobJar.getName())
+ .build())
+ .execute()
+ .returnContent()
+ .asString(StandardCharsets.UTF_8);
+
+ JarUploadResponse jarUploadResponse = parseJarUploadResponse(uploadResult);
+
+ AssertUtils.required(
+ jarUploadResponse != null && jarUploadResponse.isSuccessful(),
+ "[flink-submit] upload flink jar to flink session cluster failed, jmRestUrl="
+ + jmRestUrl
+ + ", response="
+ + jarUploadResponse);
+
+ String resp =
+ Request.post(jmRestUrl + "/jars/" + jarUploadResponse.jarId() + "/run")
+ .connectTimeout(KubernetesRetriever.FLINK_REST_AWAIT_TIMEOUT_SEC)
+ .responseTimeout(KubernetesRetriever.FLINK_CLIENT_TIMEOUT_SEC)
+ .body(new StringEntity(JsonUtils.write(new JarRunRequest(flinkConfig))))
+ .execute()
+ .returnContent()
+ .asString(StandardCharsets.UTF_8);
+
+ return parseJobId(resp);
+ }
+
+ private JarUploadResponse parseJarUploadResponse(String uploadResult) {
+ try {
+ JsonNode node = JsonUtils.read(uploadResult, JsonNode.class);
+ return new JarUploadResponse(
+ node.has("filename") ? node.get("filename").asText(null) : null,
+ node.has("status") ? node.get("status").asText(null) : null);
+ } catch (Exception e) {
+ return null;
+ }
+ }
+
+ private String parseJobId(String resp) {
+ try {
+ JsonNode node = JsonUtils.read(resp, JsonNode.class);
+ return node.has("jobid") ? node.get("jobid").asText(null) : null;
+ } catch (Exception e) {
+ return null;
+ }
+ }
+}
+
+/**
+ * refer to https://ci.apache.org/projects/flink/flink-docs-stable/docs/ops/rest_api/#jars-upload
+ */
+class JarUploadResponse {
+
+ private final String filename;
+ private final String status;
+
+ JarUploadResponse(String filename, String status) {
+ this.filename = filename;
+ this.status = status;
+ }
+
+ boolean isSuccessful() {
+ return "success".equalsIgnoreCase(status);
+ }
+
+ String jarId() {
+ return filename.substring(filename.lastIndexOf('/') + 1);
+ }
+
+ @Override
+ public String toString() {
+ return "JarUploadResponse{filename='" + filename + "', status='" + status + "'}";
+ }
+}
+
+/**
+ * refer to https://ci.apache.org/projects/flink/flink-docs-stable/docs/ops/rest_api/#jars-upload
+ */
+class JarRunRequest {
+
+ private final String entryClass;
+ private final String programArgs;
+ private final String parallelism;
+ private final String savepointPath;
+ private final boolean allowNonRestoredState;
+
+ JarRunRequest(Configuration flinkConf) {
+ this.entryClass = flinkConf.get(ApplicationConfiguration.APPLICATION_MAIN_CLASS);
+ List<String> args = flinkConf.get(ApplicationConfiguration.APPLICATION_ARGS);
+ this.programArgs = args == null ? null : String.join(" ", args);
+ this.parallelism = String.valueOf(flinkConf.get(CoreOptions.DEFAULT_PARALLELISM));
+ this.savepointPath = flinkConf.get(SavepointConfigOptions.SAVEPOINT_PATH);
+ this.allowNonRestoredState =
+ flinkConf.getBoolean(SavepointConfigOptions.SAVEPOINT_IGNORE_UNCLAIMED_STATE);
+ }
+
+ public String getEntryClass() {
+ return entryClass;
+ }
+
+ public String getProgramArgs() {
+ return programArgs;
+ }
+
+ public String getParallelism() {
+ return parallelism;
+ }
+
+ public String getSavepointPath() {
+ return savepointPath;
+ }
+
+ public boolean isAllowNonRestoredState() {
+ return allowNonRestoredState;
+ }
+}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/trait/FlinkClientTrait.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/trait/FlinkClientTrait.java
new file mode 100644
index 000000000..ca5f66a3c
--- /dev/null
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/trait/FlinkClientTrait.java
@@ -0,0 +1,884 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.client.trait;
+
+import org.apache.streampark.common.conf.ConfigKeys;
+import org.apache.streampark.common.conf.Workspace;
+import org.apache.streampark.common.constants.Constants;
+import org.apache.streampark.common.enums.ApplicationType;
+import org.apache.streampark.common.enums.FlinkDeployMode;
+import org.apache.streampark.common.enums.FlinkJobType;
+import org.apache.streampark.common.enums.FlinkRestoreMode;
+import org.apache.streampark.common.fs.FsOperator;
+import org.apache.streampark.common.util.AssertUtils;
+import org.apache.streampark.common.util.DeflaterUtils;
+import org.apache.streampark.common.util.ExceptionUtils;
+import org.apache.streampark.common.util.FlinkConfigurationUtils;
+import org.apache.streampark.common.util.LoggerSupport;
+import org.apache.streampark.common.util.SystemPropertyUtils;
+import org.apache.streampark.common.util.Utils;
+import org.apache.streampark.flink.client.bean.CancelRequest;
+import org.apache.streampark.flink.client.bean.CancelResponse;
+import org.apache.streampark.flink.client.bean.SavepointRequestTrait;
+import org.apache.streampark.flink.client.bean.SavepointResponse;
+import org.apache.streampark.flink.client.bean.SubmitRequest;
+import org.apache.streampark.flink.client.bean.SubmitResponse;
+import org.apache.streampark.flink.client.bean.TriggerSavepointRequest;
+import org.apache.streampark.flink.core.FlinkClusterClient;
+import org.apache.streampark.flink.core.conf.FlinkRunOption;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Options;
+import org.apache.commons.collections.MapUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.client.cli.CliArgsException;
+import org.apache.flink.client.cli.CliFrontend;
+import org.apache.flink.client.cli.CliFrontendParser;
+import org.apache.flink.client.cli.CustomCommandLine;
+import org.apache.flink.client.cli.ExecutionConfigAccessor;
+import org.apache.flink.client.cli.ProgramOptions;
+import org.apache.flink.client.deployment.application.ApplicationConfiguration;
+import org.apache.flink.client.program.ClusterClient;
+import org.apache.flink.client.program.PackagedProgram;
+import org.apache.flink.client.program.PackagedProgramUtils;
+import org.apache.flink.configuration.CheckpointingOptions;
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.CoreOptions;
+import org.apache.flink.configuration.DeploymentOptions;
+import org.apache.flink.configuration.ExecutionOptions;
+import org.apache.flink.configuration.GlobalConfiguration;
+import org.apache.flink.configuration.PipelineOptions;
+import org.apache.flink.configuration.PipelineOptionsInternal;
+import org.apache.flink.python.PythonOptions;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.jobgraph.SavepointConfigOptions;
+import org.apache.flink.util.FlinkException;
+import org.apache.flink.util.Preconditions;
+
+import com.google.common.collect.Lists;
+
+import java.io.File;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import scala.Tuple2;
+
+/** Base trait for Flink client implementations. */
+public abstract class FlinkClientTrait extends LoggerSupport {
+
+ private final String paramKeyFlinkConf =
+ ConfigKeys.KEY_FLINK_CONF(ConfigKeys.PARAM_PREFIX());
+ private final String paramKeyFlinkSql =
+ ConfigKeys.KEY_FLINK_SQL(ConfigKeys.PARAM_PREFIX());
+ private final String paramKeyAppConf =
+ ConfigKeys.KEY_APP_CONF(ConfigKeys.PARAM_PREFIX());
+ private final String paramKeyAppName =
+ ConfigKeys.KEY_APP_NAME(ConfigKeys.PARAM_PREFIX());
+ private final String paramKeyFlinkParallelism =
+ ConfigKeys.KEY_FLINK_PARALLELISM(ConfigKeys.PARAM_PREFIX());
+
+ protected static FlinkException asFlinkException(Throwable throwable) {
+ if (throwable instanceof FlinkException) {
+ return (FlinkException) throwable;
+ }
+ if (throwable instanceof Exception) {
+ return new FlinkException((Exception) throwable);
+ }
+ return new FlinkException(throwable.getMessage(), throwable);
+ }
+
+ @FunctionalInterface
+ protected interface FlinkCallable<T> {
+
+ T call() throws Exception;
+ }
+
+ protected static <T> T callAsFlinkException(FlinkCallable<T> callable) throws FlinkException {
+ return callAsFlinkException(callable, e -> {
+ });
+ }
+
+ protected static <T> T callAsFlinkException(
+ FlinkCallable<T> callable,
+ java.util.function.Consumer<Exception> onFailure) throws FlinkException {
+ try {
+ return callable.call();
+ } catch (FlinkException e) {
+ onFailure.accept(e);
+ throw e;
+ } catch (Exception e) {
+ onFailure.accept(e);
+ throw asFlinkException(e);
+ }
+ }
+
+ protected static <T> T callAsFlinkExceptionMapping(
+ FlinkCallable<T> callable,
+ java.util.function.Function<Exception, FlinkException> exceptionMapper) throws FlinkException {
+ try {
+ return callable.call();
+ } catch (FlinkException e) {
+ throw e;
+ } catch (Exception e) {
+ throw exceptionMapper.apply(e);
+ }
+ }
+
+ protected CancelResponse toCancelResponse(
+ CancelRequest request, JobID jobId,
+ ClusterClient<?> client) throws FlinkException {
+ return callAsFlinkException(() -> new CancelResponse(cancelJob(request, jobId, client)));
+ }
+
+ protected SavepointResponse toSavepointResponse(
+ TriggerSavepointRequest request,
+ JobID jobId,
+ ClusterClient<?> client) throws FlinkException {
+ return callAsFlinkException(
+ () -> new SavepointResponse(triggerSavepoint(request, jobId, client)));
+ }
+
+ protected void logEffectiveSubmitConfiguration(Configuration flinkConfig) {
+ logInfo(
+ String.format(
+ "%n------------------------------------------------------------------%n"
+ + "Effective submit configuration: %s%n"
+ + "------------------------------------------------------------------%n",
+ flinkConfig));
+ }
+
+ protected void logSavepointClientRequest(String operation, SavepointRequestTrait request) {
+ StringBuilder message =
+ new StringBuilder()
+ .append("\n----------------------------------------- flink job ")
+ .append(operation)
+ .append(" --------------------------------\n")
+ .append(" userFlinkHome : ")
+ .append(request.flinkVersion().getFlinkHome())
+ .append("\n")
+ .append(" flinkVersion : ")
+ .append(request.flinkVersion().version())
+ .append("\n")
+ .append(" clusterId : ")
+ .append(request.clusterId())
+ .append("\n");
+ if (request instanceof CancelRequest) {
+ CancelRequest cancelRequest = (CancelRequest) request;
+ message
+ .append(" withSavePoint : ")
+ .append(cancelRequest.withSavepoint())
+ .append("\n");
+ }
+ message
+ .append(" savePointPath : ")
+ .append(request.savepointPath())
+ .append("\n");
+ if (request instanceof CancelRequest) {
+ message
+ .append(" withDrain : ")
+ .append(((CancelRequest) request).withDrain())
+ .append("\n");
+ }
+ message
+ .append(" nativeFormat : ")
+ .append(request.nativeFormat())
+ .append("\n")
+ .append(" k8sNamespace : ")
+ .append(request.kubernetesNamespace())
+ .append("\n")
+ .append(" appId : ")
+ .append(request.clusterId())
+ .append("\n")
+ .append(" jobId : ")
+ .append(request.jobId())
+ .append("\n")
+ .append(
+ "-------------------------------------------------------------------------------------------\n");
+ logInfo(message.toString());
+ }
+
+ protected SubmitResponse submitJobGraphToCluster(
+ SubmitRequest submitRequest,
+ Configuration flinkConfig,
+ File jarFile,
+ FlinkCallable<ClusterClient<?>> clientSupplier,
+ FlinkCallable<String> clusterIdSupplier,
+ AutoCloseable... extraResources) throws FlinkException {
+ return callAsFlinkException(
+ () -> {
+ Tuple2<PackagedProgram, JobGraph> programJobGraph =
+ getJobGraph(flinkConfig, submitRequest, jarFile);
+ PackagedProgram packageProgram = programJobGraph._1();
+ JobGraph jobGraph = programJobGraph._2();
+ ClusterClient<?> client = clientSupplier.call();
+ String jobId = client.submitJob(jobGraph).get().toString();
+ SubmitResponse result =
+ new SubmitResponse(
+ clusterIdSupplier.call(),
+ flinkConfig.toMap(),
+ jobId,
+ client.getWebInterfaceURL());
+ AutoCloseable[] resources = new AutoCloseable[extraResources.length + 2];
+ resources[0] = packageProgram;
+ resources[1] = client;
+ System.arraycopy(extraResources, 0, resources, 2, extraResources.length);
+ closeSubmit(submitRequest, resources);
+ return result;
+ });
+ }
+
+ public SubmitResponse submit(SubmitRequest submitRequest) throws FlinkException {
+ logInfo(
+ "\n"
+ + "--------------------------------------- flink job start ---------------------------------------\n"
+ + " userFlinkHome : "
+ + submitRequest.flinkVersion().getFlinkHome()
+ + "\n"
+ + " flinkVersion : "
+ + submitRequest.flinkVersion().version()
+ + "\n"
+ + " appName : "
+ + submitRequest.effectiveAppName()
+ + "\n"
+ + " jobType : "
+ + submitRequest.jobType().name()
+ + "\n"
+ + " deployMode : "
+ + submitRequest.deployMode().name()
+ + "\n"
+ + " k8sNamespace : "
+ + submitRequest.kubernetesNamespace()
+ + "\n"
+ + " flinkExposedType : "
+ + submitRequest.flinkRestExposedType()
+ + "\n"
+ + " clusterId : "
+ + submitRequest.clusterId()
+ + "\n"
+ + " applicationType : "
+ + submitRequest.applicationType().getName()
+ + "\n"
+ + " savePoint : "
+ + submitRequest.savePoint()
+ + "\n"
+ + " properties : "
+ + formatProperties(submitRequest.properties())
+ + "\n"
+ + " args : "
+ + submitRequest.args()
+ + "\n"
+ + " appConf : "
+ + submitRequest.appConf()
+ + "\n"
+ + " flinkBuildResult : "
+ + submitRequest.buildResult()
+ + "\n"
+ + "-------------------------------------------------------------------------------------------\n");
+
+ Configuration flinkConfig = callAsFlinkException(() -> prepareConfig(submitRequest));
+ setConfig(submitRequest, flinkConfig);
+
+ return callAsFlinkException(
+ () -> doSubmit(submitRequest, flinkConfig),
+ e -> {
+ logSubmitFailure(submitRequest, e);
+ });
+ }
+
+ private void logSubmitFailure(SubmitRequest submitRequest, Exception e) {
+ logError(
+ "flink job "
+ + submitRequest.appName()
+ + " start failed, "
+ + "deployMode: "
+ + submitRequest.deployMode().getName()
+ + ", "
+ + "detail: "
+ + ExceptionUtils.stringifyException(e));
+ }
+
+ private Configuration prepareConfig(SubmitRequest submitRequest) throws Exception {
+ CommandLineAndConfig commandLineAndConfig = getCommandLineAndFlinkConfig(submitRequest);
+ Configuration flinkConfig = commandLineAndConfig.flinkConfig;
+ applyJobTypeConfig(submitRequest, commandLineAndConfig.commandLine, flinkConfig);
+ applyCommonPipelineConfig(submitRequest, flinkConfig);
+ applyCheckpointDefaults(submitRequest, flinkConfig);
+ applySavepointConfig(submitRequest, flinkConfig);
+ applyEnvProperties(submitRequest, flinkConfig);
+ return flinkConfig;
+ }
+
+ private void applyJobTypeConfig(
+ SubmitRequest submitRequest, CommandLine commandLine,
+ Configuration flinkConfig) throws Exception {
+ if (submitRequest.jobType() == FlinkJobType.PYFLINK) {
+ applyPyFlinkConfig(submitRequest, flinkConfig);
+ return;
+ }
+ if (submitRequest.userJarFile() != null) {
+ URI uri =
+ PackagedProgramUtils.resolveURI(
+ submitRequest.userJarFile().getAbsolutePath());
+ ProgramOptions programOptions = ProgramOptions.create(commandLine);
+ ExecutionConfigAccessor executionParameters =
+ ExecutionConfigAccessor.fromProgramOptions(
+ programOptions, Collections.singletonList(uri.toString()));
+ executionParameters.applyToConfiguration(flinkConfig);
+ }
+ }
+
+ private void applyPyFlinkConfig(SubmitRequest submitRequest, Configuration flinkConfig) throws Exception {
+ String pythonVenv = Workspace.local().APP_PYTHON_VENV();
+ AssertUtils.required(
+ FsOperator.lfs().exists(pythonVenv),
+ pythonVenv + " File does not exist");
+
+ safeSet(flinkConfig, PythonOptions.PYTHON_ARCHIVES, pythonVenv);
+ safeSet(
+ flinkConfig,
+ PythonOptions.PYTHON_CLIENT_EXECUTABLE,
+ Constants.PYTHON_EXECUTABLE);
+ safeSet(flinkConfig, PythonOptions.PYTHON_EXECUTABLE, Constants.PYTHON_EXECUTABLE);
+
+ String flinkOptPath = System.getenv(ConfigConstants.ENV_FLINK_OPT_DIR);
+ if (StringUtils.isBlank(flinkOptPath)) {
+ logWarn(
+ "Get environment variable "
+ + ConfigConstants.ENV_FLINK_OPT_DIR
+ + " fail");
+ String flinkHome = submitRequest.flinkVersion().getFlinkHome();
+ SystemPropertyUtils.setEnv(
+ ConfigConstants.ENV_FLINK_OPT_DIR, flinkHome + "/opt");
+ logInfo(
+ "Set temporary environment variables "
+ + ConfigConstants.ENV_FLINK_OPT_DIR
+ + " = "
+ + flinkHome
+ + "/opt");
+ }
+ }
+
+ private void applyCommonPipelineConfig(SubmitRequest submitRequest, Configuration flinkConfig) {
+ safeSet(flinkConfig, PipelineOptions.NAME, submitRequest.effectiveAppName());
+ safeSet(flinkConfig, DeploymentOptions.TARGET, submitRequest.deployMode().getName());
+ safeSet(flinkConfig, SavepointConfigOptions.SAVEPOINT_PATH, submitRequest.savePoint());
+ safeSet(
+ flinkConfig,
+ ApplicationConfiguration.APPLICATION_MAIN_CLASS,
+ submitRequest.appMain());
+ safeSet(
+ flinkConfig,
+ ApplicationConfiguration.APPLICATION_ARGS,
+ extractProgramArgs(submitRequest));
+ safeSet(
+ flinkConfig,
+ PipelineOptionsInternal.PIPELINE_FIXED_JOB_ID,
+ submitRequest.jobId());
+ }
+
+ private void applyCheckpointDefaults(SubmitRequest submitRequest, Configuration flinkConfig) throws Exception {
+ if (!submitRequest.hasProp(CheckpointingOptions.MAX_RETAINED_CHECKPOINTS.key())) {
+ Configuration flinkDefaultConfiguration =
+ getFlinkDefaultConfiguration(submitRequest.flinkVersion().getFlinkHome());
+ ConfigOption<Integer> retainedOption =
+ CheckpointingOptions.MAX_RETAINED_CHECKPOINTS;
+ flinkConfig.set(
+ retainedOption, flinkDefaultConfiguration.get(retainedOption));
+ }
+ }
+
+ private void applySavepointConfig(SubmitRequest submitRequest, Configuration flinkConfig) {
+ if (StringUtils.isBlank(submitRequest.savePoint())) {
+ return;
+ }
+ safeSet(
+ flinkConfig,
+ SavepointConfigOptions.SAVEPOINT_PATH,
+ submitRequest.savePoint());
+ flinkConfig.setBoolean(
+ SavepointConfigOptions.SAVEPOINT_IGNORE_UNCLAIMED_STATE,
+ submitRequest.allowNonRestoredState());
+ boolean enableRestoreMode =
+ submitRequest.restoreMode() != null
+ && submitRequest
+ .flinkVersion()
+ .checkVersion(FlinkRestoreMode.SINCE_FLINK_VERSION);
+ if (enableRestoreMode) {
+ flinkConfig.setString(
+ FlinkRestoreMode.RESTORE_MODE, submitRequest.restoreMode().getName());
+ }
+ }
+
+ private void applyEnvProperties(SubmitRequest submitRequest, Configuration flinkConfig) {
+ if (MapUtils.isEmpty(submitRequest.properties())) {
+ return;
+ }
+ if (submitRequest.hasProp(CoreOptions.FLINK_JVM_OPTIONS.key())) {
+ Object jvmOptObj =
+ submitRequest.getProp(CoreOptions.FLINK_JVM_OPTIONS.key());
+ String jvmOpt = jvmOptObj.toString();
+ if (!jvmOpt.contains("-Dfile.encoding=")) {
+ submitRequest.putProperty(
+ CoreOptions.FLINK_JVM_OPTIONS.key(), "-Dfile.encoding=UTF-8 " + jvmOpt);
+ }
+ }
+
+ for (Map.Entry<String, Object> entry : submitRequest.properties().entrySet()) {
+ if (entry.getKey().startsWith("env.")) {
+ logInfo("env opts: " + entry.getKey() + ": " + entry.getValue());
+ flinkConfig.setString(entry.getKey(), entry.getValue().toString());
+ }
+ }
+ }
+
+ public abstract void setConfig(SubmitRequest submitRequest, Configuration flinkConf);
+
+ public SavepointResponse triggerSavepoint(TriggerSavepointRequest savepointRequest) throws FlinkException {
+ logSavepointClientRequest("trigger savepoint", savepointRequest);
+ Configuration flinkConf = new Configuration();
+ return callAsFlinkException(() -> doTriggerSavepoint(savepointRequest, flinkConf));
+ }
+
+ public CancelResponse cancel(CancelRequest cancelRequest) throws FlinkException {
+ logSavepointClientRequest("cancel", cancelRequest);
+ Configuration flinkConf = new Configuration();
+ return callAsFlinkException(() -> doCancel(cancelRequest, flinkConf));
+ }
+
+ public abstract SubmitResponse doSubmit(SubmitRequest submitRequest, Configuration flinkConf) throws FlinkException;
+
+ public abstract SavepointResponse doTriggerSavepoint(
+ TriggerSavepointRequest request,
+ Configuration flinkConf) throws FlinkException;
+
+ public abstract CancelResponse doCancel(CancelRequest cancelRequest, Configuration flinkConf) throws FlinkException;
+
+ protected SubmitResponse trySubmit(
+ SubmitRequest submitRequest,
+ Configuration flinkConfig,
+ File jarFile,
+ SubmitFunc jobGraphFunc,
+ SubmitFunc restApiFunc) throws FlinkException {
+ try {
+ logInfo("[flink-submit] Submit job with JobGraph Plan.");
+ return jobGraphFunc.apply(submitRequest, flinkConfig, jarFile);
+ } catch (FlinkException e) {
+ try {
+ return restApiFunc.apply(submitRequest, flinkConfig, jarFile);
+ } catch (FlinkException fallbackException) {
+ throw new FlinkException(
+ "[flink-submit] Both JobGraph submit plan and Rest API submit plan all failed!\n"
+ + "JobGraph Submit plan failed detail:\n"
+ + "------------------------------------------------------------------\n"
+ + ExceptionUtils.stringifyException(e)
+ + "\n"
+ + "------------------------------------------------------------------\n"
+ + "\n"
+ + " RestAPI Submit plan failed detail:\n"
+ + " ------------------------------------------------------------------\n"
+ + ExceptionUtils.stringifyException(fallbackException)
+ + "\n"
+ + "------------------------------------------------------------------\n",
+ fallbackException);
+ }
+ }
+ }
+
+ public Tuple2<PackagedProgram, JobGraph> getJobGraph(
+ Configuration flinkConfig, SubmitRequest submitRequest,
+ File jarFile) throws Exception {
+ PackagedProgram.Builder builder =
+ PackagedProgram.newBuilder()
+ .setSavepointRestoreSettings(submitRequest.savepointRestoreSettings())
+ .setEntryPointClassName(
+ flinkConfig
+ .getOptional(ApplicationConfiguration.APPLICATION_MAIN_CLASS)
+ .orElseThrow(
+ () -> new IllegalStateException(
+ "Application main class is not configured")))
+ .setArguments(
+ flinkConfig
+ .getOptional(ApplicationConfiguration.APPLICATION_ARGS)
+ .orElse(Lists.newArrayList())
+ .toArray(new String[0]));
+
+ if (submitRequest.jobType() == FlinkJobType.PYFLINK) {
+ if (!submitRequest.libs().isEmpty()) {
+ // BUG: https://github.com/apache/streampark/issues/3761
+ // builder.setUserClassPaths(Lists.newArrayList(submitRequest.libs()))
+ }
+ } else {
+ builder.setJarFile(jarFile);
+ // BUG: https://github.com/apache/streampark/issues/3761
+ // .setUserClassPaths(Lists.newArrayList(submitRequest.classPaths()))
+ }
+
+ PackagedProgram packageProgram = builder.build();
+ JobGraph jobGraph =
+ PackagedProgramUtils.createJobGraph(
+ packageProgram,
+ flinkConfig,
+ getParallelism(submitRequest),
+ null,
+ false);
+
+ return new Tuple2<>(packageProgram, jobGraph);
+ }
+
+ public JobID getJobID(String jobId) throws CliArgsException {
+ try {
+ return JobID.fromHexString(jobId);
+ } catch (Exception e) {
+ throw new CliArgsException(e.getMessage());
+ }
+ }
+
+ CustomCommandLine validateAndGetActiveCommandLine(
+ List<CustomCommandLine> customCommandLines,
+ CommandLine commandLine) {
+ CommandLine line = Preconditions.checkNotNull(commandLine);
+ logInfo("Custom commandline: " + customCommandLines);
+ for (CustomCommandLine cli : customCommandLines) {
+ boolean isActive = cli.isActive(line);
+ logInfo("Checking custom commandline " + cli + ", isActive: " + isActive);
+ if (isActive) {
+ return cli;
+ }
+ }
+ throw new IllegalStateException("No valid command-line found.");
+ }
+
+ public Configuration getFlinkDefaultConfiguration(String flinkHome) {
+ try {
+ return GlobalConfiguration.loadConfiguration(flinkHome + "/conf");
+ } catch (Exception e) {
+ return new Configuration();
+ }
+ }
+
+ <T> T getOptionFromDefaultFlinkConfig(String flinkHome, ConfigOption<T> option) {
+ return getFlinkDefaultConfiguration(flinkHome).get(option);
+ }
+
+ List<CustomCommandLine> getCustomCommandLines(String flinkHome) {
+ Configuration flinkDefaultConfiguration = getFlinkDefaultConfiguration(flinkHome);
+ String confDir = flinkHome + "/conf";
+ return CliFrontend.loadCustomCommandLines(flinkDefaultConfiguration, confDir);
+ }
+
+ public Integer getParallelism(SubmitRequest submitRequest) {
+ if (submitRequest.hasProp(ConfigKeys.KEY_FLINK_PARALLELISM())) {
+ return Integer.valueOf(
+ submitRequest.getProp(ConfigKeys.KEY_FLINK_PARALLELISM()).toString());
+ }
+ return getFlinkDefaultConfiguration(submitRequest.flinkVersion().getFlinkHome())
+ .getInteger(
+ CoreOptions.DEFAULT_PARALLELISM, CoreOptions.DEFAULT_PARALLELISM.defaultValue());
+ }
+
+ Options getCommandLineOptions(String flinkHome) {
+ List<CustomCommandLine> customCommandLines = getCustomCommandLines(flinkHome);
+ Options customCommandLineOptions = new Options();
+ for (CustomCommandLine customCommandLine : customCommandLines) {
+ customCommandLine.addGeneralOptions(customCommandLineOptions);
+ customCommandLine.addRunOptions(customCommandLineOptions);
+ }
+ return FlinkRunOption.mergeOptions(
+ CliFrontendParser.getRunCommandOptions(), customCommandLineOptions);
+ }
+
+ public Configuration extractConfiguration(String flinkHome, Map<String, Object> properties) throws Exception {
+ Options commandLineOptions = getCommandLineOptions(flinkHome);
+ List<String> cliArgs = new ArrayList<>();
+ if (MapUtils.isNotEmpty(properties)) {
+ for (Map.Entry<String, Object> entry : properties.entrySet()) {
+ cliArgs.add("-D" + entry.getKey() + "=" + entry.getValue().toString().trim());
+ }
+ }
+ CommandLine commandLine =
+ FlinkRunOption.parse(
+ commandLineOptions, cliArgs.toArray(new String[0]), true);
+ CustomCommandLine activeCommandLine =
+ validateAndGetActiveCommandLine(getCustomCommandLines(flinkHome), commandLine);
+ return applyConfiguration(flinkHome, activeCommandLine, commandLine);
+ }
+
+ public String cancelJob(CancelRequest cancelRequest, JobID jobID, ClusterClient<?> client) throws Exception {
+ String savePointDir = tryGetSavepointPathIfNeed(cancelRequest);
+
+ FlinkClusterClient<?> clientWrapper = new FlinkClusterClient<>(client);
+ boolean withSavepoint = cancelRequest.withSavepoint();
+ boolean withDrain = cancelRequest.withDrain();
+
+ if (!withSavepoint && !withDrain) {
+ client.cancel(jobID).get();
+ return null;
+ }
+ return clientWrapper
+ .stopWithSavepoint(
+ jobID, cancelRequest.withDrain(), savePointDir, cancelRequest.nativeFormat())
+ .get();
+ }
+
+ public String triggerSavepoint(
+ TriggerSavepointRequest savepointRequest, JobID jobID,
+ ClusterClient<?> client) throws Exception {
+ String savepointPath = tryGetSavepointPathIfNeed(savepointRequest);
+ FlinkClusterClient<?> clientWrapper = new FlinkClusterClient<>(client);
+ return clientWrapper
+ .triggerSavepoint(jobID, savepointPath, savepointRequest.nativeFormat())
+ .get();
+ }
+
+ public void closeSubmit(SubmitRequest submitRequest, AutoCloseable... close) {
+ for (AutoCloseable resource : close) {
+ if (resource instanceof PackagedProgram) {
+ if (submitRequest.safePackageProgram()) {
+ Utils.close(resource);
+ }
+ } else {
+ Utils.close(resource);
+ }
+ }
+ }
+
+ static <T> Configuration safeSet(Configuration flinkConfig, ConfigOption<T> option, T value) {
+ if (value != null && !value.toString().isEmpty()) {
+ flinkConfig.set(option, value);
+ }
+ return flinkConfig;
+ }
+
+ private String tryGetSavepointPathIfNeed(SavepointRequestTrait request) {
+ if (!request.withSavepoint()) {
+ return null;
+ }
+ if (StringUtils.isNotBlank(request.savepointPath())) {
+ return request.savepointPath();
+ }
+ String configDir =
+ getOptionFromDefaultFlinkConfig(
+ request.flinkVersion().getFlinkHome(),
+ org.apache.flink.configuration.ConfigOptions.key(
+ CheckpointingOptions.SAVEPOINT_DIRECTORY.key())
+ .stringType()
+ .defaultValue(
+ request.deployMode() == FlinkDeployMode.YARN_APPLICATION
+ ? Workspace.remote().APP_SAVEPOINTS()
+ : null));
+
+ AssertUtils.required(
+ StringUtils.isNotBlank(configDir),
+ "[StreamPark] deployMode: "
+ + request.deployMode().getName()
+ + ", savePoint path is null or invalid.");
+ return configDir;
+ }
+
+ private CommandLineAndConfig getCommandLineAndFlinkConfig(SubmitRequest submitRequest) throws Exception {
+ Options commandLineOptions =
+ getCommandLineOptions(submitRequest.flinkVersion().getFlinkHome());
+ Map<String, Object> optionMap = buildOptionMap(submitRequest, commandLineOptions);
+ List<String> cliArgs = buildCliArgs(submitRequest, optionMap);
+ logger().info("cliArgs: " + String.join(" ", cliArgs));
+ CommandLine commandLine =
+ FlinkRunOption.parse(commandLineOptions, cliArgs.toArray(new String[0]), true);
+
+ CustomCommandLine activeCommandLine =
+ validateAndGetActiveCommandLine(
+ getCustomCommandLines(submitRequest.flinkVersion().getFlinkHome()), commandLine);
+
+ Configuration configuration =
+ applyConfiguration(
+ submitRequest.flinkVersion().getFlinkHome(), activeCommandLine, commandLine);
+
+ return new CommandLineAndConfig(commandLine, configuration);
+ }
+
+ private Map<String, Object> buildOptionMap(
+ SubmitRequest submitRequest,
+ Options commandLineOptions) {
+ Map<String, Object> optionMap = new HashMap<>();
+ Map<String, String> appOption = submitRequest.appOption();
+ for (Map.Entry<String, String> opt : appOption.entrySet()) {
+ if (!commandLineOptions.hasOption(opt.getKey())) {
+ logWarn("param:" + opt.getKey() + " is error,skip it.");
+ continue;
+ }
+ String option = commandLineOptions.getOption(opt.getKey().trim()).getOpt();
+ Object parsedValue = parseOptionValue(opt.getValue());
+ if (Boolean.TRUE.equals(parsedValue)) {
+ optionMap.put("-" + option, true);
+ } else if (!(parsedValue instanceof Boolean)) {
+ optionMap.put("-" + option, parsedValue);
+ }
+ }
+
+ if (submitRequest.savePoint() != null) {
+ optionMap.put(
+ "-" + FlinkRunOption.SAVEPOINT_PATH_OPTION.getOpt(),
+ submitRequest.savePoint());
+ }
+
+ optionMap.remove("-e");
+ optionMap.remove("--executor");
+ optionMap.remove("-t");
+ optionMap.remove("--target");
+ if (submitRequest.deployMode() != null) {
+ optionMap.put("-t", submitRequest.deployMode().getName());
+ }
+ return optionMap;
+ }
+
+ private List<String> buildCliArgs(SubmitRequest submitRequest, Map<String, Object> optionMap) {
+ List<String> array = new ArrayList<>();
+ for (Map.Entry<String, Object> opt : optionMap.entrySet()) {
+ array.add(opt.getKey());
+ if (opt.getValue() instanceof String) {
+ array.add((String) opt.getValue());
+ }
+ }
+
+ if (MapUtils.isNotEmpty(submitRequest.properties())) {
+ for (Map.Entry<String, Object> key : submitRequest.properties().entrySet()) {
+ if (!key.getKey().startsWith("env.")) {
+ logInfo(
+ "application dynamicProperties: "
+ + key.getKey()
+ + " :"
+ + key.getValue());
+ array.add("-D" + key.getKey() + "=" + key.getValue());
+ }
+ }
+ }
+ return array;
+ }
+
+ private List<String> extractProgramArgs(SubmitRequest submitRequest) {
+ List<String> programArgs = new ArrayList<>();
+ programArgs.addAll(FlinkConfigurationUtils.extractArguments(submitRequest.args()));
+
+ if (submitRequest.applicationType() == ApplicationType.STREAMPARK_FLINK) {
+ programArgs.add(paramKeyFlinkConf);
+ programArgs.add(submitRequest.flinkYaml());
+ programArgs.add(paramKeyAppName);
+ programArgs.add(DeflaterUtils.zipString(submitRequest.effectiveAppName()));
+ programArgs.add(paramKeyFlinkParallelism);
+ programArgs.add(getParallelism(submitRequest).toString());
+
+ if (submitRequest.jobType() == FlinkJobType.FLINK_SQL) {
+ programArgs.add(paramKeyFlinkSql);
+ programArgs.add(submitRequest.flinkSQL());
+ if (submitRequest.appConf() != null) {
+ programArgs.add(paramKeyAppConf);
+ programArgs.add(submitRequest.appConf());
+ }
+ } else if (shouldAddAppConf(submitRequest.appConf())) {
+ programArgs.add(paramKeyAppConf);
+ programArgs.add(submitRequest.appConf());
+ }
+ }
+
+ if (MapUtils.isNotEmpty(submitRequest.properties())
+ && submitRequest.properties().containsKey(ExecutionOptions.RUNTIME_MODE.key())) {
+ String runtimeMode =
+ submitRequest.properties().get(ExecutionOptions.RUNTIME_MODE.key()).toString();
+ programArgs.add("--" + ExecutionOptions.RUNTIME_MODE.key());
+ programArgs.add(runtimeMode);
+ }
+
+ if (submitRequest.jobType() == FlinkJobType.PYFLINK
+ && submitRequest.deployMode() != FlinkDeployMode.YARN_APPLICATION) {
+ programArgs.add("-py");
+ programArgs.add(submitRequest.userJarFile().getAbsolutePath());
+ }
+
+ return Lists.newArrayList(programArgs);
+ }
+
+ private Configuration applyConfiguration(
+ String flinkHome, CustomCommandLine activeCustomCommandLine,
+ CommandLine commandLine) throws Exception {
+ Preconditions.checkNotNull(
+ activeCustomCommandLine, "activeCustomCommandLine must not be null.");
+ Configuration configuration = new Configuration();
+ Configuration flinkDefaultConfiguration = getFlinkDefaultConfiguration(flinkHome);
+ Set<String> keys = flinkDefaultConfiguration.keySet();
+ for (String key : keys) {
+ String value = flinkDefaultConfiguration.getString(key, null);
+ if (value != null) {
+ configuration.setString(key, value);
+ }
+ }
+ configuration.addAll(activeCustomCommandLine.toConfiguration(commandLine));
+ return configuration;
+ }
+
+ private static Object parseOptionValue(String value) {
+ if ("true".equalsIgnoreCase(value) || "false".equalsIgnoreCase(value)) {
+ return Boolean.parseBoolean(value);
+ }
+ return value;
+ }
+
+ private static boolean shouldAddAppConf(String appConf) {
+ if (appConf == null) {
+ return true;
+ }
+ try {
+ return !appConf.startsWith("json:");
+ } catch (Exception e) {
+ return true;
+ }
+ }
+
+ private static String formatProperties(Map<String, Object> properties) {
+ if (MapUtils.isEmpty(properties)) {
+ return "";
+ }
+ return properties.entrySet().stream()
+ .map(e -> e.getKey() + "=" + e.getValue())
+ .collect(Collectors.joining(" "));
+ }
+
+ @FunctionalInterface
+ protected interface SubmitFunc {
+
+ SubmitResponse apply(SubmitRequest request, Configuration config, File jarFile) throws FlinkException;
+ }
+
+ private static final class CommandLineAndConfig {
+
+ final CommandLine commandLine;
+ final Configuration flinkConfig;
+
+ CommandLineAndConfig(CommandLine commandLine, Configuration flinkConfig) {
+ this.commandLine = commandLine;
+ this.flinkConfig = flinkConfig;
+ }
+ }
+}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/trait/KubernetesNativeClientTrait.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/trait/KubernetesNativeClientTrait.java
new file mode 100644
index 000000000..98f267ec0
--- /dev/null
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/trait/KubernetesNativeClientTrait.java
@@ -0,0 +1,250 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.client.trait;
+
+import org.apache.streampark.common.enums.FlinkDeployMode;
+import org.apache.streampark.common.enums.FlinkK8sRestExposedType;
+import org.apache.streampark.flink.client.bean.CancelRequest;
+import org.apache.streampark.flink.client.bean.CancelResponse;
+import org.apache.streampark.flink.client.bean.SavepointRequestTrait;
+import org.apache.streampark.flink.client.bean.SavepointResponse;
+import org.apache.streampark.flink.client.bean.SubmitRequest;
+import org.apache.streampark.flink.client.bean.TriggerSavepointRequest;
+import org.apache.streampark.flink.kubernetes.PodTemplateTool;
+import org.apache.streampark.flink.packer.pipeline.DockerImageBuildResponse;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.client.deployment.ClusterSpecification;
+import org.apache.flink.client.program.ClusterClient;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.DeploymentOptions;
+import org.apache.flink.configuration.DeploymentOptionsInternal;
+import org.apache.flink.kubernetes.KubernetesClusterClientFactory;
+import org.apache.flink.kubernetes.KubernetesClusterDescriptor;
+import org.apache.flink.kubernetes.configuration.KubernetesConfigOptions;
+import org.apache.flink.kubernetes.configuration.KubernetesConfigOptions.ServiceExposedType;
+import org.apache.flink.util.FlinkException;
+
+import javax.annotation.Nonnull;
+
+import java.util.Map;
+
+import scala.Tuple2;
+
+/** Kubernetes native mode submit support. */
+public abstract class KubernetesNativeClientTrait extends FlinkClientTrait {
+
+ @Override
+ public void setConfig(SubmitRequest submitRequest, Configuration flinkConfig) {
+ FlinkClientTrait.safeSet(
+ flinkConfig, KubernetesConfigOptions.CLUSTER_ID, submitRequest.clusterId());
+ FlinkClientTrait.safeSet(
+ flinkConfig, KubernetesConfigOptions.NAMESPACE, submitRequest.kubernetesNamespace());
+ FlinkClientTrait.safeSet(
+ flinkConfig,
+ KubernetesConfigOptions.REST_SERVICE_EXPOSED_TYPE,
+ covertToServiceExposedType(submitRequest.flinkRestExposedType()));
+
+ if (submitRequest.buildResult() != null
+ && submitRequest.deployMode() == FlinkDeployMode.KUBERNETES_NATIVE_APPLICATION) {
+ DockerImageBuildResponse buildResult =
+ (DockerImageBuildResponse) submitRequest.buildResult();
+ Map<String, String> podTemplatePaths = buildResult.podTemplatePaths();
+ if (podTemplatePaths != null) {
+ for (Map.Entry<String, String> entry : podTemplatePaths.entrySet()) {
+ if (PodTemplateTool.KUBERNETES_POD_TEMPLATE.key().equals(entry.getKey())) {
+ FlinkClientTrait.safeSet(
+ flinkConfig,
+ KubernetesConfigOptions.KUBERNETES_POD_TEMPLATE,
+ entry.getValue());
+ } else if (PodTemplateTool.KUBERNETES_JM_POD_TEMPLATE
+ .key()
+ .equals(entry.getKey())) {
+ FlinkClientTrait.safeSet(
+ flinkConfig,
+ KubernetesConfigOptions.JOB_MANAGER_POD_TEMPLATE,
+ entry.getValue());
+ } else if (PodTemplateTool.KUBERNETES_TM_POD_TEMPLATE
+ .key()
+ .equals(entry.getKey())) {
+ FlinkClientTrait.safeSet(
+ flinkConfig,
+ KubernetesConfigOptions.TASK_MANAGER_POD_TEMPLATE,
+ entry.getValue());
+ }
+ }
+ }
+ }
+
+ if (!flinkConfig.contains(DeploymentOptionsInternal.CONF_DIR)) {
+ FlinkClientTrait.safeSet(
+ flinkConfig,
+ DeploymentOptionsInternal.CONF_DIR,
+ submitRequest.flinkVersion().getFlinkHome() + "/conf");
+ }
+
+ if (flinkConfig.get(KubernetesConfigOptions.NAMESPACE).isEmpty()) {
+ flinkConfig.removeConfig(KubernetesConfigOptions.NAMESPACE);
+ }
+
+ logEffectiveSubmitConfiguration(flinkConfig);
+ }
+
+ protected void setK8sDeployTarget(Configuration flinkConf, FlinkDeployMode deployMode) {
+ FlinkClientTrait.safeSet(flinkConf, DeploymentOptions.TARGET, deployMode.getName());
+ }
+
+ @Override
+ public CancelResponse doCancel(CancelRequest cancelRequest, Configuration flinkConfig) throws FlinkException {
+ return executeClientAction(
+ cancelRequest,
+ flinkConfig,
+ (jobId, client) -> {
+ String resp = callAsFlinkException(() -> cancelJob(cancelRequest, jobId, client));
+ if (cancelRequest.deployMode() == FlinkDeployMode.KUBERNETES_NATIVE_APPLICATION) {
+ client.shutDownCluster();
+ }
+ return new CancelResponse(resp);
+ });
+ }
+
+ @Override
+ public SavepointResponse doTriggerSavepoint(
+ TriggerSavepointRequest savepointRequest,
+ Configuration flinkConfig) throws FlinkException {
+ return executeClientAction(
+ savepointRequest,
+ flinkConfig,
+ (jobId, clusterClient) -> toSavepointResponse(savepointRequest, jobId, clusterClient));
+ }
+
+ public <O> O executeClientAction(
+ SavepointRequestTrait request,
+ Configuration flinkConfig,
+ ClientAction<O> actFunc) throws FlinkException {
+ String hints =
+ "[flink-client] execute " + request.getClass().getSimpleName() + " for flink job failed,";
+ if (StringUtils.isBlank(request.clusterId())) {
+ throw new IllegalArgumentException(
+ hints
+ + " clusterId is null, mode="
+ + flinkConfig.get(DeploymentOptions.TARGET));
+ }
+
+ FlinkClientTrait.safeSet(
+ flinkConfig, KubernetesConfigOptions.CLUSTER_ID, request.clusterId());
+ FlinkClientTrait.safeSet(
+ flinkConfig, KubernetesConfigOptions.NAMESPACE, request.kubernetesNamespace());
+
+ KubernetesClusterDescriptor clusterDescriptor = null;
+ ClusterClient<String> client = null;
+
+ try {
+ clusterDescriptor = getK8sClusterDescriptor(flinkConfig);
+ client =
+ clusterDescriptor
+ .retrieve(
+ flinkConfig
+ .getOptional(KubernetesConfigOptions.CLUSTER_ID)
+ .orElseThrow(
+ () -> new IllegalStateException(
+ "Kubernetes cluster id is not configured")))
+ .getClusterClient();
+ return actFunc.apply(JobID.fromHexString(request.jobId()), client);
+ } catch (FlinkException e) {
+ logClientActionFailure(hints, flinkConfig, request, e);
+ throw e;
+ } catch (Exception e) {
+ logClientActionFailure(hints, flinkConfig, request, e);
+ throw asFlinkException(e);
+ } finally {
+ if (client != null) {
+ client.close();
+ }
+ if (clusterDescriptor != null) {
+ clusterDescriptor.close();
+ }
+ }
+ }
+
+ private void logClientActionFailure(
+ String hints,
+ Configuration flinkConfig,
+ SavepointRequestTrait request,
+ Exception e) {
+ logError(
+ hints
+ + " mode="
+ + flinkConfig.get(DeploymentOptions.TARGET)
+ + ", request="
+ + request,
+ e);
+ }
+
+ public Tuple2<KubernetesClusterDescriptor, ClusterSpecification> getK8sClusterDescriptorAndSpecification(Configuration flinkConfig) {
+ KubernetesClusterClientFactory clientFactory = new KubernetesClusterClientFactory();
+ KubernetesClusterDescriptor clusterDescriptor =
+ clientFactory.createClusterDescriptor(flinkConfig);
+ ClusterSpecification clusterSpecification =
+ clientFactory.getClusterSpecification(flinkConfig);
+ return new Tuple2<>(clusterDescriptor, clusterSpecification);
+ }
+
+ public KubernetesClusterDescriptor getK8sClusterDescriptor(Configuration flinkConfig) {
+ KubernetesClusterClientFactory clientFactory = new KubernetesClusterClientFactory();
+ return clientFactory.createClusterDescriptor(flinkConfig);
+ }
+
+ protected String flinkConfIdentifierInfo(@Nonnull Configuration conf) {
+ return "deployMode="
+ + conf.get(DeploymentOptions.TARGET)
+ + ", clusterId="
+ + conf.get(KubernetesConfigOptions.CLUSTER_ID)
+ + ", "
+ + "namespace="
+ + conf.get(KubernetesConfigOptions.NAMESPACE);
+ }
+
+ public String getDefaultKubernetesConf(String k8sConf) {
+ String homePath = System.getProperty("user.home");
+ if (k8sConf != null) {
+ return k8sConf.replace("~", homePath);
+ }
+ return homePath.concat("/.kube/config");
+ }
+
+ private ServiceExposedType covertToServiceExposedType(FlinkK8sRestExposedType exposedType) {
+ if (exposedType == FlinkK8sRestExposedType.CLUSTER_IP) {
+ return ServiceExposedType.ClusterIP;
+ }
+ if (exposedType == FlinkK8sRestExposedType.NODE_PORT) {
+ return ServiceExposedType.NodePort;
+ }
+ if (exposedType == FlinkK8sRestExposedType.LOAD_BALANCER) {
+ return ServiceExposedType.LoadBalancer;
+ }
+ return ServiceExposedType.LoadBalancer;
+ }
+
+ @FunctionalInterface
+ protected interface ClientAction<O> {
+
+ O apply(JobID jobId, ClusterClient<?> clusterClient) throws FlinkException;
+ }
+}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/trait/YarnClientTrait.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/trait/YarnClientTrait.java
new file mode 100644
index 000000000..09e152a13
--- /dev/null
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/trait/YarnClientTrait.java
@@ -0,0 +1,271 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.client.trait;
+
+import org.apache.streampark.common.util.ExceptionUtils;
+import org.apache.streampark.common.util.HadoopUtils;
+import org.apache.streampark.flink.client.bean.CancelRequest;
+import org.apache.streampark.flink.client.bean.CancelResponse;
+import org.apache.streampark.flink.client.bean.SavepointRequestTrait;
+import org.apache.streampark.flink.client.bean.SavepointResponse;
+import org.apache.streampark.flink.client.bean.SubmitRequest;
+import org.apache.streampark.flink.client.bean.TriggerSavepointRequest;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.client.deployment.ClusterSpecification;
+import org.apache.flink.client.program.ClusterClient;
+import org.apache.flink.client.program.ClusterClientProvider;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.util.FlinkException;
+import org.apache.flink.yarn.YarnClusterClientFactory;
+import org.apache.flink.yarn.YarnClusterDescriptor;
+import org.apache.flink.yarn.configuration.YarnConfigOptions;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+
+import java.lang.reflect.Method;
+import java.security.PrivilegedAction;
+
+import scala.Tuple2;
+
+/** Yarn application mode submit support. */
+public abstract class YarnClientTrait extends FlinkClientTrait {
+
+ private volatile Method deployInternalMethod;
+
+ @Override
+ public void setConfig(SubmitRequest submitRequest, Configuration flinkConfig) {
+ FlinkClientTrait.safeSet(
+ flinkConfig, YarnConfigOptions.APPLICATION_NAME, submitRequest.effectiveAppName());
+ FlinkClientTrait.safeSet(
+ flinkConfig,
+ YarnConfigOptions.APPLICATION_TYPE,
+ submitRequest.applicationType().getName());
+ FlinkClientTrait.safeSet(flinkConfig, YarnConfigOptions.APPLICATION_TAGS, "streampark");
+ }
+
+ @Override
+ public SavepointResponse doTriggerSavepoint(
+ TriggerSavepointRequest savepointRequest,
+ Configuration flinkConf) throws FlinkException {
+ return executeClientAction(
+ savepointRequest,
+ flinkConf,
+ (jobId, client) -> toSavepointResponse(savepointRequest, jobId, client));
+ }
+
+ @Override
+ public CancelResponse doCancel(CancelRequest cancelRequest, Configuration flinkConf) throws FlinkException {
+ return executeClientAction(
+ cancelRequest,
+ flinkConf,
+ (jobId, client) -> toCancelResponse(cancelRequest, jobId, client));
+ }
+
+ public ClusterClientProvider<ApplicationId> deployInternal(
+ YarnClusterDescriptor clusterDescriptor,
+ ClusterSpecification clusterSpecification,
+ String applicationName,
+ String yarnClusterEntrypoint,
+ JobGraph jobGraph,
+ Boolean detached) throws ReflectiveOperationException {
+ Method method = getDeployInternalMethod();
+ return (ClusterClientProvider<ApplicationId>) method.invoke(
+ clusterDescriptor,
+ clusterSpecification,
+ applicationName,
+ yarnClusterEntrypoint,
+ jobGraph,
+ detached);
+ }
+
+ public Tuple2<ApplicationId, YarnClusterDescriptor> getYarnClusterDescriptor(
+ Configuration flinkConfig,
+ String user) throws FlinkException {
+ return accessYarnClusterDescriptor(
+ user, () -> createYarnClusterDescriptor(flinkConfig));
+ }
+
+ public Tuple2<ApplicationId, YarnClusterDescriptor> getYarnClusterDescriptor(
+ Configuration flinkConfig) throws FlinkException {
+ return getYarnClusterDescriptor(flinkConfig, "");
+ }
+
+ public Tuple2<ClusterSpecification, YarnClusterDescriptor> getYarnClusterDeployDescriptor(
+ Configuration flinkConfig,
+ String user) throws FlinkException {
+ return accessYarnClusterDescriptor(
+ user, () -> createYarnClusterDeployDescriptor(flinkConfig));
+ }
+
+ public Tuple2<ClusterSpecification, YarnClusterDescriptor> getYarnClusterDeployDescriptor(
+ Configuration flinkConfig) throws FlinkException {
+ return getYarnClusterDeployDescriptor(flinkConfig, "");
+ }
+
+ protected void logClusterSpecification(ClusterSpecification clusterSpecification) {
+ logInfo(
+ String.format(
+ "%n------------------------<<specification>>-------------------------%n"
+ + "%s%n"
+ + "------------------------------------------------------------------%n",
+ clusterSpecification));
+ }
+
+ protected void logYarnJobStarted(Object applicationId) {
+ logInfo(
+ String.format(
+ "%n-------------------------<<applicationId>>------------------------%n"
+ + "Flink Job Started: applicationId: %s%n"
+ + "__________________________________________________________________%n",
+ applicationId));
+ }
+
+ private Tuple2<ApplicationId, YarnClusterDescriptor> createYarnClusterDescriptor(
+ Configuration flinkConfig) throws FlinkException {
+ return callAsFlinkException(
+ () -> {
+ YarnClusterClientFactory clientFactory = new YarnClusterClientFactory();
+ ApplicationId yarnClusterId = clientFactory.getClusterId(flinkConfig);
+ if (yarnClusterId == null) {
+ throw new FlinkException("Yarn cluster id is null");
+ }
+ YarnClusterDescriptor clusterDescriptor =
+ clientFactory.createClusterDescriptor(flinkConfig);
+ return new Tuple2<>(yarnClusterId, clusterDescriptor);
+ });
+ }
+
+ private Tuple2<ClusterSpecification, YarnClusterDescriptor> createYarnClusterDeployDescriptor(
+ Configuration flinkConfig) throws FlinkException {
+ return callAsFlinkException(
+ () -> {
+ YarnClusterClientFactory clientFactory = new YarnClusterClientFactory();
+ ClusterSpecification clusterSpecification =
+ clientFactory.getClusterSpecification(flinkConfig);
+ YarnClusterDescriptor clusterDescriptor =
+ clientFactory.createClusterDescriptor(flinkConfig);
+ return new Tuple2<>(clusterSpecification, clusterDescriptor);
+ });
+ }
+
+ private <O> O executeClientAction(
+ SavepointRequestTrait request, Configuration flinkConf,
+ ClientAction<O> actionFunc) throws FlinkException {
+ return callAsFlinkException(
+ () -> {
+ JobID jobID = getJobID(request.jobId());
+ FlinkClientTrait.safeSet(flinkConf, YarnConfigOptions.APPLICATION_ID, request.clusterId());
+ Tuple2<ApplicationId, YarnClusterDescriptor> descriptor =
+ getYarnClusterDescriptor(flinkConf);
+ ClusterClient<?> clusterClient =
+ descriptor._2().retrieve(descriptor._1()).getClusterClient();
+ return applyClientAction(request, actionFunc, jobID, clusterClient);
+ });
+ }
+
+ private <O> O applyClientAction(
+ SavepointRequestTrait request,
+ ClientAction<O> actionFunc,
+ JobID jobID,
+ ClusterClient<?> clusterClient) throws FlinkException {
+ return callAsFlinkExceptionMapping(
+ () -> actionFunc.apply(jobID, clusterClient),
+ e -> new FlinkException(
+ "[StreamPark] Do "
+ + request.getClass().getSimpleName()
+ + " for the job "
+ + request.jobId()
+ + " failed. "
+ + "detail: "
+ + ExceptionUtils.stringifyException(e),
+ e));
+ }
+
+ private <T> T accessYarnClusterDescriptor(String user, YarnDescriptorSupplier<T> func) throws FlinkException {
+ try {
+ return doAsYarnClusterDescriptor(user, func);
+ } catch (FlinkException e) {
+ throw new FlinkException("[StreamPark] access ClusterDescriptor error: " + e.getMessage(), e);
+ }
+ }
+
+ private Method getDeployInternalMethod() throws NoSuchMethodException {
+ if (deployInternalMethod == null) {
+ synchronized (this) {
+ if (deployInternalMethod == null) {
+ Class<?>[] paramClass =
+ new Class<?>[]{
+ ClusterSpecification.class,
+ String.class,
+ String.class,
+ JobGraph.class,
+ boolean.class
+ };
+ Method deployInternal =
+ YarnClusterDescriptor.class.getDeclaredMethod(
+ "deployInternal", paramClass);
+ deployInternal.setAccessible(true);
+ deployInternalMethod = deployInternal;
+ }
+ }
+ }
+ return deployInternalMethod;
+ }
+
+ private <T> T doAsYarnClusterDescriptor(String user, YarnDescriptorSupplier<T> func) throws FlinkException {
+ UserGroupInformation ugi = HadoopUtils.getUgi();
+ UserGroupInformation finalUgi =
+ user != null
+ && !user.isEmpty()
+ && !ugi.getShortUserName().equals(user)
+ ? UserGroupInformation.createProxyUser(user, ugi)
+ : ugi;
+
+ try {
+ return finalUgi.doAs(
+ (PrivilegedAction<T>) () -> {
+ try {
+ return func.get();
+ } catch (FlinkException e) {
+ throw new IllegalStateException(e);
+ } catch (Exception e) {
+ throw new IllegalStateException("Yarn cluster descriptor operation failed", e);
+ }
+ });
+ } catch (IllegalStateException e) {
+ Throwable cause = e.getCause() != null ? e.getCause() : e;
+ throw new FlinkException(
+ "[StreamPark] Error executing YarnClusterDescriptor operation as user " + user,
+ cause);
+ }
+ }
+
+ @FunctionalInterface
+ private interface ClientAction<O> {
+
+ O apply(JobID jobId, ClusterClient<?> client) throws FlinkException;
+ }
+
+ @FunctionalInterface
+ private interface YarnDescriptorSupplier<T> {
+
+ T get() throws FlinkException;
+ }
+}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/FlinkClientEntrypoint.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/FlinkClientEntrypoint.scala
deleted file mode 100644
index cbd442087..000000000
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/FlinkClientEntrypoint.scala
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.client
-
-import org.apache.streampark.common.enums.FlinkDeployMode
-import org.apache.streampark.flink.client.`trait`.FlinkClientTrait
-import org.apache.streampark.flink.client.bean._
-import org.apache.streampark.flink.client.impl._
-
-import FlinkDeployMode._
-
-object FlinkClientEntrypoint {
-
- private[this] val clients: Map[FlinkDeployMode, FlinkClientTrait] = Map(
- LOCAL -> LocalClient,
- REMOTE -> RemoteClient,
- YARN_APPLICATION -> YarnApplicationClient,
- YARN_SESSION -> YarnSessionClient,
- YARN_PER_JOB -> YarnPerJobClient,
- KUBERNETES_NATIVE_SESSION -> KubernetesNativeSessionClient,
- KUBERNETES_NATIVE_APPLICATION -> KubernetesNativeApplicationClient)
-
- def submit(submitRequest: SubmitRequest): SubmitResponse = {
- clients.get(submitRequest.deployMode) match {
- case Some(client) => client.submit(submitRequest)
- case _ =>
- throw new UnsupportedOperationException(
- s"Unsupported ${submitRequest.deployMode} submit ")
- }
- }
-
- def cancel(cancelRequest: CancelRequest): CancelResponse = {
- clients.get(cancelRequest.deployMode) match {
- case Some(client) => client.cancel(cancelRequest)
- case _ =>
- throw new UnsupportedOperationException(
- s"Unsupported ${cancelRequest.deployMode} cancel ")
- }
- }
-
- def triggerSavepoint(savepointRequest: TriggerSavepointRequest): SavepointResponse = {
- clients.get(savepointRequest.deployMode) match {
- case Some(client) => client.triggerSavepoint(savepointRequest)
- case _ =>
- throw new UnsupportedOperationException(
- s"Unsupported ${savepointRequest.deployMode} triggerSavepoint ")
- }
- }
-
- def deploy(deployRequest: DeployRequest): DeployResponse = {
- deployRequest.deployMode match {
- case YARN_SESSION => YarnSessionClient.deploy(deployRequest)
- case KUBERNETES_NATIVE_SESSION =>
- KubernetesNativeSessionClient.deploy(deployRequest)
- case _ =>
- throw new UnsupportedOperationException(
- s"Unsupported ${deployRequest.deployMode} deploy cluster ")
- }
- }
-
- def shutdown(shutDownRequest: ShutDownRequest): ShutDownResponse = {
- shutDownRequest.deployMode match {
- case YARN_SESSION => YarnSessionClient.shutdown(shutDownRequest)
- case KUBERNETES_NATIVE_SESSION =>
- KubernetesNativeSessionClient.shutdown(shutDownRequest)
- case _ =>
- throw new UnsupportedOperationException(
- s"Unsupported ${shutDownRequest.deployMode} shutdown cluster ")
- }
- }
-
-}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/KubernetesNativeApplicationClient.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/KubernetesNativeApplicationClient.scala
deleted file mode 100644
index 1791e5d82..000000000
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/KubernetesNativeApplicationClient.scala
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.client.impl
-
-import org.apache.streampark.common.enums.FlinkDeployMode
-import org.apache.streampark.flink.client.`trait`.KubernetesNativeClientTrait
-import org.apache.streampark.flink.client.bean._
-import org.apache.streampark.flink.packer.pipeline.DockerImageBuildResponse
-
-import com.google.common.collect.Lists
-import org.apache.commons.lang3.StringUtils
-import org.apache.flink.client.deployment.application.ApplicationConfiguration
-import org.apache.flink.configuration.{Configuration, DeploymentOptions, PipelineOptions}
-import org.apache.flink.kubernetes.configuration.KubernetesConfigOptions
-
-/**
- * Kubernetes native application mode submit.
- */
-object KubernetesNativeApplicationClient extends KubernetesNativeClientTrait {
-
- @throws[Exception]
- override def doSubmit(
- submitRequest: SubmitRequest,
- flinkConfig: Configuration): SubmitResponse = {
-
- // require parameters
- require(
- StringUtils.isNotBlank(submitRequest.clusterId),
- s"[flink-submit] submit flink job failed, clusterId is null, mode=${flinkConfig
- .get(DeploymentOptions.TARGET)}")
-
- // check the last building result
- submitRequest.checkBuildResult()
-
- val buildResult =
- submitRequest.buildResult.asInstanceOf[DockerImageBuildResponse]
-
- // add flink pipeline.jars configuration
- flinkConfig.safeSet(
- PipelineOptions.JARS,
- Lists.newArrayList(buildResult.dockerInnerMainJarPath))
-
- // add flink container image tag to flink configuration
- flinkConfig.safeSet(KubernetesConfigOptions.CONTAINER_IMAGE, buildResult.flinkImageTag)
-
- // retrieve k8s cluster and submit flink job on application mode
- val (descriptor, clusterSpecification) =
- getK8sClusterDescriptorAndSpecification(flinkConfig)
- val clusterDescriptor = descriptor
- val applicationConfig =
- ApplicationConfiguration.fromConfiguration(flinkConfig)
- val clusterClient = clusterDescriptor
- .deployApplicationCluster(clusterSpecification, applicationConfig)
- .getClusterClient
-
- val clusterId = clusterClient.getClusterId
- val result = SubmitResponse(
- clusterId,
- flinkConfig.toMap,
- submitRequest.jobId,
- clusterClient.getWebInterfaceURL)
- logInfo(s"[flink-submit] flink job has been submitted. ${flinkConfIdentifierInfo(flinkConfig)}")
-
- closeSubmit(submitRequest, clusterDescriptor, clusterClient)
- result
- }
-
- override def doCancel(cancelRequest: CancelRequest, flinkConf: Configuration): CancelResponse = {
- flinkConf.safeSet(
- DeploymentOptions.TARGET,
- FlinkDeployMode.KUBERNETES_NATIVE_APPLICATION.getName)
- executeClientAction(
- cancelRequest,
- flinkConf,
- (jobId, client) => {
- val resp = super.cancelJob(cancelRequest, jobId, client)
- client.shutDownCluster()
- CancelResponse(resp)
- })
- }
-
- override def doTriggerSavepoint(
- request: TriggerSavepointRequest,
- flinkConf: Configuration): SavepointResponse = {
- flinkConf.safeSet(
- DeploymentOptions.TARGET,
- FlinkDeployMode.KUBERNETES_NATIVE_APPLICATION.getName)
- super.doTriggerSavepoint(request, flinkConf)
- }
-}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/KubernetesNativeSessionClient.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/KubernetesNativeSessionClient.scala
deleted file mode 100644
index 8550d32bd..000000000
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/KubernetesNativeSessionClient.scala
+++ /dev/null
@@ -1,216 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.client.impl
-
-import org.apache.streampark.common.enums.FlinkDeployMode
-import org.apache.streampark.common.util.{Logger, Utils}
-import org.apache.streampark.common.util.Implicits._
-import org.apache.streampark.flink.client.`trait`.KubernetesNativeClientTrait
-import org.apache.streampark.flink.client.bean._
-import org.apache.streampark.flink.client.tool.FlinkSessionSubmitHelper
-import org.apache.streampark.flink.core.FlinkKubernetesClient
-import org.apache.streampark.flink.kubernetes.KubernetesRetriever
-import org.apache.streampark.flink.kubernetes.enums.FlinkK8sDeployMode
-import org.apache.streampark.flink.kubernetes.model.ClusterKey
-
-import org.apache.commons.lang3.StringUtils
-import org.apache.flink.client.program.ClusterClient
-import org.apache.flink.configuration._
-import org.apache.flink.kubernetes.KubernetesClusterDescriptor
-import org.apache.flink.kubernetes.configuration.{KubernetesConfigOptions, KubernetesDeploymentTarget}
-import org.apache.flink.kubernetes.configuration.KubernetesConfigOptions.ServiceExposedType
-import org.apache.flink.kubernetes.kubeclient.{FlinkKubeClient, FlinkKubeClientFactory}
-
-import java.io.File
-
-/** Kubernetes native session mode submit. */
-object KubernetesNativeSessionClient extends KubernetesNativeClientTrait with Logger {
-
- @throws[Exception]
- override def doSubmit(
- submitRequest: SubmitRequest,
- flinkConfig: Configuration): SubmitResponse = {
- // require parameters
- require(
- StringUtils.isNotBlank(submitRequest.clusterId),
- s"[flink-submit] submit flink job failed, clusterId is null, mode=${flinkConfig
- .get(DeploymentOptions.TARGET)}")
- super.trySubmit(submitRequest, flinkConfig, submitRequest.userJarFile)(
- jobGraphSubmit,
- restApiSubmit)
- }
-
- /** Submit flink session job via rest api. */
- @throws[Exception]
- def restApiSubmit(
- submitRequest: SubmitRequest,
- flinkConfig: Configuration,
- fatJar: File): SubmitResponse = {
-
- // get jm rest url of flink session cluster
- val clusterKey = ClusterKey.builder()
- .executeMode(FlinkK8sDeployMode.SESSION)
- .namespace(submitRequest.kubernetesNamespace)
- .clusterId(submitRequest.clusterId)
- .build()
- val jmRestUrl = KubernetesRetriever
- .retrieveFlinkRestUrl(clusterKey)
- .orElseThrow(() =>
- new Exception(
- s"[flink-submit] retrieve flink session rest url failed, clusterKey=$clusterKey"))
- // submit job via rest api
- val jobId =
- FlinkSessionSubmitHelper.submitViaRestApi(jmRestUrl, fatJar, flinkConfig)
- SubmitResponse(clusterKey.clusterId(), flinkConfig.toMap, jobId, jmRestUrl)
- }
-
- /** Submit flink session job with building JobGraph via ClusterClient api. */
- @throws[Exception]
- def jobGraphSubmit(
- submitRequest: SubmitRequest,
- flinkConfig: Configuration,
- jarFile: File): SubmitResponse = {
- val clusterDescriptor = getK8sClusterDescriptor(flinkConfig)
- // build JobGraph
- val packageProgramJobGraph =
- super.getJobGraph(flinkConfig, submitRequest, jarFile)
- val packageProgram = packageProgramJobGraph._1
- val jobGraph = packageProgramJobGraph._2
- // retrieve client and submit JobGraph
- val client = clusterDescriptor
- .retrieve(flinkConfig.getString(KubernetesConfigOptions.CLUSTER_ID))
- .getClusterClient
- val submitResult = client.submitJob(jobGraph)
- val jobId = submitResult.get().toString
- val result =
- SubmitResponse(client.getClusterId, flinkConfig.toMap, jobId, client.getWebInterfaceURL)
- logInfo(
- s"[flink-submit] flink job has been submitted. ${flinkConfIdentifierInfo(flinkConfig)}, jobId: $jobId")
- closeSubmit(submitRequest, packageProgram, client, client)
- result
- }
-
- override def doCancel(
- cancelRequest: CancelRequest,
- flinkConfig: Configuration): CancelResponse = {
- flinkConfig.safeSet(
- DeploymentOptions.TARGET,
- FlinkDeployMode.KUBERNETES_NATIVE_SESSION.getName)
- super.doCancel(cancelRequest, flinkConfig)
- }
-
- @throws[Exception]
- def deploy(deployRequest: DeployRequest): DeployResponse = {
- logInfo(
- s"""
- |--------------------------------------- kubernetes cluster start ---------------------------------------
- | userFlinkHome : ${deployRequest.flinkVersion.flinkHome}
- | flinkVersion : ${deployRequest.flinkVersion.version}
- | deployMode : ${deployRequest.deployMode.name()}
- | clusterId : ${deployRequest.clusterId}
- | namespace : ${deployRequest.k8sParam.kubernetesNamespace}
- | exposedType : ${deployRequest.k8sParam.flinkRestExposedType}
- | serviceAccount : ${deployRequest.k8sParam.serviceAccount}
- | flinkImage : ${deployRequest.k8sParam.flinkImage}
- | properties : ${deployRequest.properties.mkString(",")}
- |--------------------------------------------------------------------------------------------------------
- |""".stripMargin)
-
- val flinkConfig = getFlinkK8sConfig(deployRequest)
- val kubeClient = FlinkKubeClientFactory.getInstance.fromConfiguration(flinkConfig, "client")
-
- var clusterDescriptor: KubernetesClusterDescriptor = null
- var client: ClusterClient[String] = null
-
- try {
- val kubernetesClusterDescriptor = getK8sClusterDescriptorAndSpecification(flinkConfig)
- clusterDescriptor = kubernetesClusterDescriptor._1
-
- val kubeClientWrapper = new FlinkKubernetesClient(kubeClient)
- val kubeService = kubeClientWrapper.getService(deployRequest.clusterId)
-
- if (kubeService.isPresent) {
- client = clusterDescriptor.retrieve(deployRequest.clusterId).getClusterClient
- } else {
- client = clusterDescriptor
- .deploySessionCluster(kubernetesClusterDescriptor._2)
- .getClusterClient
- }
- DeployResponse(address = client.getWebInterfaceURL, clusterId = client.getClusterId)
- } catch {
- case e: Exception => DeployResponse(error = e)
- } finally {
- Utils.close(client, clusterDescriptor, kubeClient)
- }
- }
-
- def shutdown(shutDownRequest: ShutDownRequest): ShutDownResponse = {
- var kubeClient: FlinkKubeClient = null
- try {
- val flinkConfig = getFlinkK8sConfig(shutDownRequest)
- kubeClient = FlinkKubeClientFactory.getInstance.fromConfiguration(flinkConfig, "client")
- val kubeClientWrapper = new FlinkKubernetesClient(kubeClient)
-
- val stopAndCleanupState =
- shutDownRequest.clusterId != null && kubeClientWrapper
- .getService(shutDownRequest.clusterId)
- .isPresent
- if (stopAndCleanupState) {
- kubeClient.stopAndCleanupCluster(shutDownRequest.clusterId)
- ShutDownResponse(shutDownRequest.clusterId)
- } else {
- null
- }
- } catch {
- case e: Exception =>
- logError(s"shutdown flink session fail in ${shutDownRequest.deployMode} mode")
- e.printStackTrace()
- throw e
- } finally {
- Utils.close(kubeClient)
- }
- }
-
- override def doTriggerSavepoint(
- triggerSavepointRequest: TriggerSavepointRequest,
- flinkConfig: Configuration): SavepointResponse = {
- flinkConfig.safeSet(
- DeploymentOptions.TARGET,
- FlinkDeployMode.KUBERNETES_NATIVE_SESSION.getName)
- super.doTriggerSavepoint(triggerSavepointRequest, flinkConfig)
- }
-
- private[this] def getFlinkK8sConfig[R <: DeployRequestTrait](deployRequest: R): Configuration = {
- extractConfiguration(deployRequest.flinkVersion.flinkHome, deployRequest.properties)
- .safeSet(DeploymentOptions.TARGET, KubernetesDeploymentTarget.SESSION.getName)
- .safeSet(KubernetesConfigOptions.NAMESPACE, deployRequest.k8sParam.kubernetesNamespace)
- .safeSet(
- KubernetesConfigOptions.KUBERNETES_SERVICE_ACCOUNT,
- deployRequest.k8sParam.serviceAccount)
- .safeSet(KubernetesConfigOptions.CLUSTER_ID, deployRequest.clusterId)
- .safeSet(KubernetesConfigOptions.CONTAINER_IMAGE, deployRequest.k8sParam.flinkImage)
- .safeSet(
- KubernetesConfigOptions.REST_SERVICE_EXPOSED_TYPE,
- ServiceExposedType.valueOf(deployRequest.k8sParam.flinkRestExposedType.getName))
- .safeSet(
- KubernetesConfigOptions.KUBE_CONFIG_FILE,
- getDefaultKubernetesConf(deployRequest.k8sParam.kubeConf))
- .safeSet(DeploymentOptionsInternal.CONF_DIR, s"${deployRequest.flinkVersion.flinkHome}/conf")
- }
-
-}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/LocalClient.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/LocalClient.scala
deleted file mode 100644
index 3d18c9ab1..000000000
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/LocalClient.scala
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.client.impl
-
-import org.apache.streampark.flink.client.`trait`.FlinkClientTrait
-import org.apache.streampark.flink.client.bean._
-
-import org.apache.flink.client.deployment.executors.RemoteExecutor
-import org.apache.flink.client.program.MiniClusterClient
-import org.apache.flink.configuration._
-import org.apache.flink.runtime.minicluster.{MiniCluster, MiniClusterConfiguration}
-
-import java.lang.{Integer => JavaInt}
-
-object LocalClient extends FlinkClientTrait {
-
- override def setConfig(submitRequest: SubmitRequest, flinkConfig: Configuration): Unit = {
- flinkConfig.safeSet(PipelineOptions.NAME, submitRequest.effectiveAppName)
- logInfo(s"""
- |------------------------------------------------------------------
- |Effective submit configuration: $flinkConfig
- |------------------------------------------------------------------
- |""".stripMargin)
- }
-
- override def doSubmit(
- submitRequest: SubmitRequest,
- flinkConfig: Configuration): SubmitResponse = {
-
- // build JobGraph
- val programJobGraph =
- super.getJobGraph(flinkConfig, submitRequest, submitRequest.userJarFile)
- val packageProgram = programJobGraph._1
- val jobGraph = programJobGraph._2
- val client = createLocalCluster(flinkConfig)
- val jobId = client.submitJob(jobGraph).get().toString
- val resp =
- SubmitResponse(jobId, flinkConfig.toMap, jobId, client.getWebInterfaceURL)
- closeSubmit(submitRequest, packageProgram, client)
- resp
- }
-
- override def doTriggerSavepoint(
- savepointRequest: TriggerSavepointRequest,
- flinkConfig: Configuration): SavepointResponse = {
- RemoteClient.doTriggerSavepoint(savepointRequest, flinkConfig)
- }
-
- override def doCancel(
- cancelRequest: CancelRequest,
- flinkConfig: Configuration): CancelResponse = {
- RemoteClient.doCancel(cancelRequest, flinkConfig)
- }
-
- private[this] def createLocalCluster(flinkConfig: Configuration) = {
-
- flinkConfig.safeSet[JavaInt](JobManagerOptions.PORT, 0)
-
- val cluster = {
- val numTaskManagers = flinkConfig.getInteger(
- ConfigConstants.LOCAL_NUMBER_TASK_MANAGER,
- ConfigConstants.DEFAULT_LOCAL_NUMBER_TASK_MANAGER)
-
- val numSlotsPerTaskManager =
- flinkConfig.getInteger(TaskManagerOptions.NUM_TASK_SLOTS)
-
- val miniClusterConfig = new MiniClusterConfiguration.Builder()
- .setConfiguration(flinkConfig)
- .setNumSlotsPerTaskManager(numSlotsPerTaskManager)
- .setNumTaskManagers(numTaskManagers)
- .build()
-
- val cluster = new MiniCluster(miniClusterConfig)
- cluster.start()
- cluster
- }
-
- val host = "localhost"
- val port = cluster.getRestAddress.get.getPort
-
- flinkConfig
- .safeSet(JobManagerOptions.ADDRESS, host)
- .safeSet[JavaInt](JobManagerOptions.PORT, port)
- .safeSet(RestOptions.ADDRESS, host)
- .safeSet[JavaInt](RestOptions.PORT, port)
- .safeSet(DeploymentOptions.TARGET, RemoteExecutor.NAME)
-
- logInfo(s"\nStarting local Flink cluster (host: localhost, port: $port).\n")
-
- new MiniClusterClient(flinkConfig, cluster)
- }
-
-}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/RemoteClient.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/RemoteClient.scala
deleted file mode 100644
index e8c421183..000000000
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/RemoteClient.scala
+++ /dev/null
@@ -1,171 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.client.impl
-
-import org.apache.streampark.flink.client.`trait`.FlinkClientTrait
-import org.apache.streampark.flink.client.bean.{CancelRequest, CancelResponse, SavepointRequestTrait, SavepointResponse, SubmitRequest, SubmitResponse, TriggerSavepointRequest}
-import org.apache.streampark.flink.client.tool.FlinkSessionSubmitHelper
-
-import org.apache.flink.api.common.JobID
-import org.apache.flink.client.deployment.{DefaultClusterClientServiceLoader, StandaloneClusterDescriptor, StandaloneClusterId}
-import org.apache.flink.client.program.ClusterClient
-import org.apache.flink.configuration._
-
-import java.io.File
-import java.lang.{Integer => JavaInt}
-
-/** Submit Job to Remote Cluster */
-object RemoteClient extends FlinkClientTrait {
-
- override def setConfig(submitRequest: SubmitRequest, flinkConfig: Configuration): Unit = {}
-
- override def doSubmit(
- submitRequest: SubmitRequest,
- flinkConfig: Configuration): SubmitResponse = {
-
- // 2) submit job
- super.trySubmit(submitRequest, flinkConfig, submitRequest.userJarFile)(
- jobGraphSubmit,
- restApiSubmit)
- }
-
- override def doCancel(
- cancelRequest: CancelRequest,
- flinkConfig: Configuration): CancelResponse = {
- executeClientAction(
- cancelRequest,
- flinkConfig,
- (jobID, clusterClient) => {
- CancelResponse(super.cancelJob(cancelRequest, jobID, clusterClient))
- })
- }
-
- private[this] def executeClientAction[O, R <: SavepointRequestTrait](
- request: R,
- flinkConfig: Configuration,
- actFunc: (JobID, ClusterClient[_]) => O): O = {
- var client: ClusterClient[StandaloneClusterId] = null
- var standAloneDescriptor: (StandaloneClusterId, StandaloneClusterDescriptor) = null
- try {
- flinkConfig
- .safeSet(DeploymentOptions.TARGET, request.deployMode.getName)
- .safeSet(RestOptions.ADDRESS, request.properties.get(RestOptions.ADDRESS.key()).toString)
- .safeSet[JavaInt](
- RestOptions.PORT,
- request.properties.get(RestOptions.PORT.key()).toString.toInt)
- logInfo(s"""
- |------------------------------------------------------------------
- |Effective submit configuration: $flinkConfig
- |------------------------------------------------------------------
- |""".stripMargin)
- standAloneDescriptor = getStandAloneClusterDescriptor(flinkConfig)
- client = standAloneDescriptor._2
- .retrieve(standAloneDescriptor._1)
- .getClusterClient
- actFunc(JobID.fromHexString(request.jobId), client)
- } catch {
- case e: Exception =>
- logError(s"Do ${request.getClass.getSimpleName} for flink standalone job fail")
- e.printStackTrace()
- throw e
- } finally {
- if (client != null) client.close()
- if (standAloneDescriptor != null) standAloneDescriptor._2.close()
- }
- }
-
- override def doTriggerSavepoint(
- savepointRequest: TriggerSavepointRequest,
- flinkConfig: Configuration): SavepointResponse = {
- executeClientAction(
- savepointRequest,
- flinkConfig,
- (jobID, clusterClient) => {
- SavepointResponse(super.triggerSavepoint(savepointRequest, jobID, clusterClient))
- })
- }
-
- /** Submit flink session job via rest api. */
- // noinspection DuplicatedCode
- @throws[Exception]
- def restApiSubmit(
- submitRequest: SubmitRequest,
- flinkConfig: Configuration,
- fatJar: File): SubmitResponse = {
- // retrieve standalone session cluster and submit flink job on session mode
- var clusterDescriptor: StandaloneClusterDescriptor = null;
- var client: ClusterClient[StandaloneClusterId] = null
- val standAloneDescriptor = getStandAloneClusterDescriptor(flinkConfig)
- val yarnClusterId: StandaloneClusterId = standAloneDescriptor._1
- clusterDescriptor = standAloneDescriptor._2
-
- client = clusterDescriptor.retrieve(yarnClusterId).getClusterClient
- val jobId =
- FlinkSessionSubmitHelper.submitViaRestApi(client.getWebInterfaceURL, fatJar, flinkConfig)
- logInfo(
- s"${submitRequest.deployMode} mode submit by restApi, WebInterfaceURL ${client.getWebInterfaceURL}, jobId: $jobId")
- val resp =
- SubmitResponse(null, flinkConfig.toMap, jobId, client.getWebInterfaceURL)
- closeSubmit(submitRequest, client, clusterDescriptor)
- resp
- }
-
- /** Submit flink session job with building JobGraph via Standalone ClusterClient api. */
- @throws[Exception]
- def jobGraphSubmit(
- submitRequest: SubmitRequest,
- flinkConfig: Configuration,
- jarFile: File): SubmitResponse = {
-
- val standAloneDescriptor = getStandAloneClusterDescriptor(flinkConfig)
- val clusterDescriptor = standAloneDescriptor._2
- // build JobGraph
- val packageProgramJobGraph =
- super.getJobGraph(flinkConfig, submitRequest, jarFile)
- val packageProgram = packageProgramJobGraph._1
- val jobGraph = packageProgramJobGraph._2
- val client =
- clusterDescriptor.retrieve(standAloneDescriptor._1).getClusterClient
- val jobId = client.submitJob(jobGraph).get().toString
- logInfo(
- s"${submitRequest.deployMode} mode submit by jobGraph, WebInterfaceURL ${client.getWebInterfaceURL}, jobId: $jobId")
- val result =
- SubmitResponse(null, flinkConfig.toMap, jobId, client.getWebInterfaceURL)
- closeSubmit(submitRequest, packageProgram, client, clusterDescriptor)
- result
- }
-
- /**
- * create StandAloneClusterDescriptor
- *
- * @param flinkConfig
- */
- private[this] def getStandAloneClusterDescriptor(
- flinkConfig: Configuration): (StandaloneClusterId, StandaloneClusterDescriptor) = {
- val serviceLoader = new DefaultClusterClientServiceLoader
- val clientFactory = serviceLoader.getClusterClientFactory(flinkConfig)
- val standaloneClusterId: StandaloneClusterId =
- clientFactory.getClusterId(flinkConfig)
- val standaloneClusterDescriptor =
- clientFactory
- .createClusterDescriptor(flinkConfig)
- .asInstanceOf[StandaloneClusterDescriptor]
- (standaloneClusterId, standaloneClusterDescriptor)
- }
-
-}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/YarnApplicationClient.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/YarnApplicationClient.scala
deleted file mode 100644
index 966325282..000000000
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/YarnApplicationClient.scala
+++ /dev/null
@@ -1,155 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.client.impl
-
-import org.apache.streampark.common.conf.Workspace
-import org.apache.streampark.common.constants.Constants
-import org.apache.streampark.common.enums.FlinkJobType
-import org.apache.streampark.common.fs.FsOperator
-import org.apache.streampark.common.util.{AssertUtils, FileUtils, HdfsUtils}
-import org.apache.streampark.common.util.Implicits._
-import org.apache.streampark.flink.client.`trait`.YarnClientTrait
-import org.apache.streampark.flink.client.bean._
-import org.apache.streampark.flink.packer.pipeline.ShadedBuildResponse
-
-import org.apache.flink.client.deployment.application.ApplicationConfiguration
-import org.apache.flink.configuration._
-import org.apache.flink.python.PythonOptions
-import org.apache.flink.yarn.YarnClusterDescriptor
-import org.apache.flink.yarn.configuration.YarnConfigOptions
-import org.apache.hadoop.yarn.api.records.ApplicationId
-
-import java.util
-import java.util.Collections
-
-import scala.collection.mutable.ListBuffer
-
-/** yarn application mode submit */
-object YarnApplicationClient extends YarnClientTrait {
-
- private[this] lazy val workspace = Workspace.remote
-
- override def setConfig(submitRequest: SubmitRequest, flinkConfig: Configuration): Unit = {
- super.setConfig(submitRequest, flinkConfig)
- val providedLibs = {
- val array = ListBuffer(
- submitRequest.hdfsWorkspace.flinkLib,
- submitRequest.hdfsWorkspace.flinkPlugins,
- submitRequest.hdfsWorkspace.appJars)
- submitRequest.jobType match {
- case FlinkJobType.FLINK_SQL =>
- array += s"${workspace.APP_SHIMS}/flink-${submitRequest.flinkVersion.majorVersion}"
- val jobLib = s"${workspace.APP_WORKSPACE}/${submitRequest.id}/lib"
- if (HdfsUtils.exists(jobLib)) {
- array += jobLib
- }
- case _ =>
- }
- array.toList
- }
-
- flinkConfig
- // yarn.provided.lib.dirs
- .safeSet(YarnConfigOptions.PROVIDED_LIB_DIRS, providedLibs.asJava)
- // flinkDistJar
- .safeSet(YarnConfigOptions.FLINK_DIST_JAR, submitRequest.hdfsWorkspace.flinkDistJar)
- // pipeline.jars
- .safeSet(
- PipelineOptions.JARS,
- Collections.singletonList(
- submitRequest.buildResult
- .asInstanceOf[ShadedBuildResponse]
- .shadedJarPath))
- // yarn application name
- .safeSet(YarnConfigOptions.APPLICATION_NAME, submitRequest.effectiveAppName)
- // yarn application Type
- .safeSet(YarnConfigOptions.APPLICATION_TYPE, submitRequest.applicationType.getName)
-
- if (submitRequest.jobType == FlinkJobType.PYFLINK) {
- val pyVenv: String = workspace.APP_PYTHON_VENV
- AssertUtils.required(FsOperator.hdfs.exists(pyVenv), s"$pyVenv File does not exist")
-
- val localLib: String =
- s"${Workspace.local.APP_WORKSPACE}/${submitRequest.id}/lib"
- if (FileUtils.exists(localLib) && FileUtils.directoryNotBlank(localLib)) {
- flinkConfig.safeSet(PipelineOptions.JARS, util.Arrays.asList(localLib))
- }
-
- // yarn.ship-files
- val shipFiles = new util.ArrayList[String]()
- shipFiles.add(submitRequest.userJarFile.getParentFile.getAbsolutePath)
-
- flinkConfig
- .safeSet(YarnConfigOptions.SHIP_FILES, shipFiles)
- // python.files
- .safeSet(PythonOptions.PYTHON_FILES, submitRequest.userJarFile.getParentFile.getName)
- // python.archives
- .safeSet(PythonOptions.PYTHON_ARCHIVES, pyVenv)
- // python.client.executable
- .safeSet(PythonOptions.PYTHON_CLIENT_EXECUTABLE, Constants.PYTHON_EXECUTABLE)
- // python.executable
- .safeSet(PythonOptions.PYTHON_EXECUTABLE, Constants.PYTHON_EXECUTABLE)
-
- val args: util.List[String] =
- flinkConfig.get(ApplicationConfiguration.APPLICATION_ARGS)
- // Caused by: java.lang.UnsupportedOperationException
- val argsList: util.ArrayList[String] = new util.ArrayList[String](args)
- argsList.add("-pym")
- argsList.add(submitRequest.userJarFile.getName.dropRight(Constants.PYTHON_SUFFIX.length))
- flinkConfig.safeSet(ApplicationConfiguration.APPLICATION_ARGS, argsList)
- }
-
- logInfo(s"""
- |------------------------------------------------------------------
- |Effective submit configuration: $flinkConfig
- |------------------------------------------------------------------
- |""".stripMargin)
- }
-
- override def doSubmit(
- submitRequest: SubmitRequest,
- flinkConfig: Configuration): SubmitResponse = {
- val (clusterSpecification, clusterDescriptor: YarnClusterDescriptor) = getYarnClusterDeployDescriptor(flinkConfig, submitRequest.hadoopUser)
- logInfo(s"""
- |------------------------<<specification>>-------------------------
- |$clusterSpecification
- |------------------------------------------------------------------
- |""".stripMargin)
-
- val applicationConfiguration =
- ApplicationConfiguration.fromConfiguration(flinkConfig)
- var applicationId: ApplicationId = null
- var jobManagerUrl: String = null
- val clusterClient = clusterDescriptor
- .deployApplicationCluster(clusterSpecification, applicationConfiguration)
- .getClusterClient
- applicationId = clusterClient.getClusterId
- jobManagerUrl = clusterClient.getWebInterfaceURL
- logInfo(s"""
- |-------------------------<<applicationId>>------------------------
- |Flink Job Started: applicationId: $applicationId
- |__________________________________________________________________
- |""".stripMargin)
-
- val resp =
- SubmitResponse(applicationId.toString, flinkConfig.toMap, jobManagerUrl = jobManagerUrl)
- closeSubmit(submitRequest, clusterClient, clusterDescriptor)
- resp
- }
-
-}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/YarnPerJobClient.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/YarnPerJobClient.scala
deleted file mode 100644
index d19ba8787..000000000
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/YarnPerJobClient.scala
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.client.impl
-
-import org.apache.streampark.common.util.Implicits._
-import org.apache.streampark.flink.client.`trait`.YarnClientTrait
-import org.apache.streampark.flink.client.bean._
-import org.apache.streampark.flink.util.FlinkUtils
-
-import org.apache.flink.client.program.PackagedProgram
-import org.apache.flink.configuration.{Configuration, DeploymentOptions}
-import org.apache.flink.yarn.YarnClusterDescriptor
-import org.apache.flink.yarn.configuration.YarnDeploymentTarget
-import org.apache.flink.yarn.entrypoint.YarnJobClusterEntrypoint
-import org.apache.hadoop.fs.{Path => HadoopPath}
-import org.apache.hadoop.yarn.api.records.ApplicationId
-
-import java.lang.{Boolean => JavaBool}
-
-/** yarn PerJob mode submit */
-object YarnPerJobClient extends YarnClientTrait {
-
- override def setConfig(submitRequest: SubmitRequest, flinkConfig: Configuration): Unit = {
- super.setConfig(submitRequest, flinkConfig)
- // execution.target
- flinkConfig
- .safeSet(DeploymentOptions.TARGET, YarnDeploymentTarget.PER_JOB.getName)
- .safeSet(DeploymentOptions.ATTACHED, JavaBool.TRUE)
- .safeSet(DeploymentOptions.SHUTDOWN_IF_ATTACHED, JavaBool.TRUE)
-
- logInfo(s"""
- |------------------------------------------------------------------
- |Effective submit configuration: $flinkConfig
- |------------------------------------------------------------------
- |""".stripMargin)
- }
-
- override def doSubmit(
- submitRequest: SubmitRequest,
- flinkConfig: Configuration): SubmitResponse = {
- val flinkHome = submitRequest.flinkVersion.flinkHome
-
- val (clusterSpecification, clusterDescriptor: YarnClusterDescriptor) =
- getYarnClusterDeployDescriptor(flinkConfig, submitRequest.hadoopUser)
- val flinkDistJar = FlinkUtils.getFlinkDistJar(flinkHome)
- clusterDescriptor.setLocalJarPath(new HadoopPath(flinkDistJar))
- clusterDescriptor.addShipFiles(List(new HadoopPath(s"$flinkHome/lib")))
-
- var packagedProgram: PackagedProgram = null
- val clusterClient = {
- logInfo(s"""
- |------------------------<<specification>>-------------------------
- |$clusterSpecification
- |------------------------------------------------------------------
- |""".stripMargin)
-
- val programJobGraph =
- super.getJobGraph(flinkConfig, submitRequest, submitRequest.userJarFile)
- packagedProgram = programJobGraph._1
- val jobGraph = programJobGraph._2
-
- logInfo(s"""
- |-------------------------<<applicationId>>------------------------
- |jobGraph getJobID: ${jobGraph.getJobID.toString}
- |__________________________________________________________________
- |""".stripMargin)
- deployInternal(
- clusterDescriptor,
- clusterSpecification,
- submitRequest.effectiveAppName,
- classOf[YarnJobClusterEntrypoint].getName,
- jobGraph,
- true).getClusterClient
- }
- val applicationId = clusterClient.getClusterId
- val jobManagerUrl = clusterClient.getWebInterfaceURL
- logInfo(s"""
- |-------------------------<<applicationId>>------------------------
- |Flink Job Started: applicationId: $applicationId
- |__________________________________________________________________
- |""".stripMargin)
-
- val resp =
- SubmitResponse(applicationId.toString, flinkConfig.toMap, jobManagerUrl = jobManagerUrl)
- closeSubmit(submitRequest, packagedProgram, clusterClient, clusterDescriptor)
- resp
- }
-
- override def doCancel(
- cancelRequest: CancelRequest,
- flinkConfig: Configuration): CancelResponse = {
- val response = super.doCancel(cancelRequest, flinkConfig)
- val (yarnClusterId: ApplicationId, clusterDescriptor: YarnClusterDescriptor) =
- getYarnClusterDescriptor(flinkConfig)
- clusterDescriptor.killCluster(ApplicationId.fromString(cancelRequest.clusterId))
- response
- }
-
-}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/YarnSessionClient.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/YarnSessionClient.scala
deleted file mode 100644
index bb2348cf1..000000000
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/YarnSessionClient.scala
+++ /dev/null
@@ -1,209 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.client.impl
-
-import org.apache.streampark.common.util.Implicits._
-import org.apache.streampark.common.util.Utils
-import org.apache.streampark.flink.client.`trait`.YarnClientTrait
-import org.apache.streampark.flink.client.bean._
-
-import org.apache.commons.lang3.StringUtils
-import org.apache.flink.client.program.ClusterClient
-import org.apache.flink.configuration._
-import org.apache.flink.yarn.YarnClusterDescriptor
-import org.apache.flink.yarn.configuration.{YarnConfigOptions, YarnDeploymentTarget}
-import org.apache.hadoop.yarn.api.records.{ApplicationId, FinalApplicationStatus}
-import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException
-import org.apache.hadoop.yarn.util.ConverterUtils
-
-import java.util
-
-/** Submit Job to YARN Session Cluster */
-object YarnSessionClient extends YarnClientTrait {
-
- /**
- * @param submitRequest
- * @param flinkConfig
- */
- override def setConfig(submitRequest: SubmitRequest, flinkConfig: Configuration): Unit = {
- super.setConfig(submitRequest, flinkConfig)
- flinkConfig
- .safeSet(DeploymentOptions.TARGET, YarnDeploymentTarget.SESSION.getName)
- logInfo(s"""
- |------------------------------------------------------------------
- |Effective submit configuration: $flinkConfig
- |------------------------------------------------------------------
- |""".stripMargin)
- }
-
- /**
- * @param deployRequest
- * @param flinkConfig
- */
- def deployClusterConfig(deployRequest: DeployRequest, flinkConfig: Configuration): Unit = {
- val shipFiles = new util.ArrayList[String]()
- shipFiles.add(s"${deployRequest.flinkVersion.flinkHome}/lib")
- shipFiles.add(s"${deployRequest.flinkVersion.flinkHome}/plugins")
-
- flinkConfig
- // flinkDistJar
- .safeSet(YarnConfigOptions.FLINK_DIST_JAR, deployRequest.hdfsWorkspace.flinkDistJar)
- // flink lib
- .safeSet(YarnConfigOptions.SHIP_FILES, shipFiles)
- // yarnDeployment Target
- .safeSet(DeploymentOptions.TARGET, YarnDeploymentTarget.SESSION.getName)
- // conf dir
- .safeSet(DeploymentOptionsInternal.CONF_DIR, s"${deployRequest.flinkVersion.flinkHome}/conf")
-
- logInfo(s"""
- |------------------------------------------------------------------
- |Effective submit configuration: $flinkConfig
- |------------------------------------------------------------------
- |""".stripMargin)
- }
-
- @throws[Exception]
- override def doSubmit(
- submitRequest: SubmitRequest,
- flinkConfig: Configuration): SubmitResponse = {
- val (yarnClusterId: ApplicationId, clusterDescriptor: YarnClusterDescriptor) = getYarnClusterDescriptor(flinkConfig)
- val programJobGraph =
- super.getJobGraph(flinkConfig, submitRequest, submitRequest.userJarFile)
- val packageProgram = programJobGraph._1
- val jobGraph = programJobGraph._2
- val client = clusterDescriptor.retrieve(yarnClusterId).getClusterClient
- val jobId = client.submitJob(jobGraph).get().toString
- val jobManagerUrl = client.getWebInterfaceURL
-
- logInfo(s"""
- |-------------------------<<applicationId>>------------------------
- |Flink Job Started: jobId: $jobId , applicationId: ${yarnClusterId.toString}
- |__________________________________________________________________
- |""".stripMargin)
- val resp = SubmitResponse(yarnClusterId.toString, flinkConfig.toMap, jobId, jobManagerUrl)
- closeSubmit(submitRequest, packageProgram, client, clusterDescriptor)
- resp
- }
-
- override def doCancel(
- cancelRequest: CancelRequest,
- flinkConfig: Configuration): CancelResponse = {
- flinkConfig
- .safeSet(DeploymentOptions.TARGET, YarnDeploymentTarget.SESSION.getName)
- super.doCancel(cancelRequest, flinkConfig)
- }
-
- override def doTriggerSavepoint(
- request: TriggerSavepointRequest,
- flinkConfig: Configuration): SavepointResponse = {
- flinkConfig
- .safeSet(DeploymentOptions.TARGET, YarnDeploymentTarget.SESSION.getName)
- super.doTriggerSavepoint(request, flinkConfig)
- }
-
- def deploy(deployRequest: DeployRequest): DeployResponse = {
- logInfo(
- s"""
- |--------------------------------------- flink yarn sesion start ---------------------------------------
- | userFlinkHome : ${deployRequest.flinkVersion.flinkHome}
- | flinkVersion : ${deployRequest.flinkVersion.version}
- | deployMode : ${deployRequest.deployMode.name()}
- | clusterId : ${deployRequest.clusterId}
- | properties : ${deployRequest.properties.mkString(",")}
- |-------------------------------------------------------------------------------------------------------
- |""".stripMargin)
- var clusterDescriptor: YarnClusterDescriptor = null
- var client: ClusterClient[ApplicationId] = null
- try {
- val flinkConfig =
- extractConfiguration(deployRequest.flinkVersion.flinkHome, deployRequest.properties)
- deployClusterConfig(deployRequest, flinkConfig)
- val yarnClusterDescriptor = getYarnClusterDeployDescriptor(flinkConfig)
- clusterDescriptor = yarnClusterDescriptor._2
- if (StringUtils.isNotBlank(deployRequest.clusterId)) {
- try {
- val applicationStatus =
- clusterDescriptor.getYarnClient
- .getApplicationReport(ApplicationId.fromString(deployRequest.clusterId))
- .getFinalApplicationStatus
- if (FinalApplicationStatus.UNDEFINED == applicationStatus) {
- // application is running
- val yarnClient = clusterDescriptor
- .retrieve(ApplicationId.fromString(deployRequest.clusterId))
- .getClusterClient
- if (yarnClient.getWebInterfaceURL != null) {
- return DeployResponse(yarnClient.getWebInterfaceURL, yarnClient.getClusterId.toString)
- }
- }
- } catch {
- case _: ApplicationNotFoundException =>
- logInfo("this applicationId have not managed by yarn ,need deploy ...")
- }
- }
- val clientProvider = clusterDescriptor.deploySessionCluster(yarnClusterDescriptor._1)
- client = clientProvider.getClusterClient
- if (client.getWebInterfaceURL != null) {
- DeployResponse(client.getWebInterfaceURL, client.getClusterId.toString)
- } else {
- DeployResponse(error = new RuntimeException("get the cluster getWebInterfaceURL failed."))
- }
- } catch {
- case e: Exception =>
- logError(s"start flink session fail in ${deployRequest.deployMode} mode")
- throw e
- } finally {
- Utils.close(client, clusterDescriptor)
- }
- }
- def shutdown(shutDownRequest: ShutDownRequest): ShutDownResponse = {
- var clusterDescriptor: YarnClusterDescriptor = null
- var client: ClusterClient[ApplicationId] = null
- try {
- val flinkConfig = getFlinkDefaultConfiguration(shutDownRequest.flinkVersion.flinkHome)
- shutDownRequest.properties.foreach(m =>
- m._2 match {
- case v if v != null => flinkConfig.setString(m._1, m._2.toString)
- case _ =>
- })
- flinkConfig.safeSet(YarnConfigOptions.APPLICATION_ID, shutDownRequest.clusterId)
- flinkConfig.safeSet(DeploymentOptions.TARGET, YarnDeploymentTarget.SESSION.getName)
- val yarnClusterDescriptor = getYarnClusterDescriptor(flinkConfig)
- clusterDescriptor = yarnClusterDescriptor._2
- val shutDownState = FinalApplicationStatus.UNDEFINED.equals(
- clusterDescriptor.getYarnClient
- .getApplicationReport(ApplicationId.fromString(shutDownRequest.clusterId))
- .getFinalApplicationStatus)
- if (shutDownState) {
- val clientProvider =
- clusterDescriptor.retrieve(yarnClusterDescriptor._1)
- client = clientProvider.getClusterClient
- client.shutDownCluster()
- }
- logInfo(s"the ${shutDownRequest.clusterId}'s final status is ${clusterDescriptor.getYarnClient
- .getApplicationReport(ConverterUtils.toApplicationId(shutDownRequest.clusterId))
- .getFinalApplicationStatus}")
- ShutDownResponse(shutDownRequest.clusterId)
- } catch {
- case e: Exception =>
- logError(s"shutdown flink session fail in ${shutDownRequest.deployMode} mode")
- throw e
- } finally {
- Utils.close(client, clusterDescriptor)
- }
- }
-}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/tool/FlinkSessionClientHelper.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/tool/FlinkSessionClientHelper.scala
deleted file mode 100644
index bd3d8b5f5..000000000
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/tool/FlinkSessionClientHelper.scala
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.client.tool
-
-import org.apache.streampark.common.util.{AssertUtils, Logger}
-import org.apache.streampark.flink.kubernetes.KubernetesRetriever
-
-import org.apache.flink.client.deployment.application.ApplicationConfiguration
-import org.apache.flink.configuration.{Configuration, CoreOptions}
-import org.apache.flink.runtime.jobgraph.SavepointConfigOptions
-import org.apache.hc.client5.http.entity.mime.MultipartEntityBuilder
-import org.apache.hc.client5.http.fluent.Request
-import org.apache.hc.core5.http.ContentType
-import org.apache.hc.core5.http.io.entity.StringEntity
-import org.json4s.DefaultFormats
-import org.json4s.jackson.JsonMethods._
-import org.json4s.jackson.Serialization
-
-import java.io.File
-import java.nio.charset.StandardCharsets
-
-import scala.util.{Failure, Success, Try}
-
-object FlinkSessionSubmitHelper extends Logger {
-
- @transient
- implicit lazy val formats: DefaultFormats.type = org.json4s.DefaultFormats
-
- /**
- * Submit Flink Job via Rest API.
- *
- * @param jmRestUrl
- * jobmanager rest url of target flink cluster
- * @param flinkJobJar
- * flink job jar file
- * @param flinkConfig
- * flink configuration
- * @return
- * jobID of submitted flink job
- */
- @throws[Exception]
- def submitViaRestApi(jmRestUrl: String, flinkJobJar: File, flinkConfig: Configuration): String = {
- // upload flink-job jar
- val uploadResult = Request
- .post(s"$jmRestUrl/jars/upload")
- .connectTimeout(KubernetesRetriever.FLINK_REST_AWAIT_TIMEOUT_SEC)
- .responseTimeout(KubernetesRetriever.FLINK_CLIENT_TIMEOUT_SEC)
- .body(
- MultipartEntityBuilder
- .create()
- .addBinaryBody(
- "jarfile",
- flinkJobJar,
- ContentType.create("application/java-archive"),
- flinkJobJar.getName)
- .build())
- .execute
- .returnContent()
- .asString(StandardCharsets.UTF_8)
-
- val jarUploadResponse = Try(parse(uploadResult)) match {
- case Success(ok) =>
- JarUploadResponse(
- (ok \ "filename").extractOpt[String].orNull,
- (ok \ "status").extractOpt[String].orNull)
- case Failure(_) => null
- }
-
- AssertUtils.required(
- jarUploadResponse.isSuccessful,
- s"[flink-submit] upload flink jar to flink session cluster failed, jmRestUrl=$jmRestUrl, response=$jarUploadResponse")
-
- // refer to https://ci.apache.org/projects/flink/flink-docs-stable/docs/ops/rest_api/#jars-upload
- val resp = Request
- .post(s"$jmRestUrl/jars/${jarUploadResponse.jarId}/run")
- .connectTimeout(KubernetesRetriever.FLINK_REST_AWAIT_TIMEOUT_SEC)
- .responseTimeout(KubernetesRetriever.FLINK_CLIENT_TIMEOUT_SEC)
- .body(new StringEntity(Serialization.write(new JarRunRequest(flinkConfig))))
- .execute
- .returnContent()
- .asString(StandardCharsets.UTF_8)
-
- Try(parse(resp)) match {
- case Success(ok) => (ok \ "jobid").extractOpt[String].orNull
- case Failure(_) => null
- }
- }
-
-}
-
-/**
- * refer to https://ci.apache.org/projects/flink/flink-docs-stable/docs/ops/rest_api/#jars-upload
- */
-private[client] case class JarUploadResponse(filename: String, status: String) {
-
- def isSuccessful: Boolean = "success".equalsIgnoreCase(status)
-
- def jarId: String = filename.substring(filename.lastIndexOf("/") + 1)
-}
-
-/**
- * refer to https://ci.apache.org/projects/flink/flink-docs-stable/docs/ops/rest_api/#jars-upload
- */
-private[client] case class JarRunRequest(
- entryClass: String,
- programArgs: String,
- parallelism: String,
- savepointPath: String,
- allowNonRestoredState: Boolean) {
- def this(flinkConf: Configuration) {
- this(
- entryClass = flinkConf.get(ApplicationConfiguration.APPLICATION_MAIN_CLASS),
- programArgs = Option(flinkConf.get(ApplicationConfiguration.APPLICATION_ARGS))
- .map(String.join(" ", _))
- .orNull,
- parallelism = String.valueOf(flinkConf.get(CoreOptions.DEFAULT_PARALLELISM)),
- savepointPath = flinkConf.get(SavepointConfigOptions.SAVEPOINT_PATH),
- allowNonRestoredState =
- flinkConf.getBoolean(SavepointConfigOptions.SAVEPOINT_IGNORE_UNCLAIMED_STATE))
- }
-
-}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/trait/FlinkClientTrait.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/trait/FlinkClientTrait.scala
deleted file mode 100644
index 0c0b7c84c..000000000
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/trait/FlinkClientTrait.scala
+++ /dev/null
@@ -1,617 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.client.`trait`
-
-import org.apache.streampark.common.conf.ConfigKeys._
-import org.apache.streampark.common.conf.Workspace
-import org.apache.streampark.common.constants.Constants
-import org.apache.streampark.common.enums._
-import org.apache.streampark.common.fs.FsOperator
-import org.apache.streampark.common.util._
-import org.apache.streampark.common.util.Implicits._
-import org.apache.streampark.flink.client.bean._
-import org.apache.streampark.flink.core.FlinkClusterClient
-import org.apache.streampark.flink.core.conf.FlinkRunOption
-
-import com.google.common.collect.Lists
-import org.apache.commons.cli.{CommandLine, Options}
-import org.apache.commons.collections.MapUtils
-import org.apache.commons.lang3.StringUtils
-import org.apache.flink.api.common.JobID
-import org.apache.flink.client.cli._
-import org.apache.flink.client.cli.CliFrontend.loadCustomCommandLines
-import org.apache.flink.client.deployment.application.ApplicationConfiguration
-import org.apache.flink.client.program.{ClusterClient, PackagedProgram, PackagedProgramUtils}
-import org.apache.flink.configuration._
-import org.apache.flink.python.PythonOptions
-import org.apache.flink.runtime.jobgraph.{JobGraph, SavepointConfigOptions}
-import org.apache.flink.util.Preconditions.checkNotNull
-
-import java.io.File
-import java.util.Collections
-
-import scala.collection.mutable
-import scala.collection.mutable.ArrayBuffer
-import scala.util.{Failure, Success, Try}
-
-trait FlinkClientTrait extends Logger {
-
- private[client] lazy val PARAM_KEY_FLINK_CONF = KEY_FLINK_CONF(PARAM_PREFIX)
- private[client] lazy val PARAM_KEY_FLINK_SQL = KEY_FLINK_SQL(PARAM_PREFIX)
- private[client] lazy val PARAM_KEY_APP_CONF = KEY_APP_CONF(PARAM_PREFIX)
- private[client] lazy val PARAM_KEY_APP_NAME = KEY_APP_NAME(PARAM_PREFIX)
- private[client] lazy val PARAM_KEY_FLINK_PARALLELISM = KEY_FLINK_PARALLELISM(PARAM_PREFIX)
-
- @throws[Exception]
- def submit(submitRequest: SubmitRequest): SubmitResponse = {
- logInfo(
- s"""
- |--------------------------------------- flink job start ---------------------------------------
- | userFlinkHome : ${submitRequest.flinkVersion.flinkHome}
- | flinkVersion : ${submitRequest.flinkVersion.version}
- | appName : ${submitRequest.effectiveAppName}
- | jobType : ${submitRequest.jobType.name()}
- | deployMode : ${submitRequest.deployMode.name()}
- | k8sNamespace : ${submitRequest.kubernetesNamespace}
- | flinkExposedType : ${submitRequest.flinkRestExposedType}
- | clusterId : ${submitRequest.clusterId}
- | applicationType : ${submitRequest.applicationType.getName}
- | savePoint : ${submitRequest.savePoint}
- | properties : ${submitRequest.properties.mkString(" ")}
- | args : ${submitRequest.args}
- | appConf : ${submitRequest.appConf}
- | flinkBuildResult : ${submitRequest.buildResult}
- |-------------------------------------------------------------------------------------------
- |""".stripMargin)
-
- // prepare flink config
- val flinkConfig = prepareConfig(submitRequest)
-
- setConfig(submitRequest, flinkConfig)
-
- Try(doSubmit(submitRequest, flinkConfig)) match {
- case Success(resp) => resp
- case Failure(e) =>
- logError(
- s"flink job ${submitRequest.appName} start failed, " +
- s"deployMode: ${submitRequest.deployMode.getName}, " +
- s"detail: ${ExceptionUtils.stringifyException(e)}")
- throw e
- }
- }
-
- private[this] def prepareConfig(submitRequest: SubmitRequest): Configuration = {
-
- val (commandLine, flinkConfig) = getCommandLineAndFlinkConfig(submitRequest)
-
- submitRequest.jobType match {
- case FlinkJobType.PYFLINK =>
- val pythonVenv: String = Workspace.local.APP_PYTHON_VENV
- AssertUtils.required(FsOperator.lfs.exists(pythonVenv), s"$pythonVenv File does not exist")
-
- flinkConfig
- // python.archives
- .safeSet(PythonOptions.PYTHON_ARCHIVES, pythonVenv)
- // python.client.executable
- .safeSet(PythonOptions.PYTHON_CLIENT_EXECUTABLE, Constants.PYTHON_EXECUTABLE)
- // python.executable
- .safeSet(PythonOptions.PYTHON_EXECUTABLE, Constants.PYTHON_EXECUTABLE)
-
- val flinkOptPath: String =
- System.getenv(ConfigConstants.ENV_FLINK_OPT_DIR)
- if (StringUtils.isBlank(flinkOptPath)) {
- logWarn(s"Get environment variable ${ConfigConstants.ENV_FLINK_OPT_DIR} fail")
- val flinkHome = submitRequest.flinkVersion.flinkHome
- SystemPropertyUtils.setEnv(ConfigConstants.ENV_FLINK_OPT_DIR, s"$flinkHome/opt")
- logInfo(
- s"Set temporary environment variables ${ConfigConstants.ENV_FLINK_OPT_DIR} = $flinkHome/opt")
- }
- case _ =>
- if (submitRequest.userJarFile != null) {
- val uri = PackagedProgramUtils.resolveURI(submitRequest.userJarFile.getAbsolutePath)
- val programOptions = ProgramOptions.create(commandLine)
- val executionParameters = ExecutionConfigAccessor.fromProgramOptions(
- programOptions,
- Collections.singletonList(uri.toString))
- executionParameters.applyToConfiguration(flinkConfig)
- }
- }
-
- // 1) set common parameter
- flinkConfig
- .safeSet(PipelineOptions.NAME, submitRequest.effectiveAppName)
- .safeSet(DeploymentOptions.TARGET, submitRequest.deployMode.getName)
- .safeSet(SavepointConfigOptions.SAVEPOINT_PATH, submitRequest.savePoint)
- .safeSet(ApplicationConfiguration.APPLICATION_MAIN_CLASS, submitRequest.appMain)
- .safeSet(ApplicationConfiguration.APPLICATION_ARGS, extractProgramArgs(submitRequest))
- .safeSet(PipelineOptionsInternal.PIPELINE_FIXED_JOB_ID, submitRequest.jobId)
-
- if (!submitRequest.hasProp(CheckpointingOptions.MAX_RETAINED_CHECKPOINTS.key())) {
- val flinkDefaultConfiguration = getFlinkDefaultConfiguration(
- submitRequest.flinkVersion.flinkHome)
- // state.checkpoints.num-retained
- val retainedOption = CheckpointingOptions.MAX_RETAINED_CHECKPOINTS
- flinkConfig.safeSet(retainedOption, flinkDefaultConfiguration.get(retainedOption))
- }
-
- // 2) set savepoint parameter
- if (StringUtils.isNotBlank(submitRequest.savePoint)) {
- flinkConfig.safeSet(SavepointConfigOptions.SAVEPOINT_PATH, submitRequest.savePoint)
- flinkConfig.setBoolean(
- SavepointConfigOptions.SAVEPOINT_IGNORE_UNCLAIMED_STATE,
- submitRequest.allowNonRestoredState)
- val enableRestoreMode =
- submitRequest.restoreMode != null && submitRequest.flinkVersion
- .checkVersion(FlinkRestoreMode.SINCE_FLINK_VERSION)
- if (enableRestoreMode) {
- flinkConfig.setString(FlinkRestoreMode.RESTORE_MODE, submitRequest.restoreMode.getName);
- }
- }
-
- // 4) set env.xx.opts parameter
- if (MapUtils.isNotEmpty(submitRequest.properties)) {
- // file.encoding...
- if (submitRequest.hasProp(CoreOptions.FLINK_JVM_OPTIONS.key())) {
- val jvmOpt =
- submitRequest.getProp(CoreOptions.FLINK_JVM_OPTIONS.key()).toString
- if (!jvmOpt.contains("-Dfile.encoding=")) {
- // set default file.encoding
- val opt = s"-Dfile.encoding=UTF-8 $jvmOpt"
- submitRequest.properties.put(CoreOptions.FLINK_JVM_OPTIONS.key(), opt)
- }
- }
-
- submitRequest.properties
- .filter(_._1.startsWith("env."))
- .foreach(x => {
- logInfo(s"env opts: ${x._1}: ${x._2}")
- flinkConfig.setString(x._1, x._2.toString)
- })
- }
-
- flinkConfig
- }
-
- def setConfig(submitRequest: SubmitRequest, flinkConf: Configuration): Unit
-
- @throws[Exception]
- def triggerSavepoint(savepointRequest: TriggerSavepointRequest): SavepointResponse = {
- logInfo(
- s"""
- |----------------------------------------- flink job trigger savepoint ---------------------
- | userFlinkHome : ${savepointRequest.flinkVersion.flinkHome}
- | flinkVersion : ${savepointRequest.flinkVersion.version}
- | clusterId : ${savepointRequest.clusterId}
- | savePointPath : ${savepointRequest.savepointPath}
- | nativeFormat : ${savepointRequest.nativeFormat}
- | k8sNamespace : ${savepointRequest.kubernetesNamespace}
- | appId : ${savepointRequest.clusterId}
- | jobId : ${savepointRequest.jobId}
- |-------------------------------------------------------------------------------------------
- |""".stripMargin)
- val flinkConf = new Configuration()
- doTriggerSavepoint(savepointRequest, flinkConf)
- }
-
- @throws[Exception]
- def cancel(cancelRequest: CancelRequest): CancelResponse = {
- logInfo(
- s"""
- |----------------------------------------- flink job cancel --------------------------------
- | userFlinkHome : ${cancelRequest.flinkVersion.flinkHome}
- | flinkVersion : ${cancelRequest.flinkVersion.version}
- | clusterId : ${cancelRequest.clusterId}
- | withSavePoint : ${cancelRequest.withSavepoint}
- | savePointPath : ${cancelRequest.savepointPath}
- | withDrain : ${cancelRequest.withDrain}
- | nativeFormat : ${cancelRequest.nativeFormat}
- | k8sNamespace : ${cancelRequest.kubernetesNamespace}
- | appId : ${cancelRequest.clusterId}
- | jobId : ${cancelRequest.jobId}
- |-------------------------------------------------------------------------------------------
- |""".stripMargin)
- val flinkConf = new Configuration()
- doCancel(cancelRequest, flinkConf)
- }
-
- @throws[Exception]
- def doSubmit(submitRequest: SubmitRequest, flinkConf: Configuration): SubmitResponse
-
- @throws[Exception]
- def doTriggerSavepoint(
- request: TriggerSavepointRequest,
- flinkConf: Configuration): SavepointResponse
-
- @throws[Exception]
- def doCancel(cancelRequest: CancelRequest, flinkConf: Configuration): CancelResponse
-
- def trySubmit(submitRequest: SubmitRequest, flinkConfig: Configuration, jarFile: File)(
- jobGraphFunc: (SubmitRequest, Configuration, File) => SubmitResponse,
- restApiFunc: (SubmitRequest, Configuration, File) => SubmitResponse): SubmitResponse = {
- // Prioritize using JobGraph submit plan while using Rest API submit plan as backup
- Try {
- logInfo(s"[flink-submit] Submit job with JobGraph Plan.")
- jobGraphFunc(submitRequest, flinkConfig, jarFile)
- } match {
- case Failure(e) =>
- Try(restApiFunc(submitRequest, flinkConfig, jarFile)) match {
- case Success(r) => r
- case Failure(e1) =>
- throw new RuntimeException(
- s"""
- |[flink-submit] Both JobGraph submit plan and Rest API submit plan all failed!
- |JobGraph Submit plan failed detail:
- |------------------------------------------------------------------
- |${ExceptionUtils.stringifyException(e)}
- |------------------------------------------------------------------
- |
- | RestAPI Submit plan failed detail:
- | ------------------------------------------------------------------
- |${ExceptionUtils.stringifyException(e1)}
- |------------------------------------------------------------------
- |""".stripMargin)
- }
- case Success(v) => v
- }
- }
-
- private[client] def getJobGraph(
- flinkConfig: Configuration,
- submitRequest: SubmitRequest,
- jarFile: File): (PackagedProgram, JobGraph) = {
-
- val packagedProgramBuilder = {
- val builder = PackagedProgram.newBuilder
- .setSavepointRestoreSettings(submitRequest.savepointRestoreSettings)
- .setEntryPointClassName(
- flinkConfig
- .getOptional(ApplicationConfiguration.APPLICATION_MAIN_CLASS)
- .get())
- .setArguments(
- flinkConfig
- .getOptional(ApplicationConfiguration.APPLICATION_ARGS)
- .orElse(Lists.newArrayList()): _*)
-
- submitRequest.jobType match {
- case FlinkJobType.PYFLINK =>
- if (submitRequest.libs.nonEmpty) {
- // BUG: https://github.com/apache/streampark/issues/3761
- // builder.setUserClassPaths(Lists.newArrayList(submitRequest.libs: _*))
- }
- case _ =>
- builder
- .setJarFile(jarFile)
- // BUG: https://github.com/apache/streampark/issues/3761
- // .setUserClassPaths(Lists.newArrayList(submitRequest.classPaths: _*))
- }
- builder
- }
-
- val packageProgram = packagedProgramBuilder.build()
- val jobGraph = PackagedProgramUtils.createJobGraph(
- packageProgram,
- flinkConfig,
- getParallelism(submitRequest),
- null,
- false)
-
- packageProgram -> jobGraph
- }
-
- private[client] def getJobID(jobId: String) = Try(JobID.fromHexString(jobId)) match {
- case Success(id) => id
- case Failure(e) => throw new CliArgsException(e.getMessage)
- }
-
- // ----------Public Method end ------------------
-
- private[client] def validateAndGetActiveCommandLine(
- customCommandLines: JavaList[CustomCommandLine],
- commandLine: CommandLine): CustomCommandLine = {
- val line = checkNotNull(commandLine)
- logInfo(s"Custom commandline: $customCommandLines")
- for (cli <- customCommandLines) {
- val isActive = cli.isActive(line)
- logInfo(s"Checking custom commandline $cli, isActive: $isActive")
- if (isActive) return cli
- }
- throw new IllegalStateException("No valid command-line found.")
- }
-
- private[client] def getFlinkDefaultConfiguration(flinkHome: String): Configuration = {
- Try(GlobalConfiguration.loadConfiguration(s"$flinkHome/conf"))
- .getOrElse(new Configuration())
- }
-
- private[client] def getOptionFromDefaultFlinkConfig[T](
- flinkHome: String,
- option: ConfigOption[T]): T = {
- getFlinkDefaultConfiguration(flinkHome).get(option)
- }
-
- private[this] def getCustomCommandLines(flinkHome: String): JavaList[CustomCommandLine] = {
- val flinkDefaultConfiguration: Configuration = getFlinkDefaultConfiguration(flinkHome)
- // 1. find the configuration directory
- val confDir = s"$flinkHome/conf"
- // 2. load the custom command lines
- loadCustomCommandLines(flinkDefaultConfiguration, confDir)
- }
-
- private[client] def getParallelism(submitRequest: SubmitRequest): Integer = {
- if (submitRequest.hasProp(KEY_FLINK_PARALLELISM())) {
- Integer.valueOf(submitRequest.getProp(KEY_FLINK_PARALLELISM()).toString)
- } else {
- getFlinkDefaultConfiguration(submitRequest.flinkVersion.flinkHome)
- .getInteger(CoreOptions.DEFAULT_PARALLELISM, CoreOptions.DEFAULT_PARALLELISM.defaultValue())
- }
- }
-
- private[this] def getCommandLineAndFlinkConfig(
- submitRequest: SubmitRequest): (CommandLine, Configuration) = {
-
- val commandLineOptions = getCommandLineOptions(submitRequest.flinkVersion.flinkHome)
-
- // read and verify user config...
- val cliArgs = {
- val optionMap = new mutable.HashMap[String, Any]()
- submitRequest.appOption
- .foreach {
- opt =>
- val verify = commandLineOptions.hasOption(opt._1)
- if (!verify) {
- logWarn(s"param:${opt._1} is error,skip it.")
- } else {
- val option = commandLineOptions.getOption(opt._1.trim).getOpt
- Try(opt._2.toBoolean).getOrElse(opt._2) match {
- case b if b.isInstanceOf[Boolean] =>
- if (b.asInstanceOf[Boolean]) {
- optionMap += s"-$option" -> true
- }
- case v => optionMap += s"-$option" -> v
- }
- }
- }
-
- // fromSavePoint
- if (submitRequest.savePoint != null) {
- optionMap += s"-${FlinkRunOption.SAVEPOINT_PATH_OPTION.getOpt}" -> submitRequest.savePoint
- }
-
- Seq("-e", "--executor", "-t", "--target").foreach(optionMap.remove)
- if (submitRequest.deployMode != null) {
- optionMap += "-t" -> submitRequest.deployMode.getName
- }
-
- val array = new ArrayBuffer[String]()
- optionMap.foreach(opt => {
- array += opt._1
- opt._2 match {
- case v: String => array += v
- case _ =>
- }
- })
-
- // app properties
- if (MapUtils.isNotEmpty(submitRequest.properties)) {
- submitRequest.properties.foreach {
- key =>
- if (!key._1.startsWith("env.")) {
- logInfo(s"application dynamicProperties: ${key._1} :${key._2}")
- array += s"-D${key._1}=${key._2}"
- }
- }
- }
- array.toArray
- }
-
- logger.info(s"cliArgs: ${cliArgs.mkString(" ")}")
-
- val commandLine = FlinkRunOption.parse(commandLineOptions, cliArgs, true)
-
- val activeCommandLine = validateAndGetActiveCommandLine(
- getCustomCommandLines(submitRequest.flinkVersion.flinkHome),
- commandLine)
-
- val configuration =
- applyConfiguration(submitRequest.flinkVersion.flinkHome, activeCommandLine, commandLine)
-
- commandLine -> configuration
-
- }
-
- private[client] def getCommandLineOptions(flinkHome: String) = {
- val customCommandLines = getCustomCommandLines(flinkHome)
- val customCommandLineOptions = new Options
- for (customCommandLine <- customCommandLines) {
- customCommandLine.addGeneralOptions(customCommandLineOptions)
- customCommandLine.addRunOptions(customCommandLineOptions)
- }
- FlinkRunOption.mergeOptions(CliFrontendParser.getRunCommandOptions, customCommandLineOptions)
- }
-
- private[client] def extractConfiguration(
- flinkHome: String,
- properties: JavaMap[String, Any]): Configuration = {
- val commandLine = {
- val commandLineOptions = getCommandLineOptions(flinkHome)
- // read and verify user config...
- val cliArgs = {
- val array = new ArrayBuffer[String]()
- // The priority of the parameters defined on the page is greater than the app conf file, property parameters etc.
- if (MapUtils.isNotEmpty(properties)) {
- properties.foreach(x => array += s"-D${x._1}=${x._2.toString.trim}")
- }
- array.toArray
- }
- FlinkRunOption.parse(commandLineOptions, cliArgs, true)
- }
- val activeCommandLine =
- validateAndGetActiveCommandLine(getCustomCommandLines(flinkHome), commandLine)
- val flinkConfig =
- applyConfiguration(flinkHome, activeCommandLine, commandLine)
- flinkConfig
- }
-
- private[this] def extractProgramArgs(submitRequest: SubmitRequest): JavaList[String] = {
- val programArgs = new ArrayBuffer[String]()
- programArgs ++= FlinkConfigurationUtils.extractArguments(submitRequest.args)
-
- if (submitRequest.applicationType == ApplicationType.STREAMPARK_FLINK) {
-
- programArgs += PARAM_KEY_FLINK_CONF += submitRequest.flinkYaml
- programArgs += PARAM_KEY_APP_NAME += DeflaterUtils.zipString(submitRequest.effectiveAppName)
- programArgs += PARAM_KEY_FLINK_PARALLELISM += getParallelism(submitRequest).toString
-
- submitRequest.jobType match {
- case FlinkJobType.FLINK_SQL =>
- programArgs += PARAM_KEY_FLINK_SQL += submitRequest.flinkSQL
- if (submitRequest.appConf != null) {
- programArgs += PARAM_KEY_APP_CONF += submitRequest.appConf
- }
- case _
- if Try(!submitRequest.appConf.startsWith("json:"))
- .getOrElse(true) =>
- programArgs += PARAM_KEY_APP_CONF += submitRequest.appConf
- }
- }
-
- // execution.runtime-mode
- Try(
- submitRequest
- .properties(ExecutionOptions.RUNTIME_MODE.key())
- .toString) match {
- case Success(runtimeMode) =>
- programArgs += s"--${ExecutionOptions.RUNTIME_MODE.key()}"
- programArgs += runtimeMode
- case _ =>
- }
-
- if (submitRequest.jobType == FlinkJobType.PYFLINK) {
- // TODO why deployMode is not yarn-application ???
- if (submitRequest.deployMode != FlinkDeployMode.YARN_APPLICATION) {
- // python file
- programArgs.add("-py")
- programArgs.add(submitRequest.userJarFile.getAbsolutePath)
- }
- }
-
- Lists.newArrayList(programArgs: _*)
- }
-
- private[this] def applyConfiguration(
- flinkHome: String,
- activeCustomCommandLine: CustomCommandLine,
- commandLine: CommandLine): Configuration = {
-
- require(activeCustomCommandLine != null, "activeCustomCommandLine must not be null.")
- val configuration = new Configuration()
- val flinkDefaultConfiguration = getFlinkDefaultConfiguration(flinkHome)
- flinkDefaultConfiguration.keySet.foreach(key => {
- flinkDefaultConfiguration.getString(key, null) match {
- case v if v != null => configuration.setString(key, v)
- case _ =>
- }
- })
- configuration.addAll(activeCustomCommandLine.toConfiguration(commandLine))
- configuration
- }
-
- implicit private[client] class EnhanceFlinkConfiguration(flinkConfig: Configuration) {
- def safeSet[T](option: ConfigOption[T], value: T): Configuration = {
- flinkConfig match {
- case conf if value != null && value.toString.nonEmpty =>
- conf.set(option, value)
- case conf => conf
- }
- }
- }
-
- private[client] def cancelJob(
- cancelRequest: CancelRequest,
- jobID: JobID,
- client: ClusterClient[_]): String = {
-
- val savePointDir: String = tryGetSavepointPathIfNeed(cancelRequest)
-
- val clientWrapper = new FlinkClusterClient(client)
- val withSavepoint = Try(cancelRequest.withSavepoint).getOrElse(false)
- val withDrain = Try(cancelRequest.withDrain).getOrElse(false)
-
- (withSavepoint, withDrain) match {
- case (false, false) =>
- client.cancel(jobID).get()
- null
- case (_, _) =>
- clientWrapper
- .stopWithSavepoint(
- jobID,
- cancelRequest.withDrain,
- savePointDir,
- cancelRequest.nativeFormat)
- .get()
- }
- }
-
- private def tryGetSavepointPathIfNeed(request: SavepointRequestTrait): String = {
- if (!request.withSavepoint) null
- else {
- if (StringUtils.isNotBlank(request.savepointPath)) {
- request.savepointPath
- } else {
- val configDir = getOptionFromDefaultFlinkConfig[String](
- request.flinkVersion.flinkHome,
- ConfigOptions
- .key(CheckpointingOptions.SAVEPOINT_DIRECTORY.key())
- .stringType()
- .defaultValue {
- if (request.deployMode == FlinkDeployMode.YARN_APPLICATION) {
- Workspace.remote.APP_SAVEPOINTS
- } else null
- })
-
- AssertUtils.required(
- StringUtils.isNotBlank(configDir),
- s"[StreamPark] deployMode: ${request.deployMode.getName}, savePoint path is null or invalid.")
- configDir
- }
- }
- }
-
- private[client] def triggerSavepoint(
- savepointRequest: TriggerSavepointRequest,
- jobID: JobID,
- client: ClusterClient[_]): String = {
- val savepointPath = tryGetSavepointPathIfNeed(savepointRequest)
- val clientWrapper = new FlinkClusterClient(client)
- clientWrapper
- .triggerSavepoint(jobID, savepointPath, savepointRequest.nativeFormat)
- .get()
- }
-
- def closeSubmit(submitRequest: SubmitRequest, close: AutoCloseable*): Unit = {
- close.foreach(x => {
- if (x.isInstanceOf[PackagedProgram]) {
- if (submitRequest.safePackageProgram) {
- Utils.close(x)
- }
- } else {
- Utils.close(x)
- }
- })
- }
-}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/trait/KubernetesNativeClientTrait.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/trait/KubernetesNativeClientTrait.scala
deleted file mode 100644
index c77ed3af3..000000000
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/trait/KubernetesNativeClientTrait.scala
+++ /dev/null
@@ -1,190 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.client.`trait`
-
-import org.apache.streampark.common.enums.{FlinkDeployMode, FlinkK8sRestExposedType}
-import org.apache.streampark.flink.client.bean._
-import org.apache.streampark.flink.kubernetes.PodTemplateTool
-import org.apache.streampark.flink.packer.pipeline.DockerImageBuildResponse
-
-import org.apache.commons.lang3.StringUtils
-import org.apache.flink.api.common.JobID
-import org.apache.flink.client.deployment.ClusterSpecification
-import org.apache.flink.client.program.ClusterClient
-import org.apache.flink.configuration._
-import org.apache.flink.kubernetes.{KubernetesClusterClientFactory, KubernetesClusterDescriptor}
-import org.apache.flink.kubernetes.configuration.KubernetesConfigOptions
-import org.apache.flink.kubernetes.configuration.KubernetesConfigOptions.ServiceExposedType
-
-import javax.annotation.Nonnull
-
-import scala.collection.JavaConverters.mapAsScalaMapConverter
-
-/** kubernetes native mode submit */
-trait KubernetesNativeClientTrait extends FlinkClientTrait {
-
- override def setConfig(submitRequest: SubmitRequest, flinkConfig: Configuration): Unit = {
- // extract from submitRequest
- flinkConfig
- .safeSet(KubernetesConfigOptions.CLUSTER_ID, submitRequest.clusterId)
- .safeSet(KubernetesConfigOptions.NAMESPACE, submitRequest.kubernetesNamespace)
- .safeSet(
- KubernetesConfigOptions.REST_SERVICE_EXPOSED_TYPE,
- covertToServiceExposedType(submitRequest.flinkRestExposedType))
-
- if (submitRequest.buildResult != null && submitRequest.deployMode == FlinkDeployMode.KUBERNETES_NATIVE_APPLICATION) {
- val buildResult =
- submitRequest.buildResult.asInstanceOf[DockerImageBuildResponse]
- buildResult.podTemplatePaths.asScala.foreach(p => {
- if (PodTemplateTool.KUBERNETES_POD_TEMPLATE.key.equals(p._1)) {
- flinkConfig.safeSet(KubernetesConfigOptions.KUBERNETES_POD_TEMPLATE, p._2)
- } else if (PodTemplateTool.KUBERNETES_JM_POD_TEMPLATE.key.equals(p._1)) {
- flinkConfig.safeSet(KubernetesConfigOptions.JOB_MANAGER_POD_TEMPLATE, p._2)
- } else if (PodTemplateTool.KUBERNETES_TM_POD_TEMPLATE.key.equals(p._1)) {
- flinkConfig.safeSet(KubernetesConfigOptions.TASK_MANAGER_POD_TEMPLATE, p._2)
- }
- })
- }
-
- // add flink conf configuration, mainly to set the log4j configuration
- if (!flinkConfig.contains(DeploymentOptionsInternal.CONF_DIR)) {
- flinkConfig.safeSet(
- DeploymentOptionsInternal.CONF_DIR,
- s"${submitRequest.flinkVersion.flinkHome}/conf")
- }
-
- if (flinkConfig.get(KubernetesConfigOptions.NAMESPACE).isEmpty) {
- flinkConfig.removeConfig(KubernetesConfigOptions.NAMESPACE)
- }
-
- logInfo(s"""
- |------------------------------------------------------------------
- |Effective submit configuration: $flinkConfig
- |------------------------------------------------------------------
- |""".stripMargin)
- }
-
- // Tip: Perhaps it would be better to let users freely specify the savepoint directory
- @throws[Exception]
- override def doCancel(
- cancelRequest: CancelRequest,
- flinkConfig: Configuration): CancelResponse = {
- executeClientAction(
- cancelRequest,
- flinkConfig,
- (jobId, client) => {
- val resp = super.cancelJob(cancelRequest, jobId, client)
- if (cancelRequest.deployMode == FlinkDeployMode.KUBERNETES_NATIVE_APPLICATION) {
- client.shutDownCluster()
- }
- CancelResponse(resp)
- })
- }
-
- private[client] def executeClientAction[O, R <: SavepointRequestTrait](
- request: R,
- flinkConfig: Configuration,
- actFunc: (JobID, ClusterClient[_]) => O): O = {
- val hints =
- s"[flink-client] execute ${request.getClass.getSimpleName} for flink job failed,"
- require(
- StringUtils.isNotBlank(request.clusterId),
- s"$hints, clusterId is null, mode=${flinkConfig.get(DeploymentOptions.TARGET)}")
-
- flinkConfig
- .safeSet(KubernetesConfigOptions.CLUSTER_ID, request.clusterId)
- .safeSet(KubernetesConfigOptions.NAMESPACE, request.kubernetesNamespace)
-
- var clusterDescriptor: KubernetesClusterDescriptor = null
- var client: ClusterClient[String] = null
-
- try {
- clusterDescriptor = getK8sClusterDescriptor(flinkConfig)
- client = clusterDescriptor
- .retrieve(flinkConfig.getString(KubernetesConfigOptions.CLUSTER_ID))
- .getClusterClient
- actFunc(JobID.fromHexString(request.jobId), client)
- } catch {
- case e: Exception =>
- logger.error(s"$hints mode=${flinkConfig.get(DeploymentOptions.TARGET)}, request=$request")
- throw e
- } finally {
- if (client != null) client.close()
- if (clusterDescriptor != null) clusterDescriptor.close()
- }
- }
-
- @throws[Exception]
- override def doTriggerSavepoint(
- savepointRequest: TriggerSavepointRequest,
- flinkConfig: Configuration): SavepointResponse = {
- executeClientAction(
- savepointRequest,
- flinkConfig,
- (jobId, clusterClient) => {
- val actionResult =
- super.triggerSavepoint(savepointRequest, jobId, clusterClient)
- SavepointResponse(actionResult)
- })
- }
-
- // noinspection DuplicatedCode
- /*
- tips:
- The default kubernetes cluster communication information will be obtained from ./kube/conf file.
-
- If you need to customize the kubernetes cluster context, such as multiple target kubernetes clusters
- or multiple kubernetes api-server accounts, there are two ways to achieve this:
- 1. Get the KubernetesClusterDescriptor by manually, building the FlinkKubeClient and specify
- the kubernetes context contents in the FlinkKubeClient.
- 2. Specify an explicit key kubernetes.config.file in flinkConfig instead of the default value.
- */
- def getK8sClusterDescriptorAndSpecification(
- flinkConfig: Configuration): (KubernetesClusterDescriptor, ClusterSpecification) = {
- val clientFactory = new KubernetesClusterClientFactory()
- val clusterDescriptor = clientFactory.createClusterDescriptor(flinkConfig)
- val clusterSpecification =
- clientFactory.getClusterSpecification(flinkConfig)
- (clusterDescriptor, clusterSpecification)
- }
-
- def getK8sClusterDescriptor(flinkConfig: Configuration): KubernetesClusterDescriptor = {
- val clientFactory = new KubernetesClusterClientFactory()
- clientFactory.createClusterDescriptor(flinkConfig)
- }
-
- protected def flinkConfIdentifierInfo(@Nonnull conf: Configuration): String =
- s"deployMode=${conf.get(DeploymentOptions.TARGET)}, clusterId=${conf.get(
- KubernetesConfigOptions.CLUSTER_ID)}, " +
- s"namespace=${conf.get(KubernetesConfigOptions.NAMESPACE)}"
-
- private def covertToServiceExposedType(exposedType: FlinkK8sRestExposedType): ServiceExposedType =
- exposedType match {
- case FlinkK8sRestExposedType.CLUSTER_IP => ServiceExposedType.ClusterIP
- case FlinkK8sRestExposedType.LOAD_BALANCER =>
- ServiceExposedType.LoadBalancer
- case FlinkK8sRestExposedType.NODE_PORT => ServiceExposedType.NodePort
- case _ => ServiceExposedType.LoadBalancer
- }
-
- def getDefaultKubernetesConf(k8sConf: String): String = {
- val homePath: String = System.getProperty("user.home")
- if (k8sConf != null) k8sConf.replace("~", homePath)
- else homePath.concat("/.kube/config")
- }
-}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/trait/YarnClientTrait.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/trait/YarnClientTrait.scala
deleted file mode 100644
index e9c5d1b5e..000000000
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/trait/YarnClientTrait.scala
+++ /dev/null
@@ -1,199 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.client.`trait`
-
-import org.apache.streampark.common.util.{ExceptionUtils, HadoopUtils}
-import org.apache.streampark.common.util.Implicits._
-import org.apache.streampark.flink.client.bean._
-
-import org.apache.flink.api.common.JobID
-import org.apache.flink.client.deployment.ClusterSpecification
-import org.apache.flink.client.program.{ClusterClient, ClusterClientProvider}
-import org.apache.flink.configuration.Configuration
-import org.apache.flink.runtime.jobgraph.JobGraph
-import org.apache.flink.util.FlinkException
-import org.apache.flink.yarn.{YarnClusterClientFactory, YarnClusterDescriptor}
-import org.apache.flink.yarn.configuration.YarnConfigOptions
-import org.apache.hadoop.security.UserGroupInformation
-import org.apache.hadoop.yarn.api.records.ApplicationId
-
-import java.lang.{Boolean => JavaBool}
-import java.lang.reflect.Method
-import java.security.PrivilegedAction
-
-import scala.util.{Failure, Success, Try}
-
-/** yarn application mode submit */
-trait YarnClientTrait extends FlinkClientTrait {
-
- override def setConfig(submitRequest: SubmitRequest, flinkConfig: Configuration): Unit = {
- flinkConfig
- .safeSet(YarnConfigOptions.APPLICATION_NAME, submitRequest.effectiveAppName)
- .safeSet(YarnConfigOptions.APPLICATION_TYPE, submitRequest.applicationType.getName)
- .safeSet(YarnConfigOptions.APPLICATION_TAGS, "streampark")
- }
-
- private[this] def executeClientAction[R <: SavepointRequestTrait, O](
- request: R,
- flinkConf: Configuration,
- actionFunc: (JobID, ClusterClient[_]) => O): O = {
- val jobID = getJobID(request.jobId)
- flinkConf.safeSet(YarnConfigOptions.APPLICATION_ID, request.clusterId)
- // Get the ClusterClient from the YarnClusterDescriptor
- val (applicationId: ApplicationId, clusterDescriptor: YarnClusterDescriptor) = getYarnClusterDescriptor(flinkConf)
- val clusterClient = clusterDescriptor.retrieve(applicationId).getClusterClient
-
- Try {
- actionFunc(jobID, clusterClient)
- }.recover {
- case e =>
- throw new FlinkException(
- s"[StreamPark] Do ${request.getClass.getSimpleName} for the job ${request.jobId} failed. " +
- s"detail: ${ExceptionUtils.stringifyException(e)}");
- }.get
- }
-
- override def doTriggerSavepoint(
- savepointRequest: TriggerSavepointRequest,
- flinkConf: Configuration): SavepointResponse = {
- executeClientAction(
- savepointRequest,
- flinkConf,
- (jid, client) => {
- SavepointResponse(super.triggerSavepoint(savepointRequest, jid, client))
- })
- }
-
- override def doCancel(cancelRequest: CancelRequest, flinkConf: Configuration): CancelResponse = {
- executeClientAction(
- cancelRequest,
- flinkConf,
- (jid, client) => {
- CancelResponse(super.cancelJob(cancelRequest, jid, client))
- })
- }
-
- private lazy val deployInternalMethod: Method = {
- val paramClass = Array(
- classOf[ClusterSpecification],
- classOf[String],
- classOf[String],
- classOf[JobGraph],
- Boolean2boolean(true).getClass // get boolean class.
- )
- val deployInternal =
- classOf[YarnClusterDescriptor].getDeclaredMethod("deployInternal", paramClass: _*)
- deployInternal.setAccessible(true)
- deployInternal
- }
-
- private[client] def deployInternal(
- clusterDescriptor: YarnClusterDescriptor,
- clusterSpecification: ClusterSpecification,
- applicationName: String,
- yarnClusterEntrypoint: String,
- jobGraph: JobGraph,
- detached: JavaBool): ClusterClientProvider[ApplicationId] = {
- deployInternalMethod
- .invoke(
- clusterDescriptor,
- clusterSpecification,
- applicationName,
- yarnClusterEntrypoint,
- jobGraph,
- detached)
- .asInstanceOf[ClusterClientProvider[ApplicationId]]
- }
-
- /**
- * Retrieves the YarnClusterDescriptor and the application ID.
- *
- * @param flinkConfig
- * the Flink configuration
- * @return
- * a tuple containing the application ID and the YarnClusterDescriptor
- */
- private[client] def getYarnClusterDescriptor(
- flinkConfig: Configuration,
- user: String = ""): (ApplicationId, YarnClusterDescriptor) = {
- Try {
- doAsYarnClusterDescriptor[ApplicationId](
- user,
- () => {
- val clientFactory = new YarnClusterClientFactory
- // Get the cluster ID
- val yarnClusterId: ApplicationId = clientFactory.getClusterId(flinkConfig)
- require(yarnClusterId != null)
- // Create the ClusterDescriptor
- val clusterDescriptor = clientFactory.createClusterDescriptor(flinkConfig)
- (yarnClusterId, clusterDescriptor)
- })
- } match {
- case Success(result) => result
- case Failure(e) =>
- throw new IllegalArgumentException(s"[StreamPark] access ClusterDescriptor error: $e")
- }
- }
-
- /**
- * Retrieves the ClusterSpecification and the YarnClusterDescriptor for deployment.
- *
- * @param flinkConfig
- * the Flink configuration
- * @return
- * a tuple containing the ClusterSpecification and the YarnClusterDescriptor
- */
- private[client] def getYarnClusterDeployDescriptor(
- flinkConfig: Configuration,
- user: String = ""): (ClusterSpecification, YarnClusterDescriptor) = {
- Try {
- doAsYarnClusterDescriptor[ClusterSpecification](
- user,
- () => {
- val clientFactory = new YarnClusterClientFactory
- // Get the ClusterSpecification
- val clusterSpecification = clientFactory.getClusterSpecification(flinkConfig)
- // Create the ClusterDescriptor
- val clusterDescriptor = clientFactory.createClusterDescriptor(flinkConfig)
- clusterSpecification -> clusterDescriptor
- })
- } match {
- case Success(result) => result
- case Failure(e) =>
- throw new IllegalArgumentException(s"[StreamPark] access ClusterDescriptor error: $e")
- }
- }
-
- private[this] def doAsYarnClusterDescriptor[T](
- user: String,
- func: () => (T, YarnClusterDescriptor)): (T, YarnClusterDescriptor) = {
- // Wrap the operation in ugi.doAs()
- val ugi = HadoopUtils.getUgi()
- val finalUgi = if (user != null && user.nonEmpty && ugi.getShortUserName != user) UserGroupInformation.createProxyUser(user, ugi) else ugi
-
- try {
- finalUgi.doAs(new PrivilegedAction[(T, YarnClusterDescriptor)] {
- override def run(): (T, YarnClusterDescriptor) = func()
- })
- } catch {
- case e: Exception =>
- throw new RuntimeException(s"[StreamPark] Error executing YarnClusterDescriptor operation as user $user", e)
- }
- }
-
-}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/java/org/apache/streampark/flink/client/test/ParameterTestCase.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/java/org/apache/streampark/flink/client/test/ParameterTestCase.java
new file mode 100644
index 000000000..4b1976395
--- /dev/null
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/java/org/apache/streampark/flink/client/test/ParameterTestCase.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.flink.client.test;
+
+import org.apache.flink.api.java.utils.ParameterTool;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+class ParameterTestCase {
+
+ @Test
+ void readArgs() {
+ String[] arg = new String[]{"--flink.deployment.option.parallelism", "10"};
+ String[] args = new String[]{
+ "--flink.home",
+ "hdfs://nameservice1/streampark/flink/flink-1.11.1",
+ "--app.name",
+ "testApp123",
+ "--flink.deployment.option.parallelism",
+ "5"
+ };
+ ParameterTool param = ParameterTool.fromArgs(arg).mergeWith(ParameterTool.fromArgs(args));
+
+ Assertions.assertEquals(
+ "hdfs://nameservice1/streampark/flink/flink-1.11.1", param.get("flink.home"));
+ Assertions.assertEquals("testApp123", param.get("app.name"));
+ Assertions.assertEquals("5", param.get("flink.deployment.option.parallelism"));
+ }
+}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/scala/org/apache/streampark/flink/client/test/ParameterTestCase.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/scala/org/apache/streampark/flink/client/test/ParameterTestCase.scala
deleted file mode 100644
index f0dad7fcf..000000000
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/scala/org/apache/streampark/flink/client/test/ParameterTestCase.scala
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.client.test
-
-import org.apache.flink.api.java.utils.ParameterTool
-import org.junit.jupiter.api.{Assertions, Test}
-
-class ParameterTestCase {
-
- @Test def readArgs(): Unit = {
- val arg = Array("--flink.deployment.option.parallelism", "10")
- val args = Array(
- "--flink.home",
- "hdfs://nameservice1/streampark/flink/flink-1.11.1",
- "--app.name",
- "testApp123",
- "--flink.deployment.option.parallelism",
- "5")
- val param =
- ParameterTool.fromArgs(arg).mergeWith(ParameterTool.fromArgs(args))
-
- Assertions.assertEquals(
- "hdfs://nameservice1/streampark/flink/flink-1.11.1",
- param.get("flink.home"))
- Assertions.assertEquals("testApp123", param.get("app.name"))
- Assertions.assertEquals("5", param.get("flink.deployment.option.parallelism"))
- }
-
-}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/scala/org/apache/streampark/flink/client/test/YarnPerJobTestCase.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/scala/org/apache/streampark/flink/client/test/YarnPerJobTestCase.scala
deleted file mode 100644
index db540f678..000000000
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/scala/org/apache/streampark/flink/client/test/YarnPerJobTestCase.scala
+++ /dev/null
@@ -1,208 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.streampark.flink.client.test
-
-import org.apache.streampark.common.util.Implicits._
-import org.apache.streampark.common.util.Logger
-import org.apache.streampark.flink.client.bean
-import org.apache.streampark.flink.core.conf.FlinkRunOption
-
-import org.apache.commons.cli.Options
-import org.apache.flink.client.cli.{CliFrontendParser, CustomCommandLine}
-import org.apache.flink.client.cli.CliFrontend.loadCustomCommandLines
-import org.apache.flink.client.deployment.{ClusterSpecification, DefaultClusterClientServiceLoader}
-import org.apache.flink.client.deployment.application.ApplicationConfiguration
-import org.apache.flink.client.program.{ClusterClientProvider, PackagedProgram, PackagedProgramUtils}
-import org.apache.flink.configuration._
-import org.apache.flink.configuration.MemorySize.MemoryUnit
-import org.apache.flink.runtime.jobgraph.JobGraph
-import org.apache.flink.util.Preconditions.checkNotNull
-import org.apache.flink.yarn.YarnClusterDescriptor
-import org.apache.flink.yarn.configuration.YarnDeploymentTarget
-import org.apache.flink.yarn.entrypoint.YarnJobClusterEntrypoint
-import org.apache.hadoop.fs.{Path => HadoopPath}
-import org.apache.hadoop.yarn.api.records.ApplicationId
-
-import java.io.File
-import java.lang.reflect.Method
-import java.util
-
-/** perJob to submit jobs programmatically, */
-object YarnPerJobTestCase extends Logger {
-
- /** You must install and deploy flink locally, and configure FLINK_HOME */
- lazy val FLINK_HOME = {
- val flinkLocalHome = System.getenv("FLINK_HOME")
- logInfo(s"flinkHome: $flinkLocalHome")
- flinkLocalHome
- }
-
- /** SocketWindowWordCount.jar Download the built-in sample program in flink/examples */
- val userJar = s"$FLINK_HOME/examples/streaming/SocketWindowWordCount.jar"
-
- /** Parameters required to run the program */
- val programArgs = "--hostname localhost --port 9999"
-
- /** Run the specified option parameter */
- val option = "-e yarn-per-job -p 2 -n"
-
- lazy val flinkDefaultConfiguration: Configuration = {
- require(FLINK_HOME != null)
- // get flink config
- GlobalConfiguration.loadConfiguration(s"$FLINK_HOME/conf")
- }
-
- lazy val customCommandLines: util.List[CustomCommandLine] = {
- // 1. find the configuration directory
- val configurationDirectory = s"$FLINK_HOME/conf"
- // 2. load the custom command lines
- loadCustomCommandLines(flinkDefaultConfiguration, configurationDirectory)
- }
-
- /**
- * The private method deployInternal of YarnClusterDescriptor is reflected, mainly to pass in
- * applicationName. In the original method, applicationName is hard-coded.
- */
- lazy val deployInternalMethod: Method = {
- val paramClass = Array(
- classOf[ClusterSpecification],
- classOf[String],
- classOf[String],
- classOf[JobGraph],
- Boolean2boolean(true).getClass)
- val deployInternal =
- classOf[YarnClusterDescriptor].getDeclaredMethod("deployInternal", paramClass: _*)
- deployInternal.setAccessible(true)
- deployInternal
- }
-
- def deployInternal(
- clusterDescriptor: YarnClusterDescriptor,
- clusterSpecification: ClusterSpecification,
- applicationName: String,
- yarnClusterEntrypoint: String,
- jobGraph: JobGraph,
- detached: java.lang.Boolean): ClusterClientProvider[ApplicationId] = {
- deployInternalMethod
- .invoke(
- clusterDescriptor,
- clusterSpecification,
- applicationName,
- yarnClusterEntrypoint,
- jobGraph,
- detached)
- .asInstanceOf[ClusterClientProvider[ApplicationId]]
- }
-
- def main(args: Array[String]): Unit = {
-
- val commandLine = {
- val customCommandLineOptions = new Options
- for (customCommandLine <- customCommandLines) {
- customCommandLine.addGeneralOptions(customCommandLineOptions)
- customCommandLine.addRunOptions(customCommandLineOptions)
- }
- val commandLineOptions = FlinkRunOption.mergeOptions(
- CliFrontendParser.getRunCommandOptions,
- customCommandLineOptions)
- val cliArgs = option.split("\\s+")
- FlinkRunOption.parse(commandLineOptions, cliArgs, true)
- }
-
- val activeCommandLine = {
- val line = checkNotNull(commandLine)
- logInfo("Custom commandlines: $customCommandLines")
- customCommandLines.filter(_.isActive(line)).get(0)
- }
-
- val executorConfig =
- checkNotNull(activeCommandLine).toConfiguration(commandLine)
- val flinkConfig = new Configuration(executorConfig)
- flinkConfig.set(DeploymentOptions.TARGET, YarnDeploymentTarget.PER_JOB.getName)
- flinkConfig.set(
- ApplicationConfiguration.APPLICATION_ARGS,
- programArgs.split("\\s+").toList.asJava)
- flinkConfig.set(
- JobManagerOptions.TOTAL_FLINK_MEMORY,
- MemorySize.parse("1024", MemoryUnit.MEGA_BYTES))
- flinkConfig.set(
- TaskManagerOptions.TOTAL_FLINK_MEMORY,
- MemorySize.parse("1024", MemoryUnit.MEGA_BYTES))
-
- val clusterClientServiceLoader = new DefaultClusterClientServiceLoader
- val clientFactory =
- clusterClientServiceLoader.getClusterClientFactory[ApplicationId](flinkConfig)
-
- val clusterDescriptor = {
- val clusterDescriptor =
- clientFactory
- .createClusterDescriptor(flinkConfig)
- .asInstanceOf[YarnClusterDescriptor]
- val flinkDistJar =
- new File(s"$FLINK_HOME/lib")
- .list()
- .filter(_.matches("flink-dist.*\\.jar")) match {
- case Array() =>
- throw new IllegalArgumentException(
- s"[StreamPark] can no found flink-dist jar in $FLINK_HOME/lib")
- case array if array.length == 1 => s"$FLINK_HOME/lib/${array.head}"
- case more =>
- throw new IllegalArgumentException(
- s"[StreamPark] found multiple flink-dist jar in $FLINK_HOME/lib,[${more
- .mkString(",")}]")
- }
- clusterDescriptor.setLocalJarPath(new HadoopPath(flinkDistJar))
- clusterDescriptor
- }
-
- try {
- val clusterClient = {
- val clusterSpecification =
- clientFactory.getClusterSpecification(flinkConfig)
- logInfo("------------------<<specification>>------------------")
- logInfo(s"$clusterSpecification")
- logInfo("------------------------------------")
- val packagedProgram = PackagedProgram.newBuilder
- .setJarFile(new File(userJar))
- .setArguments(programArgs.split("\\s"): _*)
- .build
- val jobGraph = PackagedProgramUtils.createJobGraph(packagedProgram, flinkConfig, 1, false)
- logInfo("------------------<<jobId>>------------------")
- logger.info(s"${jobGraph.getJobID.toString}")
- logInfo("------------------------------------")
-
- deployInternal(
- clusterDescriptor,
- clusterSpecification,
- "MyJob",
- classOf[YarnJobClusterEntrypoint].getName,
- jobGraph,
- false).getClusterClient
- }
- val applicationId = clusterClient.getClusterId
- logInfo("------------------<<applicationId>>-------------------")
- logInfo(s"Flink Job Started: applicationId: $applicationId ")
- logInfo("-------------------------------------")
- bean.SubmitResponse(applicationId.toString, flinkConfig.toMap)
- } finally
- if (clusterDescriptor != null) {
- clusterDescriptor.close()
- }
- }
-
-}
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/SimpleBuildResponse.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/AbstractFlinkBuildResponse.java
similarity index 81%
copy from streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/SimpleBuildResponse.java
copy to streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/AbstractFlinkBuildResponse.java
index cee1412da..2f80f3b4d 100644
--- a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/SimpleBuildResponse.java
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/AbstractFlinkBuildResponse.java
@@ -20,16 +20,21 @@ package org.apache.streampark.flink.packer.pipeline;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
+import java.io.Serializable;
+
+/** Shared workspace and pass fields for Flink build responses. */
@JsonIgnoreProperties(ignoreUnknown = true)
-public class SimpleBuildResponse implements FlinkBuildResult {
+abstract class AbstractFlinkBuildResponse implements FlinkBuildResult, Serializable {
+
+ private static final long serialVersionUID = 1L;
private String workspacePath;
private boolean pass = true;
- public SimpleBuildResponse() {
+ protected AbstractFlinkBuildResponse() {
}
- public SimpleBuildResponse(String workspacePath, boolean pass) {
+ protected AbstractFlinkBuildResponse(String workspacePath, boolean pass) {
this.workspacePath = workspacePath;
this.pass = pass;
}
@@ -53,9 +58,4 @@ public class SimpleBuildResponse implements FlinkBuildResult {
public void setPass(boolean pass) {
this.pass = pass;
}
-
- @Override
- public String toString() {
- return "{ workspacePath: " + workspacePath + ", pass: " + pass + " }";
- }
}
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/BuildResult.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/BuildResult.java
index 56d9969fc..f6668dc06 100644
--- a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/BuildResult.java
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/BuildResult.java
@@ -17,8 +17,10 @@
package org.apache.streampark.flink.packer.pipeline;
+import java.io.Serializable;
+
/** Result of a BuildPipeline instance. */
-public interface BuildResult {
+public interface BuildResult extends Serializable {
/** is pass aka is successfully */
boolean pass();
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/DockerImageBuildResponse.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/DockerImageBuildResponse.java
index 443df4bd5..1d66f60a3 100644
--- a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/DockerImageBuildResponse.java
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/DockerImageBuildResponse.java
@@ -23,13 +23,11 @@ import com.fasterxml.jackson.annotation.JsonProperty;
import java.util.Map;
@JsonIgnoreProperties(ignoreUnknown = true)
-public class DockerImageBuildResponse implements FlinkBuildResult {
+public class DockerImageBuildResponse extends AbstractFlinkBuildResponse {
- private String workspacePath;
private String flinkImageTag;
private Map<String, String> podTemplatePaths;
private String dockerInnerMainJarPath;
- private boolean pass = true;
public DockerImageBuildResponse() {
}
@@ -48,16 +46,10 @@ public class DockerImageBuildResponse implements FlinkBuildResult {
Map<String, String> podTemplatePaths,
String dockerInnerMainJarPath,
boolean pass) {
- this.workspacePath = workspacePath;
+ super(workspacePath, pass);
this.flinkImageTag = flinkImageTag;
this.podTemplatePaths = podTemplatePaths;
this.dockerInnerMainJarPath = dockerInnerMainJarPath;
- this.pass = pass;
- }
-
- @Override
- public String workspacePath() {
- return workspacePath;
}
public String flinkImageTag() {
@@ -72,16 +64,6 @@ public class DockerImageBuildResponse implements FlinkBuildResult {
return dockerInnerMainJarPath;
}
- @Override
- public boolean pass() {
- return pass;
- }
-
- @JsonProperty("workspacePath")
- public void setWorkspacePath(String workspacePath) {
- this.workspacePath = workspacePath;
- }
-
@JsonProperty("flinkImageTag")
public void setFlinkImageTag(String flinkImageTag) {
this.flinkImageTag = flinkImageTag;
@@ -97,15 +79,10 @@ public class DockerImageBuildResponse implements FlinkBuildResult {
this.dockerInnerMainJarPath = dockerInnerMainJarPath;
}
- @JsonProperty("pass")
- public void setPass(boolean pass) {
- this.pass = pass;
- }
-
@Override
public String toString() {
return "{ workspacePath: "
- + workspacePath
+ + workspacePath()
+ ", flinkImageTag: "
+ flinkImageTag
+ ", podTemplatePaths: "
@@ -113,7 +90,7 @@ public class DockerImageBuildResponse implements FlinkBuildResult {
+ ", dockerInnerMainJarPath: "
+ dockerInnerMainJarPath
+ ", pass: "
- + pass
+ + pass()
+ " }";
}
}
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/K8sAppModeBuildResponse.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/K8sAppModeBuildResponse.java
index f8fac6c43..1124f3a9a 100644
--- a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/K8sAppModeBuildResponse.java
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/K8sAppModeBuildResponse.java
@@ -23,13 +23,11 @@ import com.fasterxml.jackson.annotation.JsonProperty;
import java.util.Set;
@JsonIgnoreProperties(ignoreUnknown = true)
-public class K8sAppModeBuildResponse implements FlinkBuildResult {
+public class K8sAppModeBuildResponse extends AbstractFlinkBuildResponse {
- private String workspacePath;
private String flinkBaseImage;
private String mainJarPath;
private Set<String> extraLibJarPaths;
- private boolean pass = false;
public K8sAppModeBuildResponse() {
}
@@ -40,16 +38,10 @@ public class K8sAppModeBuildResponse implements FlinkBuildResult {
String mainJarPath,
Set<String> extraLibJarPaths,
boolean pass) {
- this.workspacePath = workspacePath;
+ super(workspacePath, pass);
this.flinkBaseImage = flinkBaseImage;
this.mainJarPath = mainJarPath;
this.extraLibJarPaths = extraLibJarPaths;
- this.pass = pass;
- }
-
- @Override
- public String workspacePath() {
- return workspacePath;
}
public String flinkBaseImage() {
@@ -64,16 +56,6 @@ public class K8sAppModeBuildResponse implements FlinkBuildResult {
return extraLibJarPaths;
}
- @Override
- public boolean pass() {
- return pass;
- }
-
- @JsonProperty("workspacePath")
- public void setWorkspacePath(String workspacePath) {
- this.workspacePath = workspacePath;
- }
-
@JsonProperty("flinkBaseImage")
public void setFlinkBaseImage(String flinkBaseImage) {
this.flinkBaseImage = flinkBaseImage;
@@ -89,15 +71,10 @@ public class K8sAppModeBuildResponse implements FlinkBuildResult {
this.extraLibJarPaths = extraLibJarPaths;
}
- @JsonProperty("pass")
- public void setPass(boolean pass) {
- this.pass = pass;
- }
-
@Override
public String toString() {
return "{ workspacePath: "
- + workspacePath
+ + workspacePath()
+ ", flinkBaseImage: "
+ flinkBaseImage
+ ", mainJarPath: "
@@ -105,7 +82,7 @@ public class K8sAppModeBuildResponse implements FlinkBuildResult {
+ ", extraLibJarPaths: "
+ extraLibJarPaths
+ ", pass: "
- + pass
+ + pass()
+ " }";
}
}
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/ShadedBuildResponse.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/ShadedBuildResponse.java
index 9e383c5cb..f0370b54d 100644
--- a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/ShadedBuildResponse.java
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/ShadedBuildResponse.java
@@ -21,62 +21,39 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
@JsonIgnoreProperties(ignoreUnknown = true)
-public class ShadedBuildResponse implements FlinkBuildResult {
+public class ShadedBuildResponse extends AbstractFlinkBuildResponse {
- private String workspacePath;
private String shadedJarPath;
- private boolean pass = true;
public ShadedBuildResponse(String workspacePath, String shadedJarPath) {
this(workspacePath, shadedJarPath, true);
}
public ShadedBuildResponse(String workspacePath, String shadedJarPath, boolean pass) {
- this.workspacePath = workspacePath;
+ super(workspacePath, pass);
this.shadedJarPath = shadedJarPath;
- this.pass = pass;
}
public ShadedBuildResponse() {
}
- @Override
- public String workspacePath() {
- return workspacePath;
- }
-
public String shadedJarPath() {
return shadedJarPath;
}
- @Override
- public boolean pass() {
- return pass;
- }
-
- @JsonProperty("workspacePath")
- public void setWorkspacePath(String workspacePath) {
- this.workspacePath = workspacePath;
- }
-
@JsonProperty("shadedJarPath")
public void setShadedJarPath(String shadedJarPath) {
this.shadedJarPath = shadedJarPath;
}
- @JsonProperty("pass")
- public void setPass(boolean pass) {
- this.pass = pass;
- }
-
@Override
public String toString() {
return "{ workspacePath: "
- + workspacePath
+ + workspacePath()
+ ", shadedJarPath: "
+ shadedJarPath
+ ", pass: "
- + pass
+ + pass()
+ " }";
}
}
diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/SimpleBuildResponse.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/SimpleBuildResponse.java
index cee1412da..6711e85ff 100644
--- a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/SimpleBuildResponse.java
+++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/SimpleBuildResponse.java
@@ -18,44 +18,19 @@
package org.apache.streampark.flink.packer.pipeline;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonProperty;
@JsonIgnoreProperties(ignoreUnknown = true)
-public class SimpleBuildResponse implements FlinkBuildResult {
-
- private String workspacePath;
- private boolean pass = true;
+public class SimpleBuildResponse extends AbstractFlinkBuildResponse {
public SimpleBuildResponse() {
}
public SimpleBuildResponse(String workspacePath, boolean pass) {
- this.workspacePath = workspacePath;
- this.pass = pass;
- }
-
- @Override
- public String workspacePath() {
- return workspacePath;
- }
-
- @Override
- public boolean pass() {
- return pass;
- }
-
- @JsonProperty("workspacePath")
- public void setWorkspacePath(String workspacePath) {
- this.workspacePath = workspacePath;
- }
-
- @JsonProperty("pass")
- public void setPass(boolean pass) {
- this.pass = pass;
+ super(workspacePath, pass);
}
@Override
public String toString() {
- return "{ workspacePath: " + workspacePath + ", pass: " + pass + " }";
+ return "{ workspacePath: " + workspacePath() + ", pass: " + pass() + " }";
}
}