You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampark.apache.org by be...@apache.org on 2023/02/12 13:54:38 UTC
[incubator-streampark] branch release-2.0.0 updated: app status reset bug fixe
This is an automated email from the ASF dual-hosted git repository.
benjobs pushed a commit to branch release-2.0.0
in repository https://gitbox.apache.org/repos/asf/incubator-streampark.git
The following commit(s) were added to refs/heads/release-2.0.0 by this push:
new 57687f2d4 app status reset bug fixe
57687f2d4 is described below
commit 57687f2d471b56fc6ae3feb36fee6786a7ab14d7
Author: benjobs <be...@apache.org>
AuthorDate: Sun Feb 12 21:54:20 2023 +0800
app status reset bug fixe
---
.../console/core/aspect/StreamParkAspect.java | 13 +++-----
.../console/core/controller/ProjectController.java | 1 -
.../core/service/impl/AppBuildPipeServiceImpl.java | 10 ++++++
.../core/service/impl/ApplicationServiceImpl.java | 20 ++++++------
.../core/service/impl/ProjectServiceImpl.java | 30 ++++++++++++++++-
.../console/core/task/FlinkRESTAPIWatcher.java | 26 +++++++++++----
.../console/core/task/ProjectBuildTask.java | 38 ++++++++--------------
.../resources/mapper/core/ApplicationMapper.xml | 3 --
8 files changed, 86 insertions(+), 55 deletions(-)
diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/aspect/StreamParkAspect.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/aspect/StreamParkAspect.java
index e0fafc942..964dd59d5 100644
--- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/aspect/StreamParkAspect.java
+++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/aspect/StreamParkAspect.java
@@ -48,17 +48,11 @@ public class StreamParkAspect {
"execution(public"
+ " org.apache.streampark.console.base.domain.RestResponse"
+ " org.apache.streampark.console.*.controller.*.*(..))")
- public void response() {}
-
- @Pointcut("@annotation(org.apache.streampark.console.core.annotation.AppUpdated)")
- public void appUpdated() {}
-
- @Pointcut("@annotation(org.apache.streampark.console.core.annotation.ApiAccess)")
public void apiAccess() {}
@SuppressWarnings("checkstyle:SimplifyBooleanExpression")
- @Around(value = "response()")
- public RestResponse response(ProceedingJoinPoint joinPoint) throws Throwable {
+ @Around(value = "apiAccess()")
+ public RestResponse apiAccess(ProceedingJoinPoint joinPoint) throws Throwable {
MethodSignature methodSignature = (MethodSignature) joinPoint.getSignature();
log.debug("restResponse aspect, method:{}", methodSignature.getName());
Boolean isApi =
@@ -72,6 +66,9 @@ public class StreamParkAspect {
return (RestResponse) joinPoint.proceed();
}
+ @Pointcut("@annotation(org.apache.streampark.console.core.annotation.AppUpdated)")
+ public void appUpdated() {}
+
@Around("appUpdated()")
public Object appUpdated(ProceedingJoinPoint joinPoint) throws Throwable {
MethodSignature methodSignature = (MethodSignature) joinPoint.getSignature();
diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/controller/ProjectController.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/controller/ProjectController.java
index 46fd10066..3af92bdc8 100644
--- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/controller/ProjectController.java
+++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/controller/ProjectController.java
@@ -67,7 +67,6 @@ public class ProjectController {
return RestResponse.success().data(projectService.getById(id));
}
- @AppUpdated
@PostMapping("build")
@RequiresPermissions("project:build")
public RestResponse build(Long id) throws Exception {
diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/AppBuildPipeServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/AppBuildPipeServiceImpl.java
index 3f16ac791..7d9fc974d 100644
--- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/AppBuildPipeServiceImpl.java
+++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/AppBuildPipeServiceImpl.java
@@ -48,6 +48,7 @@ import org.apache.streampark.console.core.service.FlinkEnvService;
import org.apache.streampark.console.core.service.FlinkSqlService;
import org.apache.streampark.console.core.service.MessageService;
import org.apache.streampark.console.core.service.SettingService;
+import org.apache.streampark.console.core.task.FlinkRESTAPIWatcher;
import org.apache.streampark.flink.packer.docker.DockerConf;
import org.apache.streampark.flink.packer.pipeline.BuildPipeline;
import org.apache.streampark.flink.packer.pipeline.BuildResult;
@@ -114,6 +115,8 @@ public class AppBuildPipeServiceImpl
@Autowired private ApplicationService applicationService;
+ @Autowired private FlinkRESTAPIWatcher flinkRESTAPIWatcher;
+
@Autowired private ApplicationConfigService applicationConfigService;
private final ExecutorService executorService =
@@ -165,6 +168,10 @@ public class AppBuildPipeServiceImpl
app.setLaunch(LaunchState.LAUNCHING.get());
applicationService.updateLaunch(app);
+ if (flinkRESTAPIWatcher.isWatchingApp(app.getId())) {
+ flinkRESTAPIWatcher.init();
+ }
+
// 1) checkEnv
applicationService.checkEnv(app);
@@ -272,6 +279,9 @@ public class AppBuildPipeServiceImpl
app.setBuild(true);
}
applicationService.updateLaunch(app);
+ if (flinkRESTAPIWatcher.isWatchingApp(app.getId())) {
+ flinkRESTAPIWatcher.init();
+ }
}
});
// save docker resolve progress detail to cache, only for flink-k8s application mode.
diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ApplicationServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ApplicationServiceImpl.java
index e0b563ac5..00a0ffd7a 100644
--- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ApplicationServiceImpl.java
+++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ApplicationServiceImpl.java
@@ -78,6 +78,12 @@ import org.apache.streampark.console.core.service.SavePointService;
import org.apache.streampark.console.core.service.SettingService;
import org.apache.streampark.console.core.service.VariableService;
import org.apache.streampark.console.core.task.FlinkRESTAPIWatcher;
+import org.apache.streampark.flink.client.FlinkClient;
+import org.apache.streampark.flink.client.bean.CancelRequest;
+import org.apache.streampark.flink.client.bean.CancelResponse;
+import org.apache.streampark.flink.client.bean.KubernetesSubmitParam;
+import org.apache.streampark.flink.client.bean.SubmitRequest;
+import org.apache.streampark.flink.client.bean.SubmitResponse;
import org.apache.streampark.flink.core.conf.ParameterCli;
import org.apache.streampark.flink.kubernetes.FlinkK8sWatcher;
import org.apache.streampark.flink.kubernetes.IngressController;
@@ -87,12 +93,6 @@ import org.apache.streampark.flink.kubernetes.model.TrackId;
import org.apache.streampark.flink.packer.pipeline.BuildResult;
import org.apache.streampark.flink.packer.pipeline.DockerImageBuildResponse;
import org.apache.streampark.flink.packer.pipeline.ShadedBuildResponse;
-import org.apache.streampark.flink.submit.FlinkSubmitter;
-import org.apache.streampark.flink.submit.bean.CancelRequest;
-import org.apache.streampark.flink.submit.bean.CancelResponse;
-import org.apache.streampark.flink.submit.bean.KubernetesSubmitParam;
-import org.apache.streampark.flink.submit.bean.SubmitRequest;
-import org.apache.streampark.flink.submit.bean.SubmitResponse;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
@@ -1233,7 +1233,7 @@ public class ApplicationServiceImpl extends ServiceImpl<ApplicationMapper, Appli
properties);
CompletableFuture<CancelResponse> cancelFuture =
- CompletableFuture.supplyAsync(() -> FlinkSubmitter.cancel(cancelRequest), executorService);
+ CompletableFuture.supplyAsync(() -> FlinkClient.cancel(cancelRequest), executorService);
cancelFutureMap.put(application.getId(), cancelFuture);
@@ -1529,7 +1529,7 @@ public class ApplicationServiceImpl extends ServiceImpl<ApplicationMapper, Appli
extraParameter);
CompletableFuture<SubmitResponse> future =
- CompletableFuture.supplyAsync(() -> FlinkSubmitter.submit(submitRequest), executorService);
+ CompletableFuture.supplyAsync(() -> FlinkClient.submit(submitRequest), executorService);
startFutureMap.put(application.getId(), future);
@@ -1652,7 +1652,7 @@ public class ApplicationServiceImpl extends ServiceImpl<ApplicationMapper, Appli
}
Map<String, String> dynamicProperties =
- FlinkSubmitter.extractDynamicPropertiesAsJava(application.getDynamicProperties());
+ FlinkClient.extractDynamicPropertiesAsJava(application.getDynamicProperties());
properties.putAll(dynamicProperties);
ResolveOrder resolveOrder = ResolveOrder.of(application.getResolveOrder());
if (resolveOrder != null) {
@@ -1692,7 +1692,7 @@ public class ApplicationServiceImpl extends ServiceImpl<ApplicationMapper, Appli
// 1) properties have the highest priority, read the properties are set: -Dstate.savepoints.dir
String savepointPath =
- FlinkSubmitter.extractDynamicPropertiesAsJava(application.getDynamicProperties())
+ FlinkClient.extractDynamicPropertiesAsJava(application.getDynamicProperties())
.get(CheckpointingOptions.SAVEPOINT_DIRECTORY.key());
// Application conf configuration has the second priority. If it is a streampark|flinksql type
diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java
index 5b0570a7f..5dfa124d4 100644
--- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java
+++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java
@@ -39,6 +39,7 @@ import org.apache.streampark.console.core.enums.LaunchState;
import org.apache.streampark.console.core.mapper.ProjectMapper;
import org.apache.streampark.console.core.service.ApplicationService;
import org.apache.streampark.console.core.service.ProjectService;
+import org.apache.streampark.console.core.task.FlinkRESTAPIWatcher;
import org.apache.streampark.console.core.task.ProjectBuildTask;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
@@ -79,6 +80,8 @@ public class ProjectServiceImpl extends ServiceImpl<ProjectMapper, Project>
@Autowired private ApplicationService applicationService;
+ @Autowired private FlinkRESTAPIWatcher flinkRESTAPIWatcher;
+
private final ExecutorService executorService =
new ThreadPoolExecutor(
Runtime.getRuntime().availableProcessors() * 5,
@@ -195,8 +198,33 @@ public class ProjectServiceImpl extends ServiceImpl<ProjectMapper, Project>
public void build(Long id) throws Exception {
Project project = getById(id);
this.baseMapper.updateBuildState(project.getId(), BuildState.BUILDING.get());
+ String logPath = getBuildLogPath(id);
ProjectBuildTask projectBuildTask =
- new ProjectBuildTask(getBuildLogPath(id), project, baseMapper, applicationService);
+ new ProjectBuildTask(
+ logPath,
+ project,
+ buildState -> {
+ baseMapper.updateBuildState(id, buildState.get());
+ if (buildState == BuildState.SUCCESSFUL) {
+ baseMapper.updateBuildTime(id);
+ }
+ flinkRESTAPIWatcher.init();
+ },
+ fileLogger -> {
+ List<Application> applications =
+ this.applicationService.getByProjectId(project.getId());
+ applications.forEach(
+ (app) -> {
+ fileLogger.info(
+ "update deploy by project: {}, appName:{}",
+ project.getName(),
+ app.getJobName());
+ app.setLaunch(LaunchState.NEED_LAUNCH.get());
+ app.setBuild(true);
+ this.applicationService.updateLaunch(app);
+ });
+ flinkRESTAPIWatcher.init();
+ });
CompletableFuture<Void> buildTask =
CompletableFuture.runAsync(projectBuildTask, executorService);
// TODO May need to define parameters to set the build timeout in the future.
diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/task/FlinkRESTAPIWatcher.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/task/FlinkRESTAPIWatcher.java
index b9d6c0993..123e9b2de 100644
--- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/task/FlinkRESTAPIWatcher.java
+++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/task/FlinkRESTAPIWatcher.java
@@ -39,6 +39,7 @@ import org.apache.streampark.console.core.service.SavePointService;
import org.apache.streampark.console.core.service.alert.AlertService;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
+import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import lombok.extern.slf4j.Slf4j;
@@ -153,11 +154,11 @@ public class FlinkRESTAPIWatcher {
@PostConstruct
public void init() {
WATCHING_APPS.clear();
- LambdaQueryWrapper<Application> queryWrapper = new LambdaQueryWrapper<>();
- queryWrapper
- .eq(Application::getTracking, 1)
- .notIn(Application::getExecutionMode, ExecutionMode.getKubernetesMode());
- List<Application> applications = applicationService.list(queryWrapper);
+ List<Application> applications =
+ applicationService.list(
+ new LambdaQueryWrapper<Application>()
+ .eq(Application::getTracking, 1)
+ .notIn(Application::getExecutionMode, ExecutionMode.getKubernetesMode()));
applications.forEach((app) -> WATCHING_APPS.put(app.getId(), app));
}
@@ -378,15 +379,22 @@ public class FlinkRESTAPIWatcher {
NEED_RESTART_AFTER_DEPLOY (Need to rollback after deploy)
*/
if (OptionState.STARTING.equals(optionState)) {
- switch (LaunchState.of(application.getLaunch())) {
+ Application latestApp = WATCHING_APPS.get(application.getId());
+ LaunchState launchState = latestApp.getLaunchState();
+ switch (launchState) {
case NEED_RESTART:
case NEED_ROLLBACK:
- application.setLaunch(LaunchState.DONE.get());
+ LambdaUpdateWrapper<Application> updateWrapper =
+ new LambdaUpdateWrapper<Application>()
+ .eq(Application::getId, application.getId())
+ .set(Application::getLaunch, LaunchState.DONE.get());
+ applicationService.update(updateWrapper);
break;
default:
break;
}
}
+
// The current state is running, and there is a current task in the savePointCache,
// indicating that the task is doing savepoint
if (SAVEPOINT_CACHE.getIfPresent(application.getId()) != null) {
@@ -753,4 +761,8 @@ public class FlinkRESTAPIWatcher {
}
return JacksonUtils.read(result, clazz);
}
+
+ public boolean isWatchingApp(Long id) {
+ return WATCHING_APPS.containsKey(id);
+ }
}
diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/task/ProjectBuildTask.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/task/ProjectBuildTask.java
index be43cb61a..8b673fb5e 100644
--- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/task/ProjectBuildTask.java
+++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/task/ProjectBuildTask.java
@@ -20,13 +20,10 @@ package org.apache.streampark.console.core.task;
import org.apache.streampark.common.util.CommandUtils;
import org.apache.streampark.common.util.Utils;
import org.apache.streampark.console.base.util.GitUtils;
-import org.apache.streampark.console.core.entity.Application;
import org.apache.streampark.console.core.entity.Project;
import org.apache.streampark.console.core.enums.BuildState;
-import org.apache.streampark.console.core.enums.LaunchState;
-import org.apache.streampark.console.core.mapper.ProjectMapper;
-import org.apache.streampark.console.core.service.ApplicationService;
+import ch.qos.logback.classic.Logger;
import lombok.extern.slf4j.Slf4j;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.lib.StoredConfig;
@@ -36,25 +33,26 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
+import java.util.function.Consumer;
@Slf4j
public class ProjectBuildTask extends AbstractLogFileTask {
final Project project;
- final ProjectMapper baseMapper;
+ final Consumer<BuildState> stateUpdateConsumer;
- final ApplicationService applicationService;
+ final Consumer<Logger> notifyLaunchConsumer;
public ProjectBuildTask(
String logPath,
Project project,
- ProjectMapper baseMapper,
- ApplicationService applicationService) {
+ Consumer<BuildState> stateUpdateConsumer,
+ Consumer<Logger> notifyLaunchConsumer) {
super(logPath, true);
this.project = project;
- this.baseMapper = baseMapper;
- this.applicationService = applicationService;
+ this.stateUpdateConsumer = stateUpdateConsumer;
+ this.notifyLaunchConsumer = notifyLaunchConsumer;
}
@Override
@@ -64,33 +62,23 @@ public class ProjectBuildTask extends AbstractLogFileTask {
boolean cloneSuccess = cloneSourceCode(project);
if (!cloneSuccess) {
fileLogger.error("[StreamPark] clone or pull error.");
- this.baseMapper.updateBuildState(project.getId(), BuildState.FAILED.get());
+ stateUpdateConsumer.accept(BuildState.FAILED);
return;
}
boolean build = projectBuild(project);
if (!build) {
- this.baseMapper.updateBuildState(project.getId(), BuildState.FAILED.get());
+ stateUpdateConsumer.accept(BuildState.FAILED);
fileLogger.error("build error, project name: {} ", project.getName());
return;
}
- this.baseMapper.updateBuildState(project.getId(), BuildState.SUCCESSFUL.get());
- this.baseMapper.updateBuildTime(project.getId());
+ stateUpdateConsumer.accept(BuildState.SUCCESSFUL);
this.deploy(project);
- List<Application> applications = this.applicationService.getByProjectId(project.getId());
-
- applications.forEach(
- (app) -> {
- fileLogger.info(
- "update deploy by project: {}, appName:{}", project.getName(), app.getJobName());
- app.setLaunch(LaunchState.NEED_LAUNCH.get());
- app.setBuild(true);
- this.applicationService.updateLaunch(app);
- });
+ notifyLaunchConsumer.accept(fileLogger);
}
@Override
protected void processException(Throwable t) {
- this.baseMapper.updateBuildState(project.getId(), BuildState.FAILED.get());
+ stateUpdateConsumer.accept(BuildState.FAILED);
fileLogger.error("Build error, project name: {}", project.getName(), t);
}
diff --git a/streampark-console/streampark-console-service/src/main/resources/mapper/core/ApplicationMapper.xml b/streampark-console/streampark-console-service/src/main/resources/mapper/core/ApplicationMapper.xml
index 6e7d516e2..70d19275b 100644
--- a/streampark-console/streampark-console-service/src/main/resources/mapper/core/ApplicationMapper.xml
+++ b/streampark-console/streampark-console-service/src/main/resources/mapper/core/ApplicationMapper.xml
@@ -191,9 +191,6 @@
<if test="application.tracking != null">
tracking=#{application.tracking},
</if>
- <if test="application.launch != null">
- launch=#{application.launch},
- </if>
<if test="application.optionState != null">
option_state=#{application.optionState},
</if>