You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampark.apache.org by be...@apache.org on 2023/02/12 13:54:38 UTC

[incubator-streampark] branch release-2.0.0 updated: app status reset bug fixe

This is an automated email from the ASF dual-hosted git repository.

benjobs pushed a commit to branch release-2.0.0
in repository https://gitbox.apache.org/repos/asf/incubator-streampark.git


The following commit(s) were added to refs/heads/release-2.0.0 by this push:
     new 57687f2d4 app status reset bug fixe
57687f2d4 is described below

commit 57687f2d471b56fc6ae3feb36fee6786a7ab14d7
Author: benjobs <be...@apache.org>
AuthorDate: Sun Feb 12 21:54:20 2023 +0800

    app status reset bug fixe
---
 .../console/core/aspect/StreamParkAspect.java      | 13 +++-----
 .../console/core/controller/ProjectController.java |  1 -
 .../core/service/impl/AppBuildPipeServiceImpl.java | 10 ++++++
 .../core/service/impl/ApplicationServiceImpl.java  | 20 ++++++------
 .../core/service/impl/ProjectServiceImpl.java      | 30 ++++++++++++++++-
 .../console/core/task/FlinkRESTAPIWatcher.java     | 26 +++++++++++----
 .../console/core/task/ProjectBuildTask.java        | 38 ++++++++--------------
 .../resources/mapper/core/ApplicationMapper.xml    |  3 --
 8 files changed, 86 insertions(+), 55 deletions(-)

diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/aspect/StreamParkAspect.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/aspect/StreamParkAspect.java
index e0fafc942..964dd59d5 100644
--- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/aspect/StreamParkAspect.java
+++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/aspect/StreamParkAspect.java
@@ -48,17 +48,11 @@ public class StreamParkAspect {
       "execution(public"
           + " org.apache.streampark.console.base.domain.RestResponse"
           + " org.apache.streampark.console.*.controller.*.*(..))")
-  public void response() {}
-
-  @Pointcut("@annotation(org.apache.streampark.console.core.annotation.AppUpdated)")
-  public void appUpdated() {}
-
-  @Pointcut("@annotation(org.apache.streampark.console.core.annotation.ApiAccess)")
   public void apiAccess() {}
 
   @SuppressWarnings("checkstyle:SimplifyBooleanExpression")
-  @Around(value = "response()")
-  public RestResponse response(ProceedingJoinPoint joinPoint) throws Throwable {
+  @Around(value = "apiAccess()")
+  public RestResponse apiAccess(ProceedingJoinPoint joinPoint) throws Throwable {
     MethodSignature methodSignature = (MethodSignature) joinPoint.getSignature();
     log.debug("restResponse aspect, method:{}", methodSignature.getName());
     Boolean isApi =
@@ -72,6 +66,9 @@ public class StreamParkAspect {
     return (RestResponse) joinPoint.proceed();
   }
 
+  @Pointcut("@annotation(org.apache.streampark.console.core.annotation.AppUpdated)")
+  public void appUpdated() {}
+
   @Around("appUpdated()")
   public Object appUpdated(ProceedingJoinPoint joinPoint) throws Throwable {
     MethodSignature methodSignature = (MethodSignature) joinPoint.getSignature();
diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/controller/ProjectController.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/controller/ProjectController.java
index 46fd10066..3af92bdc8 100644
--- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/controller/ProjectController.java
+++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/controller/ProjectController.java
@@ -67,7 +67,6 @@ public class ProjectController {
     return RestResponse.success().data(projectService.getById(id));
   }
 
-  @AppUpdated
   @PostMapping("build")
   @RequiresPermissions("project:build")
   public RestResponse build(Long id) throws Exception {
diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/AppBuildPipeServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/AppBuildPipeServiceImpl.java
index 3f16ac791..7d9fc974d 100644
--- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/AppBuildPipeServiceImpl.java
+++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/AppBuildPipeServiceImpl.java
@@ -48,6 +48,7 @@ import org.apache.streampark.console.core.service.FlinkEnvService;
 import org.apache.streampark.console.core.service.FlinkSqlService;
 import org.apache.streampark.console.core.service.MessageService;
 import org.apache.streampark.console.core.service.SettingService;
+import org.apache.streampark.console.core.task.FlinkRESTAPIWatcher;
 import org.apache.streampark.flink.packer.docker.DockerConf;
 import org.apache.streampark.flink.packer.pipeline.BuildPipeline;
 import org.apache.streampark.flink.packer.pipeline.BuildResult;
@@ -114,6 +115,8 @@ public class AppBuildPipeServiceImpl
 
   @Autowired private ApplicationService applicationService;
 
+  @Autowired private FlinkRESTAPIWatcher flinkRESTAPIWatcher;
+
   @Autowired private ApplicationConfigService applicationConfigService;
 
   private final ExecutorService executorService =
@@ -165,6 +168,10 @@ public class AppBuildPipeServiceImpl
             app.setLaunch(LaunchState.LAUNCHING.get());
             applicationService.updateLaunch(app);
 
+            if (flinkRESTAPIWatcher.isWatchingApp(app.getId())) {
+              flinkRESTAPIWatcher.init();
+            }
+
             // 1) checkEnv
             applicationService.checkEnv(app);
 
@@ -272,6 +279,9 @@ public class AppBuildPipeServiceImpl
               app.setBuild(true);
             }
             applicationService.updateLaunch(app);
+            if (flinkRESTAPIWatcher.isWatchingApp(app.getId())) {
+              flinkRESTAPIWatcher.init();
+            }
           }
         });
     // save docker resolve progress detail to cache, only for flink-k8s application mode.
diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ApplicationServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ApplicationServiceImpl.java
index e0b563ac5..00a0ffd7a 100644
--- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ApplicationServiceImpl.java
+++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ApplicationServiceImpl.java
@@ -78,6 +78,12 @@ import org.apache.streampark.console.core.service.SavePointService;
 import org.apache.streampark.console.core.service.SettingService;
 import org.apache.streampark.console.core.service.VariableService;
 import org.apache.streampark.console.core.task.FlinkRESTAPIWatcher;
+import org.apache.streampark.flink.client.FlinkClient;
+import org.apache.streampark.flink.client.bean.CancelRequest;
+import org.apache.streampark.flink.client.bean.CancelResponse;
+import org.apache.streampark.flink.client.bean.KubernetesSubmitParam;
+import org.apache.streampark.flink.client.bean.SubmitRequest;
+import org.apache.streampark.flink.client.bean.SubmitResponse;
 import org.apache.streampark.flink.core.conf.ParameterCli;
 import org.apache.streampark.flink.kubernetes.FlinkK8sWatcher;
 import org.apache.streampark.flink.kubernetes.IngressController;
@@ -87,12 +93,6 @@ import org.apache.streampark.flink.kubernetes.model.TrackId;
 import org.apache.streampark.flink.packer.pipeline.BuildResult;
 import org.apache.streampark.flink.packer.pipeline.DockerImageBuildResponse;
 import org.apache.streampark.flink.packer.pipeline.ShadedBuildResponse;
-import org.apache.streampark.flink.submit.FlinkSubmitter;
-import org.apache.streampark.flink.submit.bean.CancelRequest;
-import org.apache.streampark.flink.submit.bean.CancelResponse;
-import org.apache.streampark.flink.submit.bean.KubernetesSubmitParam;
-import org.apache.streampark.flink.submit.bean.SubmitRequest;
-import org.apache.streampark.flink.submit.bean.SubmitResponse;
 
 import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
 import com.baomidou.mybatisplus.core.metadata.IPage;
@@ -1233,7 +1233,7 @@ public class ApplicationServiceImpl extends ServiceImpl<ApplicationMapper, Appli
             properties);
 
     CompletableFuture<CancelResponse> cancelFuture =
-        CompletableFuture.supplyAsync(() -> FlinkSubmitter.cancel(cancelRequest), executorService);
+        CompletableFuture.supplyAsync(() -> FlinkClient.cancel(cancelRequest), executorService);
 
     cancelFutureMap.put(application.getId(), cancelFuture);
 
@@ -1529,7 +1529,7 @@ public class ApplicationServiceImpl extends ServiceImpl<ApplicationMapper, Appli
             extraParameter);
 
     CompletableFuture<SubmitResponse> future =
-        CompletableFuture.supplyAsync(() -> FlinkSubmitter.submit(submitRequest), executorService);
+        CompletableFuture.supplyAsync(() -> FlinkClient.submit(submitRequest), executorService);
 
     startFutureMap.put(application.getId(), future);
 
@@ -1652,7 +1652,7 @@ public class ApplicationServiceImpl extends ServiceImpl<ApplicationMapper, Appli
     }
 
     Map<String, String> dynamicProperties =
-        FlinkSubmitter.extractDynamicPropertiesAsJava(application.getDynamicProperties());
+        FlinkClient.extractDynamicPropertiesAsJava(application.getDynamicProperties());
     properties.putAll(dynamicProperties);
     ResolveOrder resolveOrder = ResolveOrder.of(application.getResolveOrder());
     if (resolveOrder != null) {
@@ -1692,7 +1692,7 @@ public class ApplicationServiceImpl extends ServiceImpl<ApplicationMapper, Appli
 
     // 1) properties have the highest priority, read the properties are set: -Dstate.savepoints.dir
     String savepointPath =
-        FlinkSubmitter.extractDynamicPropertiesAsJava(application.getDynamicProperties())
+        FlinkClient.extractDynamicPropertiesAsJava(application.getDynamicProperties())
             .get(CheckpointingOptions.SAVEPOINT_DIRECTORY.key());
 
     // Application conf configuration has the second priority. If it is a streampark|flinksql type
diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java
index 5b0570a7f..5dfa124d4 100644
--- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java
+++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java
@@ -39,6 +39,7 @@ import org.apache.streampark.console.core.enums.LaunchState;
 import org.apache.streampark.console.core.mapper.ProjectMapper;
 import org.apache.streampark.console.core.service.ApplicationService;
 import org.apache.streampark.console.core.service.ProjectService;
+import org.apache.streampark.console.core.task.FlinkRESTAPIWatcher;
 import org.apache.streampark.console.core.task.ProjectBuildTask;
 
 import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
@@ -79,6 +80,8 @@ public class ProjectServiceImpl extends ServiceImpl<ProjectMapper, Project>
 
   @Autowired private ApplicationService applicationService;
 
+  @Autowired private FlinkRESTAPIWatcher flinkRESTAPIWatcher;
+
   private final ExecutorService executorService =
       new ThreadPoolExecutor(
           Runtime.getRuntime().availableProcessors() * 5,
@@ -195,8 +198,33 @@ public class ProjectServiceImpl extends ServiceImpl<ProjectMapper, Project>
   public void build(Long id) throws Exception {
     Project project = getById(id);
     this.baseMapper.updateBuildState(project.getId(), BuildState.BUILDING.get());
+    String logPath = getBuildLogPath(id);
     ProjectBuildTask projectBuildTask =
-        new ProjectBuildTask(getBuildLogPath(id), project, baseMapper, applicationService);
+        new ProjectBuildTask(
+            logPath,
+            project,
+            buildState -> {
+              baseMapper.updateBuildState(id, buildState.get());
+              if (buildState == BuildState.SUCCESSFUL) {
+                baseMapper.updateBuildTime(id);
+              }
+              flinkRESTAPIWatcher.init();
+            },
+            fileLogger -> {
+              List<Application> applications =
+                  this.applicationService.getByProjectId(project.getId());
+              applications.forEach(
+                  (app) -> {
+                    fileLogger.info(
+                        "update deploy by project: {}, appName:{}",
+                        project.getName(),
+                        app.getJobName());
+                    app.setLaunch(LaunchState.NEED_LAUNCH.get());
+                    app.setBuild(true);
+                    this.applicationService.updateLaunch(app);
+                  });
+              flinkRESTAPIWatcher.init();
+            });
     CompletableFuture<Void> buildTask =
         CompletableFuture.runAsync(projectBuildTask, executorService);
     // TODO May need to define parameters to set the build timeout in the future.
diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/task/FlinkRESTAPIWatcher.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/task/FlinkRESTAPIWatcher.java
index b9d6c0993..123e9b2de 100644
--- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/task/FlinkRESTAPIWatcher.java
+++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/task/FlinkRESTAPIWatcher.java
@@ -39,6 +39,7 @@ import org.apache.streampark.console.core.service.SavePointService;
 import org.apache.streampark.console.core.service.alert.AlertService;
 
 import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
+import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
 import com.github.benmanes.caffeine.cache.Cache;
 import com.github.benmanes.caffeine.cache.Caffeine;
 import lombok.extern.slf4j.Slf4j;
@@ -153,11 +154,11 @@ public class FlinkRESTAPIWatcher {
   @PostConstruct
   public void init() {
     WATCHING_APPS.clear();
-    LambdaQueryWrapper<Application> queryWrapper = new LambdaQueryWrapper<>();
-    queryWrapper
-        .eq(Application::getTracking, 1)
-        .notIn(Application::getExecutionMode, ExecutionMode.getKubernetesMode());
-    List<Application> applications = applicationService.list(queryWrapper);
+    List<Application> applications =
+        applicationService.list(
+            new LambdaQueryWrapper<Application>()
+                .eq(Application::getTracking, 1)
+                .notIn(Application::getExecutionMode, ExecutionMode.getKubernetesMode()));
     applications.forEach((app) -> WATCHING_APPS.put(app.getId(), app));
   }
 
@@ -378,15 +379,22 @@ public class FlinkRESTAPIWatcher {
      NEED_RESTART_AFTER_DEPLOY (Need to rollback after deploy)
     */
     if (OptionState.STARTING.equals(optionState)) {
-      switch (LaunchState.of(application.getLaunch())) {
+      Application latestApp = WATCHING_APPS.get(application.getId());
+      LaunchState launchState = latestApp.getLaunchState();
+      switch (launchState) {
         case NEED_RESTART:
         case NEED_ROLLBACK:
-          application.setLaunch(LaunchState.DONE.get());
+          LambdaUpdateWrapper<Application> updateWrapper =
+              new LambdaUpdateWrapper<Application>()
+                  .eq(Application::getId, application.getId())
+                  .set(Application::getLaunch, LaunchState.DONE.get());
+          applicationService.update(updateWrapper);
           break;
         default:
           break;
       }
     }
+
     // The current state is running, and there is a current task in the savePointCache,
     // indicating that the task is doing savepoint
     if (SAVEPOINT_CACHE.getIfPresent(application.getId()) != null) {
@@ -753,4 +761,8 @@ public class FlinkRESTAPIWatcher {
     }
     return JacksonUtils.read(result, clazz);
   }
+
+  public boolean isWatchingApp(Long id) {
+    return WATCHING_APPS.containsKey(id);
+  }
 }
diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/task/ProjectBuildTask.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/task/ProjectBuildTask.java
index be43cb61a..8b673fb5e 100644
--- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/task/ProjectBuildTask.java
+++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/task/ProjectBuildTask.java
@@ -20,13 +20,10 @@ package org.apache.streampark.console.core.task;
 import org.apache.streampark.common.util.CommandUtils;
 import org.apache.streampark.common.util.Utils;
 import org.apache.streampark.console.base.util.GitUtils;
-import org.apache.streampark.console.core.entity.Application;
 import org.apache.streampark.console.core.entity.Project;
 import org.apache.streampark.console.core.enums.BuildState;
-import org.apache.streampark.console.core.enums.LaunchState;
-import org.apache.streampark.console.core.mapper.ProjectMapper;
-import org.apache.streampark.console.core.service.ApplicationService;
 
+import ch.qos.logback.classic.Logger;
 import lombok.extern.slf4j.Slf4j;
 import org.eclipse.jgit.api.Git;
 import org.eclipse.jgit.lib.StoredConfig;
@@ -36,25 +33,26 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
+import java.util.function.Consumer;
 
 @Slf4j
 public class ProjectBuildTask extends AbstractLogFileTask {
 
   final Project project;
 
-  final ProjectMapper baseMapper;
+  final Consumer<BuildState> stateUpdateConsumer;
 
-  final ApplicationService applicationService;
+  final Consumer<Logger> notifyLaunchConsumer;
 
   public ProjectBuildTask(
       String logPath,
       Project project,
-      ProjectMapper baseMapper,
-      ApplicationService applicationService) {
+      Consumer<BuildState> stateUpdateConsumer,
+      Consumer<Logger> notifyLaunchConsumer) {
     super(logPath, true);
     this.project = project;
-    this.baseMapper = baseMapper;
-    this.applicationService = applicationService;
+    this.stateUpdateConsumer = stateUpdateConsumer;
+    this.notifyLaunchConsumer = notifyLaunchConsumer;
   }
 
   @Override
@@ -64,33 +62,23 @@ public class ProjectBuildTask extends AbstractLogFileTask {
     boolean cloneSuccess = cloneSourceCode(project);
     if (!cloneSuccess) {
       fileLogger.error("[StreamPark] clone or pull error.");
-      this.baseMapper.updateBuildState(project.getId(), BuildState.FAILED.get());
+      stateUpdateConsumer.accept(BuildState.FAILED);
       return;
     }
     boolean build = projectBuild(project);
     if (!build) {
-      this.baseMapper.updateBuildState(project.getId(), BuildState.FAILED.get());
+      stateUpdateConsumer.accept(BuildState.FAILED);
       fileLogger.error("build error, project name: {} ", project.getName());
       return;
     }
-    this.baseMapper.updateBuildState(project.getId(), BuildState.SUCCESSFUL.get());
-    this.baseMapper.updateBuildTime(project.getId());
+    stateUpdateConsumer.accept(BuildState.SUCCESSFUL);
     this.deploy(project);
-    List<Application> applications = this.applicationService.getByProjectId(project.getId());
-
-    applications.forEach(
-        (app) -> {
-          fileLogger.info(
-              "update deploy by project: {}, appName:{}", project.getName(), app.getJobName());
-          app.setLaunch(LaunchState.NEED_LAUNCH.get());
-          app.setBuild(true);
-          this.applicationService.updateLaunch(app);
-        });
+    notifyLaunchConsumer.accept(fileLogger);
   }
 
   @Override
   protected void processException(Throwable t) {
-    this.baseMapper.updateBuildState(project.getId(), BuildState.FAILED.get());
+    stateUpdateConsumer.accept(BuildState.FAILED);
     fileLogger.error("Build error, project name: {}", project.getName(), t);
   }
 
diff --git a/streampark-console/streampark-console-service/src/main/resources/mapper/core/ApplicationMapper.xml b/streampark-console/streampark-console-service/src/main/resources/mapper/core/ApplicationMapper.xml
index 6e7d516e2..70d19275b 100644
--- a/streampark-console/streampark-console-service/src/main/resources/mapper/core/ApplicationMapper.xml
+++ b/streampark-console/streampark-console-service/src/main/resources/mapper/core/ApplicationMapper.xml
@@ -191,9 +191,6 @@
             <if test="application.tracking != null">
                 tracking=#{application.tracking},
             </if>
-            <if test="application.launch != null">
-                launch=#{application.launch},
-            </if>
             <if test="application.optionState != null">
                 option_state=#{application.optionState},
             </if>