You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by ga...@apache.org on 2023/06/14 11:33:33 UTC
[seatunnel-web] branch add_canvas_job_define updated: [Feature][API] Add Job Defile API (#59)
This is an automated email from the ASF dual-hosted git repository.
gaojun2048 pushed a commit to branch add_canvas_job_define
in repository https://gitbox.apache.org/repos/asf/seatunnel-web.git
The following commit(s) were added to refs/heads/add_canvas_job_define by this push:
new 374c4007 [Feature][API] Add Job Defile API (#59)
374c4007 is described below
commit 374c400726f599059a795159538d8e6e4d22bf0f
Author: xiaoqigh <17...@163.com>
AuthorDate: Wed Jun 14 19:33:28 2023 +0800
[Feature][API] Add Job Defile API (#59)
---
pom.xml | 6 +
seatunnel-server/seatunnel-app/pom.xml | 5 +
.../apache/seatunnel/app/SeatunnelApplication.java | 2 +-
.../seatunnel/app/bean/engine/EngineDataType.java | 70 +++
.../apache/seatunnel/app/bean/env/JobEnvCache.java | 52 ++
.../app/controller/ConnectorController.java | 82 +++
.../seatunnel/app/controller/EngineController.java | 56 ++
.../seatunnel/app/controller/EnvController.java | 42 ++
.../app/controller/JobConfigController.java | 62 ++
.../app/controller/JobDefinitionController.java | 85 +++
.../app/controller/JobTaskController.java | 88 +++
.../controller/SeatunnelDatasourceController.java | 34 +-
.../seatunnel/app/dal/dao/IJobDefinitionDao.java | 7 +-
.../dao/IJobInstanceDao.java} | 26 +-
.../dao/IJobInstanceHistoryDao.java} | 14 +-
.../Transform.java => dal/dao/IJobLineDao.java} | 19 +-
.../Transform.java => dal/dao/IJobMetricsDao.java} | 19 +-
.../app/dal/dao/impl/DatasourceDaoImpl.java | 5 +-
.../app/dal/dao/impl/JobDefinitionDaoImpl.java | 21 +-
.../app/dal/dao/impl/JobInstanceDaoImpl.java | 61 ++
.../dal/dao/impl/JobInstanceHistoryDaoImpl.java | 46 ++
.../seatunnel/app/dal/dao/impl/JobLineDaoImpl.java | 52 ++
.../JobMetricsDaoImpl.java} | 33 +-
.../app/dal/dao/impl/TaskDefinitionDaoImpl.java | 4 +-
.../seatunnel/app/dal/entity/JobDefinition.java | 5 +-
.../{JobDefinition.java => JobInstance.java} | 27 +-
...{JobDefinition.java => JobInstanceHistory.java} | 24 +-
.../entity/{JobDefinition.java => JobLine.java} | 25 +-
.../entity/{JobDefinition.java => JobMetrics.java} | 40 +-
.../mapper/JobInstanceHistoryMapper.java} | 14 +-
.../mapper/JobInstanceMapper.java} | 14 +-
.../mapper/JobLineMapper.java} | 23 +-
.../apache/seatunnel/app/dal/mapper/JobMapper.java | 8 +-
.../mapper/JobMetricsMapper.java} | 20 +-
.../DatasourceCheckReq.java} | 14 +-
.../DatasourceReq.java} | 16 +-
.../Transform.java => DatabaseTableSchemaReq.java} | 24 +-
.../job/{transform/Transform.java => Edge.java} | 20 +-
.../job/{transform/Transform.java => JobDAG.java} | 16 +-
.../{transform/Transform.java => JobTaskInfo.java} | 22 +-
.../request/job/PluginConfig.java} | 49 +-
.../domain/request/job/transform/Transform.java | 5 +-
.../Transform.java => response/engine/Engine.java} | 15 +-
.../executor/JobExecutorRes.java} | 29 +-
.../Transform.java => response/metrics/Edge.java} | 18 +-
.../metrics/JobDAG.java} | 25 +-
.../metrics/JobPipelineDetailMetricsRes.java} | 37 +-
.../metrics/JobPipelineSummaryMetricsRes.java} | 20 +-
.../metrics/JobSummaryMetricsRes.java} | 22 +-
.../metrics/SinkMetricsInfo.java} | 14 +-
.../metrics/SourceMetricsInfo.java} | 14 +-
.../metrics/VertexInfo.java} | 21 +-
.../Transform.java => service/IEngineService.java} | 17 +-
.../IJobConfigService.java} | 17 +-
.../app/service/IJobDefinitionService.java | 9 +-
.../Transform.java => service/IJobEnvService.java} | 11 +-
.../IJobInstanceService.java} | 28 +-
.../seatunnel/app/service/IJobMetricsService.java | 67 ++
.../seatunnel/app/service/IJobTaskService.java | 39 ++
.../app/service/impl/EngineServiceImpl.java | 75 +++
.../app/service/impl/JobConfigServiceImpl.java | 104 +++
.../app/service/impl/JobDefinitionServiceImpl.java | 31 +-
.../app/service/impl/JobEnvServiceImpl.java | 41 ++
.../app/service/impl/JobInstanceServiceImpl.java | 697 +++++++++++++++++++++
.../app/service/impl/JobMetricsServiceImpl.java | 346 ++++++++++
.../app/service/impl/JobTaskServiceImpl.java | 593 ++++++++++++++++++
.../engine/SeaTunnelEngineMetricsExtractor.java | 284 +++++++++
.../thirdparty/engine/SeaTunnelEngineProxy.java | 87 +++
.../metrics/EngineMetricsExtractorFactory.java | 41 ++
.../metrics/IEngineMetricsExtractor.java} | 27 +-
.../app/utils/GlobalExceptionHandler.java | 28 +-
.../seatunnel/app/utils/SeaTunnelConfigUtil.java | 44 ++
.../seatunnel/app/utils/TaskOptionUtils.java | 67 ++
.../seatunnel/app/dal/mapper/DataSourceMapper.xml | 4 +-
.../seatunnel/app/dal/mapper/JobInstanceMapper.xml | 44 ++
.../seatunnel/app/dal/mapper/JobLineMapper.xml | 42 ++
.../apache/seatunnel/app/dal/mapper/JobMapper.xml | 17 +-
.../seatunnel/app/dal/mapper/JobMetricsMapper.xml | 79 +++
.../app/dal/mapper/ProcessTaskRelationMapper.xml | 1 +
.../main/resources/script/seatunnel_server_h2.sql | 89 ++-
.../resources/script/seatunnel_server_mysql.sql | 140 ++---
.../server/common/SeatunnelErrorEnum.java | 1 +
seatunnel-web-dist/release-docs/LICENSE | 1 +
tools/dependencies/known-dependencies.txt | 1 +
84 files changed, 4051 insertions(+), 493 deletions(-)
diff --git a/pom.xml b/pom.xml
index adde6b03..1ee1df59 100644
--- a/pom.xml
+++ b/pom.xml
@@ -524,6 +524,12 @@
<artifactId>jsr305</artifactId>
<version>${jsr305.version}</version>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-engine-client</artifactId>
+ <version>${seatunnel-framework.version}</version>
+ </dependency>
</dependencies>
</dependencyManagement>
diff --git a/seatunnel-server/seatunnel-app/pom.xml b/seatunnel-server/seatunnel-app/pom.xml
index 9580bb8a..d6aa95be 100644
--- a/seatunnel-server/seatunnel-app/pom.xml
+++ b/seatunnel-server/seatunnel-app/pom.xml
@@ -256,5 +256,10 @@
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-engine-client</artifactId>
+ </dependency>
</dependencies>
</project>
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/SeatunnelApplication.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/SeatunnelApplication.java
index 96716789..791d12b4 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/SeatunnelApplication.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/SeatunnelApplication.java
@@ -31,7 +31,7 @@ import org.springframework.transaction.annotation.EnableTransactionManagement;
@EnableConfigurationProperties
@EnableScheduling
@EnableAsync(proxyTargetClass = true)
-@MapperScan({"org.apache.seatunnel.app.dal"})
+@MapperScan({"org.apache.seatunnel.app.dal.mapper"})
public class SeatunnelApplication {
public static void main(String[] args) {
SpringApplication.run(SeatunnelApplication.class, args);
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/bean/engine/EngineDataType.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/bean/engine/EngineDataType.java
new file mode 100644
index 00000000..214f8139
--- /dev/null
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/bean/engine/EngineDataType.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.seatunnel.app.bean.engine;
+
+import org.apache.seatunnel.api.table.type.ArrayType;
+import org.apache.seatunnel.api.table.type.BasicType;
+import org.apache.seatunnel.api.table.type.DecimalType;
+import org.apache.seatunnel.api.table.type.LocalTimeType;
+import org.apache.seatunnel.api.table.type.PrimitiveByteArrayType;
+import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+public class EngineDataType {
+
+ public static DataType T_STRING = new DataType("string", BasicType.STRING_TYPE);
+ public static DataType T_BOOLEAN = new DataType("boolean", BasicType.BOOLEAN_TYPE);
+ public static DataType T_BYTE = new DataType("tinyint", BasicType.BYTE_TYPE);
+ public static DataType T_SHORT = new DataType("smallint", BasicType.SHORT_TYPE);
+ public static DataType T_INT = new DataType("int", BasicType.INT_TYPE);
+ public static DataType T_LONG = new DataType("bigint", BasicType.LONG_TYPE);
+ public static DataType T_FLOAT = new DataType("float", BasicType.FLOAT_TYPE);
+ public static DataType T_DOUBLE = new DataType("double", BasicType.DOUBLE_TYPE);
+ public static DataType T_VOID = new DataType("null", BasicType.VOID_TYPE);
+
+ public static DataType T_DECIMAL = new DataType("decimal(38, 18)", new DecimalType(38, 18));
+
+ public static DataType T_LOCAL_DATE = new DataType("date", LocalTimeType.LOCAL_DATE_TYPE);
+ public static DataType T_LOCAL_TIME = new DataType("time", LocalTimeType.LOCAL_TIME_TYPE);
+ public static DataType T_LOCAL_DATE_TIME =
+ new DataType("timestamp", LocalTimeType.LOCAL_DATE_TIME_TYPE);
+
+ public static DataType T_PRIMITIVE_BYTE_ARRAY =
+ new DataType("bytes", PrimitiveByteArrayType.INSTANCE);
+
+ public static DataType T_STRING_ARRAY =
+ new DataType("array<string>", ArrayType.STRING_ARRAY_TYPE);
+ public static DataType T_BOOLEAN_ARRAY =
+ new DataType("array<boolean>", ArrayType.BOOLEAN_ARRAY_TYPE);
+ public static DataType T_BYTE_ARRAY = new DataType("array<tinyint>", ArrayType.BYTE_ARRAY_TYPE);
+ public static DataType T_SHORT_ARRAY =
+ new DataType("array<smallint>", ArrayType.SHORT_ARRAY_TYPE);
+ public static DataType T_INT_ARRAY = new DataType("array<int>", ArrayType.INT_ARRAY_TYPE);
+ public static DataType T_LONG_ARRAY = new DataType("array<bigint>", ArrayType.LONG_ARRAY_TYPE);
+ public static DataType T_FLOAT_ARRAY = new DataType("array<float>", ArrayType.FLOAT_ARRAY_TYPE);
+ public static DataType T_DOUBLE_ARRAY =
+ new DataType("array<double>", ArrayType.DOUBLE_ARRAY_TYPE);
+
+ @Data
+ @AllArgsConstructor
+ public static class DataType {
+ String name;
+ SeaTunnelDataType<?> RawType;
+ }
+}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/bean/env/JobEnvCache.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/bean/env/JobEnvCache.java
new file mode 100644
index 00000000..c488c0e0
--- /dev/null
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/bean/env/JobEnvCache.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.seatunnel.app.bean.env;
+
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.api.env.EnvOptionRule;
+import org.apache.seatunnel.app.dynamicforms.AbstractFormOption;
+import org.apache.seatunnel.app.dynamicforms.FormStructure;
+import org.apache.seatunnel.app.thirdparty.framework.SeaTunnelOptionRuleWrapper;
+
+import org.springframework.stereotype.Component;
+
+import lombok.Data;
+import lombok.Getter;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+@Component
+@Data
+public class JobEnvCache {
+
+ @Getter private final FormStructure envFormStructure;
+
+ public JobEnvCache() {
+ OptionRule envOptionRules = EnvOptionRule.getEnvOptionRules();
+ envFormStructure =
+ SeaTunnelOptionRuleWrapper.wrapper(
+ envOptionRules.getOptionalOptions(),
+ envOptionRules.getRequiredOptions(),
+ "Env");
+ List<AbstractFormOption> collect =
+ envFormStructure.getForms().stream()
+ .filter(form -> !"parallelism".equalsIgnoreCase(form.getField()))
+ .collect(Collectors.toList());
+ envFormStructure.setForms(collect);
+ }
+}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/ConnectorController.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/ConnectorController.java
new file mode 100644
index 00000000..c05d8545
--- /dev/null
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/ConnectorController.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.app.controller;
+
+import org.apache.seatunnel.app.common.Result;
+import org.apache.seatunnel.app.domain.request.connector.ConnectorStatus;
+import org.apache.seatunnel.app.domain.response.connector.ConnectorInfo;
+import org.apache.seatunnel.app.service.IConnectorService;
+import org.apache.seatunnel.common.utils.JsonUtils;
+
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+import io.swagger.annotations.ApiOperation;
+import io.swagger.annotations.ApiParam;
+
+import javax.annotation.Resource;
+
+import java.io.IOException;
+import java.util.List;
+
+@RequestMapping("/whaletunnel/api/v1/connector")
+@RestController
+public class ConnectorController {
+
+ @Resource private IConnectorService connectorService;
+
+ @GetMapping("/sources")
+ @ApiOperation(value = "list all source connector", httpMethod = "GET")
+ public Result<List<ConnectorInfo>> listSource(
+ @RequestParam(defaultValue = "ALL") ConnectorStatus status) {
+ return Result.success(connectorService.listSources(status));
+ }
+
+ @GetMapping("/transforms")
+ @ApiOperation(value = "list all transforms", httpMethod = "GET")
+ public Result<List<ConnectorInfo>> listAllTransform() {
+ return Result.success(connectorService.listTransforms());
+ }
+
+ @GetMapping("/sinks")
+ @ApiOperation(value = "list all sink connector", httpMethod = "GET")
+ public Result<List<ConnectorInfo>> listSink(
+ @RequestParam(defaultValue = "ALL") ConnectorStatus status) {
+ return Result.success(connectorService.listSinks(status));
+ }
+
+ @GetMapping("/sync")
+ @ApiOperation(value = "sync all connector from disk", httpMethod = "GET")
+ public Result<List<ConnectorInfo>> sync() throws IOException {
+ connectorService.sync();
+ return Result.success();
+ }
+
+ @GetMapping("/form")
+ @ApiOperation(value = "get source connector form structure", httpMethod = "GET")
+ public Result<String> getConnectorFormStructure(
+ @ApiParam(value = "connector type", required = true) @RequestParam String connectorType,
+ @ApiParam(value = "connector name", required = true) @RequestParam
+ String connectorName) {
+ return Result.success(
+ JsonUtils.toJsonString(
+ connectorService.getConnectorFormStructure(connectorType, connectorName)));
+ }
+}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/EngineController.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/EngineController.java
new file mode 100644
index 00000000..0f081f2d
--- /dev/null
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/EngineController.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.seatunnel.app.controller;
+
+import org.apache.seatunnel.app.bean.engine.EngineDataType;
+import org.apache.seatunnel.app.common.Result;
+import org.apache.seatunnel.app.domain.response.engine.Engine;
+import org.apache.seatunnel.app.service.IEngineService;
+
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import io.swagger.annotations.ApiOperation;
+
+import javax.annotation.Resource;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+
+@RequestMapping("/seatunnel/api/v1/engine")
+@RestController
+public class EngineController {
+
+ @Resource private IEngineService engineService;
+
+ @GetMapping("/list")
+ @ApiOperation(value = "list all supported engines", httpMethod = "GET")
+ public Result<List<Engine>> listSupportEngines() {
+ return Result.success(engineService.listSupportEngines());
+ }
+
+ @GetMapping("/type")
+ @ApiOperation(value = "list all supported Data Type", httpMethod = "GET")
+ public Result<List<String>> listSupportDataTypes() {
+ return Result.success(
+ Arrays.stream(engineService.listSupportDataTypes())
+ .map(EngineDataType.DataType::getName)
+ .collect(Collectors.toList()));
+ }
+}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/EnvController.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/EnvController.java
new file mode 100644
index 00000000..dfd8661f
--- /dev/null
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/EnvController.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.seatunnel.app.controller;
+
+import org.apache.seatunnel.app.common.Result;
+import org.apache.seatunnel.app.service.IJobEnvService;
+import org.apache.seatunnel.common.utils.JsonUtils;
+
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import io.swagger.annotations.ApiOperation;
+
+import javax.annotation.Resource;
+
+@RequestMapping("/seatunnel/api/v1/job/env")
+@RestController
+public class EnvController {
+
+ @Resource private IJobEnvService jobEnvService;
+
+ @GetMapping("")
+ @ApiOperation(value = "get job env config parameters", httpMethod = "GET")
+ public Result<String> getJobEnvFormStructure() {
+ return Result.success(JsonUtils.toJsonString(jobEnvService.getJobEnvFormStructure()));
+ }
+}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/JobConfigController.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/JobConfigController.java
new file mode 100644
index 00000000..a3ac95fa
--- /dev/null
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/JobConfigController.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.seatunnel.app.controller;
+
+import org.apache.seatunnel.app.common.Result;
+import org.apache.seatunnel.app.domain.request.job.JobConfig;
+import org.apache.seatunnel.app.domain.response.job.JobConfigRes;
+import org.apache.seatunnel.app.service.IJobConfigService;
+
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.PutMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import io.swagger.annotations.ApiOperation;
+import io.swagger.annotations.ApiParam;
+
+import javax.annotation.Resource;
+
+@RestController
+@RequestMapping("/seatunnel/api/v1/job/config")
+public class JobConfigController {
+
+ @Resource private IJobConfigService jobConfigService;
+
+ @PutMapping("/{jobVersionId}")
+ @ApiOperation(value = "update job config", httpMethod = "PUT")
+ Result<Void> updateJobConfig(
+ @ApiParam(value = "userId", required = true) @RequestParam Integer userId,
+ @ApiParam(value = "jobVersionId", required = true) @PathVariable long jobVersionId,
+ @ApiParam(value = "jobConfig", required = true) @RequestBody JobConfig jobConfig)
+ throws JsonProcessingException {
+ jobConfigService.updateJobConfig(userId, jobVersionId, jobConfig);
+ return Result.success();
+ }
+
+ @GetMapping("/{jobVersionId}")
+ @ApiOperation(value = "get job config", httpMethod = "GET")
+ Result<JobConfigRes> getJobConfig(
+ @ApiParam(value = "jobVersionId", required = true) @PathVariable long jobVersionId)
+ throws JsonProcessingException {
+ return Result.success(jobConfigService.getJobConfig(jobVersionId));
+ }
+}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/JobDefinitionController.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/JobDefinitionController.java
new file mode 100644
index 00000000..dbe72d0f
--- /dev/null
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/JobDefinitionController.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.app.controller;
+
+import org.apache.seatunnel.app.common.Result;
+import org.apache.seatunnel.app.dal.entity.JobDefinition;
+import org.apache.seatunnel.app.domain.request.job.JobReq;
+import org.apache.seatunnel.app.domain.response.PageInfo;
+import org.apache.seatunnel.app.domain.response.job.JobDefinitionRes;
+import org.apache.seatunnel.app.service.IJobDefinitionService;
+import org.apache.seatunnel.server.common.CodeGenerateUtils;
+
+import org.springframework.web.bind.annotation.DeleteMapping;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+import io.swagger.annotations.ApiOperation;
+import io.swagger.annotations.ApiParam;
+
+import javax.annotation.Resource;
+
+@RestController
+@RequestMapping("/seatunnel/api/v1/job/definition")
+public class JobDefinitionController {
+
+ @Resource private IJobDefinitionService jobService;
+
+ /**
+ * create job definition
+ *
+ * @return created job id
+ */
+ @PostMapping
+ @ApiOperation(value = "create job definition", httpMethod = "POST")
+ Result<Long> createJobDefinition(
+ @ApiParam(value = "userId", required = true) @RequestParam Integer userId,
+ @RequestBody JobReq jobReq)
+ throws CodeGenerateUtils.CodeGenerateException {
+ return Result.success(jobService.createJob(userId, jobReq));
+ }
+
+ @GetMapping
+ @ApiOperation(value = "get job definition", httpMethod = "GET")
+ Result<PageInfo<JobDefinitionRes>> getJobDefinition(
+ @ApiParam(value = "job name") @RequestParam(required = false) String searchName,
+ @ApiParam(value = "page num", required = true) @RequestParam Integer pageNo,
+ @ApiParam(value = "page size", required = true) @RequestParam Integer pageSize,
+ @ApiParam(value = "job mode") @RequestParam(required = false) String jobMode) {
+ return Result.success(jobService.getJob(searchName, pageNo, pageSize, jobMode));
+ }
+
+ @GetMapping("/{jobId}")
+ @ApiOperation(value = "get job definition", httpMethod = "GET")
+ Result<JobDefinition> getJobDefinition(@PathVariable long jobId) {
+ return Result.success(jobService.getJobDefinitionByJobId(jobId));
+ }
+
+ @DeleteMapping
+ @ApiOperation(value = "delete job definition", httpMethod = "DELETE")
+ Result<Void> deleteJobDefinition(
+ @ApiParam(value = "id", required = true) @RequestParam long id) {
+ jobService.deleteJob(id);
+ return Result.success();
+ }
+}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/JobTaskController.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/JobTaskController.java
new file mode 100644
index 00000000..0e2fd1e9
--- /dev/null
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/JobTaskController.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.app.controller;
+
+import org.apache.seatunnel.app.common.Result;
+import org.apache.seatunnel.app.domain.request.job.JobDAG;
+import org.apache.seatunnel.app.domain.request.job.JobTaskInfo;
+import org.apache.seatunnel.app.domain.request.job.PluginConfig;
+import org.apache.seatunnel.app.domain.response.job.JobTaskCheckRes;
+import org.apache.seatunnel.app.service.IJobTaskService;
+
+import org.springframework.web.bind.annotation.DeleteMapping;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+import io.swagger.annotations.ApiOperation;
+import io.swagger.annotations.ApiParam;
+
+import javax.annotation.Resource;
+
+@RestController
+@RequestMapping("/seatunnel/api/v1/job/")
+public class JobTaskController {
+
+ @Resource private IJobTaskService jobTaskService;
+
+ @PostMapping("/dag/{jobVersionId}")
+ @ApiOperation(value = "save job dag", httpMethod = "POST")
+ Result<JobTaskCheckRes> saveJobDAG(
+ @ApiParam(value = "job version id", required = true) @PathVariable long jobVersionId,
+ @ApiParam(value = "task info", required = true) @RequestBody JobDAG jobDAG) {
+ return Result.success(jobTaskService.saveJobDAG(jobVersionId, jobDAG));
+ }
+
+ @GetMapping("/{jobVersionId}")
+ @ApiOperation(value = "get job task and dag info", httpMethod = "GET")
+ Result<JobTaskInfo> getJob(
+ @ApiParam(value = "job version id", required = true) @PathVariable long jobVersionId) {
+ return Result.success(jobTaskService.getTaskConfig(jobVersionId));
+ }
+
+ @PostMapping("/task/{jobVersionId}")
+ @ApiOperation(value = "save or update single task", httpMethod = "POST")
+ Result<Void> saveSingleTask(
+ @ApiParam(value = "job version id", required = true) @PathVariable long jobVersionId,
+ @ApiParam(value = "task info", required = true) @RequestBody
+ PluginConfig pluginConfig) {
+ jobTaskService.saveSingleTask(jobVersionId, pluginConfig);
+ return Result.success();
+ }
+
+ @GetMapping("/task/{jobVersionId}")
+ @ApiOperation(value = "get single task", httpMethod = "GET")
+ Result<PluginConfig> getSingleTask(
+ @ApiParam(value = "job version id", required = true) @PathVariable long jobVersionId,
+ @ApiParam(value = "task plugin id", required = true) @RequestParam String pluginId) {
+ return Result.success(jobTaskService.getSingleTask(jobVersionId, pluginId));
+ }
+
+ @DeleteMapping("/task/{jobVersionId}")
+ @ApiOperation(value = "delete single task", httpMethod = "DELETE")
+ Result<Void> deleteSingleTask(
+ @ApiParam(value = "job version id", required = true) @PathVariable long jobVersionId,
+ @ApiParam(value = "task plugin id", required = true) @RequestParam String pluginId) {
+ jobTaskService.deleteSingleTask(jobVersionId, pluginId);
+ return Result.success();
+ }
+}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/SeatunnelDatasourceController.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/SeatunnelDatasourceController.java
index 9729c593..85bdc091 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/SeatunnelDatasourceController.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/SeatunnelDatasourceController.java
@@ -26,6 +26,8 @@ import org.apache.seatunnel.app.dal.entity.User;
import org.apache.seatunnel.app.domain.dto.datasource.DatabaseTableFields;
import org.apache.seatunnel.app.domain.dto.datasource.DatabaseTables;
import org.apache.seatunnel.app.domain.dto.datasource.TableInfo;
+import org.apache.seatunnel.app.domain.request.datasource.DatasourceCheckReq;
+import org.apache.seatunnel.app.domain.request.datasource.DatasourceReq;
import org.apache.seatunnel.app.domain.response.PageInfo;
import org.apache.seatunnel.app.domain.response.datasource.DatasourceDetailRes;
import org.apache.seatunnel.app.domain.response.datasource.DatasourceRes;
@@ -116,18 +118,16 @@ public class SeatunnelDatasourceController extends BaseController {
@PostMapping("/create")
Result<String> createDatasource(
@ApiIgnore @RequestAttribute(value = SESSION_USER) User loginUser,
- @RequestParam("datasourceName") String datasourceName,
- @RequestParam("pluginName") String pluginName,
- @RequestParam("description") String description,
- @RequestParam("datasourceConfig") String datasourceConfig) {
+ @RequestBody DatasourceReq req) {
+ String datasourceConfig = req.getDatasourceConfig();
Map<String, String> stringStringMap = JSONUtils.toMap(datasourceConfig);
return Result.success(
datasourceService.createDatasource(
loginUser.getId(),
- datasourceName,
- pluginName,
+ req.getDatasourceName(),
+ req.getPluginName(),
DEFAULT_PLUGIN_VERSION,
- description,
+ req.getDescription(),
stringStringMap));
}
@@ -149,12 +149,14 @@ public class SeatunnelDatasourceController extends BaseController {
@PostMapping("/check/connect")
Result<Boolean> testConnect(
@ApiIgnore @RequestAttribute(value = SESSION_USER) User loginUser,
- @RequestParam("pluginName") String pluginName,
- @RequestParam("datasourceConfig") String datasourceConfig) {
- Map<String, String> stringStringMap = JSONUtils.toMap(datasourceConfig);
+ @RequestBody DatasourceCheckReq req) {
+ Map<String, String> stringStringMap = JSONUtils.toMap(req.getDatasourceConfig());
return Result.success(
datasourceService.testDatasourceConnectionAble(
- loginUser.getId(), pluginName, DEFAULT_PLUGIN_VERSION, stringStringMap));
+ loginUser.getId(),
+ req.getPluginName(),
+ DEFAULT_PLUGIN_VERSION,
+ stringStringMap));
}
@ApiOperation(value = "update datasource", notes = "update datasource")
@@ -182,17 +184,15 @@ public class SeatunnelDatasourceController extends BaseController {
Result<Boolean> updateDatasource(
@ApiIgnore @RequestAttribute(value = SESSION_USER) User loginUser,
@PathVariable("id") String id,
- @RequestParam("datasourceName") String datasourceName,
- @RequestParam("description") String description,
- @RequestParam("datasourceConfig") String datasourceConfig) {
- Map<String, String> stringStringMap = JSONUtils.toMap(datasourceConfig);
+ @RequestBody DatasourceReq req) {
+ Map<String, String> stringStringMap = JSONUtils.toMap(req.getDatasourceConfig());
Long datasourceId = Long.parseLong(id);
return Result.success(
datasourceService.updateDatasource(
loginUser.getId(),
datasourceId,
- datasourceName,
- description,
+ req.getDatasourceName(),
+ req.getDescription(),
stringStringMap));
}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/IJobDefinitionDao.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/IJobDefinitionDao.java
index 34b87b57..51b29ee2 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/IJobDefinitionDao.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/IJobDefinitionDao.java
@@ -32,10 +32,9 @@ public interface IJobDefinitionDao {
void updateJob(JobDefinition jobDefinition);
- PageInfo<JobDefinition> getJob(
- String name, Integer pageNo, Integer pageSize, List<Long> projectCodes, String jobMode);
+ PageInfo<JobDefinition> getJob(String name, Integer pageNo, Integer pageSize, String jobMode);
- List<JobDefinition> getJob(@NonNull List<Long> projectCodes, @NonNull String name);
+ List<JobDefinition> getJob(@NonNull String name);
- void delete(long id, long projectCode);
+ void delete(long id);
}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/IJobInstanceDao.java
similarity index 61%
copy from seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
copy to seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/IJobInstanceDao.java
index 769387c2..c9d9c165 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/IJobInstanceDao.java
@@ -15,12 +15,22 @@
* limitations under the License.
*/
-package org.apache.seatunnel.app.domain.request.job.transform;
-
-public enum Transform {
- REPLACE,
- COPY,
- SPLIT,
- FIELDMAPPER,
- FILTERROWKIND
+package org.apache.seatunnel.app.dal.dao;
+
+import org.apache.seatunnel.app.dal.entity.JobInstance;
+import org.apache.seatunnel.app.dal.mapper.JobInstanceMapper;
+
+import lombok.NonNull;
+
+public interface IJobInstanceDao {
+
+ JobInstance getJobInstance(@NonNull Long jobInstanceId);
+
+ JobInstance getJobInstanceByEngineId(@NonNull Long jobEngineId);
+
+ void update(@NonNull JobInstance jobInstance);
+
+ void insert(@NonNull JobInstance jobInstance);
+
+ JobInstanceMapper getJobInstanceMapper();
}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/IJobInstanceHistoryDao.java
similarity index 73%
copy from seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
copy to seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/IJobInstanceHistoryDao.java
index 769387c2..9f976de4 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/IJobInstanceHistoryDao.java
@@ -15,12 +15,12 @@
* limitations under the License.
*/
-package org.apache.seatunnel.app.domain.request.job.transform;
+package org.apache.seatunnel.app.dal.dao;
-public enum Transform {
- REPLACE,
- COPY,
- SPLIT,
- FIELDMAPPER,
- FILTERROWKIND
+import org.apache.seatunnel.app.dal.entity.JobInstanceHistory;
+
+public interface IJobInstanceHistoryDao {
+ JobInstanceHistory getByInstanceId(Long jobInstanceId, Long projectCode);
+
+ void insert(JobInstanceHistory jobInstanceHistory);
}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/IJobLineDao.java
similarity index 72%
copy from seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
copy to seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/IJobLineDao.java
index 769387c2..02a3ed39 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/IJobLineDao.java
@@ -15,12 +15,17 @@
* limitations under the License.
*/
-package org.apache.seatunnel.app.domain.request.job.transform;
+package org.apache.seatunnel.app.dal.dao;
-public enum Transform {
- REPLACE,
- COPY,
- SPLIT,
- FIELDMAPPER,
- FILTERROWKIND
+import org.apache.seatunnel.app.dal.entity.JobLine;
+
+import java.util.List;
+
+public interface IJobLineDao {
+
+ void deleteLinesByVersionId(long jobVersionId);
+
+ void insertLines(List<JobLine> lines);
+
+ List<JobLine> getLinesByVersionId(long jobVersionId);
}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/IJobMetricsDao.java
similarity index 67%
copy from seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
copy to seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/IJobMetricsDao.java
index 769387c2..9578ece4 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/IJobMetricsDao.java
@@ -14,13 +14,18 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.seatunnel.app.dal.dao;
-package org.apache.seatunnel.app.domain.request.job.transform;
+import org.apache.seatunnel.app.dal.entity.JobMetrics;
+import org.apache.seatunnel.app.dal.mapper.JobMetricsMapper;
-public enum Transform {
- REPLACE,
- COPY,
- SPLIT,
- FIELDMAPPER,
- FILTERROWKIND
+import lombok.NonNull;
+
+import java.util.List;
+
+public interface IJobMetricsDao {
+
+ List<JobMetrics> getByInstanceId(@NonNull Long projectCode, @NonNull Long jobInstanceId);
+
+ JobMetricsMapper getJobMetricsMapper();
}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/DatasourceDaoImpl.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/DatasourceDaoImpl.java
index 3e7d273d..fc5c760c 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/DatasourceDaoImpl.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/DatasourceDaoImpl.java
@@ -64,7 +64,10 @@ public class DatasourceDaoImpl implements IDatasourceDao {
@Override
public boolean checkDatasourceNameUnique(String dataSourceName, Long dataSourceId) {
- return datasourceMapper.checkDataSourceNameUnique(dataSourceName, dataSourceId) <= 0;
+ QueryWrapper<Datasource> queryWrapper = new QueryWrapper<>();
+ queryWrapper.eq("datasource_name", dataSourceName);
+ queryWrapper.ne("id", dataSourceId);
+ return datasourceMapper.selectList(queryWrapper).size() <= 0;
}
@Override
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/JobDefinitionDaoImpl.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/JobDefinitionDaoImpl.java
index b99f8c56..3d45edaf 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/JobDefinitionDaoImpl.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/JobDefinitionDaoImpl.java
@@ -57,20 +57,15 @@ public class JobDefinitionDaoImpl implements IJobDefinitionDao {
@Override
public PageInfo<JobDefinition> getJob(
- String searchName,
- Integer pageNo,
- Integer pageSize,
- List<Long> projectCodes,
- String jobMode) {
+ String searchName, Integer pageNo, Integer pageSize, String jobMode) {
IPage<JobDefinition> jobDefinitionIPage;
if (StringUtils.isEmpty(jobMode)) {
jobDefinitionIPage =
- jobMapper.queryJobListPaging(
- new Page<>(pageNo, pageSize), searchName, projectCodes);
+ jobMapper.queryJobListPaging(new Page<>(pageNo, pageSize), searchName);
} else {
jobDefinitionIPage =
jobMapper.queryJobListPagingWithJobMode(
- new Page<>(pageNo, pageSize), searchName, projectCodes, jobMode);
+ new Page<>(pageNo, pageSize), searchName, jobMode);
}
PageInfo<JobDefinition> jobs = new PageInfo<>();
jobs.setData(jobDefinitionIPage.getRecords());
@@ -81,14 +76,12 @@ public class JobDefinitionDaoImpl implements IJobDefinitionDao {
}
@Override
- public List<JobDefinition> getJob(@NonNull List<Long> projectCodes, @NonNull String name) {
- return jobMapper.queryJobList(name, projectCodes);
+ public List<JobDefinition> getJob(@NonNull String name) {
+ return jobMapper.queryJobList(name);
}
- public void delete(long id, long projectCode) {
+ public void delete(long id) {
jobMapper.delete(
- Wrappers.lambdaQuery(new JobDefinition())
- .eq(JobDefinition::getProjectCode, projectCode)
- .and(i -> i.eq(JobDefinition::getId, id)));
+ Wrappers.lambdaQuery(new JobDefinition()).and(i -> i.eq(JobDefinition::getId, id)));
}
}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/JobInstanceDaoImpl.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/JobInstanceDaoImpl.java
new file mode 100644
index 00000000..ce23e7ef
--- /dev/null
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/JobInstanceDaoImpl.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.app.dal.dao.impl;
+
+import org.apache.seatunnel.app.dal.dao.IJobInstanceDao;
+import org.apache.seatunnel.app.dal.entity.JobInstance;
+import org.apache.seatunnel.app.dal.mapper.JobInstanceMapper;
+
+import org.springframework.stereotype.Repository;
+
+import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
+import lombok.NonNull;
+
+import javax.annotation.Resource;
+
+@Repository
+public class JobInstanceDaoImpl implements IJobInstanceDao {
+ @Resource private JobInstanceMapper jobInstanceMapper;
+
+ @Override
+ public JobInstance getJobInstance(@NonNull Long jobInstanceId) {
+ return jobInstanceMapper.selectById(jobInstanceId);
+ }
+
+ @Override
+ public JobInstance getJobInstanceByEngineId(@NonNull Long jobEngineId) {
+ return jobInstanceMapper.selectOne(
+ new LambdaQueryWrapper<>(new JobInstance())
+ .eq(JobInstance::getJobEngineId, jobEngineId));
+ }
+
+ @Override
+ public void update(@NonNull JobInstance jobInstance) {
+ jobInstanceMapper.updateById(jobInstance);
+ }
+
+ @Override
+ public void insert(@NonNull JobInstance jobInstance) {
+ jobInstanceMapper.insert(jobInstance);
+ }
+
+ @Override
+ public JobInstanceMapper getJobInstanceMapper() {
+ return jobInstanceMapper;
+ }
+}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/JobInstanceHistoryDaoImpl.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/JobInstanceHistoryDaoImpl.java
new file mode 100644
index 00000000..40099f5e
--- /dev/null
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/JobInstanceHistoryDaoImpl.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.app.dal.dao.impl;
+
+import org.apache.seatunnel.app.dal.dao.IJobInstanceHistoryDao;
+import org.apache.seatunnel.app.dal.entity.JobInstanceHistory;
+import org.apache.seatunnel.app.dal.mapper.JobInstanceHistoryMapper;
+
+import org.springframework.stereotype.Repository;
+
+import com.baomidou.mybatisplus.core.toolkit.Wrappers;
+
+import javax.annotation.Resource;
+
+@Repository
+public class JobInstanceHistoryDaoImpl implements IJobInstanceHistoryDao {
+
+ @Resource private JobInstanceHistoryMapper jobInstanceHistoryMapper;
+
+ @Override
+ public JobInstanceHistory getByInstanceId(Long jobInstanceId, Long projectCode) {
+ return jobInstanceHistoryMapper.selectOne(
+ Wrappers.lambdaQuery(new JobInstanceHistory())
+ .eq(JobInstanceHistory::getId, jobInstanceId));
+ }
+
+ @Override
+ public void insert(JobInstanceHistory jobInstanceHistory) {
+ jobInstanceHistoryMapper.insert(jobInstanceHistory);
+ }
+}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/JobLineDaoImpl.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/JobLineDaoImpl.java
new file mode 100644
index 00000000..3be8e972
--- /dev/null
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/JobLineDaoImpl.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.app.dal.dao.impl;
+
+import org.apache.seatunnel.app.dal.dao.IJobLineDao;
+import org.apache.seatunnel.app.dal.entity.JobLine;
+import org.apache.seatunnel.app.dal.mapper.JobLineMapper;
+
+import org.springframework.stereotype.Service;
+
+import com.baomidou.mybatisplus.core.toolkit.Wrappers;
+
+import javax.annotation.Resource;
+
+import java.util.List;
+
+@Service
+public class JobLineDaoImpl implements IJobLineDao {
+
+ @Resource private JobLineMapper jobLineMapper;
+
+ @Override
+ public void deleteLinesByVersionId(long jobVersionId) {
+ jobLineMapper.deleteLinesByVersionId(jobVersionId);
+ }
+
+ @Override
+ public void insertLines(List<JobLine> lines) {
+ jobLineMapper.insertBatchLines(lines);
+ }
+
+ @Override
+ public List<JobLine> getLinesByVersionId(long jobVersionId) {
+ return jobLineMapper.selectList(
+ Wrappers.lambdaQuery(new JobLine()).eq(JobLine::getVersionId, jobVersionId));
+ }
+}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/IJobDefinitionDao.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/JobMetricsDaoImpl.java
similarity index 53%
copy from seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/IJobDefinitionDao.java
copy to seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/JobMetricsDaoImpl.java
index 34b87b57..0428c26d 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/IJobDefinitionDao.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/JobMetricsDaoImpl.java
@@ -14,28 +14,33 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.seatunnel.app.dal.dao.impl;
-package org.apache.seatunnel.app.dal.dao;
+import org.apache.seatunnel.app.dal.dao.IJobMetricsDao;
+import org.apache.seatunnel.app.dal.entity.JobMetrics;
+import org.apache.seatunnel.app.dal.mapper.JobMetricsMapper;
-import org.apache.seatunnel.app.dal.entity.JobDefinition;
-import org.apache.seatunnel.app.domain.response.PageInfo;
+import org.springframework.stereotype.Repository;
import lombok.NonNull;
-import java.util.List;
-
-public interface IJobDefinitionDao {
+import javax.annotation.Resource;
- void add(JobDefinition job);
-
- JobDefinition getJob(long id);
+import java.util.List;
- void updateJob(JobDefinition jobDefinition);
+@Repository
+public class JobMetricsDaoImpl implements IJobMetricsDao {
- PageInfo<JobDefinition> getJob(
- String name, Integer pageNo, Integer pageSize, List<Long> projectCodes, String jobMode);
+ @Resource private JobMetricsMapper jobMetricsMapper;
- List<JobDefinition> getJob(@NonNull List<Long> projectCodes, @NonNull String name);
+ @Override
+ public List<JobMetrics> getByInstanceId(
+ @NonNull Long projectCode, @NonNull Long jobInstanceId) {
+ return jobMetricsMapper.queryJobMetricsByInstanceId(projectCode, jobInstanceId);
+ }
- void delete(long id, long projectCode);
+ @Override
+ public JobMetricsMapper getJobMetricsMapper() {
+ return jobMetricsMapper;
+ }
}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/TaskDefinitionDaoImpl.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/TaskDefinitionDaoImpl.java
index 3da9205f..ccbd3dd3 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/TaskDefinitionDaoImpl.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/TaskDefinitionDaoImpl.java
@@ -27,7 +27,7 @@ import org.apache.seatunnel.app.dal.mapper.TaskDefinitionMapper;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Repository;
+import org.springframework.stereotype.Service;
import java.util.Collection;
import java.util.Collections;
@@ -35,7 +35,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
-@Repository
+@Service
public class TaskDefinitionDaoImpl implements TaskDefinitionDao {
@Autowired private TaskDefinitionMapper taskDefinitionMapper;
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/entity/JobDefinition.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/entity/JobDefinition.java
index f753fb58..0809907c 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/entity/JobDefinition.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/entity/JobDefinition.java
@@ -42,7 +42,7 @@ public class JobDefinition {
@TableField private String description;
- /** {@link BusinessMode} string values */
+ /** string values */
@TableField("job_type")
private String jobType;
@@ -52,9 +52,6 @@ public class JobDefinition {
@TableField("update_user_id")
private Integer updateUserId;
- @TableField("project_code")
- private Long projectCode;
-
@TableField("create_time")
private Date createTime;
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/entity/JobDefinition.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/entity/JobInstance.java
similarity index 78%
copy from seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/entity/JobDefinition.java
copy to seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/entity/JobInstance.java
index f753fb58..63419a9b 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/entity/JobDefinition.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/entity/JobInstance.java
@@ -32,19 +32,29 @@ import java.util.Date;
@Builder
@AllArgsConstructor
@NoArgsConstructor
-@TableName("t_st_job_definition")
-public class JobDefinition {
+@TableName("t_st_job_instance")
+public class JobInstance {
@TableId(value = "id", type = IdType.INPUT)
private Long id;
- @TableField private String name;
+ @TableField("job_define_id")
+ private Long jobDefineId;
- @TableField private String description;
+ @TableField("job_status")
+ private String jobStatus;
- /** {@link BusinessMode} string values */
- @TableField("job_type")
- private String jobType;
+ @TableField("job_config")
+ private String jobConfig;
+
+ @TableField("engine_name")
+ private String engineName;
+
+ @TableField("engine_version")
+ private String engineVersion;
+
+ @TableField("job_engine_id")
+ private String jobEngineId;
@TableField("create_user_id")
private Integer createUserId;
@@ -52,9 +62,6 @@ public class JobDefinition {
@TableField("update_user_id")
private Integer updateUserId;
- @TableField("project_code")
- private Long projectCode;
-
@TableField("create_time")
private Date createTime;
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/entity/JobDefinition.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/entity/JobInstanceHistory.java
similarity index 74%
copy from seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/entity/JobDefinition.java
copy to seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/entity/JobInstanceHistory.java
index f753fb58..761c96a9 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/entity/JobDefinition.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/entity/JobInstanceHistory.java
@@ -22,38 +22,22 @@ import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.AllArgsConstructor;
-import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.Date;
@Data
-@Builder
@AllArgsConstructor
@NoArgsConstructor
-@TableName("t_st_job_definition")
-public class JobDefinition {
+@TableName("t_st_job_instance_history")
+public class JobInstanceHistory {
@TableId(value = "id", type = IdType.INPUT)
private Long id;
- @TableField private String name;
-
- @TableField private String description;
-
- /** {@link BusinessMode} string values */
- @TableField("job_type")
- private String jobType;
-
- @TableField("create_user_id")
- private Integer createUserId;
-
- @TableField("update_user_id")
- private Integer updateUserId;
-
- @TableField("project_code")
- private Long projectCode;
+ @TableField("dag")
+ private String dag;
@TableField("create_time")
private Date createTime;
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/entity/JobDefinition.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/entity/JobLine.java
similarity index 76%
copy from seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/entity/JobDefinition.java
copy to seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/entity/JobLine.java
index f753fb58..fb6c7477 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/entity/JobDefinition.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/entity/JobLine.java
@@ -28,32 +28,25 @@ import lombok.NoArgsConstructor;
import java.util.Date;
+/** Save DAG task line */
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
-@TableName("t_st_job_definition")
-public class JobDefinition {
+@TableName("t_st_job_line")
+public class JobLine {
@TableId(value = "id", type = IdType.INPUT)
private Long id;
- @TableField private String name;
+ @TableField("version_id")
+ private Long versionId;
- @TableField private String description;
+ @TableField("input_plugin_id")
+ private String inputPluginId;
- /** {@link BusinessMode} string values */
- @TableField("job_type")
- private String jobType;
-
- @TableField("create_user_id")
- private Integer createUserId;
-
- @TableField("update_user_id")
- private Integer updateUserId;
-
- @TableField("project_code")
- private Long projectCode;
+ @TableField("target_plugin_id")
+ private String targetPluginId;
@TableField("create_time")
private Date createTime;
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/entity/JobDefinition.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/entity/JobMetrics.java
similarity index 69%
copy from seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/entity/JobDefinition.java
copy to seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/entity/JobMetrics.java
index f753fb58..633ad645 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/entity/JobDefinition.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/entity/JobMetrics.java
@@ -14,7 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.seatunnel.app.dal.entity;
import com.baomidou.mybatisplus.annotation.IdType;
@@ -32,19 +31,41 @@ import java.util.Date;
@Builder
@AllArgsConstructor
@NoArgsConstructor
-@TableName("t_st_job_definition")
-public class JobDefinition {
+@TableName("t_st_job_metrics")
+public class JobMetrics {
@TableId(value = "id", type = IdType.INPUT)
private Long id;
- @TableField private String name;
+ @TableField("job_instance_id")
+ private Long jobInstanceId;
+
+ @TableField("pipeline_id")
+ private Integer pipelineId;
+
+ @TableField("read_row_count")
+ private long readRowCount;
+
+ @TableField("write_row_count")
+ private long writeRowCount;
+
+ @TableField("source_table_names")
+ private String sourceTableNames;
- @TableField private String description;
+ @TableField("sink_table_names")
+ private String sinkTableNames;
- /** {@link BusinessMode} string values */
- @TableField("job_type")
- private String jobType;
+ @TableField("read_qps")
+ private long readQps;
+
+ @TableField("write_qps")
+ private long writeQps;
+
+ @TableField("record_delay")
+ private long recordDelay;
+
+ @TableField("status")
+ private String status;
@TableField("create_user_id")
private Integer createUserId;
@@ -52,9 +73,6 @@ public class JobDefinition {
@TableField("update_user_id")
private Integer updateUserId;
- @TableField("project_code")
- private Long projectCode;
-
@TableField("create_time")
private Date createTime;
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/mapper/JobInstanceHistoryMapper.java
similarity index 76%
copy from seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
copy to seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/mapper/JobInstanceHistoryMapper.java
index 769387c2..8a3f243f 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/mapper/JobInstanceHistoryMapper.java
@@ -15,12 +15,10 @@
* limitations under the License.
*/
-package org.apache.seatunnel.app.domain.request.job.transform;
+package org.apache.seatunnel.app.dal.mapper;
-public enum Transform {
- REPLACE,
- COPY,
- SPLIT,
- FIELDMAPPER,
- FILTERROWKIND
-}
+import org.apache.seatunnel.app.dal.entity.JobInstanceHistory;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+public interface JobInstanceHistoryMapper extends BaseMapper<JobInstanceHistory> {}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/mapper/JobInstanceMapper.java
similarity index 77%
copy from seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
copy to seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/mapper/JobInstanceMapper.java
index 769387c2..266cc1f4 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/mapper/JobInstanceMapper.java
@@ -15,12 +15,10 @@
* limitations under the License.
*/
-package org.apache.seatunnel.app.domain.request.job.transform;
+package org.apache.seatunnel.app.dal.mapper;
-public enum Transform {
- REPLACE,
- COPY,
- SPLIT,
- FIELDMAPPER,
- FILTERROWKIND
-}
+import org.apache.seatunnel.app.dal.entity.JobInstance;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+public interface JobInstanceMapper extends BaseMapper<JobInstance> {}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/mapper/JobLineMapper.java
similarity index 65%
copy from seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
copy to seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/mapper/JobLineMapper.java
index 769387c2..9317520a 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/mapper/JobLineMapper.java
@@ -15,12 +15,19 @@
* limitations under the License.
*/
-package org.apache.seatunnel.app.domain.request.job.transform;
-
-public enum Transform {
- REPLACE,
- COPY,
- SPLIT,
- FIELDMAPPER,
- FILTERROWKIND
+package org.apache.seatunnel.app.dal.mapper;
+
+import org.apache.seatunnel.app.dal.entity.JobLine;
+
+import org.apache.ibatis.annotations.Param;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+import java.util.List;
+
+public interface JobLineMapper extends BaseMapper<JobLine> {
+
+ void deleteLinesByVersionId(@Param("versionId") long jobVersionId);
+
+ void insertBatchLines(@Param("lines") List<JobLine> lines);
}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/mapper/JobMapper.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/mapper/JobMapper.java
index 52cfdc94..b4021283 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/mapper/JobMapper.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/mapper/JobMapper.java
@@ -29,16 +29,12 @@ import java.util.List;
public interface JobMapper extends BaseMapper<JobDefinition> {
IPage<JobDefinition> queryJobListPaging(
- IPage<JobDefinition> page,
- @Param("searchName") String searchName,
- @Param("projectCodes") List<Long> projectCodes);
+ IPage<JobDefinition> page, @Param("searchName") String searchName);
IPage<JobDefinition> queryJobListPagingWithJobMode(
IPage<JobDefinition> page,
@Param("searchName") String searchName,
- @Param("projectCodes") List<Long> projectCodes,
@Param("jobMode") String jobMode);
- List<JobDefinition> queryJobList(
- @Param("searchName") String searchName, @Param("projectCodes") List<Long> projectCodes);
+ List<JobDefinition> queryJobList(@Param("searchName") String searchName);
}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/mapper/JobMetricsMapper.java
similarity index 60%
copy from seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
copy to seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/mapper/JobMetricsMapper.java
index 769387c2..f33cd46b 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/mapper/JobMetricsMapper.java
@@ -14,13 +14,19 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.seatunnel.app.dal.mapper;
-package org.apache.seatunnel.app.domain.request.job.transform;
+import org.apache.seatunnel.app.dal.entity.JobMetrics;
-public enum Transform {
- REPLACE,
- COPY,
- SPLIT,
- FIELDMAPPER,
- FILTERROWKIND
+import org.apache.ibatis.annotations.Param;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+import java.util.List;
+
+public interface JobMetricsMapper extends BaseMapper<JobMetrics> {
+ List<JobMetrics> queryJobMetricsByInstanceId(
+ @Param("projectCode") Long projectCode, @Param("jobInstanceId") Long jobInstanceId);
+
+ void insertBatchMetrics(@Param("jobMetrics") List<JobMetrics> jobMetrics);
}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/datasource/DatasourceCheckReq.java
similarity index 80%
copy from seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
copy to seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/datasource/DatasourceCheckReq.java
index 769387c2..b8367c45 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/datasource/DatasourceCheckReq.java
@@ -15,12 +15,12 @@
* limitations under the License.
*/
-package org.apache.seatunnel.app.domain.request.job.transform;
+package org.apache.seatunnel.app.domain.request.datasource;
-public enum Transform {
- REPLACE,
- COPY,
- SPLIT,
- FIELDMAPPER,
- FILTERROWKIND
+import lombok.Data;
+
+@Data
+public class DatasourceCheckReq {
+ private String pluginName;
+ private String datasourceConfig;
}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/datasource/DatasourceReq.java
similarity index 76%
copy from seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
copy to seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/datasource/DatasourceReq.java
index 769387c2..940f94b7 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/datasource/DatasourceReq.java
@@ -15,12 +15,14 @@
* limitations under the License.
*/
-package org.apache.seatunnel.app.domain.request.job.transform;
+package org.apache.seatunnel.app.domain.request.datasource;
-public enum Transform {
- REPLACE,
- COPY,
- SPLIT,
- FIELDMAPPER,
- FILTERROWKIND
+import lombok.Data;
+
+@Data
+public class DatasourceReq {
+ private String datasourceName;
+ private String pluginName;
+ private String description;
+ private String datasourceConfig;
}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/DatabaseTableSchemaReq.java
similarity index 66%
copy from seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
copy to seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/DatabaseTableSchemaReq.java
index 769387c2..40fa9fb0 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/DatabaseTableSchemaReq.java
@@ -15,12 +15,22 @@
* limitations under the License.
*/
-package org.apache.seatunnel.app.domain.request.job.transform;
+package org.apache.seatunnel.app.domain.request.job;
-public enum Transform {
- REPLACE,
- COPY,
- SPLIT,
- FIELDMAPPER,
- FILTERROWKIND
+import org.apache.seatunnel.datasource.plugin.api.model.TableField;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.List;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class DatabaseTableSchemaReq {
+
+ private String database;
+ private String tableName;
+ private List<TableField> fields;
}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/Edge.java
similarity index 74%
copy from seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
copy to seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/Edge.java
index 769387c2..6af128fd 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/Edge.java
@@ -15,12 +15,18 @@
* limitations under the License.
*/
-package org.apache.seatunnel.app.domain.request.job.transform;
+package org.apache.seatunnel.app.domain.request.job;
-public enum Transform {
- REPLACE,
- COPY,
- SPLIT,
- FIELDMAPPER,
- FILTERROWKIND
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class Edge {
+
+ private String inputPluginId;
+
+ private String targetPluginId;
}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/JobDAG.java
similarity index 83%
copy from seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
copy to seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/JobDAG.java
index 769387c2..944ea793 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/JobDAG.java
@@ -15,12 +15,14 @@
* limitations under the License.
*/
-package org.apache.seatunnel.app.domain.request.job.transform;
+package org.apache.seatunnel.app.domain.request.job;
-public enum Transform {
- REPLACE,
- COPY,
- SPLIT,
- FIELDMAPPER,
- FILTERROWKIND
+import lombok.Data;
+
+import java.util.List;
+
+@Data
+public class JobDAG {
+
+ private List<Edge> edges;
}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/JobTaskInfo.java
similarity index 72%
copy from seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
copy to seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/JobTaskInfo.java
index 769387c2..cd50c627 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/JobTaskInfo.java
@@ -15,12 +15,20 @@
* limitations under the License.
*/
-package org.apache.seatunnel.app.domain.request.job.transform;
+package org.apache.seatunnel.app.domain.request.job;
-public enum Transform {
- REPLACE,
- COPY,
- SPLIT,
- FIELDMAPPER,
- FILTERROWKIND
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.List;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class JobTaskInfo {
+
+ private List<Edge> edges;
+
+ private List<PluginConfig> plugins;
}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/entity/JobDefinition.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/PluginConfig.java
similarity index 51%
copy from seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/entity/JobDefinition.java
copy to seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/PluginConfig.java
index f753fb58..8f989f61 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/entity/JobDefinition.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/PluginConfig.java
@@ -15,49 +15,44 @@
* limitations under the License.
*/
-package org.apache.seatunnel.app.dal.entity;
+package org.apache.seatunnel.app.domain.request.job;
+
+import org.apache.seatunnel.app.domain.request.connector.SceneMode;
+import org.apache.seatunnel.common.constants.PluginType;
-import com.baomidou.mybatisplus.annotation.IdType;
-import com.baomidou.mybatisplus.annotation.TableField;
-import com.baomidou.mybatisplus.annotation.TableId;
-import com.baomidou.mybatisplus.annotation.TableName;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
-import java.util.Date;
+import java.util.List;
+import java.util.Map;
@Data
-@Builder
-@AllArgsConstructor
@NoArgsConstructor
-@TableName("t_st_job_definition")
-public class JobDefinition {
+@AllArgsConstructor
+@Builder
+public class PluginConfig {
+
+ private String pluginId;
+
+ private String name;
- @TableId(value = "id", type = IdType.INPUT)
- private Long id;
+ private PluginType type;
- @TableField private String name;
+ private String connectorType;
- @TableField private String description;
+ private DataSourceOption tableOption;
- /** {@link BusinessMode} string values */
- @TableField("job_type")
- private String jobType;
+ private SelectTableFields selectTableFields;
- @TableField("create_user_id")
- private Integer createUserId;
+ private Map<String, Object> transformOptions;
- @TableField("update_user_id")
- private Integer updateUserId;
+ private List<DatabaseTableSchemaReq> outputSchema;
- @TableField("project_code")
- private Long projectCode;
+ private Long dataSourceId;
- @TableField("create_time")
- private Date createTime;
+ private SceneMode sceneMode;
- @TableField("update_time")
- private Date updateTime;
+ private String config;
}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
index 769387c2..954f3c50 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
@@ -20,7 +20,8 @@ package org.apache.seatunnel.app.domain.request.job.transform;
public enum Transform {
REPLACE,
COPY,
- SPLIT,
+ MULTIFIELDSPLIT,
FIELDMAPPER,
- FILTERROWKIND
+ FILTERROWKIND,
+ SPLIT
}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/response/engine/Engine.java
similarity index 79%
copy from seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
copy to seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/response/engine/Engine.java
index 769387c2..c5ed0001 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/response/engine/Engine.java
@@ -14,13 +14,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.seatunnel.app.domain.response.engine;
-package org.apache.seatunnel.app.domain.request.job.transform;
+import lombok.AllArgsConstructor;
+import lombok.Data;
-public enum Transform {
- REPLACE,
- COPY,
- SPLIT,
- FIELDMAPPER,
- FILTERROWKIND
+@Data
+@AllArgsConstructor
+public class Engine {
+ private String name;
+ private String version;
}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/response/executor/JobExecutorRes.java
similarity index 58%
copy from seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
copy to seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/response/executor/JobExecutorRes.java
index 769387c2..a20cd9f8 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/response/executor/JobExecutorRes.java
@@ -15,12 +15,25 @@
* limitations under the License.
*/
-package org.apache.seatunnel.app.domain.request.job.transform;
-
-public enum Transform {
- REPLACE,
- COPY,
- SPLIT,
- FIELDMAPPER,
- FILTERROWKIND
+package org.apache.seatunnel.app.domain.response.executor;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+@Data
+@AllArgsConstructor
+public class JobExecutorRes {
+
+ private final Long jobInstanceId;
+
+ private final String jobConfig;
+
+ /** engine name Spark/Flink/SeaTunnel */
+ private final String engine;
+
+ /** The driver run mode, only spark use now, support 'client' and 'cluster' */
+ private final String deployMode;
+
+ /** The engine run mode, for SeaTunnel Engine only support 'local' and null */
+ private final String master;
}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/response/metrics/Edge.java
similarity index 74%
copy from seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
copy to seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/response/metrics/Edge.java
index 769387c2..5f265100 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/response/metrics/Edge.java
@@ -15,12 +15,16 @@
* limitations under the License.
*/
-package org.apache.seatunnel.app.domain.request.job.transform;
+package org.apache.seatunnel.app.domain.response.metrics;
-public enum Transform {
- REPLACE,
- COPY,
- SPLIT,
- FIELDMAPPER,
- FILTERROWKIND
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class Edge {
+ private Long inputVertexId;
+ private Long targetVertexId;
}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/response/metrics/JobDAG.java
similarity index 67%
copy from seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
copy to seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/response/metrics/JobDAG.java
index 769387c2..a0856f73 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/response/metrics/JobDAG.java
@@ -15,12 +15,23 @@
* limitations under the License.
*/
-package org.apache.seatunnel.app.domain.request.job.transform;
+package org.apache.seatunnel.app.domain.response.metrics;
-public enum Transform {
- REPLACE,
- COPY,
- SPLIT,
- FIELDMAPPER,
- FILTERROWKIND
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.List;
+import java.util.Map;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class JobDAG {
+
+ private long jobId;
+
+ private Map<Integer, List<Edge>> pipelineEdges;
+
+ private Map<Integer, VertexInfo> vertexInfoMap;
}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/response/metrics/JobPipelineDetailMetricsRes.java
similarity index 58%
copy from seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
copy to seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/response/metrics/JobPipelineDetailMetricsRes.java
index 769387c2..b864817c 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/response/metrics/JobPipelineDetailMetricsRes.java
@@ -14,13 +14,36 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.seatunnel.app.domain.response.metrics;
-package org.apache.seatunnel.app.domain.request.job.transform;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
-public enum Transform {
- REPLACE,
- COPY,
- SPLIT,
- FIELDMAPPER,
- FILTERROWKIND
+@Data
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
+public class JobPipelineDetailMetricsRes {
+
+ private Long id;
+
+ private Integer pipelineId;
+
+ private long readRowCount;
+
+ private long writeRowCount;
+
+ private String sourceTableNames;
+
+ private String sinkTableNames;
+
+ private long readQps;
+
+ private long writeQps;
+
+ private long recordDelay;
+
+ private String status;
}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/response/metrics/JobPipelineSummaryMetricsRes.java
similarity index 72%
copy from seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
copy to seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/response/metrics/JobPipelineSummaryMetricsRes.java
index 769387c2..cac0499e 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/response/metrics/JobPipelineSummaryMetricsRes.java
@@ -14,13 +14,19 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.seatunnel.app.domain.response.metrics;
-package org.apache.seatunnel.app.domain.request.job.transform;
+import lombok.AllArgsConstructor;
+import lombok.Data;
-public enum Transform {
- REPLACE,
- COPY,
- SPLIT,
- FIELDMAPPER,
- FILTERROWKIND
+@Data
+@AllArgsConstructor
+public class JobPipelineSummaryMetricsRes {
+ private long pipelineId;
+
+ private long readRowCount;
+
+ private long writeRowCount;
+
+ private String status;
}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/response/metrics/JobSummaryMetricsRes.java
similarity index 70%
copy from seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
copy to seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/response/metrics/JobSummaryMetricsRes.java
index 769387c2..d4b36fc5 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/response/metrics/JobSummaryMetricsRes.java
@@ -14,13 +14,21 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.seatunnel.app.domain.response.metrics;
-package org.apache.seatunnel.app.domain.request.job.transform;
+import lombok.AllArgsConstructor;
+import lombok.Data;
-public enum Transform {
- REPLACE,
- COPY,
- SPLIT,
- FIELDMAPPER,
- FILTERROWKIND
+@Data
+@AllArgsConstructor
+public class JobSummaryMetricsRes {
+ private long jobInstanceId;
+
+ private long jobEngineId;
+
+ private long readRowCount;
+
+ private long writeRowCount;
+
+ private String status;
}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/response/metrics/SinkMetricsInfo.java
similarity index 80%
copy from seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
copy to seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/response/metrics/SinkMetricsInfo.java
index 769387c2..c159fcb9 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/response/metrics/SinkMetricsInfo.java
@@ -14,13 +14,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.seatunnel.app.domain.response.metrics;
-package org.apache.seatunnel.app.domain.request.job.transform;
+import lombok.AllArgsConstructor;
+import lombok.Data;
-public enum Transform {
- REPLACE,
- COPY,
- SPLIT,
- FIELDMAPPER,
- FILTERROWKIND
+@Data
+@AllArgsConstructor
+public class SinkMetricsInfo {
+ private long writeRowCount;
}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/response/metrics/SourceMetricsInfo.java
similarity index 79%
copy from seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
copy to seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/response/metrics/SourceMetricsInfo.java
index 769387c2..17086742 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/response/metrics/SourceMetricsInfo.java
@@ -14,13 +14,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.seatunnel.app.domain.response.metrics;
-package org.apache.seatunnel.app.domain.request.job.transform;
+import lombok.AllArgsConstructor;
+import lombok.Data;
-public enum Transform {
- REPLACE,
- COPY,
- SPLIT,
- FIELDMAPPER,
- FILTERROWKIND
+@Data
+@AllArgsConstructor
+public class SourceMetricsInfo {
+ private long readRowCount;
}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/response/metrics/VertexInfo.java
similarity index 68%
copy from seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
copy to seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/response/metrics/VertexInfo.java
index 769387c2..e2098a1c 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/response/metrics/VertexInfo.java
@@ -15,12 +15,19 @@
* limitations under the License.
*/
-package org.apache.seatunnel.app.domain.request.job.transform;
+package org.apache.seatunnel.app.domain.response.metrics;
-public enum Transform {
- REPLACE,
- COPY,
- SPLIT,
- FIELDMAPPER,
- FILTERROWKIND
+import org.apache.seatunnel.common.constants.PluginType;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class VertexInfo {
+ private long vertexId;
+ private PluginType type;
+ private String connectorType;
}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/IEngineService.java
similarity index 71%
copy from seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
copy to seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/IEngineService.java
index 769387c2..ed758cd1 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/IEngineService.java
@@ -14,13 +14,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.seatunnel.app.service;
-package org.apache.seatunnel.app.domain.request.job.transform;
+import org.apache.seatunnel.app.bean.engine.EngineDataType;
+import org.apache.seatunnel.app.domain.response.engine.Engine;
-public enum Transform {
- REPLACE,
- COPY,
- SPLIT,
- FIELDMAPPER,
- FILTERROWKIND
+import java.util.List;
+
+public interface IEngineService {
+
+ List<Engine> listSupportEngines();
+
+ EngineDataType.DataType[] listSupportDataTypes();
}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/IJobConfigService.java
similarity index 63%
copy from seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
copy to seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/IJobConfigService.java
index 769387c2..b10a448a 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/IJobConfigService.java
@@ -14,13 +14,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.seatunnel.app.service;
-package org.apache.seatunnel.app.domain.request.job.transform;
+import org.apache.seatunnel.app.domain.request.job.JobConfig;
+import org.apache.seatunnel.app.domain.response.job.JobConfigRes;
-public enum Transform {
- REPLACE,
- COPY,
- SPLIT,
- FIELDMAPPER,
- FILTERROWKIND
+import com.fasterxml.jackson.core.JsonProcessingException;
+
+public interface IJobConfigService {
+ JobConfigRes getJobConfig(long jobVersionIdId) throws JsonProcessingException;
+
+ void updateJobConfig(int userId, long jobVersionId, JobConfig jobConfig)
+ throws JsonProcessingException;
}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/IJobDefinitionService.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/IJobDefinitionService.java
index fb015003..75e96f45 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/IJobDefinitionService.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/IJobDefinitionService.java
@@ -33,13 +33,12 @@ public interface IJobDefinitionService {
long createJob(int userId, JobReq jobReq) throws CodeGenerateUtils.CodeGenerateException;
- PageInfo<JobDefinitionRes> getJob(
- String name, Integer pageNo, Integer pageSize, List<Long> projectCodes);
+ PageInfo<JobDefinitionRes> getJob(String name, Integer pageNo, Integer pageSize);
PageInfo<JobDefinitionRes> getJob(
- String name, Integer pageNo, Integer pageSize, List<Long> projectCodes, String jobMode);
+ String name, Integer pageNo, Integer pageSize, String jobMode);
- Map<Long, String> getJob(@NonNull List<Long> projectCodes, @NonNull String name);
+ Map<Long, String> getJob(@NonNull String name);
JobDefinition getJobDefinitionByJobId(long jobId);
@@ -47,5 +46,5 @@ public interface IJobDefinitionService {
boolean getUsedByDataSourceIdAndVirtualTable(long datasourceId, String tableName);
- void deleteJob(long id, long projectCode);
+ void deleteJob(long id);
}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/IJobEnvService.java
similarity index 81%
copy from seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
copy to seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/IJobEnvService.java
index 769387c2..0edc6130 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/job/transform/Transform.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/IJobEnvService.java
@@ -14,13 +14,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.seatunnel.app.service;
-package org.apache.seatunnel.app.domain.request.job.transform;
+import org.apache.seatunnel.app.dynamicforms.FormStructure;
-public enum Transform {
- REPLACE,
- COPY,
- SPLIT,
- FIELDMAPPER,
- FILTERROWKIND
+public interface IJobEnvService {
+ FormStructure getJobEnvFormStructure();
}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/IJobDefinitionDao.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/IJobInstanceService.java
similarity index 53%
copy from seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/IJobDefinitionDao.java
copy to seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/IJobInstanceService.java
index 34b87b57..a6686e72 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/IJobDefinitionDao.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/IJobInstanceService.java
@@ -15,27 +15,27 @@
* limitations under the License.
*/
-package org.apache.seatunnel.app.dal.dao;
+package org.apache.seatunnel.app.service;
-import org.apache.seatunnel.app.dal.entity.JobDefinition;
-import org.apache.seatunnel.app.domain.response.PageInfo;
+import org.apache.seatunnel.app.dal.entity.JobLine;
+import org.apache.seatunnel.app.dal.entity.JobTask;
+import org.apache.seatunnel.app.domain.response.executor.JobExecutorRes;
import lombok.NonNull;
import java.util.List;
-public interface IJobDefinitionDao {
+public interface IJobInstanceService {
+ JobExecutorRes createExecuteResource(
+ @NonNull Integer userId, @NonNull Long projectCode, @NonNull Long jobDefineId);
- void add(JobDefinition job);
+ String generateJobConfig(Long jobId, List<JobTask> tasks, List<JobLine> lines, String envStr);
- JobDefinition getJob(long id);
+ JobExecutorRes getExecuteResource(@NonNull Long jobEngineId);
- void updateJob(JobDefinition jobDefinition);
-
- PageInfo<JobDefinition> getJob(
- String name, Integer pageNo, Integer pageSize, List<Long> projectCodes, String jobMode);
-
- List<JobDefinition> getJob(@NonNull List<Long> projectCodes, @NonNull String name);
-
- void delete(long id, long projectCode);
+ void complete(
+ @NonNull Integer userId,
+ @NonNull Long projectCode,
+ @NonNull Long jobInstanceId,
+ @NonNull String jobEngineId);
}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/IJobMetricsService.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/IJobMetricsService.java
new file mode 100644
index 00000000..17f6e722
--- /dev/null
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/IJobMetricsService.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.app.service;
+
+import org.apache.seatunnel.app.dal.entity.JobInstance;
+import org.apache.seatunnel.app.domain.response.metrics.JobDAG;
+import org.apache.seatunnel.app.domain.response.metrics.JobPipelineDetailMetricsRes;
+import org.apache.seatunnel.app.domain.response.metrics.JobPipelineSummaryMetricsRes;
+import org.apache.seatunnel.app.domain.response.metrics.JobSummaryMetricsRes;
+
+import org.apache.commons.lang3.tuple.ImmutablePair;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import lombok.NonNull;
+
+import java.util.List;
+
+public interface IJobMetricsService {
+
+ List<JobPipelineSummaryMetricsRes> getJobPipelineSummaryMetrics(
+ @NonNull Integer userId,
+ @NonNull Long projectCode,
+ @NonNull Long jobInstanceId,
+ @NonNull String jobEngineId);
+
+ List<JobPipelineDetailMetricsRes> getJobPipelineDetailMetricsRes(
+ @NonNull Integer userId,
+ @NonNull Long projectCode,
+ @NonNull Long jobInstanceId,
+ @NonNull String jobEngineId);
+
+ JobDAG getJobDAG(
+ @NonNull Integer userId,
+ @NonNull Long projectCode,
+ @NonNull Long jobInstanceId,
+ @NonNull String jobEngineId)
+ throws JsonProcessingException;
+
+ ImmutablePair<Long, String> getInstanceIdAndEngineId(@NonNull String key);
+
+ void syncJobDataToDb(
+ @NonNull JobInstance jobInstance,
+ @NonNull Integer userId,
+ @NonNull Long projectCode,
+ @NonNull String jobEngineId);
+
+ JobSummaryMetricsRes getJobSummaryMetrics(
+ @NonNull Integer userId,
+ @NonNull Long projectCode,
+ @NonNull Long jobInstanceId,
+ @NonNull String jobEngineId);
+}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/IJobTaskService.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/IJobTaskService.java
new file mode 100644
index 00000000..71c7d30f
--- /dev/null
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/IJobTaskService.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.app.service;
+
+import org.apache.seatunnel.app.domain.request.job.JobDAG;
+import org.apache.seatunnel.app.domain.request.job.JobTaskInfo;
+import org.apache.seatunnel.app.domain.request.job.PluginConfig;
+import org.apache.seatunnel.app.domain.request.job.transform.TransformOptions;
+import org.apache.seatunnel.app.domain.response.job.JobTaskCheckRes;
+
+public interface IJobTaskService {
+
+ JobTaskInfo getTaskConfig(long jobVersionId);
+
+ JobTaskCheckRes saveJobDAG(long jobVersionId, JobDAG jobDAG);
+
+ void saveSingleTask(long jobVersionId, PluginConfig pluginConfig);
+
+ PluginConfig getSingleTask(long jobVersionId, String pluginId);
+
+ <T extends TransformOptions> T getTransformOptions(long jobVersionId, String pluginId);
+
+ void deleteSingleTask(long jobVersionId, String pluginId);
+}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/EngineServiceImpl.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/EngineServiceImpl.java
new file mode 100644
index 00000000..b4a6aa97
--- /dev/null
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/EngineServiceImpl.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.seatunnel.app.service.impl;
+
+import org.apache.seatunnel.app.bean.engine.EngineDataType;
+import org.apache.seatunnel.app.domain.response.engine.Engine;
+import org.apache.seatunnel.app.permission.constants.SeatunnelFuncPermissionKeyConstant;
+import org.apache.seatunnel.app.service.IEngineService;
+
+import org.springframework.stereotype.Service;
+
+import com.google.common.collect.Lists;
+
+import java.util.List;
+
+@Service
+public class EngineServiceImpl extends SeatunnelBaseServiceImpl implements IEngineService {
+
+ private final ThreadLocal<List<Engine>> engines =
+ ThreadLocal.withInitial(
+ () ->
+ Lists.newArrayList(
+ // new Engine("Spark", "2.4.0"),
+ // new Engine("Flink", "1.13.6"),
+ new Engine("SeaTunnel", "2.3.1")));
+
+ @Override
+ public List<Engine> listSupportEngines() {
+ funcPermissionCheck(SeatunnelFuncPermissionKeyConstant.ENGIN_LIST, 0);
+ return engines.get();
+ }
+
+ @Override
+ public EngineDataType.DataType[] listSupportDataTypes() {
+ funcPermissionCheck(SeatunnelFuncPermissionKeyConstant.ENGIN_TYPE, 0);
+ return new EngineDataType.DataType[] {
+ EngineDataType.T_STRING,
+ EngineDataType.T_BOOLEAN,
+ EngineDataType.T_BYTE,
+ EngineDataType.T_SHORT,
+ EngineDataType.T_INT,
+ EngineDataType.T_LONG,
+ EngineDataType.T_FLOAT,
+ EngineDataType.T_DOUBLE,
+ EngineDataType.T_VOID,
+ EngineDataType.T_DECIMAL,
+ EngineDataType.T_LOCAL_DATE,
+ EngineDataType.T_LOCAL_TIME,
+ EngineDataType.T_LOCAL_DATE_TIME,
+ EngineDataType.T_PRIMITIVE_BYTE_ARRAY,
+ EngineDataType.T_STRING_ARRAY,
+ EngineDataType.T_BOOLEAN_ARRAY,
+ EngineDataType.T_BYTE_ARRAY,
+ EngineDataType.T_SHORT_ARRAY,
+ EngineDataType.T_INT_ARRAY,
+ EngineDataType.T_LONG_ARRAY,
+ EngineDataType.T_FLOAT_ARRAY,
+ EngineDataType.T_DOUBLE_ARRAY
+ };
+ }
+}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobConfigServiceImpl.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobConfigServiceImpl.java
new file mode 100644
index 00000000..eb6e7c6d
--- /dev/null
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobConfigServiceImpl.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.seatunnel.app.service.impl;
+
+import org.apache.seatunnel.app.common.EngineType;
+import org.apache.seatunnel.app.dal.dao.IJobDefinitionDao;
+import org.apache.seatunnel.app.dal.dao.IJobVersionDao;
+import org.apache.seatunnel.app.dal.entity.JobDefinition;
+import org.apache.seatunnel.app.dal.entity.JobVersion;
+import org.apache.seatunnel.app.domain.request.job.JobConfig;
+import org.apache.seatunnel.app.domain.response.job.JobConfigRes;
+import org.apache.seatunnel.app.permission.constants.SeatunnelFuncPermissionKeyConstant;
+import org.apache.seatunnel.app.service.IJobConfigService;
+import org.apache.seatunnel.common.constants.JobMode;
+import org.apache.seatunnel.server.common.SeatunnelErrorEnum;
+import org.apache.seatunnel.server.common.SeatunnelException;
+
+import org.apache.commons.lang3.StringUtils;
+
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import javax.annotation.Resource;
+
+import java.io.IOException;
+import java.util.Map;
+
+@Service
+public class JobConfigServiceImpl extends SeatunnelBaseServiceImpl implements IJobConfigService {
+
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+ private static final String JOB_MODE = "job.mode";
+
+ @Resource private IJobVersionDao jobVersionDao;
+
+ @Resource private IJobDefinitionDao jobDefinitionDao;
+
+ @Override
+ public JobConfigRes getJobConfig(long jobVersionId) throws JsonProcessingException {
+ funcPermissionCheck(SeatunnelFuncPermissionKeyConstant.JOB_CONFIG_DETAIL, 0);
+ JobVersion jobVersion = jobVersionDao.getVersionById(jobVersionId);
+ JobDefinition jobDefinition = jobDefinitionDao.getJob(jobVersion.getJobId());
+ JobConfigRes jobConfigRes = new JobConfigRes();
+ jobConfigRes.setName(jobDefinition.getName());
+ jobConfigRes.setId(jobVersion.getId());
+ jobConfigRes.setDescription(jobDefinition.getDescription());
+ try {
+ jobConfigRes.setEnv(
+ StringUtils.isEmpty(jobVersion.getEnv())
+ ? null
+ : OBJECT_MAPPER.readValue(jobVersion.getEnv(), Map.class));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ jobConfigRes.setEngine(EngineType.valueOf(jobVersion.getEngineName()));
+ return jobConfigRes;
+ }
+
+ @Override
+ @Transactional
+ public void updateJobConfig(int userId, long jobVersionId, JobConfig jobConfig)
+ throws JsonProcessingException {
+ funcPermissionCheck(SeatunnelFuncPermissionKeyConstant.JOB_CONFIG_UPDATE, 0);
+ JobVersion version = jobVersionDao.getVersionById(jobVersionId);
+ JobDefinition jobDefinition = new JobDefinition();
+ jobDefinition.setId(version.getJobId());
+ jobDefinition.setUpdateUserId(userId);
+ jobDefinition.setName(jobConfig.getName());
+ jobDefinition.setDescription(jobConfig.getDescription());
+ jobDefinitionDao.updateJob(jobDefinition);
+ if (jobConfig.getEnv().containsKey(JOB_MODE)) {
+ JobMode jobMode = JobMode.valueOf(jobConfig.getEnv().get(JOB_MODE).toString());
+ jobVersionDao.updateVersion(
+ JobVersion.builder()
+ .jobId(version.getJobId())
+ .id(version.getId())
+ .jobMode(jobMode.name())
+ .engineName(jobConfig.getEngine().name())
+ .updateUserId(userId)
+ .env(OBJECT_MAPPER.writeValueAsString(jobConfig.getEnv()))
+ .build());
+ } else {
+ throw new SeatunnelException(SeatunnelErrorEnum.ILLEGAL_STATE, "job mode is not set");
+ }
+ }
+}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobDefinitionServiceImpl.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobDefinitionServiceImpl.java
index 906b1885..abbb0b0e 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobDefinitionServiceImpl.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobDefinitionServiceImpl.java
@@ -48,7 +48,6 @@ import javax.annotation.Resource;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -86,7 +85,6 @@ public class JobDefinitionServiceImpl extends SeatunnelBaseServiceImpl
.createUserId(userId)
.updateUserId(userId)
.jobType(jobReq.getJobType().name())
- .projectCode(jobReq.getProjectCode())
.build());
jobVersionDao.createVersion(
JobVersion.builder()
@@ -103,21 +101,13 @@ public class JobDefinitionServiceImpl extends SeatunnelBaseServiceImpl
}
@Override
- public PageInfo<JobDefinitionRes> getJob(
- String name, Integer pageNo, Integer pageSize, List<Long> projectCodes) {
- return getJob(name, pageNo, pageSize, projectCodes, null);
+ public PageInfo<JobDefinitionRes> getJob(String name, Integer pageNo, Integer pageSize) {
+ return getJob(name, pageNo, pageSize, null);
}
@Override
public PageInfo<JobDefinitionRes> getJob(
- String searchName,
- Integer pageNo,
- Integer pageSize,
- List<Long> projectCodes,
- String jobMode) {
- if (CollectionUtils.isEmpty(projectCodes)) {
- return new PageInfo<>();
- }
+ String searchName, Integer pageNo, Integer pageSize, String jobMode) {
funcPermissionCheck(SeatunnelFuncPermissionKeyConstant.JOB_DEFINITION_VIEW, 0);
if (StringUtils.isNotEmpty(jobMode)) {
try {
@@ -128,7 +118,7 @@ public class JobDefinitionServiceImpl extends SeatunnelBaseServiceImpl
}
}
PageInfo<JobDefinition> jobDefinitionPageInfo =
- jobDefinitionDao.getJob(searchName, pageNo, pageSize, projectCodes, jobMode);
+ jobDefinitionDao.getJob(searchName, pageNo, pageSize, jobMode);
List<Integer> userIds =
jobDefinitionPageInfo.getData().stream()
.map(JobDefinition::getCreateUserId)
@@ -149,7 +139,6 @@ public class JobDefinitionServiceImpl extends SeatunnelBaseServiceImpl
jobDefinitionRes.setUpdateUserId(jobDefinitionRes.getUpdateUserId());
jobDefinitionRes.setCreateTime(jobDefinition.getCreateTime());
jobDefinitionRes.setUpdateTime(jobDefinition.getUpdateTime());
- jobDefinitionRes.setProjectCode(jobDefinition.getProjectCode());
jobDefinitionResList.add(jobDefinitionRes);
}
PageInfo<JobDefinitionRes> pageInfo = new PageInfo<>();
@@ -161,12 +150,10 @@ public class JobDefinitionServiceImpl extends SeatunnelBaseServiceImpl
}
@Override
- public Map<Long, String> getJob(@NonNull List<Long> projectCodes, @NonNull String name) {
- if (CollectionUtils.isEmpty(projectCodes)) {
- return Collections.emptyMap();
- }
+ public Map<Long, String> getJob(@NonNull String name) {
+
funcPermissionCheck(SeatunnelFuncPermissionKeyConstant.JOB_DEFINITION_VIEW, 0);
- List<JobDefinition> job = jobDefinitionDao.getJob(projectCodes, name);
+ List<JobDefinition> job = jobDefinitionDao.getJob(name);
if (CollectionUtils.isEmpty(job)) {
return new HashMap<>();
}
@@ -222,8 +209,8 @@ public class JobDefinitionServiceImpl extends SeatunnelBaseServiceImpl
}
@Override
- public void deleteJob(long id, long projectCode) {
+ public void deleteJob(long id) {
funcPermissionCheck(SeatunnelFuncPermissionKeyConstant.JOB_DEFINITION_DELETE, 0);
- jobDefinitionDao.delete(id, projectCode);
+ jobDefinitionDao.delete(id);
}
}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobEnvServiceImpl.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobEnvServiceImpl.java
new file mode 100644
index 00000000..77f72921
--- /dev/null
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobEnvServiceImpl.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.seatunnel.app.service.impl;
+
+import org.apache.seatunnel.app.bean.env.JobEnvCache;
+import org.apache.seatunnel.app.dynamicforms.FormStructure;
+import org.apache.seatunnel.app.permission.constants.SeatunnelFuncPermissionKeyConstant;
+import org.apache.seatunnel.app.service.IJobEnvService;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+@Service
+public class JobEnvServiceImpl extends SeatunnelBaseServiceImpl implements IJobEnvService {
+ private final JobEnvCache jobEnvCache;
+
+ @Autowired
+ public JobEnvServiceImpl(JobEnvCache jobEnvCache) {
+ this.jobEnvCache = jobEnvCache;
+ }
+
+ @Override
+ public FormStructure getJobEnvFormStructure() {
+ funcPermissionCheck(SeatunnelFuncPermissionKeyConstant.ENV_VIEW, 0);
+ return this.jobEnvCache.getEnvFormStructure();
+ }
+}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobInstanceServiceImpl.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobInstanceServiceImpl.java
new file mode 100644
index 00000000..23db4ece
--- /dev/null
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobInstanceServiceImpl.java
@@ -0,0 +1,697 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.app.service.impl;
+
+import org.apache.seatunnel.shade.com.fasterxml.jackson.core.type.TypeReference;
+import org.apache.seatunnel.shade.com.typesafe.config.Config;
+import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory;
+import org.apache.seatunnel.shade.com.typesafe.config.ConfigObject;
+import org.apache.seatunnel.shade.com.typesafe.config.ConfigRenderOptions;
+import org.apache.seatunnel.shade.com.typesafe.config.ConfigValue;
+import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueFactory;
+import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueType;
+
+import org.apache.seatunnel.api.common.CommonOptions;
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.api.env.ParsingMode;
+import org.apache.seatunnel.app.bean.connector.ConnectorCache;
+import org.apache.seatunnel.app.config.ConnectorDataSourceMapperConfig;
+import org.apache.seatunnel.app.dal.dao.IJobDefinitionDao;
+import org.apache.seatunnel.app.dal.dao.IJobInstanceDao;
+import org.apache.seatunnel.app.dal.dao.IJobLineDao;
+import org.apache.seatunnel.app.dal.dao.IJobTaskDao;
+import org.apache.seatunnel.app.dal.dao.IJobVersionDao;
+import org.apache.seatunnel.app.dal.entity.JobDefinition;
+import org.apache.seatunnel.app.dal.entity.JobInstance;
+import org.apache.seatunnel.app.dal.entity.JobLine;
+import org.apache.seatunnel.app.dal.entity.JobTask;
+import org.apache.seatunnel.app.dal.entity.JobVersion;
+import org.apache.seatunnel.app.domain.request.connector.BusinessMode;
+import org.apache.seatunnel.app.domain.request.connector.SceneMode;
+import org.apache.seatunnel.app.domain.request.job.DataSourceOption;
+import org.apache.seatunnel.app.domain.request.job.DatabaseTableSchemaReq;
+import org.apache.seatunnel.app.domain.request.job.SelectTableFields;
+import org.apache.seatunnel.app.domain.request.job.TableSchemaReq;
+import org.apache.seatunnel.app.domain.request.job.transform.Transform;
+import org.apache.seatunnel.app.domain.request.job.transform.TransformOptions;
+import org.apache.seatunnel.app.domain.response.datasource.VirtualTableDetailRes;
+import org.apache.seatunnel.app.domain.response.executor.JobExecutorRes;
+import org.apache.seatunnel.app.domain.response.metrics.JobPipelineSummaryMetricsRes;
+import org.apache.seatunnel.app.permission.constants.SeatunnelFuncPermissionKeyConstant;
+import org.apache.seatunnel.app.service.IDatasourceService;
+import org.apache.seatunnel.app.service.IJobInstanceService;
+import org.apache.seatunnel.app.service.IJobMetricsService;
+import org.apache.seatunnel.app.service.IVirtualTableService;
+import org.apache.seatunnel.app.thirdparty.datasource.DataSourceConfigSwitcherUtils;
+import org.apache.seatunnel.app.thirdparty.transfrom.TransformConfigSwitcherUtils;
+import org.apache.seatunnel.app.utils.SeaTunnelConfigUtil;
+import org.apache.seatunnel.common.constants.PluginType;
+import org.apache.seatunnel.common.utils.ExceptionUtils;
+import org.apache.seatunnel.engine.core.job.JobStatus;
+import org.apache.seatunnel.server.common.CodeGenerateUtils;
+import org.apache.seatunnel.server.common.SeatunnelErrorEnum;
+import org.apache.seatunnel.server.common.SeatunnelException;
+
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+
+import org.springframework.stereotype.Service;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+
+import javax.annotation.Resource;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static org.apache.seatunnel.app.utils.TaskOptionUtils.getTransformOption;
+
+@Service
+@Slf4j
+public class JobInstanceServiceImpl extends SeatunnelBaseServiceImpl
+ implements IJobInstanceService {
+
+ private static final String DAG_PARSING_MODE = "dag-parsing.mode";
+
+ @Resource private ConnectorDataSourceMapperConfig dataSourceMapperConfig;
+
+ @Resource private IDatasourceService datasourceService;
+
+ @Resource private IVirtualTableService virtualTableService;
+
+ @Resource private IJobInstanceDao jobInstanceDao;
+
+ @Resource private IJobDefinitionDao jobDefinitionDao;
+
+ @Resource private ConnectorCache connectorCache;
+
+ @Resource private IJobVersionDao jobVersionDao;
+
+ @Resource private IJobTaskDao jobTaskDao;
+
+ @Resource private IJobLineDao jobLineDao;
+
+ @Resource private IJobMetricsService jobMetricsService;
+
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+ @Override
+ public JobExecutorRes createExecuteResource(
+ @NonNull Integer userId, @NonNull Long projectCode, @NonNull Long jobDefineId) {
+ funcPermissionCheck(SeatunnelFuncPermissionKeyConstant.JOB_EXECUTOR_RESOURCE, userId);
+ log.info(
+ "receive createExecuteResource request, userId:{}, projectCode:{}, jobDefineId:{}",
+ userId,
+ projectCode,
+ jobDefineId);
+ JobDefinition job = jobDefinitionDao.getJob(jobDefineId);
+ JobVersion latestVersion = jobVersionDao.getLatestVersion(job.getId());
+ JobInstance jobInstance = new JobInstance();
+ String jobConfig = createJobConfig(latestVersion);
+
+ try {
+ jobInstance.setId(CodeGenerateUtils.getInstance().genCode());
+ } catch (CodeGenerateUtils.CodeGenerateException e) {
+ throw new SeatunnelException(SeatunnelErrorEnum.JOB_RUN_GENERATE_UUID_ERROR);
+ }
+ jobInstance.setJobDefineId(job.getId());
+ jobInstance.setEngineName(latestVersion.getEngineName());
+ jobInstance.setEngineVersion(latestVersion.getEngineVersion());
+ jobInstance.setJobConfig(jobConfig);
+ jobInstance.setCreateUserId(userId);
+
+ jobInstanceDao.insert(jobInstance);
+
+ return new JobExecutorRes(
+ jobInstance.getId(),
+ jobInstance.getJobConfig(),
+ jobInstance.getEngineName(),
+ null,
+ null);
+ }
+
+ @Override
+ public String generateJobConfig(
+ Long jobId, List<JobTask> tasks, List<JobLine> lines, String envStr) {
+ checkSceneMode(tasks);
+ BusinessMode businessMode =
+ BusinessMode.valueOf(jobDefinitionDao.getJob(jobId).getJobType());
+ Config envConfig = filterEmptyValue(ConfigFactory.parseString(envStr));
+
+ Map<String, List<Config>> sourceMap = new LinkedHashMap<>();
+ Map<String, List<Config>> transformMap = new LinkedHashMap<>();
+ Map<String, List<Config>> sinkMap = new LinkedHashMap<>();
+ Map<String, JobLine> inputLines =
+ lines.stream()
+ .collect(Collectors.toMap(JobLine::getInputPluginId, Function.identity()));
+ Map<String, JobLine> targetLines =
+ lines.stream()
+ .collect(Collectors.toMap(JobLine::getTargetPluginId, Function.identity()));
+
+ for (JobTask task : tasks) {
+ PluginType pluginType = PluginType.valueOf(task.getType().toUpperCase(Locale.ROOT));
+ try {
+ String pluginId = task.getPluginId();
+ OptionRule optionRule =
+ connectorCache.getOptionRule(pluginType.getType(), task.getConnectorType());
+ Config config =
+ filterEmptyValue(
+ parseConfigWithOptionRule(
+ pluginType,
+ task.getConnectorType(),
+ task.getConfig(),
+ optionRule));
+ switch (pluginType) {
+ case SOURCE:
+ if (inputLines.containsKey(pluginId)) {
+ config =
+ addTableName(
+ CommonOptions.RESULT_TABLE_NAME.key(),
+ inputLines.get(pluginId),
+ config);
+ if (!sourceMap.containsKey(task.getConnectorType())) {
+ sourceMap.put(task.getConnectorType(), new ArrayList<>());
+ }
+
+ if (businessMode.equals(BusinessMode.DATA_REPLICA)) {
+ config =
+ config.withValue(
+ DAG_PARSING_MODE,
+ ConfigValueFactory.fromAnyRef(
+ ParsingMode.MULTIPLEX.name()));
+ }
+
+ if (task.getSceneMode()
+ .toUpperCase()
+ .equals(SceneMode.SPLIT_TABLE.name())) {
+ config =
+ config.withValue(
+ DAG_PARSING_MODE,
+ ConfigValueFactory.fromAnyRef(
+ ParsingMode.SHARDING.name()));
+ }
+
+ Config mergeConfig =
+ mergeTaskConfig(
+ task,
+ pluginType,
+ task.getConnectorType(),
+ businessMode,
+ config,
+ optionRule);
+
+ sourceMap
+ .get(task.getConnectorType())
+ .add(filterEmptyValue(mergeConfig));
+ }
+ break;
+ case TRANSFORM:
+ if (!inputLines.containsKey(pluginId)
+ && !targetLines.containsKey(pluginId)) {
+ break;
+ }
+ if (inputLines.containsKey(pluginId)) {
+ config =
+ addTableName(
+ CommonOptions.RESULT_TABLE_NAME.key(),
+ inputLines.get(pluginId),
+ config);
+ }
+ if (targetLines.containsKey(pluginId)) {
+ config =
+ addTableName(
+ CommonOptions.SOURCE_TABLE_NAME.key(),
+ targetLines.get(pluginId),
+ config);
+ }
+ if (!transformMap.containsKey(task.getConnectorType())) {
+ transformMap.put(task.getConnectorType(), new ArrayList<>());
+ }
+ List<TableSchemaReq> inputSchemas = findInputSchemas(tasks, lines, task);
+ Config transformConfig = buildTransformConfig(task, config, inputSchemas);
+ transformMap
+ .get(task.getConnectorType())
+ .add(filterEmptyValue(transformConfig));
+ break;
+ case SINK:
+ if (targetLines.containsKey(pluginId)) {
+ config =
+ addTableName(
+ CommonOptions.SOURCE_TABLE_NAME.key(),
+ targetLines.get(pluginId),
+ config);
+ if (!sinkMap.containsKey(task.getConnectorType())) {
+ sinkMap.put(task.getConnectorType(), new ArrayList<>());
+ }
+ Config mergeConfig =
+ mergeTaskConfig(
+ task,
+ pluginType,
+ task.getConnectorType(),
+ businessMode,
+ config,
+ optionRule);
+
+ sinkMap.get(task.getConnectorType()).add(filterEmptyValue(mergeConfig));
+ }
+ break;
+ default:
+ throw new SeatunnelException(
+ SeatunnelErrorEnum.UNSUPPORTED_CONNECTOR_TYPE,
+ task.getType().toUpperCase());
+ }
+ } catch (SeatunnelException e) {
+ log.error(ExceptionUtils.getMessage(e));
+ throw e;
+ } catch (Exception e) {
+ throw new SeatunnelException(
+ SeatunnelErrorEnum.ERROR_CONFIG,
+ String.format(
+ "Plugin Type: %s, Connector Type: %s, Error Info: %s",
+ pluginType, task.getConnectorType(), ExceptionUtils.getMessage(e)));
+ }
+ }
+ String sources = "";
+ if (sourceMap.size() > 0) {
+ sources = getConnectorConfig(sourceMap);
+ }
+
+ String transforms = "";
+ if (transformMap.size() > 0) {
+ transforms = getConnectorConfig(transformMap);
+ }
+
+ String sinks = "";
+ if (sinkMap.size() > 0) {
+ sinks = getConnectorConfig(sinkMap);
+ }
+ String env =
+ envConfig
+ .root()
+ .render(
+ ConfigRenderOptions.defaults()
+ .setJson(false)
+ .setComments(false)
+ .setOriginComments(false));
+ return SeaTunnelConfigUtil.generateConfig(env, sources, transforms, sinks);
+ }
+
+ @Override
+ public JobExecutorRes getExecuteResource(@NonNull Long jobEngineId) {
+ funcPermissionCheck(SeatunnelFuncPermissionKeyConstant.JOB_EXECUTOR_INSTANCE, 0);
+ JobInstance jobInstance = jobInstanceDao.getJobInstanceByEngineId(jobEngineId);
+ return new JobExecutorRes(
+ jobInstance.getId(),
+ jobInstance.getJobConfig(),
+ jobInstance.getEngineName(),
+ null,
+ null);
+ }
+
+ @Override
+ public void complete(
+ @NonNull Integer userId,
+ @NonNull Long projectCode,
+ @NonNull Long jobInstanceId,
+ @NonNull String jobEngineId) {
+ funcPermissionCheck(SeatunnelFuncPermissionKeyConstant.JOB_EXECUTOR_COMPLETE, userId);
+ JobInstance jobInstance = jobInstanceDao.getJobInstanceMapper().selectById(jobInstanceId);
+ jobMetricsService.syncJobDataToDb(jobInstance, userId, projectCode, jobEngineId);
+
+ List<JobPipelineSummaryMetricsRes> status =
+ jobMetricsService.getJobPipelineSummaryMetrics(
+ userId, projectCode, jobInstanceId, jobEngineId);
+
+ String jobStatus;
+ Set<String> statusList =
+ status.stream()
+ .map(JobPipelineSummaryMetricsRes::getStatus)
+ .map(String::toUpperCase)
+ .collect(Collectors.toSet());
+ if (statusList.size() == 1 && statusList.contains("FINISHED")) {
+ jobStatus = JobStatus.FINISHED.name();
+ } else if (statusList.contains("FAILED")) {
+ jobStatus = JobStatus.FAILED.name();
+ } else if (statusList.contains("CANCELED")) {
+ jobStatus = JobStatus.CANCELED.name();
+ } else if (statusList.contains("CANCELLING")) {
+ jobStatus = JobStatus.CANCELLING.name();
+ } else {
+ jobStatus = JobStatus.RUNNING.name();
+ }
+ jobInstance.setJobStatus(jobStatus);
+ jobInstance.setJobEngineId(jobEngineId);
+ jobInstance.setUpdateUserId(userId);
+ jobInstanceDao.update(jobInstance);
+ }
+
+ private Config buildTransformConfig(
+ JobTask task, Config config, List<TableSchemaReq> inputSchemas) {
+ try {
+ Transform transform = Transform.valueOf(task.getConnectorType().toUpperCase());
+ TransformOptions transformOption =
+ getTransformOption(transform, task.getTransformOptions());
+ return TransformConfigSwitcherUtils.mergeTransformConfig(
+ transform, inputSchemas, config, transformOption);
+ } catch (IOException e) {
+ throw new SeatunnelException(SeatunnelErrorEnum.ILLEGAL_STATE, e.getMessage());
+ }
+ }
+
+ private List<TableSchemaReq> findInputSchemas(
+ List<JobTask> tasks, List<JobLine> lines, JobTask task) {
+ ArrayList<String> outputSchemas = new ArrayList<>();
+ lines.forEach(
+ jobLine -> {
+ if (jobLine.getTargetPluginId().equals(task.getPluginId())) {
+ String inputPluginId = jobLine.getInputPluginId();
+ tasks.forEach(
+ jobTask -> {
+ if (jobTask.getPluginId().equals(inputPluginId)) {
+ outputSchemas.add(jobTask.getOutputSchema());
+ }
+ });
+ }
+ });
+
+ checkArgument(outputSchemas.size() == 1, "input schema size must be 1");
+ try {
+ List<DatabaseTableSchemaReq> databaseTableSchemaReqs =
+ OBJECT_MAPPER.readValue(
+ outputSchemas.get(0),
+ new com.fasterxml.jackson.core.type.TypeReference<
+ List<DatabaseTableSchemaReq>>() {});
+ return databaseTableSchemaReqs.stream()
+ .map(
+ databaseTableSchemaReq -> {
+ TableSchemaReq tableSchemaReq = new TableSchemaReq();
+ tableSchemaReq.setTableName(databaseTableSchemaReq.getTableName());
+ tableSchemaReq.setFields(databaseTableSchemaReq.getFields());
+ return tableSchemaReq;
+ })
+ .collect(Collectors.toList());
+
+ } catch (JsonProcessingException e) {
+ throw new SeatunnelException(SeatunnelErrorEnum.ILLEGAL_STATE, e.getMessage());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private Config mergeTaskConfig(
+ JobTask task,
+ PluginType pluginType,
+ String connectorType,
+ BusinessMode businessMode,
+ Config connectorConfig,
+ OptionRule optionRule)
+ throws JsonProcessingException {
+
+ // 在这个地方mergeConfig
+ Long datasourceInstanceId = task.getDataSourceId();
+ // datasourceInstanceId 获取pluginName;
+ String pluginName =
+ datasourceService
+ .queryDatasourceDetailById(datasourceInstanceId.toString())
+ .getPluginName();
+ // 通过datasourceInstanceId 获取配置参数
+ Config datasourceConf =
+ parseConfigWithOptionRule(
+ pluginType,
+ connectorType,
+ datasourceService.queryDatasourceConfigById(
+ datasourceInstanceId.toString()),
+ optionRule);
+
+ DataSourceOption dataSourceOption = null;
+ try {
+ dataSourceOption =
+ task.getDataSourceOption() == null
+ ? null
+ : new ObjectMapper()
+ .readValue(task.getDataSourceOption(), DataSourceOption.class);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ SelectTableFields selectTableFields = null;
+ try {
+ selectTableFields =
+ task.getSelectTableFields() == null
+ ? null
+ : new ObjectMapper()
+ .readValue(
+ task.getSelectTableFields(), SelectTableFields.class);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ SceneMode sceneMode =
+ task.getSceneMode() == null ? null : SceneMode.valueOf(task.getSceneMode());
+ VirtualTableDetailRes virtualTableDetailRes = null;
+
+ if (!SceneMode.MULTIPLE_TABLE.equals(sceneMode)
+ && dataSourceOption != null
+ && CollectionUtils.isNotEmpty(dataSourceOption.getTables())) {
+ String tableName = dataSourceOption.getTables().get(0);
+ // 判断当前tableName是否为虚拟表
+ if (virtualTableService.containsVirtualTableByTableName(tableName)) {
+ // 是虚拟表,获取虚拟表参数
+ virtualTableDetailRes = virtualTableService.queryVirtualTableByTableName(tableName);
+ }
+ }
+
+ return DataSourceConfigSwitcherUtils.mergeDatasourceConfig(
+ pluginName,
+ datasourceConf,
+ virtualTableDetailRes,
+ dataSourceOption,
+ selectTableFields,
+ businessMode,
+ pluginType,
+ connectorConfig);
+ }
+
+ private String createJobConfig(@NonNull JobVersion jobVersion) {
+ List<JobTask> tasks = jobTaskDao.getTasksByVersionId(jobVersion.getId());
+ List<JobLine> lines = jobLineDao.getLinesByVersionId(jobVersion.getId());
+ return generateJobConfig(jobVersion.getJobId(), tasks, lines, jobVersion.getEnv());
+ }
+
+ private String getConnectorConfig(Map<String, List<Config>> connectorMap) {
+ List<String> configs = new ArrayList<>();
+ ConfigRenderOptions configRenderOptions =
+ ConfigRenderOptions.defaults()
+ .setJson(false)
+ .setComments(false)
+ .setOriginComments(false);
+ for (Map.Entry<String, List<Config>> entry : connectorMap.entrySet()) {
+ for (Config c : entry.getValue()) {
+ configs.add(
+ ConfigFactory.empty()
+ .withValue(entry.getKey(), c.root())
+ .root()
+ .render(configRenderOptions));
+ }
+ }
+ return StringUtils.join(configs, "\n");
+ }
+
+ private Config addTableName(String tableName, JobLine jobLine, Config config) {
+ return config.withValue(
+ tableName, ConfigValueFactory.fromAnyRef("Table" + jobLine.getId()));
+ }
+
+ private Config filterEmptyValue(Config config) {
+ List<String> removeKeys =
+ config.entrySet().stream()
+ .filter(entry -> isEmptyValue(entry.getValue()))
+ .map(Map.Entry::getKey)
+ .collect(Collectors.toList());
+ for (String removeKey : removeKeys) {
+ config = config.withoutPath(removeKey);
+ }
+ return config;
+ }
+
+ private void checkSceneMode(List<JobTask> tasks) {
+ Set<String> sceneModes = new HashSet<>();
+ Map<PluginType, Set<Long>> dataSourceIds = new HashMap<>();
+ tasks.forEach(
+ jobTask -> {
+ PluginType pluginType =
+ PluginType.valueOf(jobTask.getType().toUpperCase(Locale.ROOT));
+ if (pluginType.equals(PluginType.SOURCE)
+ || pluginType.equals(PluginType.SINK)) {
+ dataSourceIds
+ .computeIfAbsent(pluginType, n -> new HashSet<>())
+ .add(jobTask.getDataSourceId());
+ if (pluginType.equals(PluginType.SOURCE)) {
+ sceneModes.add(jobTask.getSceneMode());
+ }
+ }
+ });
+
+ if (sceneModes.size() != 1) {
+ throw new SeatunnelException(
+ SeatunnelErrorEnum.ERROR_CONFIG,
+ String.format(
+ "Does not support multiple sceneMode in a job, sceneModes: %s",
+ String.join(", ", sceneModes)));
+ }
+ SceneMode sceneMode =
+ SceneMode.valueOf(sceneModes.iterator().next().toUpperCase(Locale.ROOT));
+ dataSourceIds.forEach(
+ (pluginType, dataSourceIdList) -> {
+ dataSourceIdList.forEach(
+ id -> {
+ String pluginName =
+ datasourceService
+ .queryDatasourceDetailById(id.toString())
+ .getPluginName();
+
+ List<SceneMode> supportedSceneMode =
+ dataSourceMapperConfig
+ .supportedSceneMode(pluginName, pluginType)
+ .orElseThrow(
+ () ->
+ new SeatunnelException(
+ SeatunnelErrorEnum
+ .ILLEGAL_STATE,
+ "Unsupported Data connector Name"));
+ if (!supportedSceneMode.contains(sceneMode)) {
+ throw new SeatunnelException(
+ SeatunnelErrorEnum.ERROR_CONFIG,
+ String.format(
+ "%s not support %s sceneMode",
+ pluginName, sceneMode));
+ }
+ });
+ });
+ }
+
+ private boolean isEmptyValue(ConfigValue value) {
+ return value.unwrapped().toString().isEmpty()
+ || value.valueType().equals(ConfigValueType.NULL);
+ }
+
+ private Config parseConfigWithOptionRule(
+ PluginType pluginType, String connectorType, String config, OptionRule optionRule) {
+ return parseConfigWithOptionRule(
+ pluginType, connectorType, ConfigFactory.parseString(config), optionRule);
+ }
+
+ private Config parseConfigWithOptionRule(
+ PluginType pluginType,
+ String connectorType,
+ Map<String, String> config,
+ OptionRule optionRule) {
+ return parseConfigWithOptionRule(
+ pluginType, connectorType, ConfigFactory.parseMap(config), optionRule);
+ }
+
+ private Config parseConfigWithOptionRule(
+ PluginType pluginType, String connectorType, Config config, OptionRule optionRule) {
+ Map<String, TypeReference<?>> typeReferenceMap = new HashMap<>();
+ optionRule
+ .getOptionalOptions()
+ .forEach(option -> typeReferenceMap.put(option.key(), option.typeReference()));
+ optionRule
+ .getRequiredOptions()
+ .forEach(
+ options -> {
+ options.getOptions()
+ .forEach(
+ option -> {
+ typeReferenceMap.put(
+ option.key(), option.typeReference());
+ });
+ });
+
+ Map<String, ConfigObject> needReplaceMap = new HashMap<>();
+ Map<String, ConfigValue> needReplaceList = new HashMap<>();
+
+ config.entrySet()
+ .forEach(
+ entry -> {
+ String key = entry.getKey();
+ ConfigValue configValue = entry.getValue();
+ try {
+ if (typeReferenceMap.containsKey(key)
+ && isComplexType(typeReferenceMap.get(key))
+ && !isEmptyValue(configValue)) {
+ String valueStr = configValue.unwrapped().toString();
+ if (typeReferenceMap
+ .get(key)
+ .getType()
+ .getTypeName()
+ .startsWith("java.util.List")
+ || typeReferenceMap
+ .get(key)
+ .getType()
+ .getTypeName()
+ .startsWith(
+ "org.apache.seatunnel.api.configuration.Options")) {
+ String valueWrapper = "{key=" + valueStr + "}";
+ ConfigValue configList =
+ ConfigFactory.parseString(valueWrapper)
+ .getList("key");
+ needReplaceList.put(key, configList);
+ } else {
+ Config configObject = ConfigFactory.parseString(valueStr);
+ needReplaceMap.put(key, configObject.root());
+ }
+ }
+ } catch (Exception e) {
+ throw new SeatunnelException(
+ SeatunnelErrorEnum.ERROR_CONFIG,
+ String.format(
+ "Plugin Type: %s, Connector Type: %s, Key: %s, Error Info: %s",
+ pluginType, connectorType, key, e.getMessage()));
+ }
+ });
+ for (Map.Entry<String, ConfigObject> entry : needReplaceMap.entrySet()) {
+ config = config.withValue(entry.getKey(), entry.getValue());
+ }
+ for (Map.Entry<String, ConfigValue> entry : needReplaceList.entrySet()) {
+ config = config.withValue(entry.getKey(), entry.getValue());
+ }
+ return config;
+ }
+
+ private boolean isComplexType(TypeReference<?> typeReference) {
+ return typeReference.getType().getTypeName().startsWith("java.util.List")
+ || typeReference.getType().getTypeName().startsWith("java.util.Map")
+ || typeReference
+ .getType()
+ .getTypeName()
+ .startsWith("org.apache.seatunnel.api.configuration.Options");
+ }
+}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobMetricsServiceImpl.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobMetricsServiceImpl.java
new file mode 100644
index 00000000..26d57f50
--- /dev/null
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobMetricsServiceImpl.java
@@ -0,0 +1,346 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.seatunnel.app.service.impl;
+
+import org.apache.seatunnel.app.dal.dao.IJobInstanceDao;
+import org.apache.seatunnel.app.dal.dao.IJobInstanceHistoryDao;
+import org.apache.seatunnel.app.dal.dao.IJobMetricsDao;
+import org.apache.seatunnel.app.dal.entity.JobInstance;
+import org.apache.seatunnel.app.dal.entity.JobInstanceHistory;
+import org.apache.seatunnel.app.dal.entity.JobMetrics;
+import org.apache.seatunnel.app.domain.response.engine.Engine;
+import org.apache.seatunnel.app.domain.response.metrics.JobDAG;
+import org.apache.seatunnel.app.domain.response.metrics.JobPipelineDetailMetricsRes;
+import org.apache.seatunnel.app.domain.response.metrics.JobPipelineSummaryMetricsRes;
+import org.apache.seatunnel.app.domain.response.metrics.JobSummaryMetricsRes;
+import org.apache.seatunnel.app.permission.constants.SeatunnelFuncPermissionKeyConstant;
+import org.apache.seatunnel.app.service.IJobMetricsService;
+import org.apache.seatunnel.app.thirdparty.metrics.EngineMetricsExtractorFactory;
+import org.apache.seatunnel.app.thirdparty.metrics.IEngineMetricsExtractor;
+import org.apache.seatunnel.server.common.CodeGenerateUtils;
+import org.apache.seatunnel.server.common.Constants;
+import org.apache.seatunnel.server.common.SeatunnelErrorEnum;
+import org.apache.seatunnel.server.common.SeatunnelException;
+
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.ImmutablePair;
+
+import org.springframework.stereotype.Service;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import lombok.NonNull;
+
+import javax.annotation.Resource;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+@Service
+public class JobMetricsServiceImpl extends SeatunnelBaseServiceImpl implements IJobMetricsService {
+
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+ @Resource private IJobMetricsDao jobMetricsDao;
+
+ @Resource private IJobInstanceHistoryDao jobInstanceHistoryDao;
+
+ @Resource private IJobInstanceDao jobInstanceDao;
+
+ @Override
+ public List<JobPipelineSummaryMetricsRes> getJobPipelineSummaryMetrics(
+ @NonNull Integer userId,
+ @NonNull Long projectCode,
+ @NonNull Long jobInstanceId,
+ @NonNull String jobEngineId) {
+ funcPermissionCheck(SeatunnelFuncPermissionKeyConstant.JOB_METRICS_SUMMARY, userId);
+ List<JobMetrics> jobPipelineDetailMetrics =
+ getJobPipelineMetrics(userId, projectCode, jobInstanceId, jobEngineId);
+ return summaryMetrics(jobPipelineDetailMetrics);
+ }
+
+ @Override
+ public JobSummaryMetricsRes getJobSummaryMetrics(
+ @NonNull Integer userId,
+ @NonNull Long projectCode,
+ @NonNull Long jobInstanceId,
+ @NonNull String jobEngineId) {
+ funcPermissionCheck(SeatunnelFuncPermissionKeyConstant.JOB_METRICS_SUMMARY, userId);
+ JobInstance jobInstance = jobInstanceDao.getJobInstance(jobInstanceId);
+ Engine engine = new Engine(jobInstance.getEngineName(), jobInstance.getEngineVersion());
+ IEngineMetricsExtractor engineMetricsExtractor =
+ (new EngineMetricsExtractorFactory(engine)).getEngineMetricsExtractor();
+ String jobStatus = engineMetricsExtractor.getJobStatus(jobEngineId);
+
+ List<JobMetrics> jobPipelineDetailMetrics =
+ getJobPipelineMetrics(userId, projectCode, jobInstanceId, jobEngineId);
+ long readCount =
+ jobPipelineDetailMetrics.stream().mapToLong(JobMetrics::getReadRowCount).sum();
+ long writeCount =
+ jobPipelineDetailMetrics.stream().mapToLong(JobMetrics::getWriteRowCount).sum();
+
+ JobSummaryMetricsRes jobSummaryMetricsRes =
+ new JobSummaryMetricsRes(
+ jobInstanceId, Long.valueOf(jobEngineId), readCount, writeCount, jobStatus);
+ return jobSummaryMetricsRes;
+ }
+
+ private List<JobMetrics> getJobPipelineDetailMetrics(
+ @NonNull JobInstance jobInstance,
+ @NonNull Integer userId,
+ @NonNull Long projectCode,
+ @NonNull String jobEngineId,
+ @NonNull String jobStatus,
+ @NonNull IEngineMetricsExtractor engineMetricsExtractor) {
+
+ // If job is not end state, get metrics from engine.
+ List<JobMetrics> jobMetrics = new ArrayList<>();
+ if (engineMetricsExtractor.isJobEndStatus(jobStatus)) {
+ jobMetrics = getJobMetricsFromDb(jobInstance, userId, projectCode, jobEngineId);
+ if (CollectionUtils.isEmpty(jobMetrics)) {
+ syncMetricsToDb(jobInstance, userId, projectCode, jobEngineId);
+ jobMetrics = getJobMetricsFromEngine(jobInstance, jobEngineId);
+ }
+ } else {
+ jobMetrics = getJobMetricsFromEngine(jobInstance, jobEngineId);
+ }
+
+ return jobMetrics;
+ }
+
+ @Override
+ public List<JobPipelineDetailMetricsRes> getJobPipelineDetailMetricsRes(
+ @NonNull Integer userId,
+ @NonNull Long projectCode,
+ @NonNull Long jobInstanceId,
+ @NonNull String jobEngineId) {
+ funcPermissionCheck(SeatunnelFuncPermissionKeyConstant.JOB_DETAIL, userId);
+ List<JobMetrics> jobPipelineDetailMetrics =
+ getJobPipelineMetrics(userId, projectCode, jobInstanceId, jobEngineId);
+ return jobPipelineDetailMetrics.stream()
+ .map(this::wrapperJobMetrics)
+ .collect(Collectors.toList());
+ }
+
+ private List<JobMetrics> getJobPipelineMetrics(
+ @NonNull Integer userId,
+ @NonNull Long projectCode,
+ @NonNull Long jobInstanceId,
+ @NonNull String jobEngineId) {
+
+ JobInstance jobInstance = jobInstanceDao.getJobInstance(jobInstanceId);
+ Engine engine = new Engine(jobInstance.getEngineName(), jobInstance.getEngineVersion());
+ IEngineMetricsExtractor engineMetricsExtractor =
+ (new EngineMetricsExtractorFactory(engine)).getEngineMetricsExtractor();
+ String jobStatus = engineMetricsExtractor.getJobStatus(jobEngineId);
+ List<JobMetrics> jobPipelineDetailMetrics =
+ getJobPipelineDetailMetrics(
+ jobInstance,
+ userId,
+ projectCode,
+ jobEngineId,
+ jobStatus,
+ engineMetricsExtractor);
+ return jobPipelineDetailMetrics;
+ }
+
+ @Override
+ public JobDAG getJobDAG(
+ @NonNull Integer userId,
+ @NonNull Long projectCode,
+ @NonNull Long jobInstanceId,
+ @NonNull String jobEngineId)
+ throws JsonProcessingException {
+ funcPermissionCheck(SeatunnelFuncPermissionKeyConstant.JOB_DAG, userId);
+ JobInstance jobInstance = jobInstanceDao.getJobInstance(jobInstanceId);
+ JobInstanceHistory history =
+ getJobHistoryFromDb(jobInstance, userId, jobEngineId, projectCode);
+ if (history != null) {
+ String dag = history.getDag();
+ try {
+ return OBJECT_MAPPER.readValue(dag, JobDAG.class);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ Engine engine = new Engine(jobInstance.getEngineName(), jobInstance.getEngineVersion());
+ IEngineMetricsExtractor engineMetricsExtractor =
+ (new EngineMetricsExtractorFactory(engine)).getEngineMetricsExtractor();
+
+ if (engineMetricsExtractor.isJobEnd(jobEngineId)) {
+ syncJobInfoToDb(jobInstance, projectCode, jobEngineId);
+ history = getJobHistoryFromDb(jobInstance, userId, jobEngineId, projectCode);
+ } else {
+ history = getJobHistoryFromEngine(jobInstance, jobEngineId);
+ }
+ if (history != null) {
+ String dag = history.getDag();
+ try {
+ return OBJECT_MAPPER.readValue(dag, JobDAG.class);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ return null;
+ }
+
+ private JobInstanceHistory getJobHistoryFromEngine(
+ @NonNull JobInstance jobInstance, String jobEngineId) {
+
+ Engine engine = new Engine(jobInstance.getEngineName(), jobInstance.getEngineVersion());
+
+ IEngineMetricsExtractor engineMetricsExtractor =
+ (new EngineMetricsExtractorFactory(engine)).getEngineMetricsExtractor();
+
+ return engineMetricsExtractor.getJobHistoryById(jobEngineId);
+ }
+
+ private JobInstanceHistory getJobHistoryFromDb(
+ @NonNull JobInstance jobInstance,
+ Integer userId,
+ String jobEngineId,
+ Long projectCode) {
+ // relation jobInstanceId and jobEngineId
+ relationJobInstanceAndJobEngineId(jobInstance, userId, jobEngineId);
+ return jobInstanceHistoryDao.getByInstanceId(jobInstance.getId(), projectCode);
+ }
+
+ @Override
+ public void syncJobDataToDb(
+ @NonNull JobInstance jobInstance,
+ @NonNull Integer userId,
+ @NonNull Long projectCode,
+ @NonNull String jobEngineId) {
+ relationJobInstanceAndJobEngineId(jobInstance, userId, jobEngineId);
+ syncMetricsToDb(jobInstance, userId, projectCode, jobEngineId);
+ syncJobInfoToDb(jobInstance, projectCode, jobEngineId);
+ }
+
+ private void syncMetricsToDb(
+ @NonNull JobInstance jobInstance,
+ @NonNull Integer userId,
+ @NonNull Long projectCode,
+ @NonNull String jobEngineId) {
+ List<JobMetrics> jobMetricsFromEngine = getJobMetricsFromEngine(jobInstance, jobEngineId);
+
+ jobMetricsFromEngine.forEach(
+ metrics -> {
+ try {
+ metrics.setId(CodeGenerateUtils.getInstance().genCode());
+ } catch (CodeGenerateUtils.CodeGenerateException e) {
+ throw new SeatunnelException(
+ SeatunnelErrorEnum.JOB_RUN_GENERATE_UUID_ERROR);
+ }
+ metrics.setJobInstanceId(jobInstance.getId());
+ metrics.setCreateUserId(userId);
+ metrics.setUpdateUserId(userId);
+ });
+
+ if (!jobMetricsFromEngine.isEmpty()) {
+ jobMetricsDao.getJobMetricsMapper().insertBatchMetrics(jobMetricsFromEngine);
+ }
+ }
+
+ private void syncJobInfoToDb(
+ @NonNull JobInstance jobInstance,
+ @NonNull Long projectCode,
+ @NonNull String jobEngineId) {
+ JobInstanceHistory jobHistoryFromEngine = getJobHistoryFromEngine(jobInstance, jobEngineId);
+
+ jobHistoryFromEngine.setId(jobInstance.getId());
+
+ jobInstanceHistoryDao.insert(jobHistoryFromEngine);
+ }
+
+ private void relationJobInstanceAndJobEngineId(
+ @NonNull JobInstance jobInstance,
+ @NonNull Integer userId,
+ @NonNull String jobEngineId) {
+ // relation jobInstanceId and jobEngineId
+ if (StringUtils.isEmpty(jobInstance.getJobEngineId())) {
+ jobInstance.setJobEngineId(jobEngineId);
+ jobInstance.setUpdateUserId(userId);
+ jobInstanceDao.update(jobInstance);
+ }
+ }
+
+ private List<JobMetrics> getJobMetricsFromEngine(
+ @NonNull JobInstance jobInstance, @NonNull String jobEngineId) {
+ Engine engine = new Engine(jobInstance.getEngineName(), jobInstance.getEngineVersion());
+
+ IEngineMetricsExtractor engineMetricsExtractor =
+ (new EngineMetricsExtractorFactory(engine)).getEngineMetricsExtractor();
+
+ return engineMetricsExtractor.getMetricsByJobEngineId(jobEngineId);
+ }
+
+ private List<JobPipelineSummaryMetricsRes> summaryMetrics(
+ @NonNull List<JobMetrics> jobPipelineDetailedMetrics) {
+ return jobPipelineDetailedMetrics.stream()
+ .map(
+ metrics ->
+ new JobPipelineSummaryMetricsRes(
+ metrics.getPipelineId(),
+ metrics.getReadRowCount(),
+ metrics.getWriteRowCount(),
+ metrics.getStatus()))
+ .collect(Collectors.toList());
+ }
+
+ private List<JobMetrics> getJobMetricsFromDb(
+ @NonNull JobInstance jobInstance,
+ @NonNull Integer userId,
+ @NonNull Long projectCode,
+ @NonNull String jobEngineId) {
+
+ // relation jobInstanceId and jobEngineId
+ relationJobInstanceAndJobEngineId(jobInstance, userId, jobEngineId);
+
+ // get metrics from db
+ return jobMetricsDao.getByInstanceId(projectCode, jobInstance.getId());
+ }
+
+ @Override
+ public ImmutablePair<Long, String> getInstanceIdAndEngineId(@NonNull String key) {
+ if (!key.contains(Constants.METRICS_QUERY_KEY_SPLIT)
+ || key.split(Constants.METRICS_QUERY_KEY_SPLIT).length != 2) {
+ throw new SeatunnelException(SeatunnelErrorEnum.JOB_METRICS_QUERY_KEY_ERROR, key);
+ }
+
+ String[] split = key.split(Constants.METRICS_QUERY_KEY_SPLIT);
+ Long jobInstanceId = Long.valueOf(split[0]);
+ String jobEngineId = split[1];
+ return new ImmutablePair<>(jobInstanceId, jobEngineId);
+ }
+
+ private JobPipelineDetailMetricsRes wrapperJobMetrics(@NonNull JobMetrics metrics) {
+ return new JobPipelineDetailMetricsRes(
+ metrics.getId(),
+ metrics.getPipelineId(),
+ metrics.getReadRowCount(),
+ metrics.getWriteRowCount(),
+ metrics.getSourceTableNames(),
+ metrics.getSinkTableNames(),
+ metrics.getReadQps(),
+ metrics.getWriteQps(),
+ metrics.getRecordDelay(),
+ metrics.getStatus());
+ }
+}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobTaskServiceImpl.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobTaskServiceImpl.java
new file mode 100644
index 00000000..e54ef57a
--- /dev/null
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobTaskServiceImpl.java
@@ -0,0 +1,593 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * <p>http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * <p>Unless required by applicable law or agreed to in writing, software distributed under the
+ * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.seatunnel.app.service.impl;
+
+import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory;
+
+import org.apache.seatunnel.app.config.ConnectorDataSourceMapperConfig;
+import org.apache.seatunnel.app.dal.dao.IJobLineDao;
+import org.apache.seatunnel.app.dal.dao.IJobTaskDao;
+import org.apache.seatunnel.app.dal.dao.IJobVersionDao;
+import org.apache.seatunnel.app.dal.entity.JobLine;
+import org.apache.seatunnel.app.dal.entity.JobTask;
+import org.apache.seatunnel.app.dal.entity.JobVersion;
+import org.apache.seatunnel.app.domain.request.connector.SceneMode;
+import org.apache.seatunnel.app.domain.request.job.DataSourceOption;
+import org.apache.seatunnel.app.domain.request.job.DatabaseTableSchemaReq;
+import org.apache.seatunnel.app.domain.request.job.Edge;
+import org.apache.seatunnel.app.domain.request.job.JobDAG;
+import org.apache.seatunnel.app.domain.request.job.JobTaskInfo;
+import org.apache.seatunnel.app.domain.request.job.PluginConfig;
+import org.apache.seatunnel.app.domain.request.job.SelectTableFields;
+import org.apache.seatunnel.app.domain.request.job.transform.Copy;
+import org.apache.seatunnel.app.domain.request.job.transform.CopyTransformOptions;
+import org.apache.seatunnel.app.domain.request.job.transform.FieldMapperTransformOptions;
+import org.apache.seatunnel.app.domain.request.job.transform.RenameField;
+import org.apache.seatunnel.app.domain.request.job.transform.SplitTransformOptions;
+import org.apache.seatunnel.app.domain.request.job.transform.Transform;
+import org.apache.seatunnel.app.domain.request.job.transform.TransformOption;
+import org.apache.seatunnel.app.domain.request.job.transform.TransformOptions;
+import org.apache.seatunnel.app.domain.response.job.JobTaskCheckRes;
+import org.apache.seatunnel.app.domain.response.job.SchemaError;
+import org.apache.seatunnel.app.domain.response.job.SchemaErrorType;
+import org.apache.seatunnel.app.permission.constants.SeatunnelFuncPermissionKeyConstant;
+import org.apache.seatunnel.app.service.IDatasourceService;
+import org.apache.seatunnel.app.service.IJobInstanceService;
+import org.apache.seatunnel.app.service.IJobTaskService;
+import org.apache.seatunnel.common.constants.PluginType;
+import org.apache.seatunnel.common.utils.ExceptionUtils;
+import org.apache.seatunnel.common.utils.SeaTunnelException;
+import org.apache.seatunnel.datasource.plugin.api.model.TableField;
+import org.apache.seatunnel.server.common.CodeGenerateUtils;
+import org.apache.seatunnel.server.common.SeatunnelErrorEnum;
+import org.apache.seatunnel.server.common.SeatunnelException;
+
+import org.apache.commons.lang3.StringUtils;
+
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import lombok.extern.slf4j.Slf4j;
+
+import javax.annotation.Resource;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.apache.seatunnel.app.utils.TaskOptionUtils.getTransformOption;
+
+@Service
+@Slf4j
+public class JobTaskServiceImpl extends SeatunnelBaseServiceImpl implements IJobTaskService {
+
+ @Resource(name = "jobLineDaoImpl")
+ private IJobLineDao jobLineDao;
+
+ @Resource(name = "jobTaskDaoImpl")
+ private IJobTaskDao jobTaskDao;
+
+ @Resource(name = "jobVersionDaoImpl")
+ private IJobVersionDao jobVersionDao;
+
+ @Resource private IDatasourceService datasourceService;
+
+ @Resource private IJobInstanceService jobInstanceService;
+
+ @Resource private ConnectorDataSourceMapperConfig connectorDataSourceMapperConfig;
+
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+ private void checkConfigIntegrity(JobVersion version, JobTaskInfo jobTaskInfo) {
+ if (StringUtils.isEmpty(version.getEnv())) {
+ throw new SeatunnelException(
+ SeatunnelErrorEnum.ERROR_CONFIG,
+ "job env can't be empty, please change config");
+ }
+ Map<String, PluginConfig> pluginConfigMap =
+ jobTaskInfo.getPlugins().stream()
+ .collect(Collectors.toMap(PluginConfig::getPluginId, Function.identity()));
+
+ List<String> allPluginIdsFromEdge =
+ Stream.concat(
+ jobTaskInfo.getEdges().stream().map(Edge::getInputPluginId),
+ jobTaskInfo.getEdges().stream().map(Edge::getTargetPluginId))
+ .collect(Collectors.toList());
+
+ jobTaskInfo.getPlugins().stream()
+ .filter(p -> !allPluginIdsFromEdge.contains(p.getPluginId()))
+ .findAny()
+ .ifPresent(
+ p -> {
+ throw new SeatunnelException(
+ SeatunnelErrorEnum.ERROR_CONFIG,
+ "plugin '" + p.getName() + "' is not used in any edge");
+ });
+
+ jobTaskInfo
+ .getEdges()
+ .forEach(
+ e -> {
+ jobTaskInfo.getPlugins().stream()
+ .filter(
+ p ->
+ Objects.equals(
+ p.getPluginId(), e.getInputPluginId()))
+ .findFirst()
+ .orElseThrow(
+ () ->
+ new SeatunnelException(
+ SeatunnelErrorEnum.ERROR_CONFIG,
+ "input plugin not found"));
+ jobTaskInfo.getPlugins().stream()
+ .filter(
+ p ->
+ Objects.equals(
+ p.getPluginId(), e.getTargetPluginId()))
+ .findFirst()
+ .orElseThrow(
+ () ->
+ new SeatunnelException(
+ SeatunnelErrorEnum.ERROR_CONFIG,
+ "target plugin not found"));
+ });
+
+ List<String> inputTransformId =
+ jobTaskInfo.getEdges().stream()
+ .map(Edge::getInputPluginId)
+ .filter(e -> pluginConfigMap.get(e).getType().equals(PluginType.TRANSFORM))
+ .collect(Collectors.toList());
+ List<String> targetTransformId =
+ jobTaskInfo.getEdges().stream()
+ .map(Edge::getTargetPluginId)
+ .filter(e -> pluginConfigMap.get(e).getType().equals(PluginType.TRANSFORM))
+ .collect(Collectors.toList());
+
+ if (!new HashSet<>(inputTransformId).containsAll(targetTransformId)
+ || !new HashSet<>(targetTransformId).containsAll(inputTransformId)) {
+ throw new SeatunnelException(
+ SeatunnelErrorEnum.ERROR_CONFIG, "transform plugin must be connected");
+ }
+
+ for (Edge edge : jobTaskInfo.getEdges()) {
+ if (!pluginTypeMatch(
+ pluginConfigMap.get(edge.getInputPluginId()).getType(),
+ pluginConfigMap.get(edge.getTargetPluginId()).getType())) {
+ throw new SeatunnelException(
+ SeatunnelErrorEnum.ERROR_CONFIG,
+ "plugin line not match, please check plugin line");
+ }
+ }
+ }
+
+ private boolean pluginTypeMatch(PluginType inputType, PluginType outputType) {
+ if (inputType == PluginType.SOURCE) {
+ return outputType != PluginType.SOURCE;
+ }
+ if (inputType == PluginType.SINK) {
+ return false;
+ }
+ if (inputType == PluginType.TRANSFORM) {
+ return outputType != PluginType.SOURCE;
+ }
+ return false;
+ }
+
+ private static void checkConfigFormat(String config) {
+ if (StringUtils.isNotEmpty(config)) {
+ ConfigFactory.parseString(config);
+ }
+ }
+
+ @Override
+ public JobTaskInfo getTaskConfig(long jobVersionId) {
+ funcPermissionCheck(SeatunnelFuncPermissionKeyConstant.JOB_TASK_DETAIL, 0);
+ List<JobTask> tasks = jobTaskDao.getTasksByVersionId(jobVersionId);
+ if (tasks.isEmpty()) {
+ return null;
+ }
+ List<JobLine> lines = jobLineDao.getLinesByVersionId(jobVersionId);
+ return new JobTaskInfo(
+ lines.stream()
+ .map(l -> new Edge(l.getInputPluginId(), l.getTargetPluginId()))
+ .collect(Collectors.toList()),
+ tasks.stream()
+ .map(
+ t -> {
+ try {
+ return getPluginConfigFromJobTask(t);
+ } catch (Exception e) {
+ throw new SeatunnelException(
+ SeatunnelErrorEnum.UNKNOWN, e.getMessage());
+ }
+ })
+ .collect(Collectors.toList()));
+ }
+
+ @Override
+ @Transactional
+ public JobTaskCheckRes saveJobDAG(long jobVersionId, JobDAG jobDAG) {
+ funcPermissionCheck(SeatunnelFuncPermissionKeyConstant.JOB_TASK_DAG_CREATE, 0);
+ JobVersion version = jobVersionDao.getVersionById(jobVersionId);
+ List<JobTask> tasks = jobTaskDao.getTasksByVersionId(jobVersionId);
+ List<PluginConfig> pluginConfigs =
+ tasks.stream()
+ .map(JobTaskServiceImpl::getPluginConfigFromJobTask)
+ .collect(Collectors.toList());
+ JobTaskInfo taskInfo = new JobTaskInfo(jobDAG.getEdges(), pluginConfigs);
+ checkConfigIntegrity(version, taskInfo);
+ List<JobLine> lines =
+ jobDAG.getEdges().stream()
+ .map(
+ e -> {
+ try {
+ return JobLine.builder()
+ .id(CodeGenerateUtils.getInstance().genCode())
+ .inputPluginId(e.getInputPluginId())
+ .targetPluginId(e.getTargetPluginId())
+ .versionId(jobVersionId)
+ .build();
+ } catch (CodeGenerateUtils.CodeGenerateException ex) {
+ throw new SeatunnelException(
+ SeatunnelErrorEnum.ILLEGAL_STATE, ex.getMessage());
+ }
+ })
+ .collect(Collectors.toList());
+
+ try {
+ JobTaskCheckRes jobTaskCheckRes = checkPluginSchemaIntegrity(taskInfo);
+ if (jobTaskCheckRes != null) {
+ return jobTaskCheckRes;
+ }
+ // check the config can be generated
+ jobInstanceService.generateJobConfig(
+ version.getJobId(), tasks, lines, version.getEnv());
+ // TODO check schema output and input matched
+ } catch (SeaTunnelException e) {
+ log.error(ExceptionUtils.getMessage(e));
+ throw e;
+ } catch (Exception e) {
+ throw new SeatunnelException(SeatunnelErrorEnum.ERROR_CONFIG, e.getMessage());
+ }
+
+ jobLineDao.deleteLinesByVersionId(jobVersionId);
+ jobLineDao.insertLines(lines);
+ return null;
+ }
+
+ private JobTaskCheckRes checkPluginSchemaIntegrity(JobTaskInfo taskInfo) throws IOException {
+
+ List<PluginConfig> source =
+ taskInfo.getPlugins().stream()
+ .filter(p -> p.getType().equals(PluginType.SOURCE))
+ .collect(Collectors.toList());
+
+ Map<String, PluginConfig> pluginMap =
+ taskInfo.getPlugins().stream()
+ .collect(Collectors.toMap(PluginConfig::getPluginId, Function.identity()));
+ Map<String, String> edgeMap =
+ taskInfo.getEdges().stream()
+ .collect(Collectors.toMap(Edge::getInputPluginId, Edge::getTargetPluginId));
+
+ for (PluginConfig config : source) {
+ PluginConfig nextConfig = pluginMap.get(edgeMap.get(config.getPluginId()));
+ JobTaskCheckRes res = checkNextTaskSchema(config, nextConfig, pluginMap, edgeMap);
+ if (res != null) {
+ return res;
+ }
+ }
+ return null;
+ }
+
+ private JobTaskCheckRes checkNextTaskSchema(
+ PluginConfig config,
+ PluginConfig nextConfig,
+ Map<String, PluginConfig> pluginMap,
+ Map<String, String> edgeMap)
+ throws IOException {
+ Map<String, Object> options = nextConfig.getTransformOptions();
+ if (options != null && !options.isEmpty()) {
+ Transform transform = Transform.valueOf(nextConfig.getConnectorType().toUpperCase());
+ String transformOptionsStr = OBJECT_MAPPER.writeValueAsString(options);
+
+ List<TransformOption> transformOptions = new ArrayList<>();
+
+ switch (transform) {
+ case FIELDMAPPER:
+ FieldMapperTransformOptions fieldMapperTransformOptions =
+ getTransformOption(transform, transformOptionsStr);
+ if (fieldMapperTransformOptions != null) {
+ fillTransformOptions(
+ transformOptions, fieldMapperTransformOptions.getRenameFields());
+ fillTransformOptions(
+ transformOptions, fieldMapperTransformOptions.getChangeOrders());
+ }
+ break;
+ case MULTIFIELDSPLIT:
+ SplitTransformOptions splitTransformOptions =
+ getTransformOption(transform, transformOptionsStr);
+ if (splitTransformOptions != null) {
+ fillTransformOptions(transformOptions, splitTransformOptions.getSplits());
+ }
+ break;
+ case COPY:
+ CopyTransformOptions copyTransformOptions =
+ getTransformOption(transform, transformOptionsStr);
+ if (copyTransformOptions != null) {
+ fillTransformOptions(transformOptions, copyTransformOptions.getCopyList());
+ }
+ break;
+ case FILTERROWKIND:
+ case REPLACE:
+ default:
+ throw new SeatunnelException(
+ SeatunnelErrorEnum.UNSUPPORTED_CONNECTOR_TYPE,
+ "Unsupported Transform Option " + transform);
+ }
+
+ if (!transformOptions.isEmpty()) {
+ DatabaseTableSchemaReq databaseTableSchemaReq = config.getOutputSchema().get(0);
+ List<String> fields =
+ databaseTableSchemaReq.getFields().stream()
+ .map(TableField::getName)
+ .collect(Collectors.toList());
+ Optional<TransformOption> transformOption =
+ transformOptions.stream()
+ .filter(option -> !fields.contains(option.getSourceFieldName()))
+ .findFirst();
+ if (transformOption.isPresent()) {
+ String sourceFiledName = transformOption.get().getSourceFieldName();
+ return new JobTaskCheckRes(
+ false,
+ nextConfig.getPluginId(),
+ new SchemaError(
+ databaseTableSchemaReq.getDatabase(),
+ databaseTableSchemaReq.getTableName(),
+ sourceFiledName,
+ SchemaErrorType.MISS_FIELD),
+ null);
+ }
+ }
+ }
+ if (edgeMap.containsKey(nextConfig.getPluginId())) {
+ return checkNextTaskSchema(
+ nextConfig,
+ pluginMap.get(edgeMap.get(nextConfig.getPluginId())),
+ pluginMap,
+ edgeMap);
+ }
+ return null;
+ }
+
+ private static void fillTransformOptions(
+ List<TransformOption> transformOptions, List<? extends TransformOption> options) {
+ if (options != null) {
+ transformOptions.addAll(options);
+ }
+ }
+
+ @Override
+ public void saveSingleTask(long jobVersionId, PluginConfig pluginConfig) {
+ funcPermissionCheck(SeatunnelFuncPermissionKeyConstant.SINGLE_TASK_CREATE, 0);
+ JobTask jobTask;
+ JobTask old = jobTaskDao.getTask(jobVersionId, pluginConfig.getPluginId());
+ try {
+ checkConfigFormat(pluginConfig.getConfig());
+ long id;
+ if (old != null) {
+ id = old.getId();
+ } else {
+ id = CodeGenerateUtils.getInstance().genCode();
+ }
+ String connectorType;
+ String transformOptionsStr = null;
+ if (pluginConfig.getType().equals(PluginType.TRANSFORM)) {
+ connectorType = pluginConfig.getConnectorType();
+ if (pluginConfig.getTransformOptions() != null) {
+ transformOptionsStr =
+ OBJECT_MAPPER.writeValueAsString(pluginConfig.getTransformOptions());
+ }
+ transformOptionCheck(connectorType, transformOptionsStr);
+ } else {
+ connectorType = getConnectorTypeFromDataSource(pluginConfig.getDataSourceId());
+ }
+ jobTask =
+ JobTask.builder()
+ .id(id)
+ .pluginId(pluginConfig.getPluginId())
+ .name(pluginConfig.getName())
+ .type(pluginConfig.getType().name().toUpperCase())
+ .dataSourceId(pluginConfig.getDataSourceId())
+ .config(pluginConfig.getConfig())
+ .sceneMode(
+ pluginConfig.getSceneMode() == null
+ ? null
+ : pluginConfig.getSceneMode().name())
+ .versionId(jobVersionId)
+ .connectorType(connectorType)
+ .dataSourceOption(
+ pluginConfig.getTableOption() == null
+ ? null
+ : OBJECT_MAPPER.writeValueAsString(
+ pluginConfig.getTableOption()))
+ .selectTableFields(
+ pluginConfig.getSelectTableFields() == null
+ ? null
+ : OBJECT_MAPPER.writeValueAsString(
+ pluginConfig.getSelectTableFields()))
+ .outputSchema(
+ pluginConfig.getOutputSchema() == null
+ ? null
+ : OBJECT_MAPPER.writeValueAsString(
+ pluginConfig.getOutputSchema()))
+ .transformOptions(transformOptionsStr)
+ .build();
+ } catch (Exception e) {
+ throw new SeatunnelException(SeatunnelErrorEnum.ILLEGAL_STATE, e.getMessage());
+ }
+ if (old != null) {
+ jobTaskDao.updateTask(jobTask);
+ } else {
+ jobTaskDao.insertTask(jobTask);
+ }
+ }
+
+ private void transformOptionCheck(String connectorType, String transformOptionsStr)
+ throws IOException {
+ Transform transform = Transform.valueOf(connectorType.toUpperCase());
+ switch (transform) {
+ case FIELDMAPPER:
+ FieldMapperTransformOptions fieldMapperTransformOptions =
+ getTransformOption(transform, transformOptionsStr);
+ if (fieldMapperTransformOptions != null) {
+ List<RenameField> renameFields = fieldMapperTransformOptions.getRenameFields();
+ checkTransformTargetFieldRepeat(
+ renameFields.stream()
+ .map(RenameField::getTargetName)
+ .collect(Collectors.toList()));
+ }
+ break;
+ case MULTIFIELDSPLIT:
+ SplitTransformOptions splitTransformOptions =
+ getTransformOption(transform, transformOptionsStr);
+ if (splitTransformOptions != null) {
+ List<String> fields =
+ splitTransformOptions.getSplits().stream()
+ .flatMap(split -> split.getOutputFields().stream())
+ .collect(Collectors.toList());
+ checkTransformTargetFieldRepeat(fields);
+ }
+ break;
+ case COPY:
+ CopyTransformOptions copyTransformOptions =
+ getTransformOption(transform, transformOptionsStr);
+ if (copyTransformOptions != null) {
+ List<String> fields =
+ copyTransformOptions.getCopyList().stream()
+ .map(Copy::getTargetFieldName)
+ .collect(Collectors.toList());
+ checkTransformTargetFieldRepeat(fields);
+ }
+ break;
+ case FILTERROWKIND:
+ case REPLACE:
+ default:
+ }
+ }
+
+ private void checkTransformTargetFieldRepeat(List<String> fields) {
+ Set<String> duplicates =
+ fields.stream()
+ .filter(i -> Collections.frequency(fields, i) > 1)
+ .collect(Collectors.toSet());
+ if (!duplicates.isEmpty()) {
+ throw new SeatunnelException(
+ SeatunnelErrorEnum.ILLEGAL_STATE,
+ "Can't convert same target name: " + new ArrayList<>(duplicates));
+ }
+ }
+
+ @Override
+ public PluginConfig getSingleTask(long jobVersionId, String pluginId) {
+ funcPermissionCheck(SeatunnelFuncPermissionKeyConstant.SINGLE_TASK_DETAIL, 0);
+ return getPluginConfigFromJobTask(jobTaskDao.getTask(jobVersionId, pluginId));
+ }
+
+ @Override
+ public <T extends TransformOptions> T getTransformOptions(long jobVersionId, String pluginId) {
+ JobTask jobTask = jobTaskDao.getTask(jobVersionId, pluginId);
+ try {
+ Transform transform = Transform.valueOf(jobTask.getConnectorType().toUpperCase());
+ return getTransformOption(transform, jobTask.getTransformOptions());
+ } catch (IOException e) {
+ throw new SeatunnelException(SeatunnelErrorEnum.ILLEGAL_STATE, e.getMessage());
+ }
+ }
+
+ @Override
+ public void deleteSingleTask(long jobVersionId, String pluginId) {
+ funcPermissionCheck(SeatunnelFuncPermissionKeyConstant.SINGLE_TASK_DELETE, 0);
+ jobTaskDao.deleteTask(jobVersionId, pluginId);
+ }
+
+ private String getConnectorTypeFromDataSource(long datasourceId) {
+ String pluginName =
+ datasourceService
+ .queryDatasourceDetailById(String.valueOf(datasourceId))
+ .getPluginName();
+ return connectorDataSourceMapperConfig
+ .findConnectorForDatasourceName(pluginName)
+ .orElseThrow(
+ () ->
+ new SeatunnelException(
+ SeatunnelErrorEnum.ILLEGAL_STATE,
+ "can not find connector for datasourceName: "
+ + pluginName));
+ }
+
+ private static PluginConfig getPluginConfigFromJobTask(JobTask jobTask) {
+ try {
+ return PluginConfig.builder()
+ .pluginId(jobTask.getPluginId())
+ .name(jobTask.getName())
+ .type(PluginType.valueOf(jobTask.getType().toUpperCase()))
+ .dataSourceId(jobTask.getDataSourceId())
+ .config(jobTask.getConfig())
+ .connectorType(jobTask.getConnectorType())
+ .sceneMode(
+ StringUtils.isEmpty(jobTask.getSceneMode())
+ ? null
+ : SceneMode.valueOf(jobTask.getSceneMode()))
+ .tableOption(
+ StringUtils.isEmpty(jobTask.getDataSourceOption())
+ ? null
+ : OBJECT_MAPPER.readValue(
+ jobTask.getDataSourceOption(), DataSourceOption.class))
+ .selectTableFields(
+ StringUtils.isEmpty(jobTask.getSelectTableFields())
+ ? null
+ : OBJECT_MAPPER.readValue(
+ jobTask.getSelectTableFields(),
+ SelectTableFields.class))
+ .outputSchema(
+ StringUtils.isEmpty(jobTask.getOutputSchema())
+ ? null
+ : OBJECT_MAPPER.readValue(
+ jobTask.getOutputSchema(),
+ new TypeReference<List<DatabaseTableSchemaReq>>() {}))
+ .transformOptions(
+ StringUtils.isEmpty(jobTask.getTransformOptions())
+ ? null
+ : OBJECT_MAPPER.readValue(
+ jobTask.getTransformOptions(),
+ new TypeReference<Map<String, Object>>() {}))
+ .config(jobTask.getConfig())
+ .build();
+ } catch (Exception e) {
+ throw new SeatunnelException(SeatunnelErrorEnum.UNKNOWN, e.getMessage());
+ }
+ }
+}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/engine/SeaTunnelEngineMetricsExtractor.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/engine/SeaTunnelEngineMetricsExtractor.java
new file mode 100644
index 00000000..bf3444ec
--- /dev/null
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/engine/SeaTunnelEngineMetricsExtractor.java
@@ -0,0 +1,284 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.seatunnel.app.thirdparty.engine;
+
+import org.apache.seatunnel.shade.com.fasterxml.jackson.core.JsonProcessingException;
+import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.JsonNode;
+import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.ObjectMapper;
+
+import org.apache.seatunnel.app.dal.entity.JobInstanceHistory;
+import org.apache.seatunnel.app.dal.entity.JobMetrics;
+import org.apache.seatunnel.app.thirdparty.metrics.IEngineMetricsExtractor;
+import org.apache.seatunnel.common.utils.ExceptionUtils;
+import org.apache.seatunnel.common.utils.JsonUtils;
+import org.apache.seatunnel.engine.core.job.JobDAGInfo;
+import org.apache.seatunnel.server.common.SeatunnelErrorEnum;
+import org.apache.seatunnel.server.common.SeatunnelException;
+
+import org.apache.commons.lang3.StringUtils;
+
+import lombok.Getter;
+import lombok.NonNull;
+import lombok.Setter;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.OptionalDouble;
+
+/** Engine metrics extractor SeaTunnel Engine implement. */
+public class SeaTunnelEngineMetricsExtractor implements IEngineMetricsExtractor {
+ @Getter @Setter private SeaTunnelEngineProxy seaTunnelEngineProxy;
+
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+ public static final String[] clusterHealthMetricsKeys =
+ new String[] {
+ "processors",
+ "load.systemAverage",
+ "physical.memory.total",
+ "physical.memory.free",
+ "swap.space.total",
+ "swap.space.free",
+ "heap.memory.used",
+ "heap.memory.free",
+ "heap.memory.total",
+ "heap.memory.max",
+ "heap.memory.used/total",
+ "heap.memory.used/max",
+ "minor.gc.count",
+ "minor.gc.time",
+ "major.gc.count",
+ "major.gc.time",
+ "thread.count",
+ "thread.peakCount",
+ "operations.completed.count",
+ "operations.running.count",
+ "operations.pending.invocations.percentage",
+ "operations.pending.invocations.count",
+ "clientEndpoint.count",
+ "connection.active.count",
+ "client.connection.count",
+ "connection.count"
+ };
+
+ private SeaTunnelEngineMetricsExtractor() {
+ this.seaTunnelEngineProxy = SeaTunnelEngineProxy.getInstance();
+ }
+
+ public static SeaTunnelEngineMetricsExtractor getInstance() {
+ return SeaTunnelEngineMetricsExtractorHolder.INSTANCE;
+ }
+
+ @Override
+ public List<JobMetrics> getMetricsByJobEngineId(@NonNull String jobEngineId) {
+ LinkedHashMap<Integer, JobMetrics> metricsMap = new LinkedHashMap();
+
+ LinkedHashMap<Integer, String> jobPipelineStatus = getJobPipelineStatus(jobEngineId);
+ try {
+ String metricsContent = seaTunnelEngineProxy.getMetricsContent(jobEngineId);
+ if (StringUtils.isEmpty(metricsContent)) {
+ return new ArrayList<>();
+ }
+
+ JsonNode jsonNode =
+ JsonUtils.stringToJsonNode(seaTunnelEngineProxy.getMetricsContent(jobEngineId));
+ JsonNode sourceReceivedCount = jsonNode.get("SourceReceivedCount");
+ if (sourceReceivedCount != null && sourceReceivedCount.isArray()) {
+ for (JsonNode node : sourceReceivedCount) {
+ Integer pipelineId = node.get("tags").get("pipelineId").asInt();
+ JobMetrics currPipelineMetrics =
+ getOrCreatePipelineMetricsMap(
+ metricsMap, jobPipelineStatus, pipelineId);
+ currPipelineMetrics.setReadRowCount(
+ currPipelineMetrics.getReadRowCount() + node.get("value").asLong());
+ }
+ }
+
+ JsonNode sinkWriteCount = jsonNode.get("SinkWriteCount");
+ if (sinkWriteCount != null && sinkWriteCount.isArray()) {
+ for (JsonNode node : jsonNode.get("SinkWriteCount")) {
+ Integer pipelineId = node.get("tags").get("pipelineId").asInt();
+ JobMetrics currPipelineMetrics =
+ getOrCreatePipelineMetricsMap(
+ metricsMap, jobPipelineStatus, pipelineId);
+ currPipelineMetrics.setWriteRowCount(
+ currPipelineMetrics.getWriteRowCount() + node.get("value").asLong());
+ }
+ }
+
+ JsonNode sinkWriteQPS = jsonNode.get("SinkWriteQPS");
+ if (sinkWriteQPS != null && sinkWriteQPS.isArray()) {
+ for (JsonNode node : jsonNode.get("SinkWriteQPS")) {
+ Integer pipelineId = node.get("tags").get("pipelineId").asInt();
+ JobMetrics currPipelineMetrics =
+ getOrCreatePipelineMetricsMap(
+ metricsMap, jobPipelineStatus, pipelineId);
+ currPipelineMetrics.setWriteQps(
+ currPipelineMetrics.getWriteQps()
+ + (new Double(node.get("value").asDouble())).longValue());
+ }
+ }
+
+ JsonNode sourceReceivedQPS = jsonNode.get("SourceReceivedQPS");
+ if (sourceReceivedQPS != null && sourceReceivedQPS.isArray()) {
+ for (JsonNode node : jsonNode.get("SourceReceivedQPS")) {
+ Integer pipelineId = node.get("tags").get("pipelineId").asInt();
+ JobMetrics currPipelineMetrics =
+ getOrCreatePipelineMetricsMap(
+ metricsMap, jobPipelineStatus, pipelineId);
+ currPipelineMetrics.setReadQps(
+ currPipelineMetrics.getReadQps()
+ + (new Double(node.get("value").asDouble())).longValue());
+ }
+ }
+
+ JsonNode cdcRecordEmitDelay = jsonNode.get("CDCRecordEmitDelay");
+ if (cdcRecordEmitDelay != null && cdcRecordEmitDelay.isArray()) {
+ Map<Integer, List<Long>> dataMap = new HashMap<>();
+ for (JsonNode node : jsonNode.get("CDCRecordEmitDelay")) {
+ Integer pipelineId = node.get("tags").get("pipelineId").asInt();
+ long value = node.get("value").asLong();
+ dataMap.computeIfAbsent(pipelineId, n -> new ArrayList<>()).add(value);
+ }
+ dataMap.forEach(
+ (key, value) -> {
+ JobMetrics currPipelineMetrics =
+ getOrCreatePipelineMetricsMap(
+ metricsMap, jobPipelineStatus, key);
+ OptionalDouble average = value.stream().mapToDouble(a -> a).average();
+ currPipelineMetrics.setRecordDelay(
+ Double.valueOf(average.isPresent() ? average.getAsDouble() : 0)
+ .longValue());
+ });
+ }
+ } catch (JsonProcessingException e) {
+ throw new SeatunnelException(
+ SeatunnelErrorEnum.LOAD_ENGINE_METRICS_JSON_ERROR,
+ "SeaTunnel",
+ ExceptionUtils.getMessage(e));
+ }
+
+ return Arrays.asList(metricsMap.values().toArray(new JobMetrics[0]));
+ }
+
+ @Override
+ public LinkedHashMap<Integer, String> getJobPipelineStatus(@NonNull String jobEngineId) {
+ String jobState = seaTunnelEngineProxy.getJobPipelineStatusStr(jobEngineId);
+ LinkedHashMap<Integer, String> pipelineStatusMap = new LinkedHashMap<>();
+ try {
+ JsonNode jsonNode = JsonUtils.stringToJsonNode(jobState);
+ Iterator<Map.Entry<String, JsonNode>> iterator =
+ jsonNode.get("pipelineStateMapperMap").fields();
+
+ while (iterator.hasNext()) {
+ Map.Entry<String, JsonNode> next = iterator.next();
+ // "PipelineLocation(jobId=650612768629587969, pipelineId=2)"
+ String pipelineLocation = next.getKey();
+ String pipelineId =
+ pipelineLocation.substring(
+ pipelineLocation.lastIndexOf("=") + 1,
+ pipelineLocation.length() - 1);
+ pipelineStatusMap.put(
+ Integer.valueOf(pipelineId),
+ next.getValue().get("pipelineStatus").asText());
+ }
+ } catch (JsonProcessingException e) {
+ throw new SeatunnelException(
+ SeatunnelErrorEnum.LOAD_ENGINE_JOB_STATUS_JSON_ERROR,
+ "SeaTunnel",
+ ExceptionUtils.getMessage(e));
+ }
+ return pipelineStatusMap;
+ }
+
+ @Override
+ public JobInstanceHistory getJobHistoryById(String jobEngineId) {
+ JobDAGInfo jobInfo = seaTunnelEngineProxy.getJobInfo(jobEngineId);
+ JobInstanceHistory jobInstanceHistory = new JobInstanceHistory();
+ try {
+ jobInstanceHistory.setDag(OBJECT_MAPPER.writeValueAsString(jobInfo));
+ } catch (JsonProcessingException e) {
+ throw new org.apache.seatunnel.common.utils.SeaTunnelException(e);
+ }
+ return jobInstanceHistory;
+ }
+
+ @Override
+ public boolean isJobEnd(@NonNull String jobEngineId) {
+ String jobStatus = seaTunnelEngineProxy.getJobStatus(jobEngineId);
+ return "finished".equalsIgnoreCase(jobStatus)
+ || "canceled".equalsIgnoreCase(jobStatus)
+ || "failed".equalsIgnoreCase(jobStatus);
+ }
+
+ @Override
+ public boolean isJobEndStatus(@NonNull String jobStatus) {
+ return "finished".equalsIgnoreCase(jobStatus)
+ || "canceled".equalsIgnoreCase(jobStatus)
+ || "failed".equalsIgnoreCase(jobStatus);
+ }
+
+ @Override
+ public String getJobStatus(@NonNull String jobEngineId) {
+ return seaTunnelEngineProxy.getJobStatus(jobEngineId);
+ }
+
+ @Override
+ public List<Map<String, String>> getClusterHealthMetrics() {
+ List<Map<String, String>> zetaClusterMetrics = new ArrayList<>();
+ Map<String, String> clusterHealthMetrics = seaTunnelEngineProxy.getClusterHealthMetrics();
+ for (Map.Entry<String, String> entry : clusterHealthMetrics.entrySet()) {
+ Map<String, String> hostMetrics = new LinkedHashMap<>();
+ String[] split = entry.getKey().split(":");
+ hostMetrics.put("host", split[0]);
+ hostMetrics.put("port", split[1]);
+
+ String value = entry.getValue();
+ value = value.replace(" ", "");
+ Map<String, String> otherMetrics = JsonUtils.toMap(value);
+ for (String key : clusterHealthMetricsKeys) {
+ hostMetrics.put(key, otherMetrics.get(key).toString());
+ }
+ zetaClusterMetrics.add(hostMetrics);
+ }
+ return zetaClusterMetrics;
+ }
+
+ private JobMetrics getOrCreatePipelineMetricsMap(
+ LinkedHashMap<Integer, JobMetrics> metricsMap,
+ LinkedHashMap<Integer, String> jobPipelineStatus,
+ Integer pipelineId) {
+ JobMetrics currPipelineMetrics = metricsMap.get(pipelineId);
+ if (currPipelineMetrics == null) {
+ currPipelineMetrics = new JobMetrics();
+ metricsMap.put(pipelineId, currPipelineMetrics);
+ currPipelineMetrics.setStatus(jobPipelineStatus.get(pipelineId));
+ currPipelineMetrics.setPipelineId(pipelineId);
+ }
+ return currPipelineMetrics;
+ }
+
+ private static class SeaTunnelEngineMetricsExtractorHolder {
+ private static final SeaTunnelEngineMetricsExtractor INSTANCE =
+ new SeaTunnelEngineMetricsExtractor();
+ }
+}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/engine/SeaTunnelEngineProxy.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/engine/SeaTunnelEngineProxy.java
new file mode 100644
index 00000000..1641f5d9
--- /dev/null
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/engine/SeaTunnelEngineProxy.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.seatunnel.app.thirdparty.engine;
+
+import org.apache.seatunnel.engine.client.SeaTunnelClient;
+import org.apache.seatunnel.engine.common.config.ConfigProvider;
+import org.apache.seatunnel.engine.core.job.JobDAGInfo;
+
+import com.hazelcast.client.config.ClientConfig;
+import lombok.NonNull;
+
+import java.util.Map;
+
+public class SeaTunnelEngineProxy {
+ ClientConfig clientConfig = null;
+
+ private SeaTunnelEngineProxy() {
+ clientConfig = ConfigProvider.locateAndGetClientConfig();
+ }
+
+ public static SeaTunnelEngineProxy getInstance() {
+ return SeaTunnelEngineProxyHolder.INSTANCE;
+ }
+
+ public String getMetricsContent(@NonNull String jobEngineId) {
+ SeaTunnelClient seaTunnelClient = new SeaTunnelClient(clientConfig);
+ try {
+ return seaTunnelClient.getJobMetrics(Long.valueOf(jobEngineId));
+ } finally {
+ seaTunnelClient.close();
+ }
+ }
+
+ public String getJobPipelineStatusStr(@NonNull String jobEngineId) {
+ SeaTunnelClient seaTunnelClient = new SeaTunnelClient(clientConfig);
+ try {
+ return seaTunnelClient.getJobDetailStatus(Long.valueOf(jobEngineId));
+ } finally {
+ seaTunnelClient.close();
+ }
+ }
+
+ public JobDAGInfo getJobInfo(@NonNull String jobEngineId) {
+ SeaTunnelClient seaTunnelClient = new SeaTunnelClient(clientConfig);
+ try {
+ return seaTunnelClient.getJobInfo(Long.valueOf(jobEngineId));
+ } finally {
+ seaTunnelClient.close();
+ }
+ }
+
+ public String getJobStatus(@NonNull String jobEngineId) {
+ SeaTunnelClient seaTunnelClient = new SeaTunnelClient(clientConfig);
+ try {
+ return seaTunnelClient.getJobStatus(Long.valueOf(jobEngineId));
+ } finally {
+ seaTunnelClient.close();
+ }
+ }
+
+ public Map<String, String> getClusterHealthMetrics() {
+ SeaTunnelClient seaTunnelClient = new SeaTunnelClient(clientConfig);
+ try {
+ return seaTunnelClient.getClusterHealthMetrics();
+ } finally {
+ seaTunnelClient.close();
+ }
+ }
+
+ private static class SeaTunnelEngineProxyHolder {
+ private static final SeaTunnelEngineProxy INSTANCE = new SeaTunnelEngineProxy();
+ }
+}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/metrics/EngineMetricsExtractorFactory.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/metrics/EngineMetricsExtractorFactory.java
new file mode 100644
index 00000000..40656093
--- /dev/null
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/metrics/EngineMetricsExtractorFactory.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.seatunnel.app.thirdparty.metrics;
+
+import org.apache.seatunnel.app.domain.response.engine.Engine;
+import org.apache.seatunnel.app.thirdparty.engine.SeaTunnelEngineMetricsExtractor;
+import org.apache.seatunnel.server.common.SeatunnelErrorEnum;
+import org.apache.seatunnel.server.common.SeatunnelException;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+@Data
+@AllArgsConstructor
+public class EngineMetricsExtractorFactory {
+
+ private final Engine engine;
+
+ public IEngineMetricsExtractor getEngineMetricsExtractor() {
+ if (engine.getName().equals("SeaTunnel")) {
+ return SeaTunnelEngineMetricsExtractor.getInstance();
+ }
+
+ throw new SeatunnelException(
+ SeatunnelErrorEnum.UNSUPPORTED_ENGINE, engine.getName(), engine.getVersion());
+ }
+}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/IJobDefinitionDao.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/metrics/IEngineMetricsExtractor.java
similarity index 53%
copy from seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/IJobDefinitionDao.java
copy to seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/metrics/IEngineMetricsExtractor.java
index 34b87b57..f3d803c7 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/IJobDefinitionDao.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/metrics/IEngineMetricsExtractor.java
@@ -14,28 +14,31 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.seatunnel.app.thirdparty.metrics;
-package org.apache.seatunnel.app.dal.dao;
-
-import org.apache.seatunnel.app.dal.entity.JobDefinition;
-import org.apache.seatunnel.app.domain.response.PageInfo;
+import org.apache.seatunnel.app.dal.entity.JobInstanceHistory;
+import org.apache.seatunnel.app.dal.entity.JobMetrics;
import lombok.NonNull;
+import java.util.LinkedHashMap;
import java.util.List;
+import java.util.Map;
+
+public interface IEngineMetricsExtractor {
-public interface IJobDefinitionDao {
+ List<JobMetrics> getMetricsByJobEngineId(@NonNull String jobEngineId);
- void add(JobDefinition job);
+ LinkedHashMap<Integer, String> getJobPipelineStatus(@NonNull String jobEngineId);
- JobDefinition getJob(long id);
+ JobInstanceHistory getJobHistoryById(String jobEngineId);
- void updateJob(JobDefinition jobDefinition);
+ /** contains finished, failed, canceled */
+ boolean isJobEnd(@NonNull String jobEngineId);
- PageInfo<JobDefinition> getJob(
- String name, Integer pageNo, Integer pageSize, List<Long> projectCodes, String jobMode);
+ boolean isJobEndStatus(@NonNull String jobStatus);
- List<JobDefinition> getJob(@NonNull List<Long> projectCodes, @NonNull String name);
+ List<Map<String, String>> getClusterHealthMetrics();
- void delete(long id, long projectCode);
+ String getJobStatus(@NonNull String jobEngineId);
}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/utils/GlobalExceptionHandler.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/utils/GlobalExceptionHandler.java
index 5b515278..839bd311 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/utils/GlobalExceptionHandler.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/utils/GlobalExceptionHandler.java
@@ -18,17 +18,17 @@
package org.apache.seatunnel.app.utils;
import org.apache.seatunnel.app.common.Result;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException;
import org.apache.seatunnel.server.common.SeatunnelErrorEnum;
import org.apache.seatunnel.server.common.SeatunnelException;
+import org.springframework.web.bind.MissingServletRequestParameterException;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.bind.annotation.RestControllerAdvice;
import io.jsonwebtoken.ExpiredJwtException;
import lombok.extern.slf4j.Slf4j;
-import java.util.Optional;
-
@RestControllerAdvice
@Slf4j
public class GlobalExceptionHandler {
@@ -37,16 +37,30 @@ public class GlobalExceptionHandler {
private Result<String> portalExceptionHandler(SeatunnelException e) {
logError(e);
- final SeatunnelException seatunnelException =
- Optional.ofNullable(e)
- .orElse(SeatunnelException.newInstance(SeatunnelErrorEnum.UNKNOWN));
+ // final SeatunnelException seatunnelException =
+ // Optional.ofNullable(e)
+ //
+ // .orElse(SeatunnelException.newInstance(SeatunnelErrorEnum.UNKNOWN));
- final String message = seatunnelException.getMessage();
- final SeatunnelErrorEnum errorEnum = seatunnelException.getErrorEnum();
+ final String message = e.getMessage();
+ final SeatunnelErrorEnum errorEnum = e.getErrorEnum();
return Result.failure(errorEnum, message);
}
+ @ExceptionHandler(value = DataSourcePluginException.class)
+ private Result<String> dsHandler(DataSourcePluginException e) {
+ logError(e);
+ final String message = e.getMessage();
+ return Result.failure(SeatunnelErrorEnum.INVALID_DATASOURCE, e.getMessage());
+ }
+
+ @ExceptionHandler(value = MissingServletRequestParameterException.class)
+ private Result<String> missParam(MissingServletRequestParameterException e) {
+ logError(e);
+ return Result.failure(SeatunnelErrorEnum.MISSING_PARAM, e.getParameterName());
+ }
+
@ExceptionHandler(value = IllegalStateException.class)
private Result<String> illegalStateExceptionHandler(IllegalStateException e) {
logError(e);
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/utils/SeaTunnelConfigUtil.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/utils/SeaTunnelConfigUtil.java
new file mode 100644
index 00000000..caf89c77
--- /dev/null
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/utils/SeaTunnelConfigUtil.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.app.utils;
+
+public class SeaTunnelConfigUtil {
+
+ private static final String CONFIG_TEMPLATE =
+ "env {\n"
+ + "env_placeholder"
+ + "}\n"
+ + "source {\n"
+ + "source_placeholder"
+ + "}\n"
+ + "transform {\n"
+ + "transform_placeholder"
+ + "}\n"
+ + "sink {\n"
+ + "sink_placeholder"
+ + "}\n";
+
+ public static String generateConfig(
+ String env, String sources, String transforms, String sinks) {
+ return CONFIG_TEMPLATE
+ .replace("env_placeholder", env)
+ .replace("source_placeholder", sources)
+ .replace("transform_placeholder", transforms)
+ .replace("sink_placeholder", sinks);
+ }
+}
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/utils/TaskOptionUtils.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/utils/TaskOptionUtils.java
new file mode 100644
index 00000000..d38be581
--- /dev/null
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/utils/TaskOptionUtils.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.app.utils;
+
+import org.apache.seatunnel.app.domain.request.job.transform.CopyTransformOptions;
+import org.apache.seatunnel.app.domain.request.job.transform.FieldMapperTransformOptions;
+import org.apache.seatunnel.app.domain.request.job.transform.SplitTransformOptions;
+import org.apache.seatunnel.app.domain.request.job.transform.Transform;
+import org.apache.seatunnel.app.domain.request.job.transform.TransformOptions;
+import org.apache.seatunnel.server.common.SeatunnelErrorEnum;
+import org.apache.seatunnel.server.common.SeatunnelException;
+
+import org.apache.commons.lang3.StringUtils;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import java.io.IOException;
+
+public class TaskOptionUtils {
+
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+ public static <T extends TransformOptions> T getTransformOption(
+ Transform transform, String transformOptionsStr) throws IOException {
+ switch (transform) {
+ case FIELDMAPPER:
+ return convertTransformStrToOptions(
+ transformOptionsStr, FieldMapperTransformOptions.class);
+ case MULTIFIELDSPLIT:
+ return convertTransformStrToOptions(
+ transformOptionsStr, SplitTransformOptions.class);
+ case COPY:
+ return convertTransformStrToOptions(
+ transformOptionsStr, CopyTransformOptions.class);
+ case FILTERROWKIND:
+ case REPLACE:
+ default:
+ return null;
+ }
+ }
+
+ public static <T extends TransformOptions> T convertTransformStrToOptions(
+ String transformOptionsStr, Class<? extends TransformOptions> optionClass)
+ throws IOException {
+ if (StringUtils.isEmpty(transformOptionsStr)) {
+ throw new SeatunnelException(
+ SeatunnelErrorEnum.ILLEGAL_STATE,
+ optionClass.getName() + " transformOptions can not be empty");
+ }
+ return (T) OBJECT_MAPPER.readValue(transformOptionsStr, optionClass);
+ }
+}
diff --git a/seatunnel-server/seatunnel-app/src/main/resources/org/apache/seatunnel/app/dal/mapper/DataSourceMapper.xml b/seatunnel-server/seatunnel-app/src/main/resources/org/apache/seatunnel/app/dal/mapper/DataSourceMapper.xml
index a5a9cce9..ef68aefb 100644
--- a/seatunnel-server/seatunnel-app/src/main/resources/org/apache/seatunnel/app/dal/mapper/DataSourceMapper.xml
+++ b/seatunnel-server/seatunnel-app/src/main/resources/org/apache/seatunnel/app/dal/mapper/DataSourceMapper.xml
@@ -27,6 +27,8 @@
<result column="create_time" jdbcType="TIMESTAMP" property="createTime"/>
<result column="update_time" jdbcType="TIMESTAMP" property="updateTime"/>
</resultMap>
+
+
<sql id="Base_Column_List">
id
, `datasource_name`, `plugin_name`, `plugin_version`, `datasource_config`, `description`, `create_user_id`, `update_user_id`, `create_time`, `update_time`
@@ -63,7 +65,7 @@
where id = #{id,jdbcType=BIGINT}
</update>
- <select id="checkDataSourceNameUnique" resultType="java.lang.Integer">
+ <select id="checkDataSourceNameUnique" resultType="int">
select count(1)
from t_st_datasource
where datasource_name = #{datasourceName} and id != #{datasourceId}
diff --git a/seatunnel-server/seatunnel-app/src/main/resources/org/apache/seatunnel/app/dal/mapper/JobInstanceMapper.xml b/seatunnel-server/seatunnel-app/src/main/resources/org/apache/seatunnel/app/dal/mapper/JobInstanceMapper.xml
new file mode 100644
index 00000000..3156d923
--- /dev/null
+++ b/seatunnel-server/seatunnel-app/src/main/resources/org/apache/seatunnel/app/dal/mapper/JobInstanceMapper.xml
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="org.apache.seatunnel.app.dal.mapper.JobInstanceMapper">
+ <resultMap id="BaseResultMap" type="org.apache.seatunnel.app.dal.entity.JobInstance">
+ <id column="id" jdbcType="BIGINT" property="id"/>
+ <result column="job_define_id" jdbcType="BIGINT" property="jobDefineId"/>
+ <result column="job_status" jdbcType="VARCHAR" property="jobStatus"/>
+ <result column="job_config" jdbcType="VARCHAR" property="jobConfig"/>
+ <result column="engine_name" jdbcType="VARCHAR" property="engineName"/>
+ <result column="engine_version" jdbcType="VARCHAR" property="engineVersion"/>
+ <result column="job_engine_id" jdbcType="VARCHAR" property="jobEngineId"/>
+ </resultMap>
+ <sql id="Base_Column_List">
+ id
+ , `job_define_id`, `job_status`, `job_config`, `engine_name`, `engine_version`, `job_engine_id`
+ </sql>
+ <insert id="insert" parameterType="org.apache.seatunnel.app.dal.entity.JobInstance">
+ insert into `t_st_job_instance` (<include refid="Base_Column_List"/>, `create_user_id`, `update_user_id`)
+ values (#{id,jdbcType=BIGINT},
+ #{jobDefineId,jdbcType=BIGINT},
+ #{jobStatus,jdbcType=VARCHAR},
+ #{jobConfig,jdbcType=VARCHAR},
+ #{engineName,jdbcType=VARCHAR},
+ #{engineVersion,jdbcType=VARCHAR},
+ #{jobEngineId,jdbcType=VARCHAR},
+ #{createUserId,jdbcType=BIGINT},
+ #{updateUserId,jdbcType=BIGINT})
+ </insert>
+</mapper>
diff --git a/seatunnel-server/seatunnel-app/src/main/resources/org/apache/seatunnel/app/dal/mapper/JobLineMapper.xml b/seatunnel-server/seatunnel-app/src/main/resources/org/apache/seatunnel/app/dal/mapper/JobLineMapper.xml
new file mode 100644
index 00000000..d6e99a5f
--- /dev/null
+++ b/seatunnel-server/seatunnel-app/src/main/resources/org/apache/seatunnel/app/dal/mapper/JobLineMapper.xml
@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="org.apache.seatunnel.app.dal.mapper.JobLineMapper">
+ <resultMap id="BaseResultMap" type="org.apache.seatunnel.app.dal.entity.JobLine">
+ <id column="id" jdbcType="BIGINT" property="id"/>
+ <result column="version_id" jdbcType="BIGINT" property="versionId"/>
+ <result column="input_plugin_id" jdbcType="BIGINT" property="inputPluginId"/>
+ <result column="target_plugin_id" jdbcType="BIGINT" property="targetPluginId"/>
+ <result column="create_time" jdbcType="TIMESTAMP" property="createTime"/>
+ <result column="update_time" jdbcType="TIMESTAMP" property="updateTime"/>
+ </resultMap>
+ <delete id="deleteLinesByVersionId">
+ delete from t_st_job_line where version_id = #{versionId}
+ </delete>
+
+ <insert id="insertBatchLines">
+ insert into t_st_job_line (id, version_id, input_plugin_id, target_plugin_id)
+ values
+ <foreach collection="lines" item="line" separator=",">
+ (#{line.id},
+ #{line.versionId},
+ #{line.inputPluginId},
+ #{line.targetPluginId})
+ </foreach>
+ </insert>
+</mapper>
diff --git a/seatunnel-server/seatunnel-app/src/main/resources/org/apache/seatunnel/app/dal/mapper/JobMapper.xml b/seatunnel-server/seatunnel-app/src/main/resources/org/apache/seatunnel/app/dal/mapper/JobMapper.xml
index ad8a419a..6e7690f7 100644
--- a/seatunnel-server/seatunnel-app/src/main/resources/org/apache/seatunnel/app/dal/mapper/JobMapper.xml
+++ b/seatunnel-server/seatunnel-app/src/main/resources/org/apache/seatunnel/app/dal/mapper/JobMapper.xml
@@ -22,7 +22,6 @@
<result column="job_type" jdbcType="VARCHAR" property="jobType"/>
<result column="create_user_id" jdbcType="INTEGER" property="createUserId"/>
<result column="update_user_id" jdbcType="INTEGER" property="updateUserId"/>
- <result column="project_code" jdbcType="BIGINT" property="projectCode"/>
<result column="create_time" jdbcType="TIMESTAMP" property="createTime"/>
<result column="update_time" jdbcType="TIMESTAMP" property="updateTime"/>
</resultMap>
@@ -40,13 +39,11 @@
select
<include refid="Base_Column_List"/>, create_time, update_time
from t_st_job_definition
- where project_code in
- <foreach collection="projectCodes" item="i" open="(" close=")" separator=",">
- #{i}
- </foreach>
+ <where>
<if test="searchName!=null and searchName != ''">
and name LIKE concat('%', #{searchName}, '%')
</if>
+ </where>
order by update_time desc
</select>
@@ -54,23 +51,23 @@
select t.*
from t_st_job_definition t
join (select * from t_st_job_version where job_mode = #{jobMode}) v on t.id = v.job_id
- where project_code in
- <foreach collection="projectCodes" item="i" open="(" close=")" separator=",">
- #{i}
- </foreach>
+ <where>
<if test="searchName!=null and searchName != ''">
and name LIKE concat('%', #{searchName}, '%')
</if>
+ </where>
order by update_time desc
</select>
<select id="queryJobList" resultType="org.apache.seatunnel.app.dal.entity.JobDefinition">
select
<include refid="Base_Column_List"/>, create_time, update_time
- from t_st_job_definition where project_code = #{projectCode}
+ from t_st_job_definition
+ <where>
<if test="searchName!=null and searchName != ''">
and name LIKE concat('%', #{searchName}, '%')
</if>
+ </where>
order by update_time desc
</select>
</mapper>
diff --git a/seatunnel-server/seatunnel-app/src/main/resources/org/apache/seatunnel/app/dal/mapper/JobMetricsMapper.xml b/seatunnel-server/seatunnel-app/src/main/resources/org/apache/seatunnel/app/dal/mapper/JobMetricsMapper.xml
new file mode 100644
index 00000000..7fdf0241
--- /dev/null
+++ b/seatunnel-server/seatunnel-app/src/main/resources/org/apache/seatunnel/app/dal/mapper/JobMetricsMapper.xml
@@ -0,0 +1,79 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="org.apache.seatunnel.app.dal.mapper.JobMetricsMapper">
+ <resultMap id="BaseResultMap" type="org.apache.seatunnel.app.dal.entity.JobMetrics">
+ <id column="id" jdbcType="BIGINT" property="id"/>
+ <result column="job_instance_id" jdbcType="BIGINT" property="jobInstanceId"/>
+ <result column="pipeline_id" jdbcType="INTEGER" property="pipelineId"/>
+ <result column="read_row_count" jdbcType="BIGINT" property="readRowCount"/>
+ <result column="write_row_count" jdbcType="BIGINT" property="writeRowCount"/>
+ <result column="source_table_names" jdbcType="VARCHAR" property="sourceTableNames"/>
+ <result column="sink_table_names" jdbcType="VARCHAR" property="sinkTableNames"/>
+ <result column="read_qps" jdbcType="BIGINT" property="readQps"/>
+ <result column="write_qps" jdbcType="BIGINT" property="writeQps"/>
+ <result column="status" jdbcType="VARCHAR" property="status"/>
+ </resultMap>
+ <sql id="Base_Column_List">
+ id
+ , `job_instance_id`, `pipeline_id`, read_row_count, write_row_count, source_table_names, sink_table_names, read_qps, write_qps, status
+ </sql>
+ <insert id="insert" parameterType="org.apache.seatunnel.app.dal.entity.JobMetrics">
+ insert into `t_st_job_metrics` (<include refid="Base_Column_List"/>, `create_user_id`, `update_user_id`)
+ values (#{id,jdbcType=BIGINT},
+ #{jobInstanceId,jdbcType=BIGINT},
+ #{pipelineId,jdbcType=INTEGER},
+ #{readRowCount,jdbcType=BIGINT},
+ #{writeRowCount,jdbcType=BIGINT},
+ #{sourceTableNames,jdbcType=VARCHAR},
+ #{sinkTableNames,jdbcType=VARCHAR},
+ #{read_qps,jdbcType=BIGINT},
+ #{write_qps,jdbcType=BIGINT},
+ #{status,jdbcType=VARCHAR},
+ #{createUserId,jdbcType=BIGINT},
+ #{updateUserId,jdbcType=BIGINT})
+ </insert>
+
+ <select id="queryJobMetricsByInstanceId" resultType="org.apache.seatunnel.app.dal.entity.JobMetrics">
+ select
+ <include refid="Base_Column_List"/>
+ from t_st_job_metrics
+ where 1=1
+ <if test="jobInstanceId != null and jobInstanceId > 0">
+ and job_instance_id = #{jobInstanceId}
+ </if>
+ </select>
+
+ <insert id="insertBatchMetrics">
+ insert into t_st_job_metrics (<include refid="Base_Column_List"/>, `create_user_id`, `update_user_id`)
+ values
+ <foreach collection="jobMetrics" item="metrics" separator=",">
+ (#{metrics.id},
+ #{metrics.jobInstanceId},
+ #{metrics.pipelineId},
+ #{metrics.readRowCount},
+ #{metrics.writeRowCount},
+ #{metrics.sourceTableNames},
+ #{metrics.sinkTableNames},
+ #{metrics.readQps},
+ #{metrics.writeQps},
+ #{metrics.status},
+ #{metrics.createUserId},
+ #{metrics.updateUserId})
+ </foreach>
+ </insert>
+</mapper>
diff --git a/seatunnel-server/seatunnel-app/src/main/resources/org/apache/seatunnel/app/dal/mapper/ProcessTaskRelationMapper.xml b/seatunnel-server/seatunnel-app/src/main/resources/org/apache/seatunnel/app/dal/mapper/ProcessTaskRelationMapper.xml
index 2e8cb977..588637d2 100644
--- a/seatunnel-server/seatunnel-app/src/main/resources/org/apache/seatunnel/app/dal/mapper/ProcessTaskRelationMapper.xml
+++ b/seatunnel-server/seatunnel-app/src/main/resources/org/apache/seatunnel/app/dal/mapper/ProcessTaskRelationMapper.xml
@@ -18,6 +18,7 @@
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
<mapper namespace="org.apache.seatunnel.app.dal.mapper.ProcessTaskRelationMapper">
+
<sql id="baseSql">
id, name, process_definition_version, project_code, process_definition_code, pre_task_code, pre_task_version,
post_task_code, post_task_version, condition_type, condition_params, create_time, update_time
diff --git a/seatunnel-server/seatunnel-app/src/main/resources/script/seatunnel_server_h2.sql b/seatunnel-server/seatunnel-app/src/main/resources/script/seatunnel_server_h2.sql
index 9c4eb741..df961780 100644
--- a/seatunnel-server/seatunnel-app/src/main/resources/script/seatunnel_server_h2.sql
+++ b/seatunnel-server/seatunnel-app/src/main/resources/script/seatunnel_server_h2.sql
@@ -305,7 +305,6 @@ CREATE TABLE IF NOT EXISTS `t_st_job_definition` (
`job_type` varchar(50),
`create_user_id` int(11) NOT NULL,
`update_user_id` int(11) NOT NULL,
- `project_code` bigint(20) NOT NULL,
`create_time` timestamp(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
`update_time` timestamp(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
PRIMARY KEY (`id`)
@@ -394,3 +393,91 @@ CREATE TABLE IF NOT EXISTS `t_st_virtual_table`
`update_time` timestamp(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
primary key (`id`)
);
+
+
+DROP TABLE IF EXISTS `t_ds_process_definition` CASCADE;
+CREATE TABLE IF NOT EXISTS `t_ds_process_definition` (
+ `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'self-increasing id',
+ `code` bigint(20) NOT NULL COMMENT 'encoding',
+ `name` varchar(255) COLLATE utf8_bin DEFAULT NULL COMMENT 'process definition name',
+ `version` int(11) DEFAULT '0' COMMENT 'process definition version',
+ `description` text COLLATE utf8_bin COMMENT 'description',
+ `release_state` tinyint(4) DEFAULT NULL COMMENT 'process definition release state:0:offline,1:online',
+ `user_id` int(11) DEFAULT NULL COMMENT 'process definition creator id',
+ `global_params` text COLLATE utf8_bin COMMENT 'global parameters',
+ `flag` tinyint(4) DEFAULT NULL COMMENT '0 not available, 1 available',
+ `locations` longtext COLLATE utf8_bin COMMENT 'Node location information',
+ `warning_group_id` int(11) DEFAULT NULL COMMENT 'alert group id',
+ `timeout` int(11) DEFAULT '0' COMMENT 'time out, unit: minute',
+ `tenant_id` int(11) NOT NULL DEFAULT '-1' COMMENT 'tenant id',
+ `execution_type` tinyint(4) DEFAULT '0' COMMENT 'execution_type 0:parallel,1:serial wait,2:serial discard,3:serial priority',
+ `create_time` datetime NOT NULL COMMENT 'create time',
+ `update_time` datetime NOT NULL COMMENT 'update time',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `idx_code` (`code`) USING BTREE
+ ) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
+
+
+
+-- ----------------------------
+-- Table structure for t_st_job_instance
+-- ----------------------------
+DROP TABLE IF EXISTS `t_st_job_instance` CASCADE;
+CREATE TABLE IF NOT EXISTS `t_st_job_instance` (
+ `id` bigint(20) NOT NULL,
+ `job_define_id` bigint(20) NOT NULL,
+ `job_status` varchar(50),
+ `job_config` text NOT NULL,
+ `engine_name` varchar(50) NOT NULL,
+ `engine_version` varchar(50) NOT NULL,
+ `job_engine_id` varchar(200),
+ `create_user_id` int(20) NOT NULL,
+ `update_user_id` int(20),
+ `create_time` timestamp(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
+ `update_time` timestamp(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
+ PRIMARY KEY (`id`)
+ ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 collate=utf8mb4_bin;
+
+
+-- ----------------------------
+-- Table structure for t_st_job_metrics
+-- ----------------------------
+DROP TABLE IF EXISTS `t_st_job_metrics` CASCADE;
+CREATE TABLE IF NOT EXISTS `t_st_job_metrics` (
+ `id` bigint(20) NOT NULL,
+ `job_instance_id` bigint(20) NOT NULL,
+ `pipeline_id` int(20) NOT NULL,
+ `read_row_count` bigint(20) NOT NULL,
+ `write_row_count` bigint(20) NOT NULL,
+ `source_table_names` varchar(200),
+ `sink_table_names` varchar(200),
+ `read_qps` bigint(20),
+ `write_qps` bigint(20),
+ `record_delay` bigint(20),
+ `status` varchar(20),
+ `create_user_id` int(20) NOT NULL,
+ `update_user_id` int(20),
+ `create_time` timestamp(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
+ `update_time` timestamp(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
+ PRIMARY KEY (`id`)
+ ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 collate=utf8mb4_bin;
+
+-- ----------------------------
+-- Records of t_st_job_metrics
+-- ----------------------------
+
+-- ----------------------------
+-- Table structure for t_st_job_instance_history
+-- ----------------------------
+DROP TABLE IF EXISTS `t_st_job_instance_history` CASCADE;
+CREATE TABLE IF NOT EXISTS `t_st_job_instance_history` (
+ `id` bigint(20) NOT NULL,
+ `dag` text NOT NULL,
+ `create_time` timestamp(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
+ `update_time` timestamp(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
+ PRIMARY KEY (`id`)
+ ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 collate=utf8mb4_bin;
+
+-- ----------------------------
+-- Records of t_st_job_instance_history
+-- ----------------------------
diff --git a/seatunnel-server/seatunnel-app/src/main/resources/script/seatunnel_server_mysql.sql b/seatunnel-server/seatunnel-app/src/main/resources/script/seatunnel_server_mysql.sql
index d0274586..bc8ff049 100644
--- a/seatunnel-server/seatunnel-app/src/main/resources/script/seatunnel_server_mysql.sql
+++ b/seatunnel-server/seatunnel-app/src/main/resources/script/seatunnel_server_mysql.sql
@@ -316,11 +316,9 @@ CREATE TABLE IF NOT EXISTS `seatunnel`.`t_st_job_definition` (
`job_type` varchar(50),
`create_user_id` int(11) NOT NULL,
`update_user_id` int(11) NOT NULL,
- `project_code` bigint(20) NOT NULL,
`create_time` timestamp(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
`update_time` timestamp(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
- PRIMARY KEY (`id`),
- key job_definition_index (project_code)
+ PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 collate=utf8mb4_bin;
@@ -409,103 +407,89 @@ CREATE TABLE IF NOT EXISTS `seatunnel`.`t_st_virtual_table`
primary key (`id`)
)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 collate=utf8mb4_bin;
-
--- ----------------------------
--- Table structure for t_st_job_definition
--- ----------------------------
-CREATE TABLE IF NOT EXISTS `t_st_job_definition` (
+DROP TABLE IF EXISTS `seatunnel`.`t_ds_process_definition`;
+CREATE TABLE IF NOT EXISTS `seatunnel`.`t_ds_process_definition` (
+ `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'self-increasing id',
+ `code` bigint(20) NOT NULL COMMENT 'encoding',
+ `name` varchar(255) COLLATE utf8_bin DEFAULT NULL COMMENT 'process definition name',
+ `version` int(11) DEFAULT '0' COMMENT 'process definition version',
+ `description` text COLLATE utf8_bin COMMENT 'description',
+ `release_state` tinyint(4) DEFAULT NULL COMMENT 'process definition release state:0:offline,1:online',
+ `user_id` int(11) DEFAULT NULL COMMENT 'process definition creator id',
+ `global_params` text COLLATE utf8_bin COMMENT 'global parameters',
+ `flag` tinyint(4) DEFAULT NULL COMMENT '0 not available, 1 available',
+ `locations` longtext COLLATE utf8_bin COMMENT 'Node location information',
+ `warning_group_id` int(11) DEFAULT NULL COMMENT 'alert group id',
+ `timeout` int(11) DEFAULT '0' COMMENT 'time out, unit: minute',
+ `tenant_id` int(11) NOT NULL DEFAULT '-1' COMMENT 'tenant id',
+ `execution_type` tinyint(4) DEFAULT '0' COMMENT 'execution_type 0:parallel,1:serial wait,2:serial discard,3:serial priority',
+ `create_time` datetime NOT NULL COMMENT 'create time',
+ `update_time` datetime NOT NULL COMMENT 'update time',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `idx_code` (`code`) USING BTREE
+) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
+
+
+-- ----------------------------
+-- Table structure for t_st_job_instance
+-- ----------------------------
+DROP TABLE IF EXISTS `seatunnel`.`t_st_job_instance`;
+CREATE TABLE IF NOT EXISTS `seatunnel`.`t_st_job_instance` (
`id` bigint(20) NOT NULL,
- `name` varchar(50) NOT NULL,
- `description` varchar(255) ,
- `job_type` varchar(50),
- `create_user_id` int(11) NOT NULL,
- `update_user_id` int(11) NOT NULL,
- `project_code` bigint(20) NOT NULL,
+ `job_define_id` bigint(20) NOT NULL,
+ `job_status` varchar(50),
+ `job_config` text NOT NULL,
+ `engine_name` varchar(50) NOT NULL,
+ `engine_version` varchar(50) NOT NULL,
+ `job_engine_id` varchar(200),
+ `create_user_id` int(20) NOT NULL,
+ `update_user_id` int(20),
`create_time` timestamp(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
`update_time` timestamp(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
- PRIMARY KEY (`id`),
- key job_definition_index (project_code)
+ PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 collate=utf8mb4_bin;
--- ----------------------------
--- Records of t_st_datasource
--- ----------------------------
-CREATE TABLE IF NOT EXISTS `t_st_datasource`
-(
- id bigint NOT NULL,
- datasource_name varchar(63) NOT NULL,
- plugin_name varchar(63) NOT NULL,
- plugin_version varchar(63) default '1.0.0' NULL,
- datasource_config varchar(1023) NOT NULL,
- description varchar(63) NULL,
- create_user_id int NOT NULL,
- update_user_id int NOT NULL,
- `create_time` timestamp(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
- `update_time` timestamp(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
- unique key t_st_datasource_datasource_name_uindex (datasource_name),
- primary key (`id`)
- )ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 collate=utf8mb4_bin;
-
-- ----------------------------
--- Table structure for t_st_job_task
+-- Table structure for t_st_job_metrics
-- ----------------------------
-CREATE TABLE IF NOT EXISTS `t_st_job_task` (
+DROP TABLE IF EXISTS `seatunnel`.`t_st_job_metrics`;
+CREATE TABLE IF NOT EXISTS `seatunnel`.`t_st_job_metrics` (
`id` bigint(20) NOT NULL,
- `version_id` bigint(20) NOT NULL,
- `plugin_id` varchar(50) NOT NULL,
- `name` varchar(50) NOT NULL,
- `config` text,
- `transform_options` varchar(5000),
- `output_schema` text,
- `connector_type` varchar(50) NOT NULL,
- `datasource_id` bigint(20),
- `datasource_option` varchar(5000),
- `select_table_fields` varchar(5000),
- `scene_mode` varchar(50),
- `type` varchar(50) NOT NULL,
+ `job_instance_id` bigint(20) NOT NULL,
+ `pipeline_id` int(20) NOT NULL,
+ `read_row_count` bigint(20) NOT NULL,
+ `write_row_count` bigint(20) NOT NULL,
+ `source_table_names` varchar(200),
+ `sink_table_names` varchar(200),
+ `read_qps` bigint(20),
+ `write_qps` bigint(20),
+ `record_delay` bigint(20),
+ `status` varchar(20),
+ `create_user_id` int(20) NOT NULL,
+ `update_user_id` int(20),
`create_time` timestamp(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
`update_time` timestamp(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
- PRIMARY KEY (`id`),
- key job_task_plugin_id_index (plugin_id)
+ PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 collate=utf8mb4_bin;
+-- ----------------------------
+-- Records of t_st_job_metrics
+-- ----------------------------
-- ----------------------------
--- Table structure for t_st_job_version
+-- Table structure for t_st_job_instance_history
-- ----------------------------
-CREATE TABLE IF NOT EXISTS `t_st_job_version` (
+DROP TABLE IF EXISTS `seatunnel`.`t_st_job_instance_history`;
+CREATE TABLE IF NOT EXISTS `seatunnel`.`t_st_job_instance_history` (
`id` bigint(20) NOT NULL,
- `job_id` bigint(20) NOT NULL,
- `name` varchar(255) NOT NULL,
- `job_mode` varchar(10) NOT NULL,
- `env` text,
- `engine_name` varchar(50) NOT NULL,
- `engine_version` varchar(50) NOT NULL,
- `create_user_id` int(11) NOT NULL,
- `update_user_id` int(11) NOT NULL,
+ `dag` text NOT NULL,
`create_time` timestamp(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
`update_time` timestamp(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 collate=utf8mb4_bin;
-- ----------------------------
--- Records of t_st_virtual_table
+-- Records of t_st_job_instance_history
-- ----------------------------
-
-CREATE TABLE IF NOT EXISTS `t_st_virtual_table`
-(
- id bigint NOT NULL,
- datasource_id bigint NOT NULL,
- virtual_database_name varchar(63) NOT NULL,
- virtual_table_name varchar(63) NOT NULL,
- table_fields varchar(1023) NOT NULL,
- virtual_table_config varchar(1023) NOT NULL,
- description varchar(63) NULL,
- create_user_id int NOT NULL,
- update_user_id int NOT NULL,
- `create_time` timestamp(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
- `update_time` timestamp(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
- primary key (`id`)
- )ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 collate=utf8mb4_bin;
diff --git a/seatunnel-server/seatunnel-server-common/src/main/java/org/apache/seatunnel/server/common/SeatunnelErrorEnum.java b/seatunnel-server/seatunnel-server-common/src/main/java/org/apache/seatunnel/server/common/SeatunnelErrorEnum.java
index fcc6c695..9f1bffa2 100644
--- a/seatunnel-server/seatunnel-server-common/src/main/java/org/apache/seatunnel/server/common/SeatunnelErrorEnum.java
+++ b/seatunnel-server/seatunnel-server-common/src/main/java/org/apache/seatunnel/server/common/SeatunnelErrorEnum.java
@@ -125,6 +125,7 @@ public enum SeatunnelErrorEnum {
"data source already used ( workflowName - taskName):{0}",
"datasource is using:{0}"),
INVALID_DATASOURCE(-70001, "Datasource [{0}] invalid", "datasource [{0}] invalid"),
+ MISSING_PARAM(1777000, "param miss [{0}]", "param miss [{0}]"),
;
private final int code;
diff --git a/seatunnel-web-dist/release-docs/LICENSE b/seatunnel-web-dist/release-docs/LICENSE
index a3f514b7..b01312a9 100644
--- a/seatunnel-web-dist/release-docs/LICENSE
+++ b/seatunnel-web-dist/release-docs/LICENSE
@@ -227,6 +227,7 @@ The text of each license is the standard Apache 2.0 license.
(Apache License, Version 2.0) SeaTunnel Common (org.apache.seatunnel:seatunnel-common:2.3.1 - https://mvnrepository.com/artifact/org.apache.seatunnel/seatunnel-common)
(Apache License, Version 2.0) SeaTunnel API (org.apache.seatunnel:seatunnel-api:2.3.1 - https://mvnrepository.com/artifact/org.apache.seatunnel/seatunnel-api)
(Apache License, Version 2.0) SeaTunnel Plugin Discovery (org.apache.seatunnel:seatunnel-plugin-discovery:2.3.1 - https://mvnrepository.com/artifact/org.apache.seatunnel/seatunnel-plugin-discovery)
+ (Apache License, Version 2.0) SeaTunnel Engine Client (org.apache.seatunnel:seatunnel-engine-client:2.3.1 - https://mvnrepository.com/artifact/org.apache.seatunnel/seatunnel-engine-client)
(The Apache Software License, Version 2.0) Guava: Google Core Libraries for Java (com.google.guava:guava:19.0 - https://github.com/google/guava/guava)
(Apache License 2.0) Hibernate Validator Engine (org.hibernate.validator:hibernate-validator:6.2.2.Final - http://hibernate.org/validator/hibernate-validator)
diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt
index 087b4495..ee30f7d9 100644
--- a/tools/dependencies/known-dependencies.txt
+++ b/tools/dependencies/known-dependencies.txt
@@ -62,6 +62,7 @@ seatunnel-scheduler-dolphinscheduler-1.0.0-SNAPSHOT.jar
seatunnel-server-common-1.0.0-SNAPSHOT.jar
seatunnel-datasource-client-1.0.0-SNAPSHOT.jar
seatunnel-spi-1.0.0-SNAPSHOT.jar
+seatunnel-engine-client-2.3.1.jar
slf4j-api-1.7.25.jar
snakeyaml-1.29.jar
spring-aop-5.3.20.jar