You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@inlong.apache.org by do...@apache.org on 2022/07/29 20:35:13 UTC
[inlong] branch master updated: [INLONG-5269][Manager] Workflow supports rapid scaling of MQ resources (#5274)
This is an automated email from the ASF dual-hosted git repository.
dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 8025b57fb [INLONG-5269][Manager] Workflow supports rapid scaling of MQ resources (#5274)
8025b57fb is described below
commit 8025b57fb56a4aa1d886dadbbd9608225bf51790
Author: healchow <he...@gmail.com>
AuthorDate: Sat Jul 30 04:35:09 2022 +0800
[INLONG-5269][Manager] Workflow supports rapid scaling of MQ resources (#5274)
---
.../apache/inlong/manager/client/BaseExample.java | 2 +-
.../manager/common/consts/InlongConstants.java | 2 +-
.../inlong/manager/common/enums/ErrorCodeEnum.java | 5 +-
...eRunnerImpl.java => ApplicationInitRunner.java} | 4 +-
.../service/core/impl/AuditServiceImpl.java | 2 +-
.../service/core/impl/DataNodeServiceImpl.java | 2 +-
.../service/core/impl/SortSourceServiceImpl.java | 45 +-
.../listener/GroupTaskListenerFactory.java | 38 +-
.../listener/StreamTaskListenerFactory.java | 20 +-
.../ConsumptionCancelProcessListener.java | 2 +-
.../ConsumptionCompleteProcessListener.java | 8 +-
.../consumption}/ConsumptionPassTaskListener.java | 2 +-
.../ConsumptionRejectProcessListener.java | 2 +-
.../group}/InitGroupCompleteListener.java | 2 +-
.../group}/InitGroupFailedListener.java | 2 +-
.../group}/InitGroupListener.java | 2 +-
.../group}/UpdateGroupCompleteListener.java | 6 +-
.../group}/UpdateGroupFailedListener.java | 2 +-
.../group}/UpdateGroupListener.java | 2 +-
.../group}/apply/AfterApprovedTaskListener.java | 2 +-
.../group}/apply/ApproveApplyProcessListener.java | 2 +-
.../group}/apply/CancelApplyProcessListener.java | 2 +-
.../group}/apply/RejectApplyProcessListener.java | 2 +-
.../listener/queue/QueueResourceListener.java | 100 ++++
.../queue/StreamQueueResourceListener.java | 112 +++++
.../sink}/SinkResourceListener.java | 14 +-
.../sink}/StreamSinkResourceListener.java | 4 +-
.../{ => listener}/sort/SortConfigListener.java | 4 +-
.../sort/StreamSortConfigListener.java | 4 +-
.../source}/AbstractSourceOperateListener.java | 2 +-
.../source}/SourceDeleteListener.java | 12 +-
.../source}/SourceRestartListener.java | 12 +-
.../source}/SourceStopListener.java | 12 +-
.../stream}/InitStreamCompleteListener.java | 2 +-
.../stream}/InitStreamFailedListener.java | 2 +-
.../stream}/InitStreamListener.java | 2 +-
.../stream}/UpdateStreamCompleteListener.java | 2 +-
.../stream}/UpdateStreamFailedListener.java | 2 +-
.../stream}/UpdateStreamListener.java | 2 +-
.../service/mq/CreatePulsarGroupTaskListener.java | 168 -------
.../mq/CreatePulsarResourceTaskListener.java | 164 -------
.../mq/CreatePulsarSubscriptionTaskListener.java | 155 -------
.../service/mq/CreatePulsarTopicTaskListener.java | 135 ------
.../service/mq/CreateTubeGroupTaskListener.java | 97 ----
.../service/mq/CreateTubeTopicTaskListener.java | 91 ----
.../mq/DeletePulsarResourceTaskListener.java | 150 ------
.../service/mq/DeletePulsarTopicTaskListener.java | 133 ------
.../resource/queue/QueueResourceOperator.java | 75 +++
.../queue/QueueResourceOperatorFactory.java} | 22 +-
.../queue/pulsar}/PulsarOperator.java | 3 +-
.../queue/pulsar/PulsarResourceOperator.java | 250 ++++++++++
.../queue/pulsar}/PulsarUtils.java | 2 +-
.../queue/tubemq}/TubeMQOperator.java | 52 +--
.../queue/tubemq/TubeMQResourceOperator.java | 108 +++++
.../resource/{ => sink}/SinkResourceOperator.java | 2 +-
.../{ => sink}/SinkResourceOperatorFactory.java | 6 +-
.../{ => sink}/ck/ClickHouseJdbcUtils.java | 2 +-
.../{ => sink}/ck/ClickHouseResourceOperator.java | 4 +-
.../{ => sink}/ck/ClickHouseSqlBuilder.java | 2 +-
.../resource/{ => sink}/es/ElasticsearchApi.java | 2 +-
.../{ => sink}/es/ElasticsearchConfig.java | 2 +-
.../es/ElasticsearchResourceOperator.java | 4 +-
.../resource/{ => sink}/hbase/HBaseApiUtils.java | 2 +-
.../{ => sink}/hbase/HBaseResourceOperator.java | 4 +-
.../resource/{ => sink}/hive/HiveJdbcUtils.java | 2 +-
.../{ => sink}/hive/HiveResourceOperator.java | 4 +-
.../resource/{ => sink}/hive/SqlBuilder.java | 2 +-
.../{ => sink}/iceberg/IcebergCatalogUtils.java | 2 +-
.../iceberg/IcebergResourceOperator.java | 4 +-
.../{ => sink}/kafka/KafkaResourceOperator.java | 4 +-
.../resource/{ => sink}/mysql/MySQLJdbcUtils.java | 516 ++++++++++-----------
.../{ => sink}/mysql/MySQLResourceOperator.java | 274 +++++------
.../resource/{ => sink}/mysql/MySQLSqlBuilder.java | 468 +++++++++----------
.../{ => sink}/oracle/OracleJdbcUtils.java | 2 +-
.../{ => sink}/oracle/OracleResourceOperator.java | 4 +-
.../{ => sink}/oracle/OracleSqlBuilder.java | 2 +-
.../{ => sink}/postgresql/PostgreSQLJdbcUtils.java | 2 +-
.../postgresql/PostgreSQLResourceOperator.java | 4 +-
.../postgresql/PostgreSQLSqlBuilder.java | 2 +-
.../sort/DefaultSortConfigOperator.java | 16 +-
.../{ => resource}/sort/SortConfigOperator.java | 2 +-
.../sort/SortConfigOperatorFactory.java | 2 +-
.../{ => resource}/sort/util/ExtractNodeUtils.java | 2 +-
.../{ => resource}/sort/util/FieldInfoUtils.java | 2 +-
.../sort/util/FieldRelationUtils.java | 2 +-
.../sort/util/FilterFunctionUtils.java | 2 +-
.../{ => resource}/sort/util/LoadNodeUtils.java | 2 +-
.../sort/util/NodeRelationUtils.java | 2 +-
.../sort/util/TransformNodeUtils.java | 2 +-
.../service/source/StreamSourceService.java | 2 +-
.../service/source/StreamSourceServiceImpl.java | 2 +-
.../ApplyConsumptionWorkflowDefinition.java | 8 +-
.../group/ApplyGroupWorkflowDefinition.java | 8 +-
.../group/CreateGroupWorkflowDefinition.java | 8 +-
.../group/DeleteGroupWorkflowDefinition.java | 8 +-
.../group/RestartGroupWorkflowDefinition.java | 8 +-
.../group/SuspendGroupWorkflowDefinition.java | 8 +-
.../stream/CreateStreamWorkflowDefinition.java | 8 +-
.../stream/DeleteStreamWorkflowDefinition.java | 8 +-
.../stream/RestartStreamWorkflowDefinition.java | 8 +-
.../stream/SuspendStreamWorkflowDefinition.java | 8 +-
.../core/impl/InlongGroupProcessOperationTest.java | 2 +-
.../service/core/sink/MySQLSinkServiceTest.java | 2 +-
.../service/core/sink/OracleSinkServiceTest.java | 2 +-
.../core/sink/PostgreSQLSinkServiceTest.java | 2 +-
.../service/mocks/MockDeleteSourceListener.java | 10 +-
.../service/mocks/MockRestartSourceListener.java | 10 +-
.../service/mocks/MockStopSourceListener.java | 10 +-
.../manager/service/mq/util/PulsarUtilsTest.java | 1 +
.../workflow/GroupTaskListenerFactoryTest.java | 16 +-
.../service/workflow/WorkflowServiceImplTest.java | 37 +-
.../workflow/event/task/QueueOperateListener.java | 14 +
.../workflow/event/task/SourceOperateListener.java | 14 +
113 files changed, 1576 insertions(+), 2035 deletions(-)
diff --git a/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/BaseExample.java b/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/BaseExample.java
index 7f35072a3..f5ea48841 100644
--- a/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/BaseExample.java
+++ b/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/BaseExample.java
@@ -83,7 +83,7 @@ public class BaseExample {
// set enable zk, create resource, lightweight mode, and cluster tag
pulsarInfo.setEnableZookeeper(InlongConstants.DISABLE_ZK);
pulsarInfo.setEnableCreateResource(InlongConstants.ENABLE_CREATE_RESOURCE);
- pulsarInfo.setLightweight(InlongConstants.NORMAL_MODE);
+ pulsarInfo.setLightweight(InlongConstants.STANDARD_MODE);
pulsarInfo.setInlongClusterTag("default_cluster");
pulsarInfo.setDailyRecords(10000000);
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
index 1b1e37fc1..3983403c9 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
@@ -32,7 +32,7 @@ public class InlongConstants {
public static final Integer DELETED_STATUS = 10;
- public static final Integer NORMAL_MODE = 0;
+ public static final Integer STANDARD_MODE = 0;
public static final Integer LIGHTWEIGHT_MODE = 1;
public static final Integer DISABLE_ZK = 0;
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/ErrorCodeEnum.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/ErrorCodeEnum.java
index e266e9e53..40ec841aa 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/ErrorCodeEnum.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/ErrorCodeEnum.java
@@ -49,7 +49,7 @@ public enum ErrorCodeEnum {
"The current inlong group status does not support modifying the MQ type"),
GROUP_NAME_UPDATE_NOT_ALLOWED(1012, "The current inlong group status does not support modifying the name"),
GROUP_INFO_INCONSISTENT(1013, "The inlong group info is inconsistent, please contact the administrator"),
- GROUP_MODE_UNSUPPORTED(1014, "The current inlong group mode only support light, normal"),
+ GROUP_MODE_UNSUPPORTED(1014, "The current inlong group mode only support lightweight, standard"),
OPT_NOT_ALLOWED_BY_STATUS(1021, "InlongGroup status %s was not allowed to add/update/delete related info"),
@@ -106,6 +106,9 @@ public enum ErrorCodeEnum {
TRANSFORM_NAME_IS_NULL(1501, "Transform name is null"),
TRANSFORM_NOT_FOUND(1502, "Transform does not exist"),
+ MQ_TYPE_IS_NULL(1600, "MQ type is null"),
+ MQ_TYPE_NOT_SUPPORT(1601, "MQ type '%s' not support"),
+
WORKFLOW_EXE_FAILED(4000, "Workflow execution exception"),
CONSUMER_GROUP_DUPLICATED(2600, "The consumer group already exists"),
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/CommandLineRunnerImpl.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/ApplicationInitRunner.java
similarity index 91%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/CommandLineRunnerImpl.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/ApplicationInitRunner.java
index 3743d3417..0735b53dd 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/CommandLineRunnerImpl.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/ApplicationInitRunner.java
@@ -24,10 +24,10 @@ import org.springframework.boot.CommandLineRunner;
import org.springframework.stereotype.Component;
/**
- * Custom Command Line Runner
+ * Init global params by Custom Command Line Runner
*/
@Component
-public class CommandLineRunnerImpl implements CommandLineRunner {
+public class ApplicationInitRunner implements CommandLineRunner {
@Autowired
private ObjectMapper objectMapper;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/AuditServiceImpl.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/AuditServiceImpl.java
index 0247b0229..ae99586a0 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/AuditServiceImpl.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/AuditServiceImpl.java
@@ -26,7 +26,7 @@ import org.apache.inlong.manager.common.pojo.audit.AuditVO;
import org.apache.inlong.manager.common.util.Preconditions;
import org.apache.inlong.manager.dao.mapper.AuditEntityMapper;
import org.apache.inlong.manager.service.core.AuditService;
-import org.apache.inlong.manager.service.resource.es.ElasticsearchApi;
+import org.apache.inlong.manager.service.resource.sink.es.ElasticsearchApi;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.index.query.BoolQueryBuilder;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/DataNodeServiceImpl.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/DataNodeServiceImpl.java
index 82a0c333e..bd099d280 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/DataNodeServiceImpl.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/DataNodeServiceImpl.java
@@ -32,7 +32,7 @@ import org.apache.inlong.manager.common.util.Preconditions;
import org.apache.inlong.manager.dao.entity.DataNodeEntity;
import org.apache.inlong.manager.dao.mapper.DataNodeEntityMapper;
import org.apache.inlong.manager.service.core.DataNodeService;
-import org.apache.inlong.manager.service.resource.hive.HiveJdbcUtils;
+import org.apache.inlong.manager.service.resource.sink.hive.HiveJdbcUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/SortSourceServiceImpl.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/SortSourceServiceImpl.java
index b98632c30..3ff56c4fc 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/SortSourceServiceImpl.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/SortSourceServiceImpl.java
@@ -64,11 +64,13 @@ public class SortSourceServiceImpl implements SortSourceService {
private static final Logger LOGGER = LoggerFactory.getLogger(SortSourceServiceImpl.class);
private static final Gson GSON = new Gson();
- private static final Set<String> SUPPORTED_MQ_TYPE = new HashSet<String>() { {
- add("PULSAR");
- add("KAFKA");
- add("TUBE");
- }};
+ private static final Set<String> SUPPORTED_MQ_TYPE = new HashSet<String>() {
+ {
+ add("PULSAR");
+ add("KAFKA");
+ add("TUBE");
+ }
+ };
private static final String KEY_SERVICE_URL = "serviceUrl";
private static final String KEY_AUTH = "authentication";
private static final String KEY_TENANT = "tenant";
@@ -79,13 +81,15 @@ public class SortSourceServiceImpl implements SortSourceService {
private static final int RESPONSE_CODE_FAIL = -1;
private static final int RESPONSE_CODE_REQ_PARAMS_ERROR = -101;
- /** key 1: cluster name, key 2: task name, value : md5 */
+ /**
+ * key 1: cluster name, key 2: task name, value : md5
+ */
private Map<String, Map<String, String>> sortSourceMd5Map = new ConcurrentHashMap<>();
- /** key 1: cluster name, key 2: task name, value : source config */
+ /**
+ * key 1: cluster name, key 2: task name, value : source config
+ */
private Map<String, Map<String, CacheZoneConfig>> sortSourceConfigMap = new ConcurrentHashMap<>();
- private long reloadInterval = 60000L;
-
@Autowired
private InlongClusterEntityMapper clusterEntityMapper;
@Autowired
@@ -180,12 +184,12 @@ public class SortSourceServiceImpl implements SortSourceService {
// convert to Map<clusterName, Map<taskName, List<groupId>>> format.
Map<String, Map<String, List<String>>> groupMap = new ConcurrentHashMap<>();
allStreamInfos.forEach(stream -> {
- Map<String, List<String>> task2groupsMap =
- groupMap.computeIfAbsent(stream.getSortClusterName(), k -> new ConcurrentHashMap<>());
- List<String> groupIdList =
- task2groupsMap.computeIfAbsent(stream.getSortTaskName(), k -> new ArrayList<>());
- groupIdList.add(stream.getGroupId());
- });
+ Map<String, List<String>> task2groupsMap =
+ groupMap.computeIfAbsent(stream.getSortClusterName(), k -> new ConcurrentHashMap<>());
+ List<String> groupIdList =
+ task2groupsMap.computeIfAbsent(stream.getSortTaskName(), k -> new ArrayList<>());
+ groupIdList.add(stream.getGroupId());
+ });
// get all groups. group by group id.
List<SortSourceGroupInfo> groupInfos = inlongGroupEntityMapper.selectAllGroups();
@@ -290,7 +294,7 @@ public class SortSourceServiceImpl implements SortSourceService {
this.getCacheZoneListByTag(backupTag2GroupInfos, allTag2ClusterInfos, group2topicProp, true);
// combine two cache zone list, and group by cache zone name.
- Map<String, CacheZone> cacheZones = Stream.of(firstTagCacheZoneList, backupTagCacheZoneList)
+ return Stream.of(firstTagCacheZoneList, backupTagCacheZoneList)
.flatMap(Collection::stream)
.collect(Collectors.toMap(
CacheZone::getZoneName,
@@ -300,7 +304,6 @@ public class SortSourceServiceImpl implements SortSourceService {
return zone1;
})
);
- return cacheZones;
}
private List<CacheZone> getCacheZoneListByTag(
@@ -318,7 +321,7 @@ public class SortSourceServiceImpl implements SortSourceService {
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
// get CacheZone list
- List<CacheZone> cacheZones = tags.stream()
+ return tags.stream()
.filter(tag2ClusterInfos::containsKey)
.flatMap(tag -> {
List<SortSourceGroupInfo> groups = tag2GroupInfos.get(tag);
@@ -336,7 +339,6 @@ public class SortSourceServiceImpl implements SortSourceService {
})
.filter(Objects::nonNull)
.collect(Collectors.toList());
- return cacheZones;
}
private CacheZone getCacheZone(
@@ -348,8 +350,8 @@ public class SortSourceServiceImpl implements SortSourceService {
// get basic Cache zone fields
Map<String, String> param = cluster.getExtParamsMap();
String serviceUrl = Optional.ofNullable(param.get(KEY_SERVICE_URL))
- .orElseThrow(() ->
- new IllegalStateException(("there is no serviceUrl for cluster " + cluster.getName())));
+ .orElseThrow(
+ () -> new IllegalStateException(("there is no serviceUrl for cluster " + cluster.getName())));
String tenant = param.get(KEY_TENANT);
String namespace = param.get(KEY_NAME_SPACE);
String authentication = Optional.ofNullable(param.get(KEY_AUTH)).orElse("");
@@ -392,6 +394,7 @@ public class SortSourceServiceImpl implements SortSourceService {
*/
private void setReloadTimer() {
ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
+ long reloadInterval = 60000L;
executorService.scheduleAtFixedRate(this::reload, reloadInterval, reloadInterval, TimeUnit.MILLISECONDS);
}
}
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/listener/GroupTaskListenerFactory.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/GroupTaskListenerFactory.java
similarity index 80%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/listener/GroupTaskListenerFactory.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/GroupTaskListenerFactory.java
index 552f5e1e9..9e0328d94 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/listener/GroupTaskListenerFactory.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/GroupTaskListenerFactory.java
@@ -15,21 +15,17 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.workflow.listener;
+package org.apache.inlong.manager.service.listener;
import com.google.common.collect.Lists;
import lombok.Data;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.inlong.manager.service.mq.CreatePulsarGroupTaskListener;
-import org.apache.inlong.manager.service.mq.CreatePulsarResourceTaskListener;
-import org.apache.inlong.manager.service.mq.CreateTubeGroupTaskListener;
-import org.apache.inlong.manager.service.mq.CreateTubeTopicTaskListener;
-import org.apache.inlong.manager.service.mq.DeletePulsarResourceTaskListener;
-import org.apache.inlong.manager.service.resource.SinkResourceListener;
-import org.apache.inlong.manager.service.sort.SortConfigListener;
-import org.apache.inlong.manager.service.source.listener.SourceDeleteListener;
-import org.apache.inlong.manager.service.source.listener.SourceRestartListener;
-import org.apache.inlong.manager.service.source.listener.SourceStopListener;
+import org.apache.inlong.manager.service.listener.queue.QueueResourceListener;
+import org.apache.inlong.manager.service.listener.sink.SinkResourceListener;
+import org.apache.inlong.manager.service.listener.sort.SortConfigListener;
+import org.apache.inlong.manager.service.listener.source.SourceDeleteListener;
+import org.apache.inlong.manager.service.listener.source.SourceRestartListener;
+import org.apache.inlong.manager.service.listener.source.SourceStopListener;
import org.apache.inlong.manager.workflow.WorkflowContext;
import org.apache.inlong.manager.workflow.definition.ServiceTaskType;
import org.apache.inlong.manager.workflow.definition.TaskListenerFactory;
@@ -67,15 +63,7 @@ public class GroupTaskListenerFactory implements PluginBinder, TaskListenerFacto
@Autowired
private SourceDeleteListener sourceDeleteListener;
@Autowired
- private CreateTubeTopicTaskListener createTubeTopicTaskListener;
- @Autowired
- private CreateTubeGroupTaskListener createTubeGroupTaskListener;
- @Autowired
- private CreatePulsarResourceTaskListener createPulsarResourceTaskListener;
- @Autowired
- private CreatePulsarGroupTaskListener createPulsarGroupTaskListener;
- @Autowired
- private DeletePulsarResourceTaskListener deletePulsarResourceTaskListener;
+ private QueueResourceListener queueResourceListener;
@Autowired
private SinkResourceListener sinkResourceListener;
@Autowired
@@ -88,11 +76,7 @@ public class GroupTaskListenerFactory implements PluginBinder, TaskListenerFacto
sourceOperateListeners.add(sourceDeleteListener);
sourceOperateListeners.add(sourceRestartListener);
queueOperateListeners = new LinkedList<>();
- queueOperateListeners.add(createTubeTopicTaskListener);
- queueOperateListeners.add(createTubeGroupTaskListener);
- queueOperateListeners.add(createPulsarResourceTaskListener);
- queueOperateListeners.add(createPulsarGroupTaskListener);
- queueOperateListeners.add(deletePulsarResourceTaskListener);
+ queueOperateListeners.add(queueResourceListener);
sortOperateListeners = new LinkedList<>();
sortOperateListeners.add(sortConfigListener);
}
@@ -122,7 +106,7 @@ public class GroupTaskListenerFactory implements PluginBinder, TaskListenerFacto
switch (taskType) {
case INIT_MQ:
case DELETE_MQ:
- List<QueueOperateListener> queueOperateListeners = getQueueOperateListener(workflowContext);
+ List<QueueOperateListener> queueOperateListeners = getQueueResourceListener(workflowContext);
return Lists.newArrayList(queueOperateListeners);
case INIT_SORT:
case STOP_SORT:
@@ -168,7 +152,7 @@ public class GroupTaskListenerFactory implements PluginBinder, TaskListenerFacto
/**
* Get queue operate listener list.
*/
- public List<QueueOperateListener> getQueueOperateListener(WorkflowContext context) {
+ public List<QueueOperateListener> getQueueResourceListener(WorkflowContext context) {
List<QueueOperateListener> listeners = new ArrayList<>();
for (QueueOperateListener listener : queueOperateListeners) {
if (listener != null && listener.accept(context)) {
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/listener/StreamTaskListenerFactory.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/StreamTaskListenerFactory.java
similarity index 89%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/listener/StreamTaskListenerFactory.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/StreamTaskListenerFactory.java
index b894817eb..d14cf9052 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/listener/StreamTaskListenerFactory.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/StreamTaskListenerFactory.java
@@ -15,15 +15,13 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.workflow.listener;
+package org.apache.inlong.manager.service.listener;
import com.google.common.collect.Lists;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.inlong.manager.service.mq.CreatePulsarSubscriptionTaskListener;
-import org.apache.inlong.manager.service.mq.CreatePulsarTopicTaskListener;
-import org.apache.inlong.manager.service.mq.DeletePulsarTopicTaskListener;
-import org.apache.inlong.manager.service.resource.StreamSinkResourceListener;
-import org.apache.inlong.manager.service.sort.StreamSortConfigListener;
+import org.apache.inlong.manager.service.listener.queue.StreamQueueResourceListener;
+import org.apache.inlong.manager.service.listener.sink.StreamSinkResourceListener;
+import org.apache.inlong.manager.service.listener.sort.StreamSortConfigListener;
import org.apache.inlong.manager.workflow.WorkflowContext;
import org.apache.inlong.manager.workflow.definition.ServiceTaskType;
import org.apache.inlong.manager.workflow.definition.TaskListenerFactory;
@@ -55,11 +53,7 @@ public class StreamTaskListenerFactory implements PluginBinder, TaskListenerFact
private List<SinkOperateListener> sinkOperateListeners;
@Autowired
- private CreatePulsarTopicTaskListener createPulsarTopicTaskListener;
- @Autowired
- private CreatePulsarSubscriptionTaskListener createPulsarSubscriptionTaskListener;
- @Autowired
- private DeletePulsarTopicTaskListener deletePulsarTopicTaskListener;
+ private StreamQueueResourceListener queueResourceListener;
@Autowired
private StreamSortConfigListener streamSortConfigListener;
@Autowired
@@ -69,9 +63,7 @@ public class StreamTaskListenerFactory implements PluginBinder, TaskListenerFact
public void init() {
sourceOperateListeners = new LinkedList<>();
queueOperateListeners = new LinkedList<>();
- queueOperateListeners.add(createPulsarTopicTaskListener);
- queueOperateListeners.add(createPulsarSubscriptionTaskListener);
- queueOperateListeners.add(deletePulsarTopicTaskListener);
+ queueOperateListeners.add(queueResourceListener);
sortOperateListeners = new LinkedList<>();
sortOperateListeners.add(streamSortConfigListener);
sinkOperateListeners = new LinkedList<>();
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/consumption/listener/ConsumptionCancelProcessListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/consumption/ConsumptionCancelProcessListener.java
similarity index 97%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/consumption/listener/ConsumptionCancelProcessListener.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/consumption/ConsumptionCancelProcessListener.java
index 5d0efa33d..5780bcd99 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/consumption/listener/ConsumptionCancelProcessListener.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/consumption/ConsumptionCancelProcessListener.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.workflow.consumption.listener;
+package org.apache.inlong.manager.service.listener.consumption;
import lombok.extern.slf4j.Slf4j;
import org.apache.inlong.manager.common.consts.InlongConstants;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/consumption/listener/ConsumptionCompleteProcessListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/consumption/ConsumptionCompleteProcessListener.java
similarity index 96%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/consumption/listener/ConsumptionCompleteProcessListener.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/consumption/ConsumptionCompleteProcessListener.java
index 9ca92a771..0f86ee8b8 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/consumption/listener/ConsumptionCompleteProcessListener.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/consumption/ConsumptionCompleteProcessListener.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.workflow.consumption.listener;
+package org.apache.inlong.manager.service.listener.consumption;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
@@ -37,9 +37,9 @@ import org.apache.inlong.manager.dao.entity.InlongGroupEntity;
import org.apache.inlong.manager.dao.mapper.ConsumptionEntityMapper;
import org.apache.inlong.manager.dao.mapper.InlongGroupEntityMapper;
import org.apache.inlong.manager.service.cluster.InlongClusterService;
-import org.apache.inlong.manager.service.mq.util.PulsarOperator;
-import org.apache.inlong.manager.service.mq.util.PulsarUtils;
-import org.apache.inlong.manager.service.mq.util.TubeMQOperator;
+import org.apache.inlong.manager.service.resource.queue.pulsar.PulsarOperator;
+import org.apache.inlong.manager.service.resource.queue.pulsar.PulsarUtils;
+import org.apache.inlong.manager.service.resource.queue.tubemq.TubeMQOperator;
import org.apache.inlong.manager.workflow.WorkflowContext;
import org.apache.inlong.manager.workflow.event.ListenerResult;
import org.apache.inlong.manager.workflow.event.process.ProcessEvent;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/consumption/listener/ConsumptionPassTaskListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/consumption/ConsumptionPassTaskListener.java
similarity index 97%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/consumption/listener/ConsumptionPassTaskListener.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/consumption/ConsumptionPassTaskListener.java
index cefa3e7d9..792ff1a03 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/consumption/listener/ConsumptionPassTaskListener.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/consumption/ConsumptionPassTaskListener.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.workflow.consumption.listener;
+package org.apache.inlong.manager.service.listener.consumption;
import com.alibaba.druid.util.StringUtils;
import lombok.extern.slf4j.Slf4j;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/consumption/listener/ConsumptionRejectProcessListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/consumption/ConsumptionRejectProcessListener.java
similarity index 97%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/consumption/listener/ConsumptionRejectProcessListener.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/consumption/ConsumptionRejectProcessListener.java
index 2f8135584..2c543ebbb 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/consumption/listener/ConsumptionRejectProcessListener.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/consumption/ConsumptionRejectProcessListener.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.workflow.consumption.listener;
+package org.apache.inlong.manager.service.listener.consumption;
import org.apache.inlong.manager.common.consts.InlongConstants;
import org.apache.inlong.manager.common.enums.ConsumptionStatus;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/listener/InitGroupCompleteListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupCompleteListener.java
similarity index 98%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/listener/InitGroupCompleteListener.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupCompleteListener.java
index eb8179e67..17fd81fa6 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/listener/InitGroupCompleteListener.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupCompleteListener.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.workflow.group.listener;
+package org.apache.inlong.manager.service.listener.group;
import lombok.extern.slf4j.Slf4j;
import org.apache.inlong.manager.common.consts.InlongConstants;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/listener/InitGroupFailedListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupFailedListener.java
similarity index 97%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/listener/InitGroupFailedListener.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupFailedListener.java
index fc0c1caa3..518e290bf 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/listener/InitGroupFailedListener.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupFailedListener.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.workflow.group.listener;
+package org.apache.inlong.manager.service.listener.group;
import lombok.extern.slf4j.Slf4j;
import org.apache.inlong.manager.common.enums.GroupStatus;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/listener/InitGroupListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupListener.java
similarity index 97%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/listener/InitGroupListener.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupListener.java
index f2594bd6e..ad036a6c0 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/listener/InitGroupListener.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupListener.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.workflow.group.listener;
+package org.apache.inlong.manager.service.listener.group;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/listener/UpdateGroupCompleteListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/UpdateGroupCompleteListener.java
similarity index 95%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/listener/UpdateGroupCompleteListener.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/UpdateGroupCompleteListener.java
index d1c5aafa8..bb24ff242 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/listener/UpdateGroupCompleteListener.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/UpdateGroupCompleteListener.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.workflow.group.listener;
+package org.apache.inlong.manager.service.listener.group;
import lombok.extern.slf4j.Slf4j;
import org.apache.inlong.manager.common.consts.InlongConstants;
@@ -70,7 +70,7 @@ public class UpdateGroupCompleteListener implements ProcessEventListener {
groupService.updateStatus(groupId, GroupStatus.DELETED.getCode(), operator);
break;
default:
- log.warn("Unsupported operate={} for inlong group", operateType);
+ log.warn("unsupported operate={} for inlong group", operateType);
break;
}
InlongGroupInfo groupInfo = form.getGroupInfo();
@@ -97,7 +97,7 @@ public class UpdateGroupCompleteListener implements ProcessEventListener {
sourceService.logicDeleteAll(groupId, null, operator);
break;
default:
- log.warn("Unsupported operate={} for inlong group", operateType);
+ log.warn("unsupported operate={} for inlong group", operateType);
break;
}
}
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/listener/UpdateGroupFailedListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/UpdateGroupFailedListener.java
similarity index 97%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/listener/UpdateGroupFailedListener.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/UpdateGroupFailedListener.java
index cb93afece..05baf071d 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/listener/UpdateGroupFailedListener.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/UpdateGroupFailedListener.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.workflow.group.listener;
+package org.apache.inlong.manager.service.listener.group;
import lombok.extern.slf4j.Slf4j;
import org.apache.inlong.manager.common.enums.GroupStatus;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/listener/UpdateGroupListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/UpdateGroupListener.java
similarity index 97%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/listener/UpdateGroupListener.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/UpdateGroupListener.java
index 24442fa02..f540ea18a 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/listener/UpdateGroupListener.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/UpdateGroupListener.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.workflow.group.listener;
+package org.apache.inlong.manager.service.listener.group;
import lombok.extern.slf4j.Slf4j;
import org.apache.inlong.manager.common.enums.GroupOperateType;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/listener/apply/AfterApprovedTaskListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/apply/AfterApprovedTaskListener.java
similarity index 97%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/listener/apply/AfterApprovedTaskListener.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/apply/AfterApprovedTaskListener.java
index f3bd67f55..ab12e45f3 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/listener/apply/AfterApprovedTaskListener.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/apply/AfterApprovedTaskListener.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.workflow.group.listener.apply;
+package org.apache.inlong.manager.service.listener.group.apply;
import lombok.extern.slf4j.Slf4j;
import org.apache.inlong.manager.common.exceptions.WorkflowListenerException;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/listener/apply/ApproveApplyProcessListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/apply/ApproveApplyProcessListener.java
similarity index 97%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/listener/apply/ApproveApplyProcessListener.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/apply/ApproveApplyProcessListener.java
index 831faabd1..99b059114 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/listener/apply/ApproveApplyProcessListener.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/apply/ApproveApplyProcessListener.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.workflow.group.listener.apply;
+package org.apache.inlong.manager.service.listener.group.apply;
import lombok.extern.slf4j.Slf4j;
import org.apache.inlong.manager.common.exceptions.WorkflowListenerException;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/listener/apply/CancelApplyProcessListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/apply/CancelApplyProcessListener.java
similarity index 97%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/listener/apply/CancelApplyProcessListener.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/apply/CancelApplyProcessListener.java
index 582d95d37..aa065bab4 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/listener/apply/CancelApplyProcessListener.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/apply/CancelApplyProcessListener.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.workflow.group.listener.apply;
+package org.apache.inlong.manager.service.listener.group.apply;
import lombok.extern.slf4j.Slf4j;
import org.apache.inlong.manager.common.enums.GroupStatus;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/listener/apply/RejectApplyProcessListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/apply/RejectApplyProcessListener.java
similarity index 97%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/listener/apply/RejectApplyProcessListener.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/apply/RejectApplyProcessListener.java
index 1a903537c..b233a8576 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/listener/apply/RejectApplyProcessListener.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/apply/RejectApplyProcessListener.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.workflow.group.listener.apply;
+package org.apache.inlong.manager.service.listener.group.apply;
import lombok.extern.slf4j.Slf4j;
import org.apache.inlong.manager.common.enums.GroupStatus;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/queue/QueueResourceListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/queue/QueueResourceListener.java
new file mode 100644
index 000000000..f2ebdfed0
--- /dev/null
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/queue/QueueResourceListener.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.manager.service.listener.queue;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.inlong.manager.common.consts.InlongConstants;
+import org.apache.inlong.manager.common.enums.GroupOperateType;
+import org.apache.inlong.manager.common.enums.MQType;
+import org.apache.inlong.manager.common.exceptions.WorkflowListenerException;
+import org.apache.inlong.manager.common.pojo.group.InlongGroupInfo;
+import org.apache.inlong.manager.common.pojo.workflow.form.process.GroupResourceProcessForm;
+import org.apache.inlong.manager.service.group.InlongGroupService;
+import org.apache.inlong.manager.service.resource.queue.QueueResourceOperator;
+import org.apache.inlong.manager.service.resource.queue.QueueResourceOperatorFactory;
+import org.apache.inlong.manager.workflow.WorkflowContext;
+import org.apache.inlong.manager.workflow.event.ListenerResult;
+import org.apache.inlong.manager.workflow.event.task.QueueOperateListener;
+import org.apache.inlong.manager.workflow.event.task.TaskEvent;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+/**
+ * Create message queue resources,
+ * such as Pulsar Topic and Subscription, TubeMQ Topic and ConsumerGroup, etc.
+ */
+@Slf4j
+@Service
+public class QueueResourceListener implements QueueOperateListener {
+
+ @Autowired
+ private InlongGroupService groupService;
+ @Autowired
+ private QueueResourceOperatorFactory queueOperatorFactory;
+
+ @Override
+ public TaskEvent event() {
+ return TaskEvent.COMPLETE;
+ }
+
+ @Override
+ public boolean accept(WorkflowContext context) {
+ if (!isGroupProcessForm(context)) {
+ return false;
+ }
+ GroupResourceProcessForm processForm = (GroupResourceProcessForm) context.getProcessForm();
+ return InlongConstants.STANDARD_MODE.equals(processForm.getGroupInfo().getLightweight());
+ }
+
+ @Override
+ public ListenerResult listen(WorkflowContext context) throws WorkflowListenerException {
+ GroupResourceProcessForm groupProcessForm = (GroupResourceProcessForm) context.getProcessForm();
+ final String groupId = groupProcessForm.getInlongGroupId();
+ // ensure the inlong group exists
+ InlongGroupInfo groupInfo = groupService.get(groupId);
+ if (groupInfo == null) {
+ String msg = "inlong group not found with groupId=" + groupId;
+ log.error(msg);
+ throw new WorkflowListenerException(msg);
+ }
+
+ if (InlongConstants.DISABLE_CREATE_RESOURCE.equals(groupInfo.getEnableCreateResource())) {
+ log.warn("skip to execute QueueResourceListener as disable create resource for groupId={}", groupId);
+ return ListenerResult.success("skip - disable create resource");
+ }
+
+ QueueResourceOperator queueOperator = queueOperatorFactory.getInstance(MQType.forType(groupInfo.getMqType()));
+ GroupOperateType operateType = groupProcessForm.getGroupOperateType();
+ String operator = context.getOperator();
+ switch (operateType) {
+ case INIT:
+ queueOperator.createQueueForGroup(groupInfo, operator);
+ break;
+ case DELETE:
+ queueOperator.deleteQueueForGroup(groupInfo, operator);
+ break;
+ default:
+ log.warn("unsupported operate={} for inlong group", operateType);
+ break;
+ }
+
+ log.info("success to execute QueueResourceListener for groupId={}, operateType={}", groupId, operateType);
+ return ListenerResult.success("success");
+ }
+
+}
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/queue/StreamQueueResourceListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/queue/StreamQueueResourceListener.java
new file mode 100644
index 000000000..fec7fd137
--- /dev/null
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/queue/StreamQueueResourceListener.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.manager.service.listener.queue;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.inlong.manager.common.consts.InlongConstants;
+import org.apache.inlong.manager.common.enums.GroupOperateType;
+import org.apache.inlong.manager.common.enums.MQType;
+import org.apache.inlong.manager.common.exceptions.WorkflowListenerException;
+import org.apache.inlong.manager.common.pojo.group.InlongGroupInfo;
+import org.apache.inlong.manager.common.pojo.stream.InlongStreamInfo;
+import org.apache.inlong.manager.common.pojo.workflow.form.process.ProcessForm;
+import org.apache.inlong.manager.common.pojo.workflow.form.process.StreamResourceProcessForm;
+import org.apache.inlong.manager.service.core.InlongStreamService;
+import org.apache.inlong.manager.service.group.InlongGroupService;
+import org.apache.inlong.manager.service.resource.queue.QueueResourceOperator;
+import org.apache.inlong.manager.service.resource.queue.QueueResourceOperatorFactory;
+import org.apache.inlong.manager.workflow.WorkflowContext;
+import org.apache.inlong.manager.workflow.event.ListenerResult;
+import org.apache.inlong.manager.workflow.event.task.QueueOperateListener;
+import org.apache.inlong.manager.workflow.event.task.TaskEvent;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+/**
+ * Create task listener for Pulsar Topic
+ */
+@Slf4j
+@Component
+public class StreamQueueResourceListener implements QueueOperateListener {
+
+ @Autowired
+ private InlongGroupService groupService;
+ @Autowired
+ private InlongStreamService streamService;
+ @Autowired
+ private QueueResourceOperatorFactory queueOperatorFactory;
+
+ @Override
+ public TaskEvent event() {
+ return TaskEvent.COMPLETE;
+ }
+
+ @Override
+ public boolean accept(WorkflowContext context) {
+ ProcessForm processForm = context.getProcessForm();
+ if (!(processForm instanceof StreamResourceProcessForm)) {
+ return false;
+ }
+ StreamResourceProcessForm streamProcessForm = (StreamResourceProcessForm) processForm;
+ return InlongConstants.STANDARD_MODE.equals(streamProcessForm.getGroupInfo().getLightweight());
+ }
+
+ @Override
+ public ListenerResult listen(WorkflowContext context) throws WorkflowListenerException {
+ StreamResourceProcessForm streamProcessForm = (StreamResourceProcessForm) context.getProcessForm();
+ final String groupId = streamProcessForm.getInlongGroupId();
+ InlongStreamInfo streamInfo = streamProcessForm.getStreamInfo();
+ if (streamInfo == null) {
+ String msg = "inlong stream cannot be null in StreamResourceProcessForm with groupId=" + groupId;
+ log.error(msg);
+ throw new WorkflowListenerException(msg);
+ }
+
+ // ensure the inlong group exists
+ InlongGroupInfo groupInfo = groupService.get(groupId);
+ if (groupInfo == null) {
+ String msg = "inlong group not found with groupId=" + groupId;
+ log.error(msg);
+ throw new WorkflowListenerException(msg);
+ }
+
+ if (InlongConstants.DISABLE_CREATE_RESOURCE.equals(groupInfo.getEnableCreateResource())) {
+ log.warn("skip to execute StreamQueueResourceListener as disable create resource for groupId={}", groupId);
+ return ListenerResult.success("skip - disable create resource");
+ }
+
+ QueueResourceOperator queueOperator = queueOperatorFactory.getInstance(MQType.forType(groupInfo.getMqType()));
+ GroupOperateType operateType = streamProcessForm.getGroupOperateType();
+ String operator = context.getOperator();
+ switch (operateType) {
+ case INIT:
+ queueOperator.createQueueForStream(groupInfo, streamInfo, operator);
+ break;
+ case DELETE:
+ queueOperator.deleteQueueForStream(groupInfo, streamInfo, operator);
+ break;
+ default:
+ log.warn("unsupported operate={} for inlong group", operateType);
+ break;
+ }
+
+ log.info("success to execute StreamQueueResourceListener for groupId={}, operateType={}", groupId, operateType);
+ return ListenerResult.success("success");
+ }
+
+}
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/SinkResourceListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/sink/SinkResourceListener.java
similarity index 86%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/SinkResourceListener.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/sink/SinkResourceListener.java
index 3b18acb37..268ae3b43 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/SinkResourceListener.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/sink/SinkResourceListener.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.resource;
+package org.apache.inlong.manager.service.listener.sink;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
@@ -25,6 +25,8 @@ import org.apache.inlong.manager.common.pojo.sink.SinkInfo;
import org.apache.inlong.manager.common.pojo.stream.InlongStreamInfo;
import org.apache.inlong.manager.common.pojo.workflow.form.process.GroupResourceProcessForm;
import org.apache.inlong.manager.dao.mapper.StreamSinkEntityMapper;
+import org.apache.inlong.manager.service.resource.sink.SinkResourceOperator;
+import org.apache.inlong.manager.service.resource.sink.SinkResourceOperatorFactory;
import org.apache.inlong.manager.workflow.WorkflowContext;
import org.apache.inlong.manager.workflow.event.ListenerResult;
import org.apache.inlong.manager.workflow.event.task.SinkOperateListener;
@@ -47,7 +49,7 @@ public class SinkResourceListener implements SinkOperateListener {
@Autowired
private StreamSinkEntityMapper sinkMapper;
@Autowired
- private SinkResourceOperatorFactory resourceOperatorFactory;
+ private SinkResourceOperatorFactory sinkOperatorFactory;
@Override
public TaskEvent event() {
@@ -75,10 +77,10 @@ public class SinkResourceListener implements SinkOperateListener {
return ListenerResult.success();
}
- for (SinkInfo sinkConfig : needCreateList) {
- String sinkType = sinkConfig.getSinkType();
- SinkResourceOperator resourceOperator = resourceOperatorFactory.getInstance(SinkType.forType(sinkType));
- resourceOperator.createSinkResource(sinkConfig);
+ for (SinkInfo sinkInfo : needCreateList) {
+ String sinkType = sinkInfo.getSinkType();
+ SinkResourceOperator resourceOperator = sinkOperatorFactory.getInstance(SinkType.forType(sinkType));
+ resourceOperator.createSinkResource(sinkInfo);
}
log.info("success to create sink resources for group [" + groupId + "] and stream " + streamIdList);
return ListenerResult.success();
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/StreamSinkResourceListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/sink/StreamSinkResourceListener.java
similarity index 94%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/StreamSinkResourceListener.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/sink/StreamSinkResourceListener.java
index 7d9ebe220..01bcff447 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/StreamSinkResourceListener.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/sink/StreamSinkResourceListener.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.resource;
+package org.apache.inlong.manager.service.listener.sink;
import com.google.common.collect.Lists;
import lombok.extern.slf4j.Slf4j;
@@ -27,6 +27,8 @@ import org.apache.inlong.manager.common.pojo.stream.InlongStreamInfo;
import org.apache.inlong.manager.common.pojo.workflow.form.process.ProcessForm;
import org.apache.inlong.manager.common.pojo.workflow.form.process.StreamResourceProcessForm;
import org.apache.inlong.manager.dao.mapper.StreamSinkEntityMapper;
+import org.apache.inlong.manager.service.resource.sink.SinkResourceOperator;
+import org.apache.inlong.manager.service.resource.sink.SinkResourceOperatorFactory;
import org.apache.inlong.manager.workflow.WorkflowContext;
import org.apache.inlong.manager.workflow.event.ListenerResult;
import org.apache.inlong.manager.workflow.event.task.SinkOperateListener;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/SortConfigListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/sort/SortConfigListener.java
similarity index 96%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/SortConfigListener.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/sort/SortConfigListener.java
index b97d13e72..94b089ce2 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/SortConfigListener.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/sort/SortConfigListener.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.sort;
+package org.apache.inlong.manager.service.listener.sort;
import org.apache.inlong.manager.common.consts.InlongConstants;
import org.apache.inlong.manager.common.enums.GroupOperateType;
@@ -26,6 +26,8 @@ import org.apache.inlong.manager.common.pojo.stream.InlongStreamInfo;
import org.apache.inlong.manager.common.pojo.workflow.form.process.GroupResourceProcessForm;
import org.apache.inlong.manager.common.pojo.workflow.form.process.ProcessForm;
import org.apache.inlong.manager.common.pojo.workflow.form.process.StreamResourceProcessForm;
+import org.apache.inlong.manager.service.resource.sort.SortConfigOperator;
+import org.apache.inlong.manager.service.resource.sort.SortConfigOperatorFactory;
import org.apache.inlong.manager.workflow.WorkflowContext;
import org.apache.inlong.manager.workflow.event.ListenerResult;
import org.apache.inlong.manager.workflow.event.task.SortOperateListener;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/StreamSortConfigListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/sort/StreamSortConfigListener.java
similarity index 96%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/StreamSortConfigListener.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/sort/StreamSortConfigListener.java
index 6f4293a65..71d514040 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/StreamSortConfigListener.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/sort/StreamSortConfigListener.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.sort;
+package org.apache.inlong.manager.service.listener.sort;
import org.apache.commons.collections.CollectionUtils;
import org.apache.inlong.manager.common.consts.InlongConstants;
@@ -28,6 +28,8 @@ import org.apache.inlong.manager.common.pojo.stream.InlongStreamInfo;
import org.apache.inlong.manager.common.pojo.workflow.form.process.GroupResourceProcessForm;
import org.apache.inlong.manager.common.pojo.workflow.form.process.ProcessForm;
import org.apache.inlong.manager.common.pojo.workflow.form.process.StreamResourceProcessForm;
+import org.apache.inlong.manager.service.resource.sort.SortConfigOperator;
+import org.apache.inlong.manager.service.resource.sort.SortConfigOperatorFactory;
import org.apache.inlong.manager.workflow.WorkflowContext;
import org.apache.inlong.manager.workflow.event.ListenerResult;
import org.apache.inlong.manager.workflow.event.task.SortOperateListener;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/listener/AbstractSourceOperateListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/source/AbstractSourceOperateListener.java
similarity index 99%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/listener/AbstractSourceOperateListener.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/source/AbstractSourceOperateListener.java
index 9ec754ceb..3524f4679 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/listener/AbstractSourceOperateListener.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/source/AbstractSourceOperateListener.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.source.listener;
+package org.apache.inlong.manager.service.listener.source;
import com.google.common.collect.Lists;
import lombok.SneakyThrows;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/listener/SourceDeleteListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/source/SourceDeleteListener.java
similarity index 79%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/listener/SourceDeleteListener.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/source/SourceDeleteListener.java
index 0277cc9d4..626bac84e 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/listener/SourceDeleteListener.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/source/SourceDeleteListener.java
@@ -15,13 +15,13 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.source.listener;
+package org.apache.inlong.manager.service.listener.source;
import lombok.extern.slf4j.Slf4j;
+import org.apache.inlong.manager.common.consts.InlongConstants;
import org.apache.inlong.manager.common.enums.GroupOperateType;
import org.apache.inlong.manager.common.pojo.source.SourceRequest;
import org.apache.inlong.manager.common.pojo.workflow.form.process.GroupResourceProcessForm;
-import org.apache.inlong.manager.common.pojo.workflow.form.process.ProcessForm;
import org.apache.inlong.manager.workflow.WorkflowContext;
import org.springframework.stereotype.Component;
@@ -39,12 +39,12 @@ public class SourceDeleteListener extends AbstractSourceOperateListener {
@Override
public boolean accept(WorkflowContext context) {
- ProcessForm processForm = context.getProcessForm();
- if (!(processForm instanceof GroupResourceProcessForm)) {
+ if (!isGroupProcessForm(context)) {
return false;
}
- GroupResourceProcessForm groupResourceForm = (GroupResourceProcessForm) processForm;
- return groupResourceForm.getGroupOperateType() == GroupOperateType.DELETE;
+ GroupResourceProcessForm processForm = (GroupResourceProcessForm) context.getProcessForm();
+ return InlongConstants.STANDARD_MODE.equals(processForm.getGroupInfo().getLightweight())
+ && processForm.getGroupOperateType() == GroupOperateType.DELETE;
}
@Override
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/listener/SourceRestartListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/source/SourceRestartListener.java
similarity index 78%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/listener/SourceRestartListener.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/source/SourceRestartListener.java
index ea5071dfd..2f1c0c004 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/listener/SourceRestartListener.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/source/SourceRestartListener.java
@@ -15,12 +15,12 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.source.listener;
+package org.apache.inlong.manager.service.listener.source;
+import org.apache.inlong.manager.common.consts.InlongConstants;
import org.apache.inlong.manager.common.enums.GroupOperateType;
import org.apache.inlong.manager.common.pojo.source.SourceRequest;
import org.apache.inlong.manager.common.pojo.workflow.form.process.GroupResourceProcessForm;
-import org.apache.inlong.manager.common.pojo.workflow.form.process.ProcessForm;
import org.apache.inlong.manager.workflow.WorkflowContext;
import org.springframework.stereotype.Component;
@@ -37,12 +37,12 @@ public class SourceRestartListener extends AbstractSourceOperateListener {
@Override
public boolean accept(WorkflowContext context) {
- ProcessForm processForm = context.getProcessForm();
- if (!(processForm instanceof GroupResourceProcessForm)) {
+ if (!isGroupProcessForm(context)) {
return false;
}
- GroupResourceProcessForm groupResourceForm = (GroupResourceProcessForm) processForm;
- return groupResourceForm.getGroupOperateType() == GroupOperateType.RESTART;
+ GroupResourceProcessForm processForm = (GroupResourceProcessForm) context.getProcessForm();
+ return InlongConstants.STANDARD_MODE.equals(processForm.getGroupInfo().getLightweight())
+ && processForm.getGroupOperateType() == GroupOperateType.RESTART;
}
@Override
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/listener/SourceStopListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/source/SourceStopListener.java
similarity index 78%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/listener/SourceStopListener.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/source/SourceStopListener.java
index 4d985ba10..5e740b7d1 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/listener/SourceStopListener.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/source/SourceStopListener.java
@@ -15,12 +15,12 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.source.listener;
+package org.apache.inlong.manager.service.listener.source;
+import org.apache.inlong.manager.common.consts.InlongConstants;
import org.apache.inlong.manager.common.enums.GroupOperateType;
import org.apache.inlong.manager.common.pojo.source.SourceRequest;
import org.apache.inlong.manager.common.pojo.workflow.form.process.GroupResourceProcessForm;
-import org.apache.inlong.manager.common.pojo.workflow.form.process.ProcessForm;
import org.apache.inlong.manager.workflow.WorkflowContext;
import org.springframework.stereotype.Component;
@@ -37,12 +37,12 @@ public class SourceStopListener extends AbstractSourceOperateListener {
@Override
public boolean accept(WorkflowContext context) {
- ProcessForm processForm = context.getProcessForm();
- if (!(processForm instanceof GroupResourceProcessForm)) {
+ if (!isGroupProcessForm(context)) {
return false;
}
- GroupResourceProcessForm groupResourceForm = (GroupResourceProcessForm) processForm;
- return groupResourceForm.getGroupOperateType() == GroupOperateType.SUSPEND;
+ GroupResourceProcessForm processForm = (GroupResourceProcessForm) context.getProcessForm();
+ return InlongConstants.STANDARD_MODE.equals(processForm.getGroupInfo().getLightweight())
+ && processForm.getGroupOperateType() == GroupOperateType.SUSPEND;
}
@Override
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/stream/listener/InitStreamCompleteListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/stream/InitStreamCompleteListener.java
similarity index 97%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/stream/listener/InitStreamCompleteListener.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/stream/InitStreamCompleteListener.java
index e890f64d0..08f84c11e 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/stream/listener/InitStreamCompleteListener.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/stream/InitStreamCompleteListener.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.workflow.stream.listener;
+package org.apache.inlong.manager.service.listener.stream;
import lombok.extern.slf4j.Slf4j;
import org.apache.inlong.manager.common.enums.SourceStatus;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/stream/listener/InitStreamFailedListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/stream/InitStreamFailedListener.java
similarity index 97%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/stream/listener/InitStreamFailedListener.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/stream/InitStreamFailedListener.java
index 9ebb4e31c..53f4a9331 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/stream/listener/InitStreamFailedListener.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/stream/InitStreamFailedListener.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.workflow.stream.listener;
+package org.apache.inlong.manager.service.listener.stream;
import lombok.extern.slf4j.Slf4j;
import org.apache.inlong.manager.common.enums.StreamStatus;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/stream/listener/InitStreamListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/stream/InitStreamListener.java
similarity index 97%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/stream/listener/InitStreamListener.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/stream/InitStreamListener.java
index 2cc1a3246..33e8ad2d3 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/stream/listener/InitStreamListener.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/stream/InitStreamListener.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.workflow.stream.listener;
+package org.apache.inlong.manager.service.listener.stream;
import org.apache.inlong.manager.common.enums.ErrorCodeEnum;
import org.apache.inlong.manager.common.enums.StreamStatus;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/stream/listener/UpdateStreamCompleteListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/stream/UpdateStreamCompleteListener.java
similarity index 97%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/stream/listener/UpdateStreamCompleteListener.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/stream/UpdateStreamCompleteListener.java
index 225671651..2340c2389 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/stream/listener/UpdateStreamCompleteListener.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/stream/UpdateStreamCompleteListener.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.workflow.stream.listener;
+package org.apache.inlong.manager.service.listener.stream;
import lombok.extern.slf4j.Slf4j;
import org.apache.inlong.manager.common.enums.GroupOperateType;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/stream/listener/UpdateStreamFailedListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/stream/UpdateStreamFailedListener.java
similarity index 97%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/stream/listener/UpdateStreamFailedListener.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/stream/UpdateStreamFailedListener.java
index c57f871c0..3399bcc54 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/stream/listener/UpdateStreamFailedListener.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/stream/UpdateStreamFailedListener.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.workflow.stream.listener;
+package org.apache.inlong.manager.service.listener.stream;
import lombok.extern.slf4j.Slf4j;
import org.apache.inlong.manager.common.enums.StreamStatus;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/stream/listener/UpdateStreamListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/stream/UpdateStreamListener.java
similarity index 97%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/stream/listener/UpdateStreamListener.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/stream/UpdateStreamListener.java
index 4d1917640..d448ec217 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/stream/listener/UpdateStreamListener.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/stream/UpdateStreamListener.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.workflow.stream.listener;
+package org.apache.inlong.manager.service.listener.stream;
import org.apache.inlong.manager.common.enums.GroupOperateType;
import org.apache.inlong.manager.common.enums.StreamStatus;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/mq/CreatePulsarGroupTaskListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/mq/CreatePulsarGroupTaskListener.java
deleted file mode 100644
index e04db57ea..000000000
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/mq/CreatePulsarGroupTaskListener.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.manager.service.mq;
-
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.inlong.manager.common.consts.InlongConstants;
-import org.apache.inlong.manager.common.enums.ClusterType;
-import org.apache.inlong.manager.common.enums.GroupOperateType;
-import org.apache.inlong.manager.common.enums.MQType;
-import org.apache.inlong.manager.common.exceptions.WorkflowListenerException;
-import org.apache.inlong.manager.common.pojo.cluster.ClusterInfo;
-import org.apache.inlong.manager.common.pojo.cluster.pulsar.PulsarClusterInfo;
-import org.apache.inlong.manager.common.pojo.group.InlongGroupInfo;
-import org.apache.inlong.manager.common.pojo.group.pulsar.InlongPulsarInfo;
-import org.apache.inlong.manager.common.pojo.pulsar.PulsarTopicBean;
-import org.apache.inlong.manager.common.pojo.stream.InlongStreamBriefInfo;
-import org.apache.inlong.manager.common.pojo.workflow.form.process.GroupResourceProcessForm;
-import org.apache.inlong.manager.common.pojo.workflow.form.process.ProcessForm;
-import org.apache.inlong.manager.service.cluster.InlongClusterService;
-import org.apache.inlong.manager.service.core.ConsumptionService;
-import org.apache.inlong.manager.service.core.InlongStreamService;
-import org.apache.inlong.manager.service.group.InlongGroupService;
-import org.apache.inlong.manager.service.mq.util.PulsarOperator;
-import org.apache.inlong.manager.service.mq.util.PulsarUtils;
-import org.apache.inlong.manager.workflow.WorkflowContext;
-import org.apache.inlong.manager.workflow.event.ListenerResult;
-import org.apache.inlong.manager.workflow.event.task.QueueOperateListener;
-import org.apache.inlong.manager.workflow.event.task.TaskEvent;
-import org.apache.pulsar.client.admin.PulsarAdmin;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-import java.util.List;
-
-/**
- * Create default subscription group for Pulsar
- */
-@Component
-@Slf4j
-public class CreatePulsarGroupTaskListener implements QueueOperateListener {
-
- @Autowired
- private InlongGroupService groupService;
- @Autowired
- private InlongStreamService streamService;
- @Autowired
- private InlongClusterService clusterService;
- @Autowired
- private ConsumptionService consumptionService;
- @Autowired
- private PulsarOperator pulsarOperator;
-
- @Override
- public TaskEvent event() {
- return TaskEvent.COMPLETE;
- }
-
- @Override
- public boolean accept(WorkflowContext context) {
- ProcessForm processForm = context.getProcessForm();
- if (!(processForm instanceof GroupResourceProcessForm)) {
- return false;
- }
- GroupResourceProcessForm form = (GroupResourceProcessForm) processForm;
- GroupOperateType operateType = form.getGroupOperateType();
- if (operateType != GroupOperateType.INIT) {
- return false;
- }
- String groupId = form.getInlongGroupId();
- InlongGroupInfo groupInfo = form.getGroupInfo();
- MQType mqType = MQType.forType(groupInfo.getMqType());
- if (mqType == MQType.PULSAR || mqType == MQType.TDMQ_PULSAR) {
- InlongPulsarInfo pulsarInfo = (InlongPulsarInfo) groupInfo;
- boolean enable = InlongConstants.ENABLE_CREATE_RESOURCE.equals(pulsarInfo.getEnableCreateResource());
- if (enable) {
- log.info("need to create pulsar resource as the createResource was true for groupId [{}]", groupId);
- return true;
- } else {
- log.info("skip to create pulsar resource as the createResource was false for groupId [{}]", groupId);
- return false;
- }
- }
-
- log.warn("skip to create pulsar subscription as the mq type is {} for groupId [{}]", mqType, groupId);
- return false;
- }
-
- @Override
- public ListenerResult listen(WorkflowContext context) throws WorkflowListenerException {
- GroupResourceProcessForm form = (GroupResourceProcessForm) context.getProcessForm();
- final String groupId = form.getInlongGroupId();
- InlongGroupInfo groupInfo = groupService.get(groupId);
- if (groupInfo == null) {
- log.error("inlong group not found with groupId={}", groupId);
- throw new WorkflowListenerException("inlong group not found with groupId=" + groupId);
- }
-
- // For Pulsar, each Stream corresponds to a Topic
- List<InlongStreamBriefInfo> streamInfo = streamService.getTopicList(groupId);
- if (streamInfo == null || streamInfo.isEmpty()) {
- log.warn("inlong stream is empty for groupId={}, skip to create pulsar subscription", groupId);
- return ListenerResult.success();
- }
-
- String clusterTag = groupInfo.getInlongClusterTag();
- ClusterInfo clusterInfo = clusterService.getOne(clusterTag, null, ClusterType.PULSAR);
- PulsarClusterInfo pulsarCluster = (PulsarClusterInfo) clusterInfo;
- try (PulsarAdmin pulsarAdmin = PulsarUtils.getPulsarAdmin(pulsarCluster)) {
- String tenant = pulsarCluster.getTenant();
- if (StringUtils.isEmpty(tenant)) {
- tenant = InlongConstants.DEFAULT_PULSAR_TENANT;
- }
- String namespace = groupInfo.getMqResource();
-
- for (InlongStreamBriefInfo stream : streamInfo) {
- PulsarTopicBean topicBean = new PulsarTopicBean();
- topicBean.setTenant(tenant);
- topicBean.setNamespace(namespace);
- String topic = stream.getMqResource();
- topicBean.setTopicName(topic);
-
- // Create a subscription in the Pulsar cluster you need to ensure that the Topic exists
- try {
- boolean exist = pulsarOperator.topicIsExists(pulsarAdmin, tenant, namespace, topic);
- if (!exist) {
- String topicFull = tenant + "/" + namespace + "/" + topic;
- String serviceUrl = pulsarCluster.getAdminUrl();
- log.error("topic={} not exists in {}", topicFull, serviceUrl);
- throw new WorkflowListenerException("topic=" + topicFull + " not exists in " + serviceUrl);
- }
-
- // Consumer naming rules: clusterTag_topicName_consumer_group
- String subscription = clusterTag + "_" + topic + "_consumer_group";
- pulsarOperator.createSubscription(pulsarAdmin, topicBean, subscription);
-
- // Insert the consumption data into the consumption table
- consumptionService.saveSortConsumption(groupInfo, topic, subscription);
- } catch (Exception e) {
- log.error("create pulsar subscription error for groupId={}", groupId);
- throw new WorkflowListenerException("create pulsar subscription error: " + e.getMessage());
- }
- }
- } catch (Exception e) {
- log.error("create pulsar subscription error for groupId={}", groupId);
- throw new WorkflowListenerException("create pulsar subscription error: " + e.getMessage());
- }
-
- log.info("success to create pulsar subscription for groupId={}", groupId);
- return ListenerResult.success();
- }
-
-}
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/mq/CreatePulsarResourceTaskListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/mq/CreatePulsarResourceTaskListener.java
deleted file mode 100644
index 41a97b2c2..000000000
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/mq/CreatePulsarResourceTaskListener.java
+++ /dev/null
@@ -1,164 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.manager.service.mq;
-
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.inlong.manager.common.consts.InlongConstants;
-import org.apache.inlong.manager.common.enums.ClusterType;
-import org.apache.inlong.manager.common.enums.GroupOperateType;
-import org.apache.inlong.manager.common.enums.MQType;
-import org.apache.inlong.manager.common.exceptions.WorkflowListenerException;
-import org.apache.inlong.manager.common.pojo.cluster.ClusterInfo;
-import org.apache.inlong.manager.common.pojo.cluster.pulsar.PulsarClusterInfo;
-import org.apache.inlong.manager.common.pojo.group.InlongGroupInfo;
-import org.apache.inlong.manager.common.pojo.group.pulsar.InlongPulsarInfo;
-import org.apache.inlong.manager.common.pojo.pulsar.PulsarTopicBean;
-import org.apache.inlong.manager.common.pojo.stream.InlongStreamBriefInfo;
-import org.apache.inlong.manager.common.pojo.workflow.form.process.GroupResourceProcessForm;
-import org.apache.inlong.manager.common.pojo.workflow.form.process.ProcessForm;
-import org.apache.inlong.manager.common.util.Preconditions;
-import org.apache.inlong.manager.service.cluster.InlongClusterService;
-import org.apache.inlong.manager.service.core.InlongStreamService;
-import org.apache.inlong.manager.service.group.InlongGroupService;
-import org.apache.inlong.manager.service.mq.util.PulsarOperator;
-import org.apache.inlong.manager.service.mq.util.PulsarUtils;
-import org.apache.inlong.manager.workflow.WorkflowContext;
-import org.apache.inlong.manager.workflow.event.ListenerResult;
-import org.apache.inlong.manager.workflow.event.task.QueueOperateListener;
-import org.apache.inlong.manager.workflow.event.task.TaskEvent;
-import org.apache.pulsar.client.admin.PulsarAdmin;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-import java.util.List;
-
-/**
- * Create Pulsar tenant, namespace and topic
- */
-@Slf4j
-@Component
-public class CreatePulsarResourceTaskListener implements QueueOperateListener {
-
- @Autowired
- private InlongGroupService groupService;
- @Autowired
- private InlongStreamService streamService;
- @Autowired
- private InlongClusterService clusterService;
- @Autowired
- private PulsarOperator pulsarOperator;
-
- @Override
- public TaskEvent event() {
- return TaskEvent.COMPLETE;
- }
-
- @Override
- public boolean accept(WorkflowContext context) {
- ProcessForm processForm = context.getProcessForm();
- if (!(processForm instanceof GroupResourceProcessForm)) {
- return false;
- }
- GroupResourceProcessForm form = (GroupResourceProcessForm) processForm;
- GroupOperateType operateType = form.getGroupOperateType();
- if (operateType != GroupOperateType.INIT) {
- return false;
- }
- String groupId = form.getInlongGroupId();
- InlongGroupInfo groupInfo = form.getGroupInfo();
- MQType mqType = MQType.forType(groupInfo.getMqType());
- if (mqType == MQType.PULSAR || mqType == MQType.TDMQ_PULSAR) {
- InlongPulsarInfo pulsarInfo = (InlongPulsarInfo) groupInfo;
- boolean enable = InlongConstants.ENABLE_CREATE_RESOURCE.equals(pulsarInfo.getEnableCreateResource());
- if (enable) {
- log.info("need to create pulsar resource as the createResource was true for groupId [{}]", groupId);
- return true;
- } else {
- log.info("skip to create pulsar resource as the createResource was false for groupId [{}]", groupId);
- return false;
- }
- }
-
- log.warn("skip to create pulsar subscription as the mq type is {} for groupId [{}]", mqType, groupId);
- return false;
- }
-
- @Override
- public ListenerResult listen(WorkflowContext context) throws WorkflowListenerException {
- GroupResourceProcessForm form = (GroupResourceProcessForm) context.getProcessForm();
- String groupId = form.getInlongGroupId();
- log.info("begin to create pulsar resource for groupId={}", groupId);
-
- InlongGroupInfo groupInfo = groupService.get(groupId);
- if (groupInfo == null) {
- throw new WorkflowListenerException("inlong group or pulsar cluster not found for groupId=" + groupId);
- }
-
- try {
- InlongPulsarInfo pulsarInfo = (InlongPulsarInfo) groupInfo;
- ClusterInfo clusterInfo = clusterService.getOne(groupInfo.getInlongClusterTag(), null,
- ClusterType.PULSAR);
- this.createPulsarProcess(pulsarInfo, (PulsarClusterInfo) clusterInfo);
- } catch (Exception e) {
- log.error("create pulsar resource error for groupId={}", groupId, e);
- throw new WorkflowListenerException("create pulsar resource error for groupId=" + groupId);
- }
-
- log.info("success to create pulsar resource for groupId={}", groupId);
- return ListenerResult.success();
- }
-
- /**
- * Create Pulsar tenant, namespace and topic
- */
- private void createPulsarProcess(InlongPulsarInfo pulsarInfo, PulsarClusterInfo pulsarCluster) throws Exception {
- String groupId = pulsarInfo.getInlongGroupId();
- log.info("begin to create pulsar resource for groupId={} in cluster={}", groupId, pulsarCluster);
-
- String namespace = pulsarInfo.getMqResource();
- Preconditions.checkNotNull(namespace, "pulsar namespace cannot be empty for groupId=" + groupId);
- String queueModule = pulsarInfo.getQueueModule();
- Preconditions.checkNotNull(queueModule, "queue module cannot be empty for groupId=" + groupId);
-
- try (PulsarAdmin pulsarAdmin = PulsarUtils.getPulsarAdmin(pulsarCluster)) {
- // create pulsar tenant
- String tenant = pulsarCluster.getTenant();
- if (StringUtils.isEmpty(tenant)) {
- tenant = InlongConstants.DEFAULT_PULSAR_TENANT;
- }
- pulsarOperator.createTenant(pulsarAdmin, tenant);
-
- // create pulsar namespace
- pulsarOperator.createNamespace(pulsarAdmin, pulsarInfo, tenant, namespace);
-
- // create pulsar topic
- Integer partitionNum = pulsarInfo.getPartitionNum();
- List<InlongStreamBriefInfo> streamTopicList = streamService.getTopicList(groupId);
- PulsarTopicBean topicBean = PulsarTopicBean.builder()
- .tenant(tenant).namespace(namespace).numPartitions(partitionNum).queueModule(queueModule).build();
-
- for (InlongStreamBriefInfo streamInfo : streamTopicList) {
- topicBean.setTopicName(streamInfo.getMqResource());
- pulsarOperator.createTopic(pulsarAdmin, topicBean);
- }
- }
- log.info("finish to create pulsar resource for groupId={}, cluster={}", groupId, pulsarCluster);
- }
-
-}
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/mq/CreatePulsarSubscriptionTaskListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/mq/CreatePulsarSubscriptionTaskListener.java
deleted file mode 100644
index 7cbe50fd6..000000000
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/mq/CreatePulsarSubscriptionTaskListener.java
+++ /dev/null
@@ -1,155 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.manager.service.mq;
-
-import lombok.extern.slf4j.Slf4j;
-import org.apache.inlong.manager.common.consts.InlongConstants;
-import org.apache.inlong.manager.common.enums.ClusterType;
-import org.apache.inlong.manager.common.enums.GroupOperateType;
-import org.apache.inlong.manager.common.enums.MQType;
-import org.apache.inlong.manager.common.exceptions.BusinessException;
-import org.apache.inlong.manager.common.exceptions.WorkflowListenerException;
-import org.apache.inlong.manager.common.pojo.cluster.ClusterInfo;
-import org.apache.inlong.manager.common.pojo.cluster.pulsar.PulsarClusterInfo;
-import org.apache.inlong.manager.common.pojo.group.InlongGroupInfo;
-import org.apache.inlong.manager.common.pojo.group.pulsar.InlongPulsarInfo;
-import org.apache.inlong.manager.common.pojo.pulsar.PulsarTopicBean;
-import org.apache.inlong.manager.common.pojo.stream.InlongStreamInfo;
-import org.apache.inlong.manager.common.pojo.workflow.form.process.ProcessForm;
-import org.apache.inlong.manager.common.pojo.workflow.form.process.StreamResourceProcessForm;
-import org.apache.inlong.manager.service.cluster.InlongClusterService;
-import org.apache.inlong.manager.service.core.ConsumptionService;
-import org.apache.inlong.manager.service.mq.util.PulsarOperator;
-import org.apache.inlong.manager.service.mq.util.PulsarUtils;
-import org.apache.inlong.manager.service.sink.StreamSinkService;
-import org.apache.inlong.manager.workflow.WorkflowContext;
-import org.apache.inlong.manager.workflow.event.ListenerResult;
-import org.apache.inlong.manager.workflow.event.task.QueueOperateListener;
-import org.apache.inlong.manager.workflow.event.task.TaskEvent;
-import org.apache.pulsar.client.admin.PulsarAdmin;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-import java.util.List;
-
-/**
- * Create a subscription group for a single inlong stream
- */
-@Slf4j
-@Component
-public class CreatePulsarSubscriptionTaskListener implements QueueOperateListener {
-
- @Autowired
- private InlongClusterService clusterService;
- @Autowired
- private PulsarOperator pulsarOperator;
- @Autowired
- private StreamSinkService sinkService;
- @Autowired
- private ConsumptionService consumptionService;
-
- @Override
- public TaskEvent event() {
- return TaskEvent.COMPLETE;
- }
-
- @Override
- public boolean accept(WorkflowContext context) {
- ProcessForm processForm = context.getProcessForm();
- if (!(processForm instanceof StreamResourceProcessForm)) {
- return false;
- }
- StreamResourceProcessForm streamResourceProcessForm = (StreamResourceProcessForm) processForm;
- GroupOperateType operateType = streamResourceProcessForm.getGroupOperateType();
- if (operateType != GroupOperateType.INIT) {
- return false;
- }
-
- InlongGroupInfo groupInfo = streamResourceProcessForm.getGroupInfo();
- MQType mqType = MQType.forType(groupInfo.getMqType());
- String groupId = groupInfo.getInlongGroupId();
- String streamId = streamResourceProcessForm.getStreamInfo().getInlongStreamId();
- if (mqType == MQType.PULSAR || mqType == MQType.TDMQ_PULSAR) {
- InlongPulsarInfo pulsarInfo = (InlongPulsarInfo) groupInfo;
- boolean enable = InlongConstants.ENABLE_CREATE_RESOURCE.equals(pulsarInfo.getEnableCreateResource());
- if (enable) {
- log.info("need to create pulsar topic as the createResource was true for groupId [{}] streamId [{}]",
- groupId, streamId);
- return true;
- } else {
- log.info("skip to create pulsar topic as the createResource was false for groupId [{}] streamId [{}]",
- groupId, streamId);
- return false;
- }
- }
- return false;
- }
-
- @Override
- public ListenerResult listen(WorkflowContext context) throws WorkflowListenerException {
- StreamResourceProcessForm form = (StreamResourceProcessForm) context.getProcessForm();
- InlongGroupInfo groupInfo = form.getGroupInfo();
- InlongStreamInfo streamInfo = form.getStreamInfo();
- final String groupId = streamInfo.getInlongGroupId();
- final String streamId = streamInfo.getInlongStreamId();
-
- String clusterTag = groupInfo.getInlongClusterTag();
- ClusterInfo clusterInfo = clusterService.getOne(clusterTag, null, ClusterType.PULSAR);
- PulsarClusterInfo pulsarCluster = (PulsarClusterInfo) clusterInfo;
- try (PulsarAdmin pulsarAdmin = PulsarUtils.getPulsarAdmin(pulsarCluster)) {
- // Query data sink info based on groupId and streamId
- List<String> sinkTypeList = sinkService.getSinkTypeList(groupId, streamId);
- if (sinkTypeList == null || sinkTypeList.size() == 0) {
- log.warn("sink info is empty for groupId={}, streamId={}, skip to create pulsar group",
- groupId, streamId);
- return ListenerResult.success();
- }
-
- String tenant = pulsarCluster.getTenant();
- String namespace = groupInfo.getMqResource();
- String topic = streamInfo.getMqResource();
- PulsarTopicBean topicBean = new PulsarTopicBean();
- topicBean.setTenant(tenant);
- topicBean.setNamespace(namespace);
- topicBean.setTopicName(topic);
-
- // Create a subscription in the Pulsar cluster, you need to ensure that the Topic exists
- boolean exist = pulsarOperator.topicIsExists(pulsarAdmin, tenant, namespace, topic);
- if (!exist) {
- String fullTopic = tenant + "/" + namespace + "/" + topic;
- String msg = String.format("topic=%s not exists in %s", fullTopic, pulsarCluster.getAdminUrl());
- log.error(msg);
- throw new BusinessException(msg);
- }
-
- // Consumer naming rules: clusterTag_topicName_consumer_group
- String subscription = clusterTag + "_" + topic + "_consumer_group";
- pulsarOperator.createSubscription(pulsarAdmin, topicBean, subscription);
-
- // Insert the consumption data into the consumption table
- consumptionService.saveSortConsumption(groupInfo, topic, subscription);
- } catch (Exception e) {
- log.error("failed to create pulsar subscription for groupId=" + groupId + " streamId=" + streamId, e);
- throw new WorkflowListenerException("failed to create pulsar subscription: " + e.getMessage());
- }
-
- log.info("success to create pulsar subscription for groupId={}, streamId={}", groupId, streamId);
- return ListenerResult.success();
- }
-
-}
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/mq/CreatePulsarTopicTaskListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/mq/CreatePulsarTopicTaskListener.java
deleted file mode 100644
index 6ecafd029..000000000
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/mq/CreatePulsarTopicTaskListener.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.manager.service.mq;
-
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.inlong.manager.common.consts.InlongConstants;
-import org.apache.inlong.manager.common.enums.ClusterType;
-import org.apache.inlong.manager.common.enums.GroupOperateType;
-import org.apache.inlong.manager.common.enums.MQType;
-import org.apache.inlong.manager.common.exceptions.WorkflowListenerException;
-import org.apache.inlong.manager.common.pojo.cluster.ClusterInfo;
-import org.apache.inlong.manager.common.pojo.cluster.pulsar.PulsarClusterInfo;
-import org.apache.inlong.manager.common.pojo.group.InlongGroupInfo;
-import org.apache.inlong.manager.common.pojo.group.pulsar.InlongPulsarInfo;
-import org.apache.inlong.manager.common.pojo.pulsar.PulsarTopicBean;
-import org.apache.inlong.manager.common.pojo.stream.InlongStreamInfo;
-import org.apache.inlong.manager.common.pojo.workflow.form.process.ProcessForm;
-import org.apache.inlong.manager.common.pojo.workflow.form.process.StreamResourceProcessForm;
-import org.apache.inlong.manager.service.cluster.InlongClusterService;
-import org.apache.inlong.manager.service.mq.util.PulsarOperator;
-import org.apache.inlong.manager.service.mq.util.PulsarUtils;
-import org.apache.inlong.manager.workflow.WorkflowContext;
-import org.apache.inlong.manager.workflow.event.ListenerResult;
-import org.apache.inlong.manager.workflow.event.task.QueueOperateListener;
-import org.apache.inlong.manager.workflow.event.task.TaskEvent;
-import org.apache.pulsar.client.admin.PulsarAdmin;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-/**
- * Create task listener for Pulsar Topic
- */
-@Slf4j
-@Component
-public class CreatePulsarTopicTaskListener implements QueueOperateListener {
-
- @Autowired
- private InlongClusterService clusterService;
- @Autowired
- private PulsarOperator pulsarOperator;
-
- @Override
- public TaskEvent event() {
- return TaskEvent.COMPLETE;
- }
-
- @Override
- public boolean accept(WorkflowContext context) {
- ProcessForm processForm = context.getProcessForm();
- if (!(processForm instanceof StreamResourceProcessForm)) {
- return false;
- }
- StreamResourceProcessForm streamResourceProcessForm = (StreamResourceProcessForm) processForm;
- GroupOperateType operateType = streamResourceProcessForm.getGroupOperateType();
- if (operateType != GroupOperateType.INIT) {
- return false;
- }
-
- InlongGroupInfo groupInfo = streamResourceProcessForm.getGroupInfo();
- MQType mqType = MQType.forType(groupInfo.getMqType());
- String groupId = groupInfo.getInlongGroupId();
- String streamId = streamResourceProcessForm.getStreamInfo().getInlongStreamId();
- if (mqType == MQType.PULSAR || mqType == MQType.TDMQ_PULSAR) {
- InlongPulsarInfo pulsarInfo = (InlongPulsarInfo) groupInfo;
- boolean enable = InlongConstants.ENABLE_CREATE_RESOURCE.equals(pulsarInfo.getEnableCreateResource());
- if (enable) {
- log.info("need to create pulsar topic as the createResource was true for groupId [{}] streamId [{}]",
- groupId, streamId);
- return true;
- } else {
- log.info("skip to create pulsar topic as the createResource was false for groupId [{}] streamId [{}]",
- groupId, streamId);
- return false;
- }
- }
- return false;
- }
-
- @Override
- public ListenerResult listen(WorkflowContext context) throws WorkflowListenerException {
- StreamResourceProcessForm form = (StreamResourceProcessForm) context.getProcessForm();
- InlongGroupInfo groupInfo = form.getGroupInfo();
- InlongStreamInfo streamInfo = form.getStreamInfo();
- final String groupId = streamInfo.getInlongGroupId();
- final String streamId = streamInfo.getInlongStreamId();
- log.info("begin to create pulsar topic for groupId={}, streamId={}", groupId, streamId);
-
- try {
- InlongPulsarInfo pulsarInfo = (InlongPulsarInfo) groupInfo;
- String pulsarTopic = streamInfo.getMqResource();
- String clusterTag = pulsarInfo.getInlongClusterTag();
- ClusterInfo clusterInfo = clusterService.getOne(clusterTag, null, ClusterType.PULSAR);
- PulsarClusterInfo pulsarCluster = (PulsarClusterInfo) clusterInfo;
- String tenant = pulsarCluster.getTenant();
- if (StringUtils.isEmpty(tenant)) {
- tenant = InlongConstants.DEFAULT_PULSAR_TENANT;
- }
-
- try (PulsarAdmin pulsarAdmin = PulsarUtils.getPulsarAdmin(pulsarCluster)) {
- PulsarTopicBean topicBean = PulsarTopicBean.builder()
- .tenant(tenant)
- .namespace(pulsarInfo.getMqResource())
- .topicName(pulsarTopic)
- .queueModule(pulsarInfo.getQueueModule())
- .numPartitions(pulsarInfo.getPartitionNum())
- .build();
- pulsarOperator.createTopic(pulsarAdmin, topicBean);
- }
- } catch (Exception e) {
- String msg = String.format("failed to create pulsar topic for groupId=%s, streamId=%s", groupId, streamId);
- log.error(msg, e);
- throw new WorkflowListenerException(msg);
- }
-
- log.info("success to create pulsar topic for groupId={}, streamId={}", groupId, streamId);
- return ListenerResult.success();
- }
-
-}
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/mq/CreateTubeGroupTaskListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/mq/CreateTubeGroupTaskListener.java
deleted file mode 100644
index 187fdb845..000000000
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/mq/CreateTubeGroupTaskListener.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.manager.service.mq;
-
-import lombok.extern.slf4j.Slf4j;
-import org.apache.inlong.manager.common.enums.ClusterType;
-import org.apache.inlong.manager.common.enums.MQType;
-import org.apache.inlong.manager.common.exceptions.WorkflowListenerException;
-import org.apache.inlong.manager.common.pojo.cluster.tube.TubeClusterInfo;
-import org.apache.inlong.manager.common.pojo.workflow.form.process.GroupResourceProcessForm;
-import org.apache.inlong.manager.common.pojo.workflow.form.process.ProcessForm;
-import org.apache.inlong.manager.dao.entity.InlongGroupEntity;
-import org.apache.inlong.manager.dao.mapper.InlongGroupEntityMapper;
-import org.apache.inlong.manager.service.cluster.InlongClusterService;
-import org.apache.inlong.manager.service.mq.util.TubeMQOperator;
-import org.apache.inlong.manager.workflow.WorkflowContext;
-import org.apache.inlong.manager.workflow.event.ListenerResult;
-import org.apache.inlong.manager.workflow.event.task.QueueOperateListener;
-import org.apache.inlong.manager.workflow.event.task.TaskEvent;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-/**
- * Event listener of create tube consumer group.
- */
-@Component
-@Slf4j
-public class CreateTubeGroupTaskListener implements QueueOperateListener {
-
- @Autowired
- private InlongGroupEntityMapper groupMapper;
- @Autowired
- private InlongClusterService clusterService;
- @Autowired
- private TubeMQOperator tubeMQOperator;
-
- @Override
- public TaskEvent event() {
- return TaskEvent.COMPLETE;
- }
-
- @Override
- public boolean accept(WorkflowContext context) {
- ProcessForm processForm = context.getProcessForm();
- if (!(processForm instanceof GroupResourceProcessForm)) {
- return false;
- }
- GroupResourceProcessForm form = (GroupResourceProcessForm) processForm;
- String groupId = form.getInlongGroupId();
- MQType mqType = MQType.forType(form.getGroupInfo().getMqType());
- if (mqType == MQType.TUBE) {
- log.info("need to create tube resource for groupId [{}]", groupId);
- return true;
- }
-
- log.warn("skip to to create tube resource as the mq type is {} for groupId [{}]", mqType, groupId);
- return false;
- }
-
- @Override
- public ListenerResult listen(WorkflowContext context) throws WorkflowListenerException {
- GroupResourceProcessForm form = (GroupResourceProcessForm) context.getProcessForm();
- String groupId = form.getInlongGroupId();
- log.info("begin to create tube consumer group for groupId {}", groupId);
-
- InlongGroupEntity groupEntity = groupMapper.selectByGroupId(groupId);
- String clusterTag = groupEntity.getInlongClusterTag();
- TubeClusterInfo tubeCluster = (TubeClusterInfo) clusterService.getOne(clusterTag, null, ClusterType.TUBE);
-
- String topicName = groupEntity.getMqResource();
- // Consumer naming rules: clusterTag_topicName_consumer_group
- String consumeGroup = clusterTag + "_" + topicName + "_consumer_group";
- try {
- tubeMQOperator.createConsumerGroup(tubeCluster, topicName, consumeGroup, context.getOperator());
- } catch (Exception e) {
- throw new WorkflowListenerException("create tube consumer group for groupId=" + groupId + " error", e);
- }
- log.info("finish to create tube consumer group for groupId={}", groupId);
- return ListenerResult.success();
- }
-
-}
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/mq/CreateTubeTopicTaskListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/mq/CreateTubeTopicTaskListener.java
deleted file mode 100644
index f4d6dd204..000000000
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/mq/CreateTubeTopicTaskListener.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.manager.service.mq;
-
-import lombok.extern.slf4j.Slf4j;
-import org.apache.inlong.manager.common.enums.ClusterType;
-import org.apache.inlong.manager.common.enums.MQType;
-import org.apache.inlong.manager.common.exceptions.WorkflowListenerException;
-import org.apache.inlong.manager.common.pojo.cluster.ClusterInfo;
-import org.apache.inlong.manager.common.pojo.cluster.tube.TubeClusterInfo;
-import org.apache.inlong.manager.common.pojo.group.InlongGroupInfo;
-import org.apache.inlong.manager.common.pojo.workflow.form.process.GroupResourceProcessForm;
-import org.apache.inlong.manager.common.pojo.workflow.form.process.ProcessForm;
-import org.apache.inlong.manager.service.cluster.InlongClusterService;
-import org.apache.inlong.manager.service.mq.util.TubeMQOperator;
-import org.apache.inlong.manager.workflow.WorkflowContext;
-import org.apache.inlong.manager.workflow.event.ListenerResult;
-import org.apache.inlong.manager.workflow.event.task.QueueOperateListener;
-import org.apache.inlong.manager.workflow.event.task.TaskEvent;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-/**
- * Create a listener for MQ resource tasks
- */
-@Component
-@Slf4j
-public class CreateTubeTopicTaskListener implements QueueOperateListener {
-
- @Autowired
- private InlongClusterService clusterService;
- @Autowired
- private TubeMQOperator tubeMQOperator;
-
- @Override
- public TaskEvent event() {
- return TaskEvent.COMPLETE;
- }
-
- @Override
- public boolean accept(WorkflowContext context) {
- ProcessForm processForm = context.getProcessForm();
- if (!(processForm instanceof GroupResourceProcessForm)) {
- return false;
- }
- GroupResourceProcessForm form = (GroupResourceProcessForm) processForm;
- String groupId = form.getInlongGroupId();
- MQType mqType = MQType.forType(form.getGroupInfo().getMqType());
- if (mqType == MQType.TUBE) {
- log.info("need to create tube resource for groupId [{}]", groupId);
- return true;
- }
-
- log.warn("skip to to create tube resource as the mq type is {} for groupId [{}]", mqType, groupId);
- return false;
- }
-
- @Override
- public ListenerResult listen(WorkflowContext context) throws WorkflowListenerException {
- GroupResourceProcessForm form = (GroupResourceProcessForm) context.getProcessForm();
- log.info("begin create tube topic for groupId={}", form.getInlongGroupId());
-
- String groupId = form.getInlongGroupId();
- try {
- InlongGroupInfo groupInfo = form.getGroupInfo();
- String clusterTag = groupInfo.getInlongClusterTag();
- ClusterInfo clusterInfo = clusterService.getOne(clusterTag, null, ClusterType.TUBE);
- tubeMQOperator.createTopic((TubeClusterInfo) clusterInfo, groupInfo.getMqResource(), context.getOperator());
- log.info("finish to create tube topic for groupId={}", groupId);
- } catch (Exception e) {
- log.error("create tube topic for groupId={} error, exception {} ", groupId, e.getMessage(), e);
- }
- return ListenerResult.success();
- }
-
-}
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/mq/DeletePulsarResourceTaskListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/mq/DeletePulsarResourceTaskListener.java
deleted file mode 100644
index 978d75fbd..000000000
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/mq/DeletePulsarResourceTaskListener.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.manager.service.mq;
-
-import lombok.extern.slf4j.Slf4j;
-import org.apache.inlong.manager.common.consts.InlongConstants;
-import org.apache.inlong.manager.common.enums.ClusterType;
-import org.apache.inlong.manager.common.enums.GroupOperateType;
-import org.apache.inlong.manager.common.enums.MQType;
-import org.apache.inlong.manager.common.exceptions.WorkflowListenerException;
-import org.apache.inlong.manager.common.pojo.cluster.ClusterInfo;
-import org.apache.inlong.manager.common.pojo.cluster.pulsar.PulsarClusterInfo;
-import org.apache.inlong.manager.common.pojo.group.InlongGroupInfo;
-import org.apache.inlong.manager.common.pojo.group.pulsar.InlongPulsarInfo;
-import org.apache.inlong.manager.common.pojo.pulsar.PulsarTopicBean;
-import org.apache.inlong.manager.common.pojo.stream.InlongStreamBriefInfo;
-import org.apache.inlong.manager.common.pojo.workflow.form.process.GroupResourceProcessForm;
-import org.apache.inlong.manager.common.pojo.workflow.form.process.ProcessForm;
-import org.apache.inlong.manager.common.util.Preconditions;
-import org.apache.inlong.manager.service.cluster.InlongClusterService;
-import org.apache.inlong.manager.service.core.InlongStreamService;
-import org.apache.inlong.manager.service.group.InlongGroupService;
-import org.apache.inlong.manager.service.mq.util.PulsarOperator;
-import org.apache.inlong.manager.service.mq.util.PulsarUtils;
-import org.apache.inlong.manager.workflow.WorkflowContext;
-import org.apache.inlong.manager.workflow.event.ListenerResult;
-import org.apache.inlong.manager.workflow.event.task.QueueOperateListener;
-import org.apache.inlong.manager.workflow.event.task.TaskEvent;
-import org.apache.pulsar.client.admin.PulsarAdmin;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-import java.util.List;
-
-/**
- * Delete Pulsar tenant, namespace and topic after group is operated to delete
- */
-@Slf4j
-@Component
-public class DeletePulsarResourceTaskListener implements QueueOperateListener {
-
- @Autowired
- private InlongGroupService groupService;
- @Autowired
- private InlongStreamService streamService;
- @Autowired
- private InlongClusterService clusterService;
- @Autowired
- private PulsarOperator pulsarOperator;
-
- @Override
- public TaskEvent event() {
- return TaskEvent.COMPLETE;
- }
-
- @Override
- public boolean accept(WorkflowContext context) {
- ProcessForm processForm = context.getProcessForm();
- if (!(processForm instanceof GroupResourceProcessForm)) {
- return false;
- }
- GroupResourceProcessForm form = (GroupResourceProcessForm) processForm;
- GroupOperateType operateType = form.getGroupOperateType();
- if (operateType != GroupOperateType.DELETE) {
- return false;
- }
- String groupId = form.getInlongGroupId();
- InlongGroupInfo groupInfo = form.getGroupInfo();
- MQType mqType = MQType.forType(groupInfo.getMqType());
- if (mqType == MQType.PULSAR || mqType == MQType.TDMQ_PULSAR) {
- InlongPulsarInfo pulsarInfo = (InlongPulsarInfo) groupInfo;
- boolean enable = InlongConstants.ENABLE_CREATE_RESOURCE.equals(pulsarInfo.getEnableCreateResource());
- if (enable) {
- log.info("need to delete pulsar resource as the createResource was true for groupId [{}]", groupId);
- return true;
- } else {
- log.info("skip to delete pulsar resource as the createResource was false for groupId [{}]", groupId);
- return false;
- }
- }
-
- log.warn("skip to delete pulsar subscription as the mq type is {} for groupId [{}]", mqType, groupId);
- return false;
- }
-
- @Override
- public ListenerResult listen(WorkflowContext context) throws Exception {
- GroupResourceProcessForm form = (GroupResourceProcessForm) context.getProcessForm();
- String groupId = form.getInlongGroupId();
- log.info("begin to delete pulsar resource for groupId={}", groupId);
- InlongGroupInfo groupInfo = groupService.get(groupId);
- if (groupInfo == null) {
- throw new WorkflowListenerException("inlong group or pulsar cluster not found for groupId=" + groupId);
- }
- try {
- InlongPulsarInfo pulsarInfo = (InlongPulsarInfo) groupInfo;
- ClusterInfo clusterInfo = clusterService.getOne(groupInfo.getInlongClusterTag(), null,
- ClusterType.PULSAR);
- deletePulsarProcess(pulsarInfo, (PulsarClusterInfo) clusterInfo);
- } catch (Exception e) {
- log.error("delete pulsar resource error for groupId={}", groupId, e);
- throw new WorkflowListenerException("delete pulsar resource error for groupId=" + groupId);
- }
-
- log.info("success to delete pulsar resource for groupId={}", groupId);
- return ListenerResult.success();
- }
-
- /**
- * Delete all Pulsar topics forcefully
- */
- private void deletePulsarProcess(InlongPulsarInfo pulsarInfo, PulsarClusterInfo pulsarCluster) throws Exception {
- String groupId = pulsarInfo.getInlongGroupId();
- log.info("begin to delete pulsar resource for groupId={} in cluster={}", groupId, pulsarCluster);
- final String tenant = pulsarCluster.getTenant();
- final String namespace = pulsarInfo.getMqResource();
- Preconditions.checkNotNull(namespace, "pulsar namespace cannot be empty for groupId=" + groupId);
- try (PulsarAdmin pulsarAdmin = PulsarUtils.getPulsarAdmin(pulsarCluster)) {
- List<InlongStreamBriefInfo> streamTopicList = streamService.getTopicList(groupId);
- for (InlongStreamBriefInfo streamInfo : streamTopicList) {
- final String topic = streamInfo.getMqResource();
- if (topic == null) {
- continue;
- }
- PulsarTopicBean topicBean = PulsarTopicBean.builder()
- .tenant(tenant)
- .namespace(namespace)
- .topicName(topic)
- .build();
- pulsarOperator.forceDeleteTopic(pulsarAdmin, topicBean);
- }
- }
- log.info("finish to delete pulsar resource for groupId={}, cluster={}", groupId, pulsarCluster);
- }
-}
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/mq/DeletePulsarTopicTaskListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/mq/DeletePulsarTopicTaskListener.java
deleted file mode 100644
index 4b77a86af..000000000
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/mq/DeletePulsarTopicTaskListener.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.manager.service.mq;
-
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.inlong.manager.common.consts.InlongConstants;
-import org.apache.inlong.manager.common.enums.ClusterType;
-import org.apache.inlong.manager.common.enums.GroupOperateType;
-import org.apache.inlong.manager.common.enums.MQType;
-import org.apache.inlong.manager.common.exceptions.WorkflowListenerException;
-import org.apache.inlong.manager.common.pojo.cluster.ClusterInfo;
-import org.apache.inlong.manager.common.pojo.cluster.pulsar.PulsarClusterInfo;
-import org.apache.inlong.manager.common.pojo.group.InlongGroupInfo;
-import org.apache.inlong.manager.common.pojo.group.pulsar.InlongPulsarInfo;
-import org.apache.inlong.manager.common.pojo.pulsar.PulsarTopicBean;
-import org.apache.inlong.manager.common.pojo.stream.InlongStreamInfo;
-import org.apache.inlong.manager.common.pojo.workflow.form.process.ProcessForm;
-import org.apache.inlong.manager.common.pojo.workflow.form.process.StreamResourceProcessForm;
-import org.apache.inlong.manager.service.cluster.InlongClusterService;
-import org.apache.inlong.manager.service.mq.util.PulsarOperator;
-import org.apache.inlong.manager.service.mq.util.PulsarUtils;
-import org.apache.inlong.manager.workflow.WorkflowContext;
-import org.apache.inlong.manager.workflow.event.ListenerResult;
-import org.apache.inlong.manager.workflow.event.task.QueueOperateListener;
-import org.apache.inlong.manager.workflow.event.task.TaskEvent;
-import org.apache.pulsar.client.admin.PulsarAdmin;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-/**
- * Delete pulsar topic after stream is operated to delete.
- */
-@Slf4j
-@Component
-public class DeletePulsarTopicTaskListener implements QueueOperateListener {
-
- @Autowired
- private InlongClusterService clusterService;
- @Autowired
- private PulsarOperator pulsarOperator;
-
- @Override
- public TaskEvent event() {
- return TaskEvent.COMPLETE;
- }
-
- @Override
- public boolean accept(WorkflowContext context) {
- ProcessForm processForm = context.getProcessForm();
- if (!(processForm instanceof StreamResourceProcessForm)) {
- return false;
- }
- StreamResourceProcessForm streamResourceProcessForm = (StreamResourceProcessForm) processForm;
- GroupOperateType operateType = streamResourceProcessForm.getGroupOperateType();
- if (operateType != GroupOperateType.DELETE) {
- return false;
- }
-
- InlongGroupInfo groupInfo = streamResourceProcessForm.getGroupInfo();
- MQType mqType = MQType.forType(groupInfo.getMqType());
- String groupId = groupInfo.getInlongGroupId();
- String streamId = streamResourceProcessForm.getStreamInfo().getInlongStreamId();
- if (mqType == MQType.PULSAR || mqType == MQType.TDMQ_PULSAR) {
- InlongPulsarInfo pulsarInfo = (InlongPulsarInfo) groupInfo;
- boolean enable = InlongConstants.ENABLE_CREATE_RESOURCE.equals(pulsarInfo.getEnableCreateResource());
- if (enable) {
- log.info("need to delete pulsar topic as the createResource was true for groupId [{}] streamId [{}]",
- groupId, streamId);
- return true;
- } else {
- log.info("skip to delete pulsar topic as the createResource was false for groupId [{}] streamId [{}]",
- groupId, streamId);
- return false;
- }
- }
- return false;
- }
-
- @Override
- public ListenerResult listen(WorkflowContext context) throws Exception {
- StreamResourceProcessForm form = (StreamResourceProcessForm) context.getProcessForm();
- InlongGroupInfo groupInfo = form.getGroupInfo();
- InlongStreamInfo streamInfo = form.getStreamInfo();
- final String groupId = streamInfo.getInlongGroupId();
- final String streamId = streamInfo.getInlongStreamId();
- log.info("begin to delete pulsar topic for groupId={}, streamId={}", groupId, streamId);
-
- try {
- InlongPulsarInfo pulsarInfo = (InlongPulsarInfo) groupInfo;
- String pulsarTopic = streamInfo.getMqResource();
- String clusterTag = pulsarInfo.getInlongClusterTag();
- ClusterInfo clusterInfo = clusterService.getOne(clusterTag, null, ClusterType.PULSAR);
- PulsarClusterInfo pulsarCluster = (PulsarClusterInfo) clusterInfo;
- String tenant = pulsarCluster.getTenant();
- if (StringUtils.isEmpty(tenant)) {
- tenant = InlongConstants.DEFAULT_PULSAR_TENANT;
- }
-
- try (PulsarAdmin pulsarAdmin = PulsarUtils.getPulsarAdmin(pulsarCluster)) {
- PulsarTopicBean topicBean = PulsarTopicBean.builder()
- .tenant(tenant)
- .namespace(pulsarInfo.getMqResource())
- .topicName(pulsarTopic)
- .build();
- pulsarOperator.forceDeleteTopic(pulsarAdmin, topicBean);
- }
- } catch (Exception e) {
- String msg = String.format("failed to delete pulsar topic for groupId=%s, streamId=%s", groupId, streamId);
- log.error(msg, e);
- throw new WorkflowListenerException(msg);
- }
-
- log.info("success to delete pulsar topic for groupId={}, streamId={}", groupId, streamId);
- return ListenerResult.success();
- }
-
-}
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/queue/QueueResourceOperator.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/queue/QueueResourceOperator.java
new file mode 100644
index 000000000..afc731d9d
--- /dev/null
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/queue/QueueResourceOperator.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.manager.service.resource.queue;
+
+import org.apache.inlong.manager.common.enums.MQType;
+import org.apache.inlong.manager.common.pojo.group.InlongGroupInfo;
+import org.apache.inlong.manager.common.pojo.stream.InlongStreamInfo;
+
+import javax.validation.constraints.NotBlank;
+import javax.validation.constraints.NotNull;
+
+/**
+ * Interface of the message queue resource operator
+ */
+public interface QueueResourceOperator {
+
+ /**
+ * Determines whether the current instance matches the specified type.
+ */
+ boolean accept(MQType mqType);
+
+ /**
+ * Create message queue resource for Inlong Group.
+ *
+ * @param groupInfo inlong group info
+ * @param operator operator name
+ */
+ default void createQueueForGroup(@NotNull InlongGroupInfo groupInfo, @NotBlank String operator) {
+ }
+
+ /**
+ * Delete message queue resource for Inlong Group.
+ *
+ * @param groupInfo inlong group info
+ * @param operator operator name
+ */
+ default void deleteQueueForGroup(InlongGroupInfo groupInfo, String operator) {
+ }
+
+ /**
+ * Create message queue resource for Inlong Stream.
+ *
+ * @param groupInfo inlong group info
+ * @param streamInfo inlong stream info
+ * @param operator operator name
+ */
+ default void createQueueForStream(InlongGroupInfo groupInfo, InlongStreamInfo streamInfo, String operator) {
+ }
+
+ /**
+ * Delete message queue resource for Inlong Stream.
+ *
+ * @param groupInfo inlong group info
+ * @param streamInfo inlong stream info
+ * @param operator operator name
+ */
+ default void deleteQueueForStream(InlongGroupInfo groupInfo, InlongStreamInfo streamInfo, String operator) {
+ }
+
+}
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/SortConfigOperatorFactory.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/queue/QueueResourceOperatorFactory.java
similarity index 61%
copy from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/SortConfigOperatorFactory.java
copy to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/queue/QueueResourceOperatorFactory.java
index fcd8ed81d..67c3053a1 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/SortConfigOperatorFactory.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/queue/QueueResourceOperatorFactory.java
@@ -15,8 +15,10 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.sort;
+package org.apache.inlong.manager.service.resource.queue;
+import org.apache.inlong.manager.common.enums.ErrorCodeEnum;
+import org.apache.inlong.manager.common.enums.MQType;
import org.apache.inlong.manager.common.exceptions.BusinessException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@@ -24,25 +26,23 @@ import org.springframework.stereotype.Service;
import java.util.List;
/**
- * Factory for {@link SortConfigOperator}.
+ * Factory for {@link QueueResourceOperator}.
*/
@Service
-public class SortConfigOperatorFactory {
+public class QueueResourceOperatorFactory {
@Autowired
- private List<SortConfigOperator> operatorList;
+ private List<QueueResourceOperator> operatorList;
/**
- * Get a Sort config operator instance.
- *
- * @param enableZk is the inlong group enable the ZooKeeper, 1: enable, 0: disable
+ * Get a message queue resource operator instance via the given mqType
*/
- public SortConfigOperator getInstance(Integer enableZk) {
+ public QueueResourceOperator getInstance(MQType mqType) {
return operatorList.stream()
- .filter(inst -> inst.accept(enableZk))
+ .filter(inst -> inst.accept(mqType))
.findFirst()
- .orElseThrow(() -> new BusinessException("not found any instance of SortConfigOperator when enableZk="
- + enableZk));
+ .orElseThrow(() -> new BusinessException(ErrorCodeEnum.MQ_TYPE_NOT_SUPPORT,
+ String.format(ErrorCodeEnum.MQ_TYPE_NOT_SUPPORT.getMessage(), mqType)));
}
}
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/mq/util/PulsarOperator.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/queue/pulsar/PulsarOperator.java
similarity index 99%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/mq/util/PulsarOperator.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/queue/pulsar/PulsarOperator.java
index 049ea33bd..35e004e8d 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/mq/util/PulsarOperator.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/queue/pulsar/PulsarOperator.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.mq.util;
+package org.apache.inlong.manager.service.resource.queue.pulsar;
import com.google.common.collect.Sets;
import org.apache.commons.lang3.StringUtils;
@@ -140,7 +140,6 @@ public class PulsarOperator {
*/
public void createTopic(PulsarAdmin pulsarAdmin, PulsarTopicBean topicBean) throws PulsarAdminException {
Preconditions.checkNotNull(topicBean, "pulsar topic info cannot be empty");
-
String tenant = topicBean.getTenant();
String namespace = topicBean.getNamespace();
String topic = topicBean.getTopicName();
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/queue/pulsar/PulsarResourceOperator.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/queue/pulsar/PulsarResourceOperator.java
new file mode 100644
index 000000000..a9f6ad691
--- /dev/null
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/queue/pulsar/PulsarResourceOperator.java
@@ -0,0 +1,250 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.manager.service.resource.queue.pulsar;
+
+import com.google.common.base.Objects;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.inlong.manager.common.consts.InlongConstants;
+import org.apache.inlong.manager.common.enums.ClusterType;
+import org.apache.inlong.manager.common.enums.GroupStatus;
+import org.apache.inlong.manager.common.enums.MQType;
+import org.apache.inlong.manager.common.exceptions.WorkflowListenerException;
+import org.apache.inlong.manager.common.pojo.cluster.ClusterInfo;
+import org.apache.inlong.manager.common.pojo.cluster.pulsar.PulsarClusterInfo;
+import org.apache.inlong.manager.common.pojo.group.InlongGroupInfo;
+import org.apache.inlong.manager.common.pojo.group.pulsar.InlongPulsarInfo;
+import org.apache.inlong.manager.common.pojo.pulsar.PulsarTopicBean;
+import org.apache.inlong.manager.common.pojo.stream.InlongStreamBriefInfo;
+import org.apache.inlong.manager.common.pojo.stream.InlongStreamInfo;
+import org.apache.inlong.manager.common.util.Preconditions;
+import org.apache.inlong.manager.service.cluster.InlongClusterService;
+import org.apache.inlong.manager.service.core.ConsumptionService;
+import org.apache.inlong.manager.service.core.InlongStreamService;
+import org.apache.inlong.manager.service.resource.queue.QueueResourceOperator;
+import org.apache.pulsar.client.admin.PulsarAdmin;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
+
+/**
+ * Operator for create Pulsar Tenant, Namespace, Topic and Subscription
+ */
+@Slf4j
+@Service
+public class PulsarResourceOperator implements QueueResourceOperator {
+
+ @Autowired
+ private InlongStreamService streamService;
+ @Autowired
+ private InlongClusterService clusterService;
+ @Autowired
+ private ConsumptionService consumptionService;
+ @Autowired
+ private PulsarOperator pulsarOperator;
+
+ @Override
+ public boolean accept(MQType mqType) {
+ return MQType.PULSAR == mqType || MQType.TDMQ_PULSAR == mqType;
+ }
+
+ @Override
+ public void createQueueForGroup(InlongGroupInfo groupInfo, String operator) {
+ Preconditions.checkNotNull(groupInfo, "inlong group info cannot be null");
+ Preconditions.checkNotNull(operator, "operator cannot be null");
+
+ String groupId = groupInfo.getInlongGroupId();
+ log.info("begin to create pulsar resource for groupId={}", groupId);
+
+ // get pulsar cluster via the inlong cluster tag from the inlong group
+ String clusterTag = groupInfo.getInlongClusterTag();
+ PulsarClusterInfo pulsarCluster = (PulsarClusterInfo) clusterService.getOne(clusterTag, null,
+ ClusterType.PULSAR);
+ try (PulsarAdmin pulsarAdmin = PulsarUtils.getPulsarAdmin(pulsarCluster)) {
+ // 1. create pulsar tenant and namespace
+ String tenant = pulsarCluster.getTenant();
+ if (StringUtils.isEmpty(tenant)) {
+ tenant = InlongConstants.DEFAULT_PULSAR_TENANT;
+ }
+ String namespace = groupInfo.getMqResource();
+ InlongPulsarInfo pulsarInfo = (InlongPulsarInfo) groupInfo;
+ // if the group was not successful, need create tenant and namespace
+ if (!Objects.equal(GroupStatus.CONFIG_SUCCESSFUL.getCode(), groupInfo.getStatus())) {
+ pulsarOperator.createTenant(pulsarAdmin, tenant);
+ log.info("success to create pulsar tenant for groupId={}, tenant={}", groupId, tenant);
+ pulsarOperator.createNamespace(pulsarAdmin, pulsarInfo, tenant, namespace);
+ log.info("success to create pulsar namespace for groupId={}, namespace={}", groupId, namespace);
+ }
+
+ // 2. create Pulsar Topic - each Inlong Stream corresponds to a Pulsar Topic
+ List<InlongStreamBriefInfo> streamInfoList = streamService.getTopicList(groupId);
+ if (streamInfoList == null || streamInfoList.isEmpty()) {
+ log.warn("skip to create pulsar topic and subscription as no streams for groupId={}", groupId);
+ return;
+ }
+ for (InlongStreamBriefInfo streamInfo : streamInfoList) {
+ this.createPulsarTopic(groupInfo, pulsarCluster, streamInfo.getMqResource());
+ }
+ } catch (Exception e) {
+ String msg = String.format("failed to create pulsar resource for groupId=%s", groupId);
+ log.error(msg, e);
+ throw new WorkflowListenerException(msg + ": " + e.getMessage());
+ }
+
+ log.info("success to create pulsar resource for groupId={}, cluster={}", groupId, pulsarCluster);
+ }
+
+ @Override
+ public void deleteQueueForGroup(InlongGroupInfo groupInfo, String operator) {
+ Preconditions.checkNotNull(groupInfo, "inlong group info cannot be null");
+
+ String groupId = groupInfo.getInlongGroupId();
+ log.info("begin to delete pulsar resource for groupId={}", groupId);
+
+ ClusterInfo clusterInfo = clusterService.getOne(groupInfo.getInlongClusterTag(), null, ClusterType.PULSAR);
+ try {
+ List<InlongStreamBriefInfo> streamInfoList = streamService.getTopicList(groupId);
+ if (streamInfoList == null || streamInfoList.isEmpty()) {
+ log.warn("skip to create pulsar topic and subscription as no streams for groupId={}", groupId);
+ return;
+ }
+ for (InlongStreamBriefInfo streamInfo : streamInfoList) {
+ this.deletePulsarTopic(groupInfo, (PulsarClusterInfo) clusterInfo, streamInfo.getMqResource());
+ }
+ } catch (Exception e) {
+ log.error("failed to delete pulsar resource for groupId=" + groupId, e);
+ throw new WorkflowListenerException("failed to delete pulsar resource: " + e.getMessage());
+ }
+
+ log.info("success to delete pulsar resource for groupId={}, cluster={}", groupId, clusterInfo);
+ }
+
+ @Override
+ public void createQueueForStream(InlongGroupInfo groupInfo, InlongStreamInfo streamInfo, String operator) {
+ Preconditions.checkNotNull(groupInfo, "inlong group info cannot be null");
+ Preconditions.checkNotNull(streamInfo, "inlong stream info cannot be null");
+ Preconditions.checkNotNull(operator, "operator cannot be null");
+
+ String groupId = streamInfo.getInlongGroupId();
+ String streamId = streamInfo.getInlongStreamId();
+ log.info("begin to create pulsar resource for groupId={}, streamId={}", groupId, streamId);
+
+ try {
+ // get pulsar cluster via the inlong cluster tag from the inlong group
+ String clusterTag = groupInfo.getInlongClusterTag();
+ ClusterInfo clusterInfo = clusterService.getOne(clusterTag, null, ClusterType.PULSAR);
+ // create pulsar topic
+ this.createPulsarTopic(groupInfo, (PulsarClusterInfo) clusterInfo, streamInfo.getMqResource());
+ } catch (Exception e) {
+ String msg = String.format("failed to create pulsar topic for groupId=%s, streamId=%s", groupId, streamId);
+ log.error(msg, e);
+ throw new WorkflowListenerException(msg + ": " + e.getMessage());
+ }
+
+ log.info("success to create pulsar resource for groupId={}, streamId={}", groupId, streamId);
+ }
+
+ @Override
+ public void deleteQueueForStream(InlongGroupInfo groupInfo, InlongStreamInfo streamInfo, String operator) {
+ Preconditions.checkNotNull(groupInfo, "inlong group info cannot be null");
+ Preconditions.checkNotNull(streamInfo, "inlong stream info cannot be null");
+
+ String groupId = streamInfo.getInlongGroupId();
+ String streamId = streamInfo.getInlongStreamId();
+ log.info("begin to delete pulsar resource for groupId={} streamId={}", groupId, streamId);
+
+ try {
+ ClusterInfo clusterInfo = clusterService.getOne(groupInfo.getInlongClusterTag(), null, ClusterType.PULSAR);
+ this.deletePulsarTopic(groupInfo, (PulsarClusterInfo) clusterInfo, streamInfo.getMqResource());
+ log.info("success to delete pulsar topic for groupId={}, streamId={}", groupId, streamId);
+ } catch (Exception e) {
+ String msg = String.format("failed to delete pulsar topic for groupId=%s, streamId=%s", groupId, streamId);
+ log.error(msg, e);
+ throw new WorkflowListenerException(msg);
+ }
+
+ log.info("success to delete pulsar resource for groupId={}, streamId={}", groupId, streamId);
+ }
+
+ /**
+ * Create Pulsar Topic and Subscription, and save the consumer group info.
+ */
+ private void createPulsarTopic(InlongGroupInfo groupInfo, PulsarClusterInfo pulsarCluster, String topicName)
+ throws Exception {
+ try (PulsarAdmin pulsarAdmin = PulsarUtils.getPulsarAdmin(pulsarCluster)) {
+ // 1. create pulsar topic
+ InlongPulsarInfo pulsarInfo = (InlongPulsarInfo) groupInfo;
+ String tenant = pulsarCluster.getTenant();
+ if (StringUtils.isEmpty(tenant)) {
+ tenant = InlongConstants.DEFAULT_PULSAR_TENANT;
+ }
+ String namespace = pulsarInfo.getMqResource();
+ PulsarTopicBean topicBean = PulsarTopicBean.builder()
+ .tenant(tenant)
+ .namespace(namespace)
+ .topicName(topicName)
+ .queueModule(pulsarInfo.getQueueModule())
+ .numPartitions(pulsarInfo.getPartitionNum())
+ .build();
+ pulsarOperator.createTopic(pulsarAdmin, topicBean);
+
+ // 2. create a subscription for the pulsar topic
+ boolean exist = pulsarOperator.topicIsExists(pulsarAdmin, tenant, namespace, topicName);
+ if (!exist) {
+ String topicFullName = tenant + "/" + namespace + "/" + topicName;
+ String serviceUrl = pulsarCluster.getAdminUrl();
+ log.error("topic={} not exists in {}", topicFullName, serviceUrl);
+ throw new WorkflowListenerException("topic=" + topicFullName + " not exists in " + serviceUrl);
+ }
+
+ // subscription naming rules: clusterTag_topicName_consumer_group
+ String subscription = groupInfo.getInlongClusterTag() + "_" + topicName + "_consumer_group";
+ pulsarOperator.createSubscription(pulsarAdmin, topicBean, subscription);
+ String groupId = groupInfo.getInlongGroupId();
+ log.info("success to create pulsar subscription for groupId={}, topic={}, subs={}",
+ groupId, topicName, subscription);
+
+ // 3. insert the consumer group info into the consumption table
+ consumptionService.saveSortConsumption(groupInfo, topicName, subscription);
+ log.info("success to save consume for groupId={}, topic={}, subs={}", groupId, topicName, subscription);
+ }
+ }
+
+ /**
+ * Delete Pulsar Topic and Subscription, and delete the consumer group info.
+ */
+ private void deletePulsarTopic(InlongGroupInfo groupInfo, PulsarClusterInfo pulsarCluster, String topicName)
+ throws Exception {
+ try (PulsarAdmin pulsarAdmin = PulsarUtils.getPulsarAdmin(pulsarCluster)) {
+ // 1. delete pulsar topic
+ String tenant = pulsarCluster.getTenant();
+ if (StringUtils.isEmpty(tenant)) {
+ tenant = InlongConstants.DEFAULT_PULSAR_TENANT;
+ }
+ String namespace = groupInfo.getMqResource();
+ PulsarTopicBean topicBean = PulsarTopicBean.builder()
+ .tenant(tenant)
+ .namespace(namespace)
+ .topicName(topicName)
+ .build();
+ pulsarOperator.forceDeleteTopic(pulsarAdmin, topicBean);
+ }
+ }
+
+}
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/mq/util/PulsarUtils.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/queue/pulsar/PulsarUtils.java
similarity index 98%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/mq/util/PulsarUtils.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/queue/pulsar/PulsarUtils.java
index be8853837..ffdb82230 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/mq/util/PulsarUtils.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/queue/pulsar/PulsarUtils.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.mq.util;
+package org.apache.inlong.manager.service.resource.queue.pulsar;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/mq/util/TubeMQOperator.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/queue/tubemq/TubeMQOperator.java
similarity index 81%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/mq/util/TubeMQOperator.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/queue/tubemq/TubeMQOperator.java
index 92d4e3eae..02f8697f1 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/mq/util/TubeMQOperator.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/queue/tubemq/TubeMQOperator.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.mq.util;
+package org.apache.inlong.manager.service.resource.queue.tubemq;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
@@ -67,64 +67,64 @@ public class TubeMQOperator {
private RestTemplate restTemplate;
/**
- * Create topic for the given tube cluster.
+ * Create topic for the given tubemq cluster.
*/
public void createTopic(@Nonnull TubeClusterInfo tubeCluster, String topicName, String operator) {
String masterUrl = tubeCluster.getMasterWebUrl();
- LOGGER.info("begin to create tube topic {} in master {}", topicName, masterUrl);
+ LOGGER.info("begin to create tubemq topic {} in master {}", topicName, masterUrl);
if (StringUtils.isEmpty(masterUrl) || StringUtils.isEmpty(topicName)) {
- throw new BusinessException("tube master url or tube topic cannot be null");
+ throw new BusinessException("tubemq master url or tubemq topic cannot be null");
}
if (this.isTopicExist(masterUrl, topicName)) {
- LOGGER.warn("tube topic {} already exists in {}, skip to create", topicName, masterUrl);
+ LOGGER.warn("tubemq topic {} already exists in {}, skip to create", topicName, masterUrl);
return;
}
this.createTopicOpt(masterUrl, topicName, tubeCluster.getToken(), operator);
- LOGGER.info("success to create tube topic {} in {}", topicName, masterUrl);
+ LOGGER.info("success to create tubemq topic {} in {}", topicName, masterUrl);
}
/**
- * Create consumer group for the given tube topic and cluster.
+ * Create consumer group for the given tubemq topic and cluster.
*/
public void createConsumerGroup(TubeClusterInfo tubeCluster, String topic, String consumerGroup, String operator) {
String masterUrl = tubeCluster.getMasterWebUrl();
LOGGER.info("begin to create consumer group {} for topic {} in master {}", consumerGroup, topic, masterUrl);
if (StringUtils.isEmpty(masterUrl) || StringUtils.isEmpty(consumerGroup) || StringUtils.isEmpty(topic)) {
- throw new BusinessException("tube master url, consumer group, or tube topic cannot be null");
+ throw new BusinessException("tubemq master url, consumer group, or tubemq topic cannot be null");
}
if (!this.isTopicExist(masterUrl, topic)) {
- LOGGER.warn("cannot create tube consumer group {}, as the topic {} not exists in master {}",
+ LOGGER.warn("cannot create tubemq consumer group {}, as the topic {} not exists in master {}",
consumerGroup, topic, masterUrl);
return;
}
if (this.isConsumerGroupExist(masterUrl, topic, consumerGroup)) {
- LOGGER.warn("tube consumer group {} already exists for topic {} in master {}, skip to create",
+ LOGGER.warn("tubemq consumer group {} already exists for topic {} in master {}, skip to create",
consumerGroup, topic, masterUrl);
return;
}
this.createConsumerGroupOpt(masterUrl, topic, consumerGroup, tubeCluster.getToken(), operator);
- LOGGER.info("success to create tube consumer group {} for topic {} in {}", consumerGroup, topic, masterUrl);
+ LOGGER.info("success to create tubemq consumer group {} for topic {} in {}", consumerGroup, topic, masterUrl);
}
/**
* Check if the topic is exists in the TubeMQ.
*/
public boolean isTopicExist(String masterUrl, String topicName) {
- LOGGER.info("begin to check if the tube topic {} exists", topicName);
+ LOGGER.info("begin to check if the tubemq topic {} exists", topicName);
String url = masterUrl + QUERY_TOPIC_PATH + TOPIC_NAME + topicName;
try {
TopicResponse topicView = HttpUtils.request(restTemplate, url, HttpMethod.GET,
null, new HttpHeaders(), TopicResponse.class);
if (CollectionUtils.isEmpty(topicView.getData())) {
- LOGGER.warn("tube topic {} not exists in {}", topicName, url);
+ LOGGER.warn("tubemq topic {} not exists in {}", topicName, url);
return false;
}
- LOGGER.info("tube topic {} exists in {}", topicName, url);
+ LOGGER.info("tubemq topic {} exists in {}", topicName, url);
return true;
} catch (Exception e) {
String msg = String.format("failed to check if the topic %s exist in ", topicName);
@@ -143,10 +143,10 @@ public class TubeMQOperator {
ConsumerGroupResponse response = HttpUtils.request(restTemplate, url, HttpMethod.GET,
null, new HttpHeaders(), ConsumerGroupResponse.class);
if (CollectionUtils.isEmpty(response.getData())) {
- LOGGER.warn("tube consumer group {} not exists for topic {} in {}", consumerGroup, topicName, url);
+ LOGGER.warn("tubemq consumer group {} not exists for topic {} in {}", consumerGroup, topicName, url);
return false;
}
- LOGGER.info("tube consumer group {} exists for topic {} in {}", consumerGroup, topicName, url);
+ LOGGER.info("tubemq consumer group {} exists for topic {} in {}", consumerGroup, topicName, url);
return true;
} catch (Exception e) {
String msg = String.format("failed to check if the consumer group %s for topic %s exist in ",
@@ -165,7 +165,7 @@ public class TubeMQOperator {
TubeBrokerInfo brokerInfo = HttpUtils.request(restTemplate, url, HttpMethod.GET,
null, new HttpHeaders(), TubeBrokerInfo.class);
if (brokerInfo.getErrCode() != SUCCESS_CODE) {
- String msg = "failed to query tube broker from %s, error: %s";
+ String msg = "failed to query tubemq broker from %s, error: %s";
LOGGER.error(String.format(msg, url, brokerInfo.getErrMsg()));
throw new BusinessException(String.format(msg, masterUrl, brokerInfo.getErrMsg()));
}
@@ -173,11 +173,11 @@ public class TubeMQOperator {
// is success, divide the broker by status
brokerInfo.divideBrokerListByStatus();
if (LOGGER.isDebugEnabled()) {
- LOGGER.debug("success to query tube broker from {}, result {}", url, brokerInfo.getData());
+ LOGGER.debug("success to query tubemq broker from {}, result {}", url, brokerInfo.getData());
}
return brokerInfo;
} catch (Exception e) {
- String msg = "failed to query tube broker from %s";
+ String msg = "failed to query tubemq broker from %s";
LOGGER.error(String.format(msg, url), e);
throw new BusinessException(String.format(msg, masterUrl) + ", error: " + e.getMessage());
}
@@ -187,7 +187,7 @@ public class TubeMQOperator {
* Create topic operation.
*/
private void createTopicOpt(String masterUrl, String topicName, String token, String operator) {
- LOGGER.info(String.format("begin to create tube topic %s in master %s", topicName, masterUrl));
+ LOGGER.info(String.format("begin to create tubemq topic %s in master %s", topicName, masterUrl));
TubeBrokerInfo brokerView = this.getBrokerInfo(masterUrl);
List<Integer> allBrokers = brokerView.getAllBrokerIdList();
if (CollectionUtils.isEmpty(allBrokers)) {
@@ -204,15 +204,15 @@ public class TubeMQOperator {
TubeHttpResponse response = HttpUtils.request(restTemplate, url, HttpMethod.GET,
null, new HttpHeaders(), TubeHttpResponse.class);
if (response.getErrCode() != SUCCESS_CODE) {
- String msg = String.format("failed to create tube topic %s, error: %s",
+ String msg = String.format("failed to create tubemq topic %s, error: %s",
topicName, response.getErrMsg());
LOGGER.error(msg + " in {} for brokers {}", masterUrl, allBrokers);
throw new BusinessException(msg);
}
- LOGGER.info("success to create tube topic {} in {}", topicName, url);
+ LOGGER.info("success to create tubemq topic {} in {}", topicName, url);
} catch (Exception e) {
- String msg = String.format("failed to create tube topic %s in %s", topicName, masterUrl);
+ String msg = String.format("failed to create tubemq topic %s in %s", topicName, masterUrl);
LOGGER.error(msg, e);
throw new BusinessException(msg + ", error: " + e.getMessage());
}
@@ -233,14 +233,14 @@ public class TubeMQOperator {
TubeHttpResponse response = HttpUtils.request(restTemplate, url, HttpMethod.GET,
null, new HttpHeaders(), TubeHttpResponse.class);
if (response.getErrCode() != SUCCESS_CODE) {
- String msg = String.format("failed to create tube consumer group %s for topic %s, error: %s",
+ String msg = String.format("failed to create tubemq consumer group %s for topic %s, error: %s",
consumerGroup, topicName, response.getErrMsg());
LOGGER.error(msg + ", url {}", url);
throw new BusinessException(msg);
}
- LOGGER.info("success to create tube topic {} in {}", topicName, url);
+ LOGGER.info("success to create tubemq topic {} in {}", topicName, url);
} catch (Exception e) {
- String msg = String.format("failed to create tube topic %s in %s", topicName, masterUrl);
+ String msg = String.format("failed to create tubemq topic %s in %s", topicName, masterUrl);
LOGGER.error(msg, e);
throw new BusinessException(msg + ", error: " + e.getMessage());
}
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/queue/tubemq/TubeMQResourceOperator.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/queue/tubemq/TubeMQResourceOperator.java
new file mode 100644
index 000000000..663ada741
--- /dev/null
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/queue/tubemq/TubeMQResourceOperator.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.manager.service.resource.queue.tubemq;
+
+import com.google.common.base.Objects;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.inlong.manager.common.enums.ClusterType;
+import org.apache.inlong.manager.common.enums.GroupStatus;
+import org.apache.inlong.manager.common.enums.MQType;
+import org.apache.inlong.manager.common.exceptions.WorkflowListenerException;
+import org.apache.inlong.manager.common.pojo.cluster.tube.TubeClusterInfo;
+import org.apache.inlong.manager.common.pojo.group.InlongGroupInfo;
+import org.apache.inlong.manager.common.pojo.stream.InlongStreamInfo;
+import org.apache.inlong.manager.common.util.Preconditions;
+import org.apache.inlong.manager.service.cluster.InlongClusterService;
+import org.apache.inlong.manager.service.core.ConsumptionService;
+import org.apache.inlong.manager.service.resource.queue.QueueResourceOperator;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+/**
+ * Operator for create TubeMQ Topic and ConsumerGroup
+ */
+@Slf4j
+@Service
+public class TubeMQResourceOperator implements QueueResourceOperator {
+
+ @Autowired
+ private InlongClusterService clusterService;
+ @Autowired
+ private ConsumptionService consumptionService;
+ @Autowired
+ private TubeMQOperator tubeMQOperator;
+
+ @Override
+ public boolean accept(MQType mqType) {
+ return MQType.TUBE == mqType;
+ }
+
+ @Override
+ public void createQueueForGroup(InlongGroupInfo groupInfo, String operator) {
+ Preconditions.checkNotNull(groupInfo, "inlong group info cannot be null");
+ Preconditions.checkNotNull(operator, "operator cannot be null");
+
+ String groupId = groupInfo.getInlongGroupId();
+ log.info("begin to create pulsar resource for groupId={}", groupId);
+
+ // if the group was successful, no need re-create topic and consumer group
+ if (Objects.equal(GroupStatus.CONFIG_SUCCESSFUL.getCode(), groupInfo.getStatus())) {
+ log.info("skip to create tubemq resource as the status of groupId={} was successful", groupId);
+ }
+
+ try {
+ // 1. create tubemq topic
+ String clusterTag = groupInfo.getInlongClusterTag();
+ TubeClusterInfo tubeCluster = (TubeClusterInfo) clusterService.getOne(clusterTag, null, ClusterType.TUBE);
+ String topicName = groupInfo.getMqResource();
+ tubeMQOperator.createTopic(tubeCluster, topicName, operator);
+ log.info("success to create tubemq topic for groupId={}", groupId);
+
+ // 2. create tubemq consumer group
+ // consumer naming rules: clusterTag_topicName_consumer_group
+ String consumeGroup = clusterTag + "_" + topicName + "_consumer_group";
+ tubeMQOperator.createConsumerGroup(tubeCluster, topicName, consumeGroup, operator);
+ log.info("success to create tubemq consumer group for groupId={}", groupId);
+
+ // insert the consumer group info into the consumption table
+ consumptionService.saveSortConsumption(groupInfo, topicName, consumeGroup);
+ log.info("success to save consume for groupId={}, topic={}, consumer={}", groupId, topicName, consumeGroup);
+
+ log.info("success to create tubemq resource for groupId={}, cluster={}", groupId, tubeCluster);
+ } catch (Exception e) {
+ log.error("failed to create tubemq resource for groupId=" + groupId, e);
+ throw new WorkflowListenerException("failed to create tubemq resource: " + e.getMessage());
+ }
+ }
+
+ @Override
+ public void deleteQueueForGroup(InlongGroupInfo groupInfo, String operator) {
+ // currently, not support delete tubemq resource for group
+ }
+
+ @Override
+ public void createQueueForStream(InlongGroupInfo groupInfo, InlongStreamInfo streamInfo, String operator) {
+ // currently, not support create tubemq resource for stream
+ }
+
+ @Override
+ public void deleteQueueForStream(InlongGroupInfo groupInfo, InlongStreamInfo streamInfo, String operator) {
+ // currently, not support delete tubemq resource for stream
+ }
+
+}
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/SinkResourceOperator.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/SinkResourceOperator.java
similarity index 96%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/SinkResourceOperator.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/SinkResourceOperator.java
index 85d823106..af8cd8e25 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/SinkResourceOperator.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/SinkResourceOperator.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.resource;
+package org.apache.inlong.manager.service.resource.sink;
import org.apache.inlong.manager.common.enums.SinkType;
import org.apache.inlong.manager.common.pojo.sink.SinkInfo;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/SinkResourceOperatorFactory.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/SinkResourceOperatorFactory.java
similarity index 90%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/SinkResourceOperatorFactory.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/SinkResourceOperatorFactory.java
index b0f55acc1..37e4436e5 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/SinkResourceOperatorFactory.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/SinkResourceOperatorFactory.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.resource;
+package org.apache.inlong.manager.service.resource.sink;
import org.apache.inlong.manager.common.enums.ErrorCodeEnum;
import org.apache.inlong.manager.common.enums.SinkType;
@@ -33,13 +33,13 @@ import java.util.Optional;
public class SinkResourceOperatorFactory {
@Autowired
- private List<SinkResourceOperator> operationList;
+ private List<SinkResourceOperator> operatorList;
/**
* Get a sink resource operator instance via the given sinkType
*/
public SinkResourceOperator getInstance(SinkType sinkType) {
- Optional<SinkResourceOperator> instance = operationList.stream()
+ Optional<SinkResourceOperator> instance = operatorList.stream()
.filter(inst -> inst.accept(sinkType))
.findFirst();
if (!instance.isPresent()) {
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/ck/ClickHouseJdbcUtils.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/ck/ClickHouseJdbcUtils.java
similarity index 99%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/ck/ClickHouseJdbcUtils.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/ck/ClickHouseJdbcUtils.java
index e5a594ad9..3b3aba1c3 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/ck/ClickHouseJdbcUtils.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/ck/ClickHouseJdbcUtils.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.resource.ck;
+package org.apache.inlong.manager.service.resource.sink.ck;
import org.apache.commons.lang3.StringUtils;
import org.apache.inlong.manager.common.pojo.sink.ck.ClickHouseColumnInfo;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/ck/ClickHouseResourceOperator.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/ck/ClickHouseResourceOperator.java
similarity index 97%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/ck/ClickHouseResourceOperator.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/ck/ClickHouseResourceOperator.java
index f055d1344..f02fe98d8 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/ck/ClickHouseResourceOperator.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/ck/ClickHouseResourceOperator.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.resource.ck;
+package org.apache.inlong.manager.service.resource.sink.ck;
import org.apache.commons.collections.CollectionUtils;
import org.apache.inlong.manager.common.consts.InlongConstants;
@@ -28,7 +28,7 @@ import org.apache.inlong.manager.common.pojo.sink.SinkInfo;
import org.apache.inlong.manager.common.pojo.sink.ck.ClickHouseSinkDTO;
import org.apache.inlong.manager.dao.entity.StreamSinkFieldEntity;
import org.apache.inlong.manager.dao.mapper.StreamSinkFieldEntityMapper;
-import org.apache.inlong.manager.service.resource.SinkResourceOperator;
+import org.apache.inlong.manager.service.resource.sink.SinkResourceOperator;
import org.apache.inlong.manager.service.sink.StreamSinkService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/ck/ClickHouseSqlBuilder.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/ck/ClickHouseSqlBuilder.java
similarity index 98%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/ck/ClickHouseSqlBuilder.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/ck/ClickHouseSqlBuilder.java
index 485d6a689..8ce5496f9 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/ck/ClickHouseSqlBuilder.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/ck/ClickHouseSqlBuilder.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.resource.ck;
+package org.apache.inlong.manager.service.resource.sink.ck;
import org.apache.commons.lang3.StringUtils;
import org.apache.inlong.manager.common.pojo.sink.ck.ClickHouseColumnInfo;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/es/ElasticsearchApi.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/es/ElasticsearchApi.java
similarity index 99%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/es/ElasticsearchApi.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/es/ElasticsearchApi.java
index bb31d1f2f..1bf6fc614 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/es/ElasticsearchApi.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/es/ElasticsearchApi.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.resource.es;
+package org.apache.inlong.manager.service.resource.sink.es;
import java.io.IOException;
import java.util.ArrayList;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/es/ElasticsearchConfig.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/es/ElasticsearchConfig.java
similarity index 98%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/es/ElasticsearchConfig.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/es/ElasticsearchConfig.java
index bd0b94f45..6cccff706 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/es/ElasticsearchConfig.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/es/ElasticsearchConfig.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.resource.es;
+package org.apache.inlong.manager.service.resource.sink.es;
import java.util.ArrayList;
import java.util.List;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/es/ElasticsearchResourceOperator.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/es/ElasticsearchResourceOperator.java
similarity index 97%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/es/ElasticsearchResourceOperator.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/es/ElasticsearchResourceOperator.java
index 8e3407ba2..d33ac3884 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/es/ElasticsearchResourceOperator.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/es/ElasticsearchResourceOperator.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.resource.es;
+package org.apache.inlong.manager.service.resource.sink.es;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
@@ -28,7 +28,7 @@ import org.apache.inlong.manager.common.pojo.sink.es.ElasticsearchFieldInfo;
import org.apache.inlong.manager.common.pojo.sink.es.ElasticsearchSinkDTO;
import org.apache.inlong.manager.dao.entity.StreamSinkFieldEntity;
import org.apache.inlong.manager.dao.mapper.StreamSinkFieldEntityMapper;
-import org.apache.inlong.manager.service.resource.SinkResourceOperator;
+import org.apache.inlong.manager.service.resource.sink.SinkResourceOperator;
import org.apache.inlong.manager.service.sink.StreamSinkService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/hbase/HBaseApiUtils.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/hbase/HBaseApiUtils.java
similarity index 98%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/hbase/HBaseApiUtils.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/hbase/HBaseApiUtils.java
index 2ae2fad05..0cf1fed23 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/hbase/HBaseApiUtils.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/hbase/HBaseApiUtils.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.resource.hbase;
+package org.apache.inlong.manager.service.resource.sink.hbase;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/hbase/HBaseResourceOperator.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/hbase/HBaseResourceOperator.java
similarity index 97%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/hbase/HBaseResourceOperator.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/hbase/HBaseResourceOperator.java
index 4346fff41..c1dde9482 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/hbase/HBaseResourceOperator.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/hbase/HBaseResourceOperator.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.resource.hbase;
+package org.apache.inlong.manager.service.resource.sink.hbase;
import org.apache.commons.collections.CollectionUtils;
import org.apache.inlong.manager.common.consts.InlongConstants;
@@ -28,7 +28,7 @@ import org.apache.inlong.manager.common.pojo.sink.hbase.HBaseSinkDTO;
import org.apache.inlong.manager.common.pojo.sink.hbase.HBaseTableInfo;
import org.apache.inlong.manager.dao.entity.StreamSinkFieldEntity;
import org.apache.inlong.manager.dao.mapper.StreamSinkFieldEntityMapper;
-import org.apache.inlong.manager.service.resource.SinkResourceOperator;
+import org.apache.inlong.manager.service.resource.sink.SinkResourceOperator;
import org.apache.inlong.manager.service.sink.StreamSinkService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/hive/HiveJdbcUtils.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/hive/HiveJdbcUtils.java
similarity index 98%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/hive/HiveJdbcUtils.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/hive/HiveJdbcUtils.java
index 5b7af48b7..cc7524b76 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/hive/HiveJdbcUtils.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/hive/HiveJdbcUtils.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.resource.hive;
+package org.apache.inlong.manager.service.resource.sink.hive;
import org.apache.commons.lang3.StringUtils;
import org.apache.hive.jdbc.HiveDatabaseMetaData;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/hive/HiveResourceOperator.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/hive/HiveResourceOperator.java
similarity index 97%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/hive/HiveResourceOperator.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/hive/HiveResourceOperator.java
index 40b4349ff..664e93b2e 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/hive/HiveResourceOperator.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/hive/HiveResourceOperator.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.resource.hive;
+package org.apache.inlong.manager.service.resource.sink.hive;
import org.apache.commons.collections.CollectionUtils;
import org.apache.inlong.manager.common.consts.InlongConstants;
@@ -28,7 +28,7 @@ import org.apache.inlong.manager.common.pojo.sink.hive.HiveSinkDTO;
import org.apache.inlong.manager.common.pojo.sink.hive.HiveTableInfo;
import org.apache.inlong.manager.dao.entity.StreamSinkFieldEntity;
import org.apache.inlong.manager.dao.mapper.StreamSinkFieldEntityMapper;
-import org.apache.inlong.manager.service.resource.SinkResourceOperator;
+import org.apache.inlong.manager.service.resource.sink.SinkResourceOperator;
import org.apache.inlong.manager.service.sink.StreamSinkService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/hive/SqlBuilder.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/hive/SqlBuilder.java
similarity index 98%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/hive/SqlBuilder.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/hive/SqlBuilder.java
index bfcbf82d1..15f4be4db 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/hive/SqlBuilder.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/hive/SqlBuilder.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.resource.hive;
+package org.apache.inlong.manager.service.resource.sink.hive;
import org.apache.commons.lang3.StringUtils;
import org.apache.inlong.manager.common.pojo.sink.hive.HiveColumnInfo;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/iceberg/IcebergCatalogUtils.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/iceberg/IcebergCatalogUtils.java
similarity index 99%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/iceberg/IcebergCatalogUtils.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/iceberg/IcebergCatalogUtils.java
index a8125cfe6..1a229ee42 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/iceberg/IcebergCatalogUtils.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/iceberg/IcebergCatalogUtils.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.resource.iceberg;
+package org.apache.inlong.manager.service.resource.sink.iceberg;
import org.apache.commons.lang3.StringUtils;
import org.apache.iceberg.PartitionSpec;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/iceberg/IcebergResourceOperator.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/iceberg/IcebergResourceOperator.java
similarity index 97%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/iceberg/IcebergResourceOperator.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/iceberg/IcebergResourceOperator.java
index 2beafec60..3852f0de2 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/iceberg/IcebergResourceOperator.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/iceberg/IcebergResourceOperator.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.resource.iceberg;
+package org.apache.inlong.manager.service.resource.sink.iceberg;
import org.apache.commons.collections.CollectionUtils;
import org.apache.inlong.manager.common.consts.InlongConstants;
@@ -28,7 +28,7 @@ import org.apache.inlong.manager.common.pojo.sink.iceberg.IcebergSinkDTO;
import org.apache.inlong.manager.common.pojo.sink.iceberg.IcebergTableInfo;
import org.apache.inlong.manager.dao.entity.StreamSinkFieldEntity;
import org.apache.inlong.manager.dao.mapper.StreamSinkFieldEntityMapper;
-import org.apache.inlong.manager.service.resource.SinkResourceOperator;
+import org.apache.inlong.manager.service.resource.sink.SinkResourceOperator;
import org.apache.inlong.manager.service.sink.StreamSinkService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/kafka/KafkaResourceOperator.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/kafka/KafkaResourceOperator.java
similarity index 97%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/kafka/KafkaResourceOperator.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/kafka/KafkaResourceOperator.java
index 0956ab8b4..dda48b900 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/kafka/KafkaResourceOperator.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/kafka/KafkaResourceOperator.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.resource.kafka;
+package org.apache.inlong.manager.service.resource.sink.kafka;
import org.apache.inlong.manager.common.enums.SinkStatus;
import org.apache.inlong.manager.common.enums.SinkType;
@@ -23,7 +23,7 @@ import org.apache.inlong.manager.common.exceptions.WorkflowException;
import org.apache.inlong.manager.common.pojo.sink.SinkInfo;
import org.apache.inlong.manager.common.pojo.sink.kafka.KafkaSinkDTO;
import org.apache.inlong.manager.common.util.Preconditions;
-import org.apache.inlong.manager.service.resource.SinkResourceOperator;
+import org.apache.inlong.manager.service.resource.sink.SinkResourceOperator;
import org.apache.inlong.manager.service.sink.StreamSinkService;
import org.apache.kafka.clients.admin.Admin;
import org.apache.kafka.clients.admin.AdminClientConfig;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/mysql/MySQLJdbcUtils.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/mysql/MySQLJdbcUtils.java
similarity index 96%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/mysql/MySQLJdbcUtils.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/mysql/MySQLJdbcUtils.java
index 5774cbf51..e3e0a9fee 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/mysql/MySQLJdbcUtils.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/mysql/MySQLJdbcUtils.java
@@ -1,258 +1,258 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.manager.service.resource.mysql;
-
-import com.google.common.collect.Lists;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.inlong.manager.common.pojo.sink.mysql.MySQLColumnInfo;
-import org.apache.inlong.manager.common.pojo.sink.mysql.MySQLTableInfo;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.ResultSet;
-import java.sql.Statement;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Utils for MySQL JDBC.
- */
-public class MySQLJdbcUtils {
-
- private static final String MYSQL_JDBC_PREFIX = "jdbc:mysql";
-
- private static final String MYSQL_DRIVER_CLASS = "com.mysql.cj.jdbc.Driver";
-
- private static final Logger LOG = LoggerFactory.getLogger(MySQLJdbcUtils.class);
-
- /**
- * Get MySQL connection from the url and user.
- *
- * @param url jdbc url, such as jdbc:mysql://host:port/database
- * @param user Username for JDBC URL
- * @param password User password
- * @return {@link Connection}
- * @throws Exception on get connection error
- */
- public static Connection getConnection(String url, String user, String password)
- throws Exception {
- if (StringUtils.isBlank(url) || !url.startsWith(MYSQL_JDBC_PREFIX)) {
- throw new Exception("MySQL server URL was invalid, it should start with jdbc:mysql");
- }
- Connection conn;
- try {
- Class.forName(MYSQL_DRIVER_CLASS);
- conn = DriverManager.getConnection(url, user, password);
- } catch (Exception e) {
- String errorMsg = "get MySQL connection error, please check MySQL JDBC url, username or password!";
- LOG.error(errorMsg, e);
- throw new Exception(errorMsg + " other error msg: " + e.getMessage());
- }
- if (conn == null) {
- throw new Exception("get MySQL connection failed, please contact administrator.");
- }
- LOG.info("get MySQL connection success, url={}", url);
- return conn;
- }
-
- /**
- * Execute SQL command on MySQL.
- *
- * @param conn JDBC Connection {@link Connection}
- * @param sql SQL string to be executed
- * @throws Exception on execute SQL error
- */
- public static void executeSql(Connection conn, String sql) throws Exception {
- Statement stmt = conn.createStatement();
- LOG.info("execute sql [{}] success !", sql);
- stmt.execute(sql);
- stmt.close();
- }
-
- /**
- * Execute query SQL on MySQL.
- *
- * @param conn JDBC Connection {@link Connection}
- * @param sql SQL string to be executed
- * @return {@link ResultSet}
- * @throws Exception on execute query SQL error
- */
- public static ResultSet executeQuerySql(Connection conn, String sql)
- throws Exception {
- Statement stmt = conn.createStatement();
- LOG.info("execute sql [{}] success !", sql);
- return stmt.executeQuery(sql);
- }
-
- /**
- * Execute batch query SQL on MySQL.
- *
- * @param conn JDBC Connection {@link Connection}
- * @param sqls SQL string to be executed
- * @throws Exception on get execute SQL batch error
- */
- public static void executeSqlBatch(Connection conn, List<String> sqls)
- throws Exception {
- Statement stmt = conn.createStatement();
- for (String entry : sqls) {
- stmt.execute(entry);
- }
- stmt.close();
- LOG.info("execute sql [{}] success! ", sqls);
- }
-
- /**
- * Create MySQL database
- *
- * @param conn JDBC Connection {@link Connection}
- * @param dbName database name
- * @throws Exception on create database error
- */
- public static void createDb(Connection conn, String dbName) throws Exception {
- String checkDbSql = MySQLSqlBuilder.getCheckDatabase(dbName);
- ResultSet resultSet = executeQuerySql(conn, checkDbSql);
- if (resultSet != null) {
- if (!resultSet.next()) {
- String createDbSql = MySQLSqlBuilder.buildCreateDbSql(dbName);
- executeSql(conn, createDbSql);
- LOG.info("execute sql [{}] success! ", createDbSql);
- } else {
- LOG.info("The database [{}] are exists !", dbName);
- }
- resultSet.close();
- }
- }
-
- /**
- * Create MySQL table by MySQLTableInfo
- *
- * @param conn JDBC Connection {@link Connection}
- * @param tableInfo MySQL table info {@link MySQLTableInfo}
- * @throws Exception on create table error
- */
- public static void createTable(Connection conn, MySQLTableInfo tableInfo)
- throws Exception {
- if (checkTablesExist(conn, tableInfo.getDbName(), tableInfo.getTableName())) {
- LOG.info("The table [{}] are exists !", tableInfo.getTableName());
- } else {
- String createTableSql = MySQLSqlBuilder.buildCreateTableSql(tableInfo);
- executeSql(conn, createTableSql);
- LOG.info("execute sql [{}] success! ", createTableSql);
- }
- }
-
- /**
- * Check tables from the MySQL information_schema.
- *
- * @param conn JDBC Connection {@link Connection}
- * @param dbName MySQL database name
- * @param tableName MySQL table name
- * @return true if table exist, otherwise false
- * @throws Exception on check table exist error
- */
- public static boolean checkTablesExist(Connection conn, String dbName, String tableName)
- throws Exception {
- boolean result = false;
- String checkTableSql = MySQLSqlBuilder.getCheckTable(dbName, tableName);
- ResultSet resultSet = executeQuerySql(conn, checkTableSql);
- if (resultSet != null) {
- if (resultSet.next()) {
- LOG.info("check table exist for db={} table={}, result={}", dbName, tableName, result);
- return true;
- }
- }
- resultSet.close();
- return result;
- }
-
- /**
- * Check whether the column exists in the table.
- *
- * @param conn JDBC Connection {@link Connection}
- * @param dbName MySQL database name
- * @param tableName MySQL table name
- * @param column MySQL table column name
- * @return true if column exist in the table, otherwise false
- * @throws Exception on check column exist error
- */
- public static boolean checkColumnExist(Connection conn, String dbName, String tableName, String column)
- throws Exception {
- boolean result = false;
- String checkTableSql = MySQLSqlBuilder.getCheckColumn(dbName, tableName, column);
- ResultSet resultSet = executeQuerySql(conn, checkTableSql);
- if (resultSet != null) {
- if (resultSet.next()) {
- LOG.info("check column exist for db={} table={}, result={} column={}",
- dbName, tableName, result, column);
- return true;
- }
- }
- resultSet.close();
- return result;
- }
-
- /**
- * Query all columns of the tableName.
- *
- * @param conn JDBC Connection {@link Connection}
- * @param dbName MySQL database name
- * @param tableName MySQL table name
- * @return {@link List}
- * @throws Exception on get columns error
- */
- public static List<MySQLColumnInfo> getColumns(Connection conn, String dbName, String tableName)
- throws Exception {
- String querySql = MySQLSqlBuilder.buildDescTableSql(dbName, tableName);
- Statement stmt = conn.createStatement();
- ResultSet rs = stmt.executeQuery(querySql);
- List<MySQLColumnInfo> columnList = new ArrayList<>();
- while (rs.next()) {
- MySQLColumnInfo columnInfo = new MySQLColumnInfo();
- columnInfo.setName(rs.getString(1));
- columnInfo.setType(rs.getString(2));
- columnInfo.setComment(rs.getString(3));
- columnList.add(columnInfo);
- }
- rs.close();
- return columnList;
- }
-
- /**
- * Add columns for MySQL table.
- *
- * @param conn JDBC Connection {@link Connection}
- * @param dbName MySQL database name
- * @param tableName MySQL table name
- * @param columns MySQL columns to be added
- * @throws Exception on add columns error
- */
- public static void addColumns(Connection conn, String dbName, String tableName, List<MySQLColumnInfo> columns)
- throws Exception {
- List<MySQLColumnInfo> columnInfos = Lists.newArrayList();
-
- for (MySQLColumnInfo columnInfo : columns) {
- if (!checkColumnExist(conn, dbName, tableName, columnInfo.getName())) {
- columnInfos.add(columnInfo);
- }
- }
- List<String> addColumnSql = MySQLSqlBuilder.buildAddColumnsSql(dbName, tableName, columnInfos);
- executeSqlBatch(conn, addColumnSql);
- }
-}
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.manager.service.resource.sink.mysql;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.inlong.manager.common.pojo.sink.mysql.MySQLColumnInfo;
+import org.apache.inlong.manager.common.pojo.sink.mysql.MySQLTableInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Utils for MySQL JDBC.
+ */
+public class MySQLJdbcUtils {
+
+ private static final String MYSQL_JDBC_PREFIX = "jdbc:mysql";
+
+ private static final String MYSQL_DRIVER_CLASS = "com.mysql.cj.jdbc.Driver";
+
+ private static final Logger LOG = LoggerFactory.getLogger(MySQLJdbcUtils.class);
+
+ /**
+ * Get MySQL connection from the url and user.
+ *
+ * @param url jdbc url, such as jdbc:mysql://host:port/database
+ * @param user Username for JDBC URL
+ * @param password User password
+ * @return {@link Connection}
+ * @throws Exception on get connection error
+ */
+ public static Connection getConnection(String url, String user, String password)
+ throws Exception {
+ if (StringUtils.isBlank(url) || !url.startsWith(MYSQL_JDBC_PREFIX)) {
+ throw new Exception("MySQL server URL was invalid, it should start with jdbc:mysql");
+ }
+ Connection conn;
+ try {
+ Class.forName(MYSQL_DRIVER_CLASS);
+ conn = DriverManager.getConnection(url, user, password);
+ } catch (Exception e) {
+ String errorMsg = "get MySQL connection error, please check MySQL JDBC url, username or password!";
+ LOG.error(errorMsg, e);
+ throw new Exception(errorMsg + " other error msg: " + e.getMessage());
+ }
+ if (conn == null) {
+ throw new Exception("get MySQL connection failed, please contact administrator.");
+ }
+ LOG.info("get MySQL connection success, url={}", url);
+ return conn;
+ }
+
+ /**
+ * Execute SQL command on MySQL.
+ *
+ * @param conn JDBC Connection {@link Connection}
+ * @param sql SQL string to be executed
+ * @throws Exception on execute SQL error
+ */
+ public static void executeSql(Connection conn, String sql) throws Exception {
+ Statement stmt = conn.createStatement();
+ LOG.info("execute sql [{}] success !", sql);
+ stmt.execute(sql);
+ stmt.close();
+ }
+
+ /**
+ * Execute query SQL on MySQL.
+ *
+ * @param conn JDBC Connection {@link Connection}
+ * @param sql SQL string to be executed
+ * @return {@link ResultSet}
+ * @throws Exception on execute query SQL error
+ */
+ public static ResultSet executeQuerySql(Connection conn, String sql)
+ throws Exception {
+ Statement stmt = conn.createStatement();
+ LOG.info("execute sql [{}] success !", sql);
+ return stmt.executeQuery(sql);
+ }
+
+ /**
+ * Execute batch query SQL on MySQL.
+ *
+ * @param conn JDBC Connection {@link Connection}
+ * @param sqls SQL string to be executed
+ * @throws Exception on get execute SQL batch error
+ */
+ public static void executeSqlBatch(Connection conn, List<String> sqls)
+ throws Exception {
+ Statement stmt = conn.createStatement();
+ for (String entry : sqls) {
+ stmt.execute(entry);
+ }
+ stmt.close();
+ LOG.info("execute sql [{}] success! ", sqls);
+ }
+
+ /**
+ * Create MySQL database
+ *
+ * @param conn JDBC Connection {@link Connection}
+ * @param dbName database name
+ * @throws Exception on create database error
+ */
+ public static void createDb(Connection conn, String dbName) throws Exception {
+ String checkDbSql = MySQLSqlBuilder.getCheckDatabase(dbName);
+ ResultSet resultSet = executeQuerySql(conn, checkDbSql);
+ if (resultSet != null) {
+ if (!resultSet.next()) {
+ String createDbSql = MySQLSqlBuilder.buildCreateDbSql(dbName);
+ executeSql(conn, createDbSql);
+ LOG.info("execute sql [{}] success! ", createDbSql);
+ } else {
+ LOG.info("The database [{}] are exists !", dbName);
+ }
+ resultSet.close();
+ }
+ }
+
+ /**
+ * Create MySQL table by MySQLTableInfo
+ *
+ * @param conn JDBC Connection {@link Connection}
+ * @param tableInfo MySQL table info {@link MySQLTableInfo}
+ * @throws Exception on create table error
+ */
+ public static void createTable(Connection conn, MySQLTableInfo tableInfo)
+ throws Exception {
+ if (checkTablesExist(conn, tableInfo.getDbName(), tableInfo.getTableName())) {
+ LOG.info("The table [{}] are exists !", tableInfo.getTableName());
+ } else {
+ String createTableSql = MySQLSqlBuilder.buildCreateTableSql(tableInfo);
+ executeSql(conn, createTableSql);
+ LOG.info("execute sql [{}] success! ", createTableSql);
+ }
+ }
+
+ /**
+ * Check tables from the MySQL information_schema.
+ *
+ * @param conn JDBC Connection {@link Connection}
+ * @param dbName MySQL database name
+ * @param tableName MySQL table name
+ * @return true if table exist, otherwise false
+ * @throws Exception on check table exist error
+ */
+ public static boolean checkTablesExist(Connection conn, String dbName, String tableName)
+ throws Exception {
+ boolean result = false;
+ String checkTableSql = MySQLSqlBuilder.getCheckTable(dbName, tableName);
+ ResultSet resultSet = executeQuerySql(conn, checkTableSql);
+ if (resultSet != null) {
+ if (resultSet.next()) {
+ LOG.info("check table exist for db={} table={}, result={}", dbName, tableName, result);
+ return true;
+ }
+ }
+ resultSet.close();
+ return result;
+ }
+
+ /**
+ * Check whether the column exists in the table.
+ *
+ * @param conn JDBC Connection {@link Connection}
+ * @param dbName MySQL database name
+ * @param tableName MySQL table name
+ * @param column MySQL table column name
+ * @return true if column exist in the table, otherwise false
+ * @throws Exception on check column exist error
+ */
+ public static boolean checkColumnExist(Connection conn, String dbName, String tableName, String column)
+ throws Exception {
+ boolean result = false;
+ String checkTableSql = MySQLSqlBuilder.getCheckColumn(dbName, tableName, column);
+ ResultSet resultSet = executeQuerySql(conn, checkTableSql);
+ if (resultSet != null) {
+ if (resultSet.next()) {
+ LOG.info("check column exist for db={} table={}, result={} column={}",
+ dbName, tableName, result, column);
+ return true;
+ }
+ }
+ resultSet.close();
+ return result;
+ }
+
+ /**
+ * Query all columns of the tableName.
+ *
+ * @param conn JDBC Connection {@link Connection}
+ * @param dbName MySQL database name
+ * @param tableName MySQL table name
+ * @return {@link List}
+ * @throws Exception on get columns error
+ */
+ public static List<MySQLColumnInfo> getColumns(Connection conn, String dbName, String tableName)
+ throws Exception {
+ String querySql = MySQLSqlBuilder.buildDescTableSql(dbName, tableName);
+ Statement stmt = conn.createStatement();
+ ResultSet rs = stmt.executeQuery(querySql);
+ List<MySQLColumnInfo> columnList = new ArrayList<>();
+ while (rs.next()) {
+ MySQLColumnInfo columnInfo = new MySQLColumnInfo();
+ columnInfo.setName(rs.getString(1));
+ columnInfo.setType(rs.getString(2));
+ columnInfo.setComment(rs.getString(3));
+ columnList.add(columnInfo);
+ }
+ rs.close();
+ return columnList;
+ }
+
+ /**
+ * Add columns for MySQL table.
+ *
+ * @param conn JDBC Connection {@link Connection}
+ * @param dbName MySQL database name
+ * @param tableName MySQL table name
+ * @param columns MySQL columns to be added
+ * @throws Exception on add columns error
+ */
+ public static void addColumns(Connection conn, String dbName, String tableName, List<MySQLColumnInfo> columns)
+ throws Exception {
+ List<MySQLColumnInfo> columnInfos = Lists.newArrayList();
+
+ for (MySQLColumnInfo columnInfo : columns) {
+ if (!checkColumnExist(conn, dbName, tableName, columnInfo.getName())) {
+ columnInfos.add(columnInfo);
+ }
+ }
+ List<String> addColumnSql = MySQLSqlBuilder.buildAddColumnsSql(dbName, tableName, columnInfos);
+ executeSqlBatch(conn, addColumnSql);
+ }
+}
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/mysql/MySQLResourceOperator.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/mysql/MySQLResourceOperator.java
similarity index 95%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/mysql/MySQLResourceOperator.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/mysql/MySQLResourceOperator.java
index f20049e7d..99b06a80b 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/mysql/MySQLResourceOperator.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/mysql/MySQLResourceOperator.java
@@ -1,137 +1,137 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.manager.service.resource.mysql;
-
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.inlong.manager.common.consts.InlongConstants;
-import org.apache.inlong.manager.common.enums.SinkStatus;
-import org.apache.inlong.manager.common.enums.SinkType;
-import org.apache.inlong.manager.common.exceptions.WorkflowException;
-import org.apache.inlong.manager.common.pojo.sink.SinkInfo;
-import org.apache.inlong.manager.common.pojo.sink.mysql.MySQLColumnInfo;
-import org.apache.inlong.manager.common.pojo.sink.mysql.MySQLSinkDTO;
-import org.apache.inlong.manager.common.pojo.sink.mysql.MySQLTableInfo;
-import org.apache.inlong.manager.dao.entity.StreamSinkFieldEntity;
-import org.apache.inlong.manager.dao.mapper.StreamSinkFieldEntityMapper;
-import org.apache.inlong.manager.service.resource.SinkResourceOperator;
-import org.apache.inlong.manager.service.sink.StreamSinkService;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Service;
-
-import java.sql.Connection;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * MySQL's resource operator.
- */
-@Service
-public class MySQLResourceOperator implements SinkResourceOperator {
-
- private static final Logger LOG = LoggerFactory.getLogger(MySQLResourceOperator.class);
-
- @Autowired
- private StreamSinkService sinkService;
-
- @Autowired
- private StreamSinkFieldEntityMapper fieldEntityMapper;
-
- @Override
- public Boolean accept(SinkType sinkType) {
- return SinkType.MYSQL == sinkType;
- }
-
- @Override
- public void createSinkResource(SinkInfo sinkInfo) {
- LOG.info("begin to create MySQL resources sinkId={}", sinkInfo.getId());
- if (SinkStatus.CONFIG_SUCCESSFUL.getCode().equals(sinkInfo.getStatus())) {
- LOG.warn("MySQL resource [" + sinkInfo.getId() + "] already success, skip to create");
- return;
- } else if (InlongConstants.DISABLE_CREATE_RESOURCE.equals(sinkInfo.getEnableCreateResource())) {
- LOG.warn("create resource was disabled, skip to create for [" + sinkInfo.getId() + "]");
- return;
- }
- this.createTable(sinkInfo);
- }
-
- /**
- * Create MySQL table by SinkInfo.
- *
- * @param sinkInfo {@link SinkInfo}
- */
- private void createTable(SinkInfo sinkInfo) {
- LOG.info("begin to create MySQL table for sinkId={}", sinkInfo.getId());
- List<StreamSinkFieldEntity> fieldList = fieldEntityMapper.selectBySinkId(sinkInfo.getId());
- if (CollectionUtils.isEmpty(fieldList)) {
- LOG.warn("no MySQL fields found, skip to create table for sinkId={}", sinkInfo.getId());
- }
- // set columns
- List<MySQLColumnInfo> columnList = new ArrayList<>();
- for (StreamSinkFieldEntity field : fieldList) {
- MySQLColumnInfo columnInfo = new MySQLColumnInfo();
- columnInfo.setName(field.getFieldName());
- columnInfo.setType(field.getFieldType());
- columnInfo.setComment(field.getFieldComment());
- columnList.add(columnInfo);
- }
-
- MySQLSinkDTO mySQLSink = MySQLSinkDTO.getFromJson(sinkInfo.getExtParams());
- MySQLTableInfo tableInfo = MySQLSinkDTO.getTableInfo(mySQLSink, columnList);
- String url = mySQLSink.getJdbcUrl();
- String user = mySQLSink.getUsername();
- String password = mySQLSink.getPassword();
-
- String dbName = tableInfo.getDbName();
- String tableName = tableInfo.getTableName();
- Connection conn = null;
- try {
- conn = MySQLJdbcUtils.getConnection(url, user, password);
- // 1. create database if not exists
- MySQLJdbcUtils.createDb(conn, dbName);
- // 2. table not exists, create it
- MySQLJdbcUtils.createTable(conn, tableInfo);
- // 3. table exists, add columns - skip the exists columns
- MySQLJdbcUtils.addColumns(conn, dbName, tableName, columnList);
- // 4. update the sink status to success
- String info = "success to create MySQL resource";
- sinkService.updateStatus(sinkInfo.getId(), SinkStatus.CONFIG_SUCCESSFUL.getCode(), info);
- LOG.info(info + " for sinkInfo={}", sinkInfo);
- // 5. close connection.
- conn.close();
- } catch (Throwable e) {
- String errMsg = "create MySQL table failed: " + e.getMessage();
- LOG.error(errMsg, e);
- sinkService.updateStatus(sinkInfo.getId(), SinkStatus.CONFIG_FAILED.getCode(), errMsg);
- throw new WorkflowException(errMsg);
- } finally {
- try {
- if (null != conn) {
- conn.close();
- conn = null;
- }
- } catch (Throwable e) {
- String errMsg = "close MySQL connection failed: " + e.getMessage();
- throw new WorkflowException(errMsg);
- }
- }
- LOG.info("success create MySQL table for data sink [" + sinkInfo.getId() + "]");
- }
-
-}
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.manager.service.resource.sink.mysql;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.inlong.manager.common.consts.InlongConstants;
+import org.apache.inlong.manager.common.enums.SinkStatus;
+import org.apache.inlong.manager.common.enums.SinkType;
+import org.apache.inlong.manager.common.exceptions.WorkflowException;
+import org.apache.inlong.manager.common.pojo.sink.SinkInfo;
+import org.apache.inlong.manager.common.pojo.sink.mysql.MySQLColumnInfo;
+import org.apache.inlong.manager.common.pojo.sink.mysql.MySQLSinkDTO;
+import org.apache.inlong.manager.common.pojo.sink.mysql.MySQLTableInfo;
+import org.apache.inlong.manager.dao.entity.StreamSinkFieldEntity;
+import org.apache.inlong.manager.dao.mapper.StreamSinkFieldEntityMapper;
+import org.apache.inlong.manager.service.resource.sink.SinkResourceOperator;
+import org.apache.inlong.manager.service.sink.StreamSinkService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.sql.Connection;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * MySQL's resource operator.
+ */
+@Service
+public class MySQLResourceOperator implements SinkResourceOperator {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MySQLResourceOperator.class);
+
+ @Autowired
+ private StreamSinkService sinkService;
+
+ @Autowired
+ private StreamSinkFieldEntityMapper fieldEntityMapper;
+
+ @Override
+ public Boolean accept(SinkType sinkType) {
+ return SinkType.MYSQL == sinkType;
+ }
+
+ @Override
+ public void createSinkResource(SinkInfo sinkInfo) {
+ LOG.info("begin to create MySQL resources sinkId={}", sinkInfo.getId());
+ if (SinkStatus.CONFIG_SUCCESSFUL.getCode().equals(sinkInfo.getStatus())) {
+ LOG.warn("MySQL resource [" + sinkInfo.getId() + "] already success, skip to create");
+ return;
+ } else if (InlongConstants.DISABLE_CREATE_RESOURCE.equals(sinkInfo.getEnableCreateResource())) {
+ LOG.warn("create resource was disabled, skip to create for [" + sinkInfo.getId() + "]");
+ return;
+ }
+ this.createTable(sinkInfo);
+ }
+
+ /**
+ * Create MySQL table by SinkInfo.
+ *
+ * @param sinkInfo {@link SinkInfo}
+ */
+ private void createTable(SinkInfo sinkInfo) {
+ LOG.info("begin to create MySQL table for sinkId={}", sinkInfo.getId());
+ List<StreamSinkFieldEntity> fieldList = fieldEntityMapper.selectBySinkId(sinkInfo.getId());
+ if (CollectionUtils.isEmpty(fieldList)) {
+ LOG.warn("no MySQL fields found, skip to create table for sinkId={}", sinkInfo.getId());
+ }
+ // set columns
+ List<MySQLColumnInfo> columnList = new ArrayList<>();
+ for (StreamSinkFieldEntity field : fieldList) {
+ MySQLColumnInfo columnInfo = new MySQLColumnInfo();
+ columnInfo.setName(field.getFieldName());
+ columnInfo.setType(field.getFieldType());
+ columnInfo.setComment(field.getFieldComment());
+ columnList.add(columnInfo);
+ }
+
+ MySQLSinkDTO mySQLSink = MySQLSinkDTO.getFromJson(sinkInfo.getExtParams());
+ MySQLTableInfo tableInfo = MySQLSinkDTO.getTableInfo(mySQLSink, columnList);
+ String url = mySQLSink.getJdbcUrl();
+ String user = mySQLSink.getUsername();
+ String password = mySQLSink.getPassword();
+
+ String dbName = tableInfo.getDbName();
+ String tableName = tableInfo.getTableName();
+ Connection conn = null;
+ try {
+ conn = MySQLJdbcUtils.getConnection(url, user, password);
+ // 1. create database if not exists
+ MySQLJdbcUtils.createDb(conn, dbName);
+ // 2. table not exists, create it
+ MySQLJdbcUtils.createTable(conn, tableInfo);
+ // 3. table exists, add columns - skip the exists columns
+ MySQLJdbcUtils.addColumns(conn, dbName, tableName, columnList);
+ // 4. update the sink status to success
+ String info = "success to create MySQL resource";
+ sinkService.updateStatus(sinkInfo.getId(), SinkStatus.CONFIG_SUCCESSFUL.getCode(), info);
+ LOG.info(info + " for sinkInfo={}", sinkInfo);
+ // 5. close connection.
+ conn.close();
+ } catch (Throwable e) {
+ String errMsg = "create MySQL table failed: " + e.getMessage();
+ LOG.error(errMsg, e);
+ sinkService.updateStatus(sinkInfo.getId(), SinkStatus.CONFIG_FAILED.getCode(), errMsg);
+ throw new WorkflowException(errMsg);
+ } finally {
+ try {
+ if (null != conn) {
+ conn.close();
+ conn = null;
+ }
+ } catch (Throwable e) {
+ String errMsg = "close MySQL connection failed: " + e.getMessage();
+ throw new WorkflowException(errMsg);
+ }
+ }
+ LOG.info("success create MySQL table for data sink [" + sinkInfo.getId() + "]");
+ }
+
+}
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/mysql/MySQLSqlBuilder.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/mysql/MySQLSqlBuilder.java
similarity index 96%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/mysql/MySQLSqlBuilder.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/mysql/MySQLSqlBuilder.java
index 0f4a8c162..4c659f398 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/mysql/MySQLSqlBuilder.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/mysql/MySQLSqlBuilder.java
@@ -1,234 +1,234 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.manager.service.resource.mysql;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.inlong.manager.common.pojo.sink.mysql.MySQLColumnInfo;
-import org.apache.inlong.manager.common.pojo.sink.mysql.MySQLTableInfo;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Builder the SQL string for MySQL
- */
-public class MySQLSqlBuilder {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(MySQLSqlBuilder.class);
-
- /**
- * Build SQL to check whether the database exists.
- *
- * @param dbName MySQL database name
- * @return the check database SQL string
- */
- public static String getCheckDatabase(String dbName) {
- StringBuilder sqlBuilder = new StringBuilder();
- sqlBuilder.append("SELECT schema_name ")
- .append(" FROM information_schema.schemata ")
- .append("WHERE schema_name = '")
- .append(dbName)
- .append("';");
- LOGGER.info("check database sql: {}", sqlBuilder);
- return sqlBuilder.toString();
- }
-
- /**
- * Build SQL to check whether the table exists.
- *
- * @param dbName MySQL database name
- * @param tableName MySQL table name
- * @return the check table SQL string
- */
- public static String getCheckTable(String dbName, String tableName) {
- StringBuilder sqlBuilder = new StringBuilder();
- sqlBuilder.append("select table_schema,table_name ")
- .append(" from information_schema.tables where table_schema = '")
- .append(dbName)
- .append("' and table_name = '")
- .append(tableName)
- .append("' ;");
- LOGGER.info("check table sql: {}", sqlBuilder);
- return sqlBuilder.toString();
- }
-
- /**
- * Build SQL to check whether the column exists.
- *
- * @param dbName MySQL database name
- * @param tableName MySQL table name
- * @param columnName MySQL column name
- * @return the check column SQL string
- */
- public static String getCheckColumn(String dbName, String tableName, String columnName) {
- StringBuilder sqlBuilder = new StringBuilder();
- sqlBuilder.append("SELECT COLUMN_NAME,COLUMN_TYPE,COLUMN_COMMENT ")
- .append(" from information_schema.COLUMNS where table_schema='")
- .append(dbName)
- .append("' and table_name = '")
- .append(tableName)
- .append("' and column_name = '")
- .append(columnName)
- .append("';");
- LOGGER.info("check table sql: {}", sqlBuilder);
- return sqlBuilder.toString();
- }
-
- /**
- * Build create database SQL.
- *
- * @param dbName MySQL database name
- * @return the create database SQL string
- */
- public static String buildCreateDbSql(String dbName) {
- String sql = "CREATE DATABASE " + dbName;
- LOGGER.info("create db sql: {}", sql);
- return sql;
- }
-
- /**
- * Build create table SQL by MySQLTableInfo.
- *
- * @param table MySQL table info {@link MySQLTableInfo}
- * @return the create table SQL String
- */
- public static String buildCreateTableSql(MySQLTableInfo table) {
- StringBuilder sql = new StringBuilder();
- // Support _ beginning with underscore
- sql.append("CREATE TABLE ").append(table.getDbName())
- .append(".")
- .append(table.getTableName());
-
- // Construct columns and partition columns
- sql.append(buildCreateColumnsSql(table));
-
- if (StringUtils.isEmpty(table.getEngine())) {
- sql.append(" ENGINE=InnoDB ");
- } else {
- sql.append(" ENGINE=")
- .append(table.getEngine())
- .append(" ");
- }
- if (!StringUtils.isEmpty(table.getCharset())) {
- sql.append(" DEFAULT CHARSET=")
- .append(table.getCharset())
- .append(" ");
- }
-
- LOGGER.info("create table sql: {}", sql);
- return sql.toString();
- }
-
- /**
- * Build add columns SQL.
- *
- * @param tableName MySQL table name
- * @param columnList MySQL column list {@link List}
- * @return add column SQL string list
- */
- public static List<String> buildAddColumnsSql(String dbName, String tableName,
- List<MySQLColumnInfo> columnList) {
- List<String> columnInfoList = getColumnsInfo(columnList);
- List<String> resultList = new ArrayList<>();
- StringBuilder sqlBuilder = new StringBuilder();
- for (String columnInfo : columnInfoList) {
- sqlBuilder.append("ALTER TABLE ")
- .append(dbName)
- .append(".")
- .append(tableName)
- .append(" ADD COLUMN ")
- .append(columnInfo)
- .append(";");
- resultList.add(sqlBuilder.toString());
- sqlBuilder.delete(0, sqlBuilder.length());
- }
-
- LOGGER.info("add columns sql={}", resultList);
- return resultList;
- }
-
- /**
- * Build create column SQL.
- *
- * @param table MySQL table info {@link MySQLTableInfo}
- * @return create column SQL string
- */
- private static String buildCreateColumnsSql(MySQLTableInfo table) {
- StringBuilder sql = new StringBuilder();
- sql.append(" (");
- List<String> columnList = getColumnsInfo(table.getColumns());
- sql.append(StringUtils.join(columnList, ","));
- if (!StringUtils.isEmpty(table.getPrimaryKey())) {
- sql.append(", PRIMARY KEY (")
- .append(table.getPrimaryKey())
- .append(")");
- }
- sql.append(") ");
- LOGGER.info("create columns sql={}", sql);
- return sql.toString();
- }
-
- /**
- * Build column info by MySQLColumnInfo list.
- *
- * @param columns MySQL column info {@link MySQLColumnInfo} list
- * @return the SQL list
- */
- private static List<String> getColumnsInfo(List<MySQLColumnInfo> columns) {
- List<String> columnList = new ArrayList<>();
- for (MySQLColumnInfo columnInfo : columns) {
- // Construct columns and partition columns
- StringBuilder columnBuilder = new StringBuilder();
- columnBuilder.append("`")
- .append(columnInfo.getName())
- .append("`")
- .append(" ")
- .append(columnInfo.getType());
- if (!StringUtils.isEmpty(columnInfo.getComment())) {
- columnBuilder.append(" COMMENT '")
- .append(columnInfo.getComment())
- .append("'");
- }
- columnBuilder.append(" ");
- columnList.add(columnBuilder.toString());
- }
- return columnList;
- }
-
- /**
- * Build query table SQL.
- *
- * @param dbName MySQL database name
- * @param tableName MySQL table name
- * @return desc table SQL string
- */
- public static String buildDescTableSql(String dbName, String tableName) {
- StringBuilder sql = new StringBuilder();
- sql.append("SELECT COLUMN_NAME,COLUMN_TYPE,COLUMN_COMMENT ")
- .append(" from information_schema.COLUMNS where table_schema='")
- .append(dbName)
- .append("' and table_name = '")
- .append(tableName)
- .append("';");
- LOGGER.info("desc table sql={}", sql);
- return sql.toString();
- }
-
-}
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.manager.service.resource.sink.mysql;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.inlong.manager.common.pojo.sink.mysql.MySQLColumnInfo;
+import org.apache.inlong.manager.common.pojo.sink.mysql.MySQLTableInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Builder the SQL string for MySQL
+ */
+public class MySQLSqlBuilder {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(MySQLSqlBuilder.class);
+
+ /**
+ * Build SQL to check whether the database exists.
+ *
+ * @param dbName MySQL database name
+ * @return the check database SQL string
+ */
+ public static String getCheckDatabase(String dbName) {
+ StringBuilder sqlBuilder = new StringBuilder();
+ sqlBuilder.append("SELECT schema_name ")
+ .append(" FROM information_schema.schemata ")
+ .append("WHERE schema_name = '")
+ .append(dbName)
+ .append("';");
+ LOGGER.info("check database sql: {}", sqlBuilder);
+ return sqlBuilder.toString();
+ }
+
+ /**
+ * Build SQL to check whether the table exists.
+ *
+ * @param dbName MySQL database name
+ * @param tableName MySQL table name
+ * @return the check table SQL string
+ */
+ public static String getCheckTable(String dbName, String tableName) {
+ StringBuilder sqlBuilder = new StringBuilder();
+ sqlBuilder.append("select table_schema,table_name ")
+ .append(" from information_schema.tables where table_schema = '")
+ .append(dbName)
+ .append("' and table_name = '")
+ .append(tableName)
+ .append("' ;");
+ LOGGER.info("check table sql: {}", sqlBuilder);
+ return sqlBuilder.toString();
+ }
+
+ /**
+ * Build SQL to check whether the column exists.
+ *
+ * @param dbName MySQL database name
+ * @param tableName MySQL table name
+ * @param columnName MySQL column name
+ * @return the check column SQL string
+ */
+ public static String getCheckColumn(String dbName, String tableName, String columnName) {
+ StringBuilder sqlBuilder = new StringBuilder();
+ sqlBuilder.append("SELECT COLUMN_NAME,COLUMN_TYPE,COLUMN_COMMENT ")
+ .append(" from information_schema.COLUMNS where table_schema='")
+ .append(dbName)
+ .append("' and table_name = '")
+ .append(tableName)
+ .append("' and column_name = '")
+ .append(columnName)
+ .append("';");
+ LOGGER.info("check table sql: {}", sqlBuilder);
+ return sqlBuilder.toString();
+ }
+
+ /**
+ * Build create database SQL.
+ *
+ * @param dbName MySQL database name
+ * @return the create database SQL string
+ */
+ public static String buildCreateDbSql(String dbName) {
+ String sql = "CREATE DATABASE " + dbName;
+ LOGGER.info("create db sql: {}", sql);
+ return sql;
+ }
+
+ /**
+ * Build create table SQL by MySQLTableInfo.
+ *
+ * @param table MySQL table info {@link MySQLTableInfo}
+ * @return the create table SQL String
+ */
+ public static String buildCreateTableSql(MySQLTableInfo table) {
+ StringBuilder sql = new StringBuilder();
+ // Support _ beginning with underscore
+ sql.append("CREATE TABLE ").append(table.getDbName())
+ .append(".")
+ .append(table.getTableName());
+
+ // Construct columns and partition columns
+ sql.append(buildCreateColumnsSql(table));
+
+ if (StringUtils.isEmpty(table.getEngine())) {
+ sql.append(" ENGINE=InnoDB ");
+ } else {
+ sql.append(" ENGINE=")
+ .append(table.getEngine())
+ .append(" ");
+ }
+ if (!StringUtils.isEmpty(table.getCharset())) {
+ sql.append(" DEFAULT CHARSET=")
+ .append(table.getCharset())
+ .append(" ");
+ }
+
+ LOGGER.info("create table sql: {}", sql);
+ return sql.toString();
+ }
+
+ /**
+ * Build add columns SQL.
+ *
+ * @param tableName MySQL table name
+ * @param columnList MySQL column list {@link List}
+ * @return add column SQL string list
+ */
+ public static List<String> buildAddColumnsSql(String dbName, String tableName,
+ List<MySQLColumnInfo> columnList) {
+ List<String> columnInfoList = getColumnsInfo(columnList);
+ List<String> resultList = new ArrayList<>();
+ StringBuilder sqlBuilder = new StringBuilder();
+ for (String columnInfo : columnInfoList) {
+ sqlBuilder.append("ALTER TABLE ")
+ .append(dbName)
+ .append(".")
+ .append(tableName)
+ .append(" ADD COLUMN ")
+ .append(columnInfo)
+ .append(";");
+ resultList.add(sqlBuilder.toString());
+ sqlBuilder.delete(0, sqlBuilder.length());
+ }
+
+ LOGGER.info("add columns sql={}", resultList);
+ return resultList;
+ }
+
+ /**
+ * Build create column SQL.
+ *
+ * @param table MySQL table info {@link MySQLTableInfo}
+ * @return create column SQL string
+ */
+ private static String buildCreateColumnsSql(MySQLTableInfo table) {
+ StringBuilder sql = new StringBuilder();
+ sql.append(" (");
+ List<String> columnList = getColumnsInfo(table.getColumns());
+ sql.append(StringUtils.join(columnList, ","));
+ if (!StringUtils.isEmpty(table.getPrimaryKey())) {
+ sql.append(", PRIMARY KEY (")
+ .append(table.getPrimaryKey())
+ .append(")");
+ }
+ sql.append(") ");
+ LOGGER.info("create columns sql={}", sql);
+ return sql.toString();
+ }
+
+ /**
+ * Build column info by MySQLColumnInfo list.
+ *
+ * @param columns MySQL column info {@link MySQLColumnInfo} list
+ * @return the SQL list
+ */
+ private static List<String> getColumnsInfo(List<MySQLColumnInfo> columns) {
+ List<String> columnList = new ArrayList<>();
+ for (MySQLColumnInfo columnInfo : columns) {
+ // Construct columns and partition columns
+ StringBuilder columnBuilder = new StringBuilder();
+ columnBuilder.append("`")
+ .append(columnInfo.getName())
+ .append("`")
+ .append(" ")
+ .append(columnInfo.getType());
+ if (!StringUtils.isEmpty(columnInfo.getComment())) {
+ columnBuilder.append(" COMMENT '")
+ .append(columnInfo.getComment())
+ .append("'");
+ }
+ columnBuilder.append(" ");
+ columnList.add(columnBuilder.toString());
+ }
+ return columnList;
+ }
+
+ /**
+ * Build query table SQL.
+ *
+ * @param dbName MySQL database name
+ * @param tableName MySQL table name
+ * @return desc table SQL string
+ */
+ public static String buildDescTableSql(String dbName, String tableName) {
+ StringBuilder sql = new StringBuilder();
+ sql.append("SELECT COLUMN_NAME,COLUMN_TYPE,COLUMN_COMMENT ")
+ .append(" from information_schema.COLUMNS where table_schema='")
+ .append(dbName)
+ .append("' and table_name = '")
+ .append(tableName)
+ .append("';");
+ LOGGER.info("desc table sql={}", sql);
+ return sql.toString();
+ }
+
+}
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/oracle/OracleJdbcUtils.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/oracle/OracleJdbcUtils.java
similarity index 99%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/oracle/OracleJdbcUtils.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/oracle/OracleJdbcUtils.java
index 0127ad479..4a1723ee9 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/oracle/OracleJdbcUtils.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/oracle/OracleJdbcUtils.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.resource.oracle;
+package org.apache.inlong.manager.service.resource.sink.oracle;
import org.apache.commons.lang3.StringUtils;
import org.apache.inlong.manager.common.pojo.sink.oracle.OracleColumnInfo;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/oracle/OracleResourceOperator.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/oracle/OracleResourceOperator.java
similarity index 97%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/oracle/OracleResourceOperator.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/oracle/OracleResourceOperator.java
index b4041e6e2..543ec71d6 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/oracle/OracleResourceOperator.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/oracle/OracleResourceOperator.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.resource.oracle;
+package org.apache.inlong.manager.service.resource.sink.oracle;
import org.apache.commons.collections.CollectionUtils;
import org.apache.inlong.manager.common.consts.InlongConstants;
@@ -28,7 +28,7 @@ import org.apache.inlong.manager.common.pojo.sink.oracle.OracleSinkDTO;
import org.apache.inlong.manager.common.pojo.sink.oracle.OracleTableInfo;
import org.apache.inlong.manager.dao.entity.StreamSinkFieldEntity;
import org.apache.inlong.manager.dao.mapper.StreamSinkFieldEntityMapper;
-import org.apache.inlong.manager.service.resource.SinkResourceOperator;
+import org.apache.inlong.manager.service.resource.sink.SinkResourceOperator;
import org.apache.inlong.manager.service.sink.StreamSinkService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/oracle/OracleSqlBuilder.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/oracle/OracleSqlBuilder.java
similarity index 99%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/oracle/OracleSqlBuilder.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/oracle/OracleSqlBuilder.java
index dcac4b6bb..e611fd49f 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/oracle/OracleSqlBuilder.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/oracle/OracleSqlBuilder.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.resource.oracle;
+package org.apache.inlong.manager.service.resource.sink.oracle;
import org.apache.commons.lang3.StringUtils;
import org.apache.inlong.manager.common.pojo.sink.oracle.OracleColumnInfo;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/postgresql/PostgreSQLJdbcUtils.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/postgresql/PostgreSQLJdbcUtils.java
similarity index 99%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/postgresql/PostgreSQLJdbcUtils.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/postgresql/PostgreSQLJdbcUtils.java
index 21f736b3f..ac73df377 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/postgresql/PostgreSQLJdbcUtils.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/postgresql/PostgreSQLJdbcUtils.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.resource.postgresql;
+package org.apache.inlong.manager.service.resource.sink.postgresql;
import org.apache.commons.lang3.StringUtils;
import org.apache.inlong.manager.common.pojo.sink.postgresql.PostgreSQLColumnInfo;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/postgresql/PostgreSQLResourceOperator.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/postgresql/PostgreSQLResourceOperator.java
similarity index 97%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/postgresql/PostgreSQLResourceOperator.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/postgresql/PostgreSQLResourceOperator.java
index 99d05712f..a4d4d1a31 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/postgresql/PostgreSQLResourceOperator.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/postgresql/PostgreSQLResourceOperator.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.resource.postgresql;
+package org.apache.inlong.manager.service.resource.sink.postgresql;
import org.apache.commons.collections.CollectionUtils;
import org.apache.inlong.manager.common.consts.InlongConstants;
@@ -28,7 +28,7 @@ import org.apache.inlong.manager.common.pojo.sink.postgresql.PostgreSQLSinkDTO;
import org.apache.inlong.manager.common.pojo.sink.postgresql.PostgreSQLTableInfo;
import org.apache.inlong.manager.dao.entity.StreamSinkFieldEntity;
import org.apache.inlong.manager.dao.mapper.StreamSinkFieldEntityMapper;
-import org.apache.inlong.manager.service.resource.SinkResourceOperator;
+import org.apache.inlong.manager.service.resource.sink.SinkResourceOperator;
import org.apache.inlong.manager.service.sink.StreamSinkService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/postgresql/PostgreSQLSqlBuilder.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/postgresql/PostgreSQLSqlBuilder.java
similarity index 98%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/postgresql/PostgreSQLSqlBuilder.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/postgresql/PostgreSQLSqlBuilder.java
index 30f2d8a4a..808868aea 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/postgresql/PostgreSQLSqlBuilder.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/postgresql/PostgreSQLSqlBuilder.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.resource.postgresql;
+package org.apache.inlong.manager.service.resource.sink.postgresql;
import org.apache.commons.lang3.StringUtils;
import org.apache.inlong.manager.common.pojo.sink.postgresql.PostgreSQLColumnInfo;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/DefaultSortConfigOperator.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/DefaultSortConfigOperator.java
similarity index 94%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/DefaultSortConfigOperator.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/DefaultSortConfigOperator.java
index 0045dc8e3..23d9ae402 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/DefaultSortConfigOperator.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/DefaultSortConfigOperator.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.sort;
+package org.apache.inlong.manager.service.resource.sort;
import com.google.common.collect.Lists;
import org.apache.commons.collections.CollectionUtils;
@@ -30,10 +30,10 @@ import org.apache.inlong.manager.common.pojo.stream.InlongStreamInfo;
import org.apache.inlong.manager.common.pojo.stream.StreamField;
import org.apache.inlong.manager.common.pojo.transform.TransformResponse;
import org.apache.inlong.manager.service.sink.StreamSinkService;
-import org.apache.inlong.manager.service.sort.util.ExtractNodeUtils;
-import org.apache.inlong.manager.service.sort.util.LoadNodeUtils;
-import org.apache.inlong.manager.service.sort.util.NodeRelationUtils;
-import org.apache.inlong.manager.service.sort.util.TransformNodeUtils;
+import org.apache.inlong.manager.service.resource.sort.util.ExtractNodeUtils;
+import org.apache.inlong.manager.service.resource.sort.util.LoadNodeUtils;
+import org.apache.inlong.manager.service.resource.sort.util.NodeRelationUtils;
+import org.apache.inlong.manager.service.resource.sort.util.TransformNodeUtils;
import org.apache.inlong.manager.service.source.StreamSourceService;
import org.apache.inlong.manager.service.transform.StreamTransformService;
import org.apache.inlong.sort.protocol.GroupInfo;
@@ -85,7 +85,7 @@ public class DefaultSortConfigOperator implements SortConfigOperator {
if (InlongConstants.LIGHTWEIGHT_MODE.equals(groupInfo.getLightweight())) {
configInfo = this.getLightweightGroupInfo(groupInfo, streamInfos);
} else {
- configInfo = this.getNormalGroupInfo(groupInfo, streamInfos);
+ configInfo = this.getStandardGroupInfo(groupInfo, streamInfos);
}
String dataflow = OBJECT_MAPPER.writeValueAsString(configInfo);
@@ -138,11 +138,11 @@ public class DefaultSortConfigOperator implements SortConfigOperator {
}
/**
- * Get Sort GroupInfo of normal inlong group.
+ * Get Sort GroupInfo of STANDARD inlong group.
*
* @see org.apache.inlong.sort.protocol.GroupInfo
*/
- private GroupInfo getNormalGroupInfo(InlongGroupInfo groupInfo, List<InlongStreamInfo> streamInfoList) {
+ private GroupInfo getStandardGroupInfo(InlongGroupInfo groupInfo, List<InlongStreamInfo> streamInfoList) {
// get source info
Map<String, List<StreamSource>> sourceMap = sourceService.getSourcesMap(groupInfo, streamInfoList);
// get sink info
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/SortConfigOperator.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/SortConfigOperator.java
similarity index 96%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/SortConfigOperator.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/SortConfigOperator.java
index f8c9de5b4..45f348e55 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/SortConfigOperator.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/SortConfigOperator.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.sort;
+package org.apache.inlong.manager.service.resource.sort;
import org.apache.inlong.manager.common.pojo.group.InlongGroupInfo;
import org.apache.inlong.manager.common.pojo.stream.InlongStreamInfo;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/SortConfigOperatorFactory.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/SortConfigOperatorFactory.java
similarity index 96%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/SortConfigOperatorFactory.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/SortConfigOperatorFactory.java
index fcd8ed81d..d5e8db3ef 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/SortConfigOperatorFactory.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/SortConfigOperatorFactory.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.sort;
+package org.apache.inlong.manager.service.resource.sort;
import org.apache.inlong.manager.common.exceptions.BusinessException;
import org.springframework.beans.factory.annotation.Autowired;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/ExtractNodeUtils.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/util/ExtractNodeUtils.java
similarity index 99%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/ExtractNodeUtils.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/util/ExtractNodeUtils.java
index 42f1b1128..6f6a33577 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/ExtractNodeUtils.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/util/ExtractNodeUtils.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.sort.util;
+package org.apache.inlong.manager.service.resource.sort.util;
import com.google.common.base.Splitter;
import com.google.common.collect.Lists;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/FieldInfoUtils.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/util/FieldInfoUtils.java
similarity index 99%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/FieldInfoUtils.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/util/FieldInfoUtils.java
index 0d1073318..b6df4dc02 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/FieldInfoUtils.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/util/FieldInfoUtils.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.sort.util;
+package org.apache.inlong.manager.service.resource.sort.util;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/FieldRelationUtils.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/util/FieldRelationUtils.java
similarity index 99%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/FieldRelationUtils.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/util/FieldRelationUtils.java
index a672a98fb..761ba5249 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/FieldRelationUtils.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/util/FieldRelationUtils.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.sort.util;
+package org.apache.inlong.manager.service.resource.sort.util;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/FilterFunctionUtils.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/util/FilterFunctionUtils.java
similarity index 99%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/FilterFunctionUtils.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/util/FilterFunctionUtils.java
index 6b86ff2a1..2792494ff 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/FilterFunctionUtils.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/util/FilterFunctionUtils.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.sort.util;
+package org.apache.inlong.manager.service.resource.sort.util;
import com.google.common.collect.Lists;
import org.apache.inlong.manager.common.enums.TransformType;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/LoadNodeUtils.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/util/LoadNodeUtils.java
similarity index 99%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/LoadNodeUtils.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/util/LoadNodeUtils.java
index 3061ff97b..8291b483b 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/LoadNodeUtils.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/util/LoadNodeUtils.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.sort.util;
+package org.apache.inlong.manager.service.resource.sort.util;
import com.google.common.collect.Lists;
import org.apache.commons.collections.CollectionUtils;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/NodeRelationUtils.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/util/NodeRelationUtils.java
similarity index 99%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/NodeRelationUtils.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/util/NodeRelationUtils.java
index 233c33dc4..5aa956ec8 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/NodeRelationUtils.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/util/NodeRelationUtils.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.sort.util;
+package org.apache.inlong.manager.service.resource.sort.util;
import com.google.common.collect.Lists;
import lombok.extern.slf4j.Slf4j;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/TransformNodeUtils.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/util/TransformNodeUtils.java
similarity index 98%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/TransformNodeUtils.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/util/TransformNodeUtils.java
index 05cb6367f..200d21c6f 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/TransformNodeUtils.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/util/TransformNodeUtils.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.service.sort.util;
+package org.apache.inlong.manager.service.resource.sort.util;
import com.google.common.collect.Lists;
import lombok.extern.slf4j.Slf4j;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/StreamSourceService.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/StreamSourceService.java
index ff616fe53..32f1253b4 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/StreamSourceService.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/StreamSourceService.java
@@ -62,7 +62,7 @@ public interface StreamSourceService {
* Get the StreamSource Map by the inlong group info and inlong stream info list.
* <p/>
* If the group mode is LIGHTWEIGHT, means not using any MQ as a cached source, then just get all related sources.
- * Otherwise, if the group mode is NORMAL, need get the cached MQ sources.
+ * Otherwise, if the group mode is STANDARD, need get the cached MQ sources.
*
* @param groupInfo inlong group info
* @param streamInfos inlong stream info list
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/StreamSourceServiceImpl.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/StreamSourceServiceImpl.java
index 734b3539d..2db50f3ef 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/StreamSourceServiceImpl.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/StreamSourceServiceImpl.java
@@ -156,7 +156,7 @@ public class StreamSourceServiceImpl implements StreamSourceService {
.collect(Collectors.groupingBy(StreamSource::getInlongStreamId, HashMap::new,
Collectors.toCollection(ArrayList::new)));
} else {
- // if the group mode is NORMAL, needs to get the cached MQ sources
+ // if the group mode is STANDARD, needs to get the cached MQ sources
String sourceType = groupInfo.getMqType();
StreamSourceOperator sourceOperator = operatorFactory.getInstance(SourceType.forType(sourceType));
result = sourceOperator.getSourcesMap(groupInfo, streamInfos, streamSources);
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/consumption/ApplyConsumptionWorkflowDefinition.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/consumption/ApplyConsumptionWorkflowDefinition.java
index 031e1a06f..a1cbd303a 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/consumption/ApplyConsumptionWorkflowDefinition.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/consumption/ApplyConsumptionWorkflowDefinition.java
@@ -25,10 +25,10 @@ import org.apache.inlong.manager.common.pojo.workflow.form.task.ConsumptionAppro
import org.apache.inlong.manager.service.core.WorkflowApproverService;
import org.apache.inlong.manager.service.group.InlongGroupService;
import org.apache.inlong.manager.service.workflow.WorkflowDefinition;
-import org.apache.inlong.manager.service.workflow.consumption.listener.ConsumptionCancelProcessListener;
-import org.apache.inlong.manager.service.workflow.consumption.listener.ConsumptionCompleteProcessListener;
-import org.apache.inlong.manager.service.workflow.consumption.listener.ConsumptionPassTaskListener;
-import org.apache.inlong.manager.service.workflow.consumption.listener.ConsumptionRejectProcessListener;
+import org.apache.inlong.manager.service.listener.consumption.ConsumptionCancelProcessListener;
+import org.apache.inlong.manager.service.listener.consumption.ConsumptionCompleteProcessListener;
+import org.apache.inlong.manager.service.listener.consumption.ConsumptionPassTaskListener;
+import org.apache.inlong.manager.service.listener.consumption.ConsumptionRejectProcessListener;
import org.apache.inlong.manager.workflow.WorkflowContext;
import org.apache.inlong.manager.workflow.definition.EndEvent;
import org.apache.inlong.manager.workflow.definition.StartEvent;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/ApplyGroupWorkflowDefinition.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/ApplyGroupWorkflowDefinition.java
index efa52b637..66ae6ebb4 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/ApplyGroupWorkflowDefinition.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/ApplyGroupWorkflowDefinition.java
@@ -22,10 +22,10 @@ import org.apache.inlong.manager.common.pojo.workflow.form.process.ApplyGroupPro
import org.apache.inlong.manager.common.pojo.workflow.form.task.InlongGroupApproveForm;
import org.apache.inlong.manager.service.core.WorkflowApproverService;
import org.apache.inlong.manager.service.workflow.WorkflowDefinition;
-import org.apache.inlong.manager.service.workflow.group.listener.apply.AfterApprovedTaskListener;
-import org.apache.inlong.manager.service.workflow.group.listener.apply.ApproveApplyProcessListener;
-import org.apache.inlong.manager.service.workflow.group.listener.apply.CancelApplyProcessListener;
-import org.apache.inlong.manager.service.workflow.group.listener.apply.RejectApplyProcessListener;
+import org.apache.inlong.manager.service.listener.group.apply.AfterApprovedTaskListener;
+import org.apache.inlong.manager.service.listener.group.apply.ApproveApplyProcessListener;
+import org.apache.inlong.manager.service.listener.group.apply.CancelApplyProcessListener;
+import org.apache.inlong.manager.service.listener.group.apply.RejectApplyProcessListener;
import org.apache.inlong.manager.workflow.definition.EndEvent;
import org.apache.inlong.manager.workflow.definition.StartEvent;
import org.apache.inlong.manager.workflow.definition.UserTask;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/CreateGroupWorkflowDefinition.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/CreateGroupWorkflowDefinition.java
index b582658ca..fc835a20f 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/CreateGroupWorkflowDefinition.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/CreateGroupWorkflowDefinition.java
@@ -21,10 +21,10 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.inlong.manager.common.enums.ProcessName;
import org.apache.inlong.manager.common.pojo.workflow.form.process.GroupResourceProcessForm;
import org.apache.inlong.manager.service.workflow.WorkflowDefinition;
-import org.apache.inlong.manager.service.workflow.group.listener.InitGroupCompleteListener;
-import org.apache.inlong.manager.service.workflow.group.listener.InitGroupFailedListener;
-import org.apache.inlong.manager.service.workflow.group.listener.InitGroupListener;
-import org.apache.inlong.manager.service.workflow.listener.GroupTaskListenerFactory;
+import org.apache.inlong.manager.service.listener.group.InitGroupCompleteListener;
+import org.apache.inlong.manager.service.listener.group.InitGroupFailedListener;
+import org.apache.inlong.manager.service.listener.group.InitGroupListener;
+import org.apache.inlong.manager.service.listener.GroupTaskListenerFactory;
import org.apache.inlong.manager.workflow.definition.EndEvent;
import org.apache.inlong.manager.workflow.definition.ServiceTask;
import org.apache.inlong.manager.workflow.definition.ServiceTaskType;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/DeleteGroupWorkflowDefinition.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/DeleteGroupWorkflowDefinition.java
index 3c17ecad3..f83a2a7a0 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/DeleteGroupWorkflowDefinition.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/DeleteGroupWorkflowDefinition.java
@@ -21,10 +21,10 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.inlong.manager.common.pojo.workflow.form.process.GroupResourceProcessForm;
import org.apache.inlong.manager.common.enums.ProcessName;
import org.apache.inlong.manager.service.workflow.WorkflowDefinition;
-import org.apache.inlong.manager.service.workflow.group.listener.UpdateGroupCompleteListener;
-import org.apache.inlong.manager.service.workflow.group.listener.UpdateGroupFailedListener;
-import org.apache.inlong.manager.service.workflow.group.listener.UpdateGroupListener;
-import org.apache.inlong.manager.service.workflow.listener.GroupTaskListenerFactory;
+import org.apache.inlong.manager.service.listener.group.UpdateGroupCompleteListener;
+import org.apache.inlong.manager.service.listener.group.UpdateGroupFailedListener;
+import org.apache.inlong.manager.service.listener.group.UpdateGroupListener;
+import org.apache.inlong.manager.service.listener.GroupTaskListenerFactory;
import org.apache.inlong.manager.workflow.definition.EndEvent;
import org.apache.inlong.manager.workflow.definition.ServiceTask;
import org.apache.inlong.manager.workflow.definition.ServiceTaskType;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/RestartGroupWorkflowDefinition.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/RestartGroupWorkflowDefinition.java
index 4449b78e5..dffbaea67 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/RestartGroupWorkflowDefinition.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/RestartGroupWorkflowDefinition.java
@@ -21,10 +21,10 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.inlong.manager.common.pojo.workflow.form.process.GroupResourceProcessForm;
import org.apache.inlong.manager.common.enums.ProcessName;
import org.apache.inlong.manager.service.workflow.WorkflowDefinition;
-import org.apache.inlong.manager.service.workflow.group.listener.UpdateGroupCompleteListener;
-import org.apache.inlong.manager.service.workflow.group.listener.UpdateGroupFailedListener;
-import org.apache.inlong.manager.service.workflow.group.listener.UpdateGroupListener;
-import org.apache.inlong.manager.service.workflow.listener.GroupTaskListenerFactory;
+import org.apache.inlong.manager.service.listener.group.UpdateGroupCompleteListener;
+import org.apache.inlong.manager.service.listener.group.UpdateGroupFailedListener;
+import org.apache.inlong.manager.service.listener.group.UpdateGroupListener;
+import org.apache.inlong.manager.service.listener.GroupTaskListenerFactory;
import org.apache.inlong.manager.workflow.definition.EndEvent;
import org.apache.inlong.manager.workflow.definition.ServiceTask;
import org.apache.inlong.manager.workflow.definition.ServiceTaskType;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/SuspendGroupWorkflowDefinition.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/SuspendGroupWorkflowDefinition.java
index d9aaf588e..6e1ffe331 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/SuspendGroupWorkflowDefinition.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/group/SuspendGroupWorkflowDefinition.java
@@ -21,10 +21,10 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.inlong.manager.common.pojo.workflow.form.process.GroupResourceProcessForm;
import org.apache.inlong.manager.common.enums.ProcessName;
import org.apache.inlong.manager.service.workflow.WorkflowDefinition;
-import org.apache.inlong.manager.service.workflow.group.listener.UpdateGroupCompleteListener;
-import org.apache.inlong.manager.service.workflow.group.listener.UpdateGroupFailedListener;
-import org.apache.inlong.manager.service.workflow.group.listener.UpdateGroupListener;
-import org.apache.inlong.manager.service.workflow.listener.GroupTaskListenerFactory;
+import org.apache.inlong.manager.service.listener.group.UpdateGroupCompleteListener;
+import org.apache.inlong.manager.service.listener.group.UpdateGroupFailedListener;
+import org.apache.inlong.manager.service.listener.group.UpdateGroupListener;
+import org.apache.inlong.manager.service.listener.GroupTaskListenerFactory;
import org.apache.inlong.manager.workflow.definition.EndEvent;
import org.apache.inlong.manager.workflow.definition.ServiceTask;
import org.apache.inlong.manager.workflow.definition.ServiceTaskType;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/stream/CreateStreamWorkflowDefinition.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/stream/CreateStreamWorkflowDefinition.java
index 0183448ca..e9a7409ac 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/stream/CreateStreamWorkflowDefinition.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/stream/CreateStreamWorkflowDefinition.java
@@ -21,10 +21,10 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.inlong.manager.common.pojo.workflow.form.process.StreamResourceProcessForm;
import org.apache.inlong.manager.common.enums.ProcessName;
import org.apache.inlong.manager.service.workflow.WorkflowDefinition;
-import org.apache.inlong.manager.service.workflow.listener.StreamTaskListenerFactory;
-import org.apache.inlong.manager.service.workflow.stream.listener.InitStreamCompleteListener;
-import org.apache.inlong.manager.service.workflow.stream.listener.InitStreamFailedListener;
-import org.apache.inlong.manager.service.workflow.stream.listener.InitStreamListener;
+import org.apache.inlong.manager.service.listener.StreamTaskListenerFactory;
+import org.apache.inlong.manager.service.listener.stream.InitStreamCompleteListener;
+import org.apache.inlong.manager.service.listener.stream.InitStreamFailedListener;
+import org.apache.inlong.manager.service.listener.stream.InitStreamListener;
import org.apache.inlong.manager.workflow.definition.EndEvent;
import org.apache.inlong.manager.workflow.definition.ServiceTask;
import org.apache.inlong.manager.workflow.definition.ServiceTaskType;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/stream/DeleteStreamWorkflowDefinition.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/stream/DeleteStreamWorkflowDefinition.java
index 886f35138..3bad0833f 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/stream/DeleteStreamWorkflowDefinition.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/stream/DeleteStreamWorkflowDefinition.java
@@ -21,10 +21,10 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.inlong.manager.common.pojo.workflow.form.process.StreamResourceProcessForm;
import org.apache.inlong.manager.common.enums.ProcessName;
import org.apache.inlong.manager.service.workflow.WorkflowDefinition;
-import org.apache.inlong.manager.service.workflow.listener.StreamTaskListenerFactory;
-import org.apache.inlong.manager.service.workflow.stream.listener.UpdateStreamCompleteListener;
-import org.apache.inlong.manager.service.workflow.stream.listener.UpdateStreamFailedListener;
-import org.apache.inlong.manager.service.workflow.stream.listener.UpdateStreamListener;
+import org.apache.inlong.manager.service.listener.StreamTaskListenerFactory;
+import org.apache.inlong.manager.service.listener.stream.UpdateStreamCompleteListener;
+import org.apache.inlong.manager.service.listener.stream.UpdateStreamFailedListener;
+import org.apache.inlong.manager.service.listener.stream.UpdateStreamListener;
import org.apache.inlong.manager.workflow.definition.EndEvent;
import org.apache.inlong.manager.workflow.definition.ServiceTask;
import org.apache.inlong.manager.workflow.definition.ServiceTaskType;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/stream/RestartStreamWorkflowDefinition.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/stream/RestartStreamWorkflowDefinition.java
index ca6f8f24b..9d764175f 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/stream/RestartStreamWorkflowDefinition.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/stream/RestartStreamWorkflowDefinition.java
@@ -21,10 +21,10 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.inlong.manager.common.pojo.workflow.form.process.StreamResourceProcessForm;
import org.apache.inlong.manager.common.enums.ProcessName;
import org.apache.inlong.manager.service.workflow.WorkflowDefinition;
-import org.apache.inlong.manager.service.workflow.listener.StreamTaskListenerFactory;
-import org.apache.inlong.manager.service.workflow.stream.listener.UpdateStreamCompleteListener;
-import org.apache.inlong.manager.service.workflow.stream.listener.UpdateStreamFailedListener;
-import org.apache.inlong.manager.service.workflow.stream.listener.UpdateStreamListener;
+import org.apache.inlong.manager.service.listener.StreamTaskListenerFactory;
+import org.apache.inlong.manager.service.listener.stream.UpdateStreamCompleteListener;
+import org.apache.inlong.manager.service.listener.stream.UpdateStreamFailedListener;
+import org.apache.inlong.manager.service.listener.stream.UpdateStreamListener;
import org.apache.inlong.manager.workflow.definition.EndEvent;
import org.apache.inlong.manager.workflow.definition.ServiceTask;
import org.apache.inlong.manager.workflow.definition.ServiceTaskType;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/stream/SuspendStreamWorkflowDefinition.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/stream/SuspendStreamWorkflowDefinition.java
index 5b0e9992d..059d2a009 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/stream/SuspendStreamWorkflowDefinition.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/stream/SuspendStreamWorkflowDefinition.java
@@ -21,10 +21,10 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.inlong.manager.common.pojo.workflow.form.process.StreamResourceProcessForm;
import org.apache.inlong.manager.common.enums.ProcessName;
import org.apache.inlong.manager.service.workflow.WorkflowDefinition;
-import org.apache.inlong.manager.service.workflow.listener.StreamTaskListenerFactory;
-import org.apache.inlong.manager.service.workflow.stream.listener.UpdateStreamCompleteListener;
-import org.apache.inlong.manager.service.workflow.stream.listener.UpdateStreamFailedListener;
-import org.apache.inlong.manager.service.workflow.stream.listener.UpdateStreamListener;
+import org.apache.inlong.manager.service.listener.StreamTaskListenerFactory;
+import org.apache.inlong.manager.service.listener.stream.UpdateStreamCompleteListener;
+import org.apache.inlong.manager.service.listener.stream.UpdateStreamFailedListener;
+import org.apache.inlong.manager.service.listener.stream.UpdateStreamListener;
import org.apache.inlong.manager.workflow.definition.EndEvent;
import org.apache.inlong.manager.workflow.definition.ServiceTask;
import org.apache.inlong.manager.workflow.definition.ServiceTaskType;
diff --git a/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/core/impl/InlongGroupProcessOperationTest.java b/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/core/impl/InlongGroupProcessOperationTest.java
index bf9590264..f7bad8f2f 100644
--- a/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/core/impl/InlongGroupProcessOperationTest.java
+++ b/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/core/impl/InlongGroupProcessOperationTest.java
@@ -28,7 +28,7 @@ import org.apache.inlong.manager.service.ServiceBaseTest;
import org.apache.inlong.manager.service.core.operation.InlongGroupProcessOperation;
import org.apache.inlong.manager.service.group.InlongGroupService;
import org.apache.inlong.manager.service.mocks.MockPlugin;
-import org.apache.inlong.manager.service.workflow.listener.GroupTaskListenerFactory;
+import org.apache.inlong.manager.service.listener.GroupTaskListenerFactory;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
diff --git a/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/core/sink/MySQLSinkServiceTest.java b/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/core/sink/MySQLSinkServiceTest.java
index 91578766b..6f600015a 100644
--- a/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/core/sink/MySQLSinkServiceTest.java
+++ b/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/core/sink/MySQLSinkServiceTest.java
@@ -27,7 +27,7 @@ import org.apache.inlong.manager.common.pojo.sink.mysql.MySQLSinkRequest;
import org.apache.inlong.manager.common.pojo.sink.mysql.MySQLTableInfo;
import org.apache.inlong.manager.service.ServiceBaseTest;
import org.apache.inlong.manager.service.core.impl.InlongStreamServiceTest;
-import org.apache.inlong.manager.service.resource.mysql.MySQLJdbcUtils;
+import org.apache.inlong.manager.service.resource.sink.mysql.MySQLJdbcUtils;
import org.apache.inlong.manager.service.sink.StreamSinkService;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Disabled;
diff --git a/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/core/sink/OracleSinkServiceTest.java b/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/core/sink/OracleSinkServiceTest.java
index ea0d32f77..c11933789 100644
--- a/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/core/sink/OracleSinkServiceTest.java
+++ b/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/core/sink/OracleSinkServiceTest.java
@@ -28,7 +28,7 @@ import org.apache.inlong.manager.common.pojo.sink.oracle.OracleTableInfo;
import org.apache.inlong.manager.common.util.CommonBeanUtils;
import org.apache.inlong.manager.service.ServiceBaseTest;
import org.apache.inlong.manager.service.core.impl.InlongStreamServiceTest;
-import org.apache.inlong.manager.service.resource.oracle.OracleJdbcUtils;
+import org.apache.inlong.manager.service.resource.sink.oracle.OracleJdbcUtils;
import org.apache.inlong.manager.service.sink.StreamSinkService;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Disabled;
diff --git a/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/core/sink/PostgreSQLSinkServiceTest.java b/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/core/sink/PostgreSQLSinkServiceTest.java
index f40790483..18d3e032e 100644
--- a/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/core/sink/PostgreSQLSinkServiceTest.java
+++ b/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/core/sink/PostgreSQLSinkServiceTest.java
@@ -28,7 +28,7 @@ import org.apache.inlong.manager.common.pojo.sink.postgresql.PostgreSQLSinkReque
import org.apache.inlong.manager.common.pojo.sink.postgresql.PostgreSQLTableInfo;
import org.apache.inlong.manager.service.ServiceBaseTest;
import org.apache.inlong.manager.service.core.impl.InlongStreamServiceTest;
-import org.apache.inlong.manager.service.resource.postgresql.PostgreSQLJdbcUtils;
+import org.apache.inlong.manager.service.resource.sink.postgresql.PostgreSQLJdbcUtils;
import org.apache.inlong.manager.service.sink.StreamSinkService;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Disabled;
diff --git a/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/mocks/MockDeleteSourceListener.java b/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/mocks/MockDeleteSourceListener.java
index 60d3e2b07..c5f87e1a2 100644
--- a/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/mocks/MockDeleteSourceListener.java
+++ b/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/mocks/MockDeleteSourceListener.java
@@ -17,9 +17,9 @@
package org.apache.inlong.manager.service.mocks;
+import org.apache.inlong.manager.common.consts.InlongConstants;
import org.apache.inlong.manager.common.enums.GroupOperateType;
import org.apache.inlong.manager.common.pojo.workflow.form.process.GroupResourceProcessForm;
-import org.apache.inlong.manager.common.pojo.workflow.form.process.ProcessForm;
import org.apache.inlong.manager.workflow.WorkflowContext;
import org.apache.inlong.manager.workflow.event.ListenerResult;
import org.apache.inlong.manager.workflow.event.task.SourceOperateListener;
@@ -37,12 +37,12 @@ public class MockDeleteSourceListener implements SourceOperateListener {
@Override
public boolean accept(WorkflowContext context) {
- ProcessForm processForm = context.getProcessForm();
- if (!(processForm instanceof GroupResourceProcessForm)) {
+ if (!isGroupProcessForm(context)) {
return false;
}
- GroupResourceProcessForm form = (GroupResourceProcessForm) processForm;
- return form.getGroupOperateType() == GroupOperateType.DELETE;
+ GroupResourceProcessForm processForm = (GroupResourceProcessForm) context.getProcessForm();
+ return InlongConstants.STANDARD_MODE.equals(processForm.getGroupInfo().getLightweight())
+ && processForm.getGroupOperateType() == GroupOperateType.DELETE;
}
@Override
diff --git a/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/mocks/MockRestartSourceListener.java b/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/mocks/MockRestartSourceListener.java
index a6fa79114..4adf578b5 100644
--- a/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/mocks/MockRestartSourceListener.java
+++ b/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/mocks/MockRestartSourceListener.java
@@ -17,9 +17,9 @@
package org.apache.inlong.manager.service.mocks;
+import org.apache.inlong.manager.common.consts.InlongConstants;
import org.apache.inlong.manager.common.enums.GroupOperateType;
import org.apache.inlong.manager.common.pojo.workflow.form.process.GroupResourceProcessForm;
-import org.apache.inlong.manager.common.pojo.workflow.form.process.ProcessForm;
import org.apache.inlong.manager.workflow.WorkflowContext;
import org.apache.inlong.manager.workflow.event.ListenerResult;
import org.apache.inlong.manager.workflow.event.task.SourceOperateListener;
@@ -37,12 +37,12 @@ public class MockRestartSourceListener implements SourceOperateListener {
@Override
public boolean accept(WorkflowContext context) {
- ProcessForm processForm = context.getProcessForm();
- if (!(processForm instanceof GroupResourceProcessForm)) {
+ if (!isGroupProcessForm(context)) {
return false;
}
- GroupResourceProcessForm form = (GroupResourceProcessForm) processForm;
- return form.getGroupOperateType() == GroupOperateType.RESTART;
+ GroupResourceProcessForm processForm = (GroupResourceProcessForm) context.getProcessForm();
+ return InlongConstants.STANDARD_MODE.equals(processForm.getGroupInfo().getLightweight())
+ && processForm.getGroupOperateType() == GroupOperateType.RESTART;
}
@Override
diff --git a/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/mocks/MockStopSourceListener.java b/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/mocks/MockStopSourceListener.java
index ae554323d..009015352 100644
--- a/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/mocks/MockStopSourceListener.java
+++ b/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/mocks/MockStopSourceListener.java
@@ -17,9 +17,9 @@
package org.apache.inlong.manager.service.mocks;
+import org.apache.inlong.manager.common.consts.InlongConstants;
import org.apache.inlong.manager.common.enums.GroupOperateType;
import org.apache.inlong.manager.common.pojo.workflow.form.process.GroupResourceProcessForm;
-import org.apache.inlong.manager.common.pojo.workflow.form.process.ProcessForm;
import org.apache.inlong.manager.workflow.WorkflowContext;
import org.apache.inlong.manager.workflow.event.ListenerResult;
import org.apache.inlong.manager.workflow.event.task.SourceOperateListener;
@@ -37,12 +37,12 @@ public class MockStopSourceListener implements SourceOperateListener {
@Override
public boolean accept(WorkflowContext context) {
- ProcessForm processForm = context.getProcessForm();
- if (!(processForm instanceof GroupResourceProcessForm)) {
+ if (!isGroupProcessForm(context)) {
return false;
}
- GroupResourceProcessForm form = (GroupResourceProcessForm) processForm;
- return form.getGroupOperateType() == GroupOperateType.SUSPEND;
+ GroupResourceProcessForm processForm = (GroupResourceProcessForm) context.getProcessForm();
+ return InlongConstants.STANDARD_MODE.equals(processForm.getGroupInfo().getLightweight())
+ && processForm.getGroupOperateType() == GroupOperateType.SUSPEND;
}
@Override
diff --git a/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/mq/util/PulsarUtilsTest.java b/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/mq/util/PulsarUtilsTest.java
index a99f4a1e4..42f42c449 100644
--- a/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/mq/util/PulsarUtilsTest.java
+++ b/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/mq/util/PulsarUtilsTest.java
@@ -17,6 +17,7 @@
package org.apache.inlong.manager.service.mq.util;
+import org.apache.inlong.manager.service.resource.queue.pulsar.PulsarUtils;
import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.client.admin.internal.PulsarAdminImpl;
import org.apache.pulsar.client.api.Authentication;
diff --git a/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/workflow/GroupTaskListenerFactoryTest.java b/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/workflow/GroupTaskListenerFactoryTest.java
index 46e7a937b..f58280c4a 100644
--- a/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/workflow/GroupTaskListenerFactoryTest.java
+++ b/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/workflow/GroupTaskListenerFactoryTest.java
@@ -22,11 +22,7 @@ import org.apache.inlong.manager.common.pojo.group.pulsar.InlongPulsarInfo;
import org.apache.inlong.manager.common.pojo.group.tube.InlongTubeInfo;
import org.apache.inlong.manager.common.pojo.workflow.form.process.GroupResourceProcessForm;
import org.apache.inlong.manager.service.ServiceBaseTest;
-import org.apache.inlong.manager.service.mq.CreatePulsarGroupTaskListener;
-import org.apache.inlong.manager.service.mq.CreatePulsarResourceTaskListener;
-import org.apache.inlong.manager.service.mq.CreateTubeGroupTaskListener;
-import org.apache.inlong.manager.service.mq.CreateTubeTopicTaskListener;
-import org.apache.inlong.manager.service.workflow.listener.GroupTaskListenerFactory;
+import org.apache.inlong.manager.service.listener.GroupTaskListenerFactory;
import org.apache.inlong.manager.workflow.WorkflowContext;
import org.apache.inlong.manager.workflow.event.task.QueueOperateListener;
import org.junit.jupiter.api.Assertions;
@@ -52,21 +48,17 @@ public class GroupTaskListenerFactoryTest extends ServiceBaseTest {
processForm.setGroupInfo(pulsarInfo);
WorkflowContext context = new WorkflowContext();
context.setProcessForm(processForm);
- List<QueueOperateListener> queueOperateListeners = groupTaskListenerFactory.getQueueOperateListener(context);
+ List<QueueOperateListener> queueOperateListeners = groupTaskListenerFactory.getQueueResourceListener(context);
if (queueOperateListeners.size() == 0) {
return;
}
- Assertions.assertEquals(2, queueOperateListeners.size());
- Assertions.assertTrue(queueOperateListeners.get(0) instanceof CreatePulsarResourceTaskListener);
- Assertions.assertTrue(queueOperateListeners.get(1) instanceof CreatePulsarGroupTaskListener);
+ Assertions.assertEquals(1, queueOperateListeners.size());
// check tube listener
InlongTubeInfo tubeInfo = new InlongTubeInfo();
tubeInfo.setMqType(MQType.TUBE.getType());
- queueOperateListeners = groupTaskListenerFactory.getQueueOperateListener(context);
+ queueOperateListeners = groupTaskListenerFactory.getQueueResourceListener(context);
Assertions.assertEquals(2, queueOperateListeners.size());
- Assertions.assertTrue(queueOperateListeners.get(0) instanceof CreateTubeTopicTaskListener);
- Assertions.assertTrue(queueOperateListeners.get(1) instanceof CreateTubeGroupTaskListener);
}
}
diff --git a/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/workflow/WorkflowServiceImplTest.java b/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/workflow/WorkflowServiceImplTest.java
index 416fd171d..b134a2817 100644
--- a/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/workflow/WorkflowServiceImplTest.java
+++ b/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/workflow/WorkflowServiceImplTest.java
@@ -30,12 +30,9 @@ import org.apache.inlong.manager.dao.mapper.WorkflowTaskEntityMapper;
import org.apache.inlong.manager.service.ServiceBaseTest;
import org.apache.inlong.manager.service.core.InlongStreamService;
import org.apache.inlong.manager.service.group.InlongGroupService;
-import org.apache.inlong.manager.service.mq.CreatePulsarGroupTaskListener;
-import org.apache.inlong.manager.service.mq.CreatePulsarResourceTaskListener;
-import org.apache.inlong.manager.service.mq.CreateTubeGroupTaskListener;
-import org.apache.inlong.manager.service.mq.CreateTubeTopicTaskListener;
-import org.apache.inlong.manager.service.resource.SinkResourceListener;
-import org.apache.inlong.manager.service.workflow.listener.GroupTaskListenerFactory;
+import org.apache.inlong.manager.service.listener.GroupTaskListenerFactory;
+import org.apache.inlong.manager.service.listener.queue.QueueResourceListener;
+import org.apache.inlong.manager.service.listener.sink.SinkResourceListener;
import org.apache.inlong.manager.workflow.WorkflowContext;
import org.apache.inlong.manager.workflow.core.ProcessService;
import org.apache.inlong.manager.workflow.definition.ServiceTask;
@@ -100,29 +97,11 @@ public class WorkflowServiceImplTest extends ServiceBaseTest {
* Mock the task listener factory
*/
public void mockTaskListenerFactory() {
- CreateTubeGroupTaskListener createTubeGroupListener = mock(CreateTubeGroupTaskListener.class);
- when(createTubeGroupListener.listen(any(WorkflowContext.class))).thenReturn(ListenerResult.success());
- when(createTubeGroupListener.name()).thenReturn(SinkResourceListener.class.getSimpleName());
- when(createTubeGroupListener.event()).thenReturn(TaskEvent.COMPLETE);
- taskListenerFactory.setCreateTubeGroupTaskListener(createTubeGroupListener);
-
- CreateTubeTopicTaskListener createTubeTopicListener = mock(CreateTubeTopicTaskListener.class);
- when(createTubeTopicListener.listen(any(WorkflowContext.class))).thenReturn(ListenerResult.success());
- when(createTubeTopicListener.name()).thenReturn(CreateTubeTopicTaskListener.class.getSimpleName());
- when(createTubeTopicListener.event()).thenReturn(TaskEvent.COMPLETE);
- taskListenerFactory.setCreateTubeTopicTaskListener(createTubeTopicListener);
-
- CreatePulsarResourceTaskListener createPulsarResourceListener = mock(CreatePulsarResourceTaskListener.class);
- when(createPulsarResourceListener.listen(any(WorkflowContext.class))).thenReturn(ListenerResult.success());
- when(createPulsarResourceListener.name()).thenReturn(CreatePulsarResourceTaskListener.class.getSimpleName());
- when(createPulsarResourceListener.event()).thenReturn(TaskEvent.COMPLETE);
- taskListenerFactory.setCreatePulsarResourceTaskListener(createPulsarResourceListener);
-
- CreatePulsarGroupTaskListener createPulsarGroupListener = mock(CreatePulsarGroupTaskListener.class);
- when(createPulsarGroupListener.listen(any(WorkflowContext.class))).thenReturn(ListenerResult.success());
- when(createPulsarGroupListener.name()).thenReturn(CreatePulsarGroupTaskListener.class.getSimpleName());
- when(createPulsarGroupListener.event()).thenReturn(TaskEvent.COMPLETE);
- taskListenerFactory.setCreatePulsarGroupTaskListener(createPulsarGroupListener);
+ QueueResourceListener queueResourceListener = mock(QueueResourceListener.class);
+ when(queueResourceListener.listen(any(WorkflowContext.class))).thenReturn(ListenerResult.success());
+ when(queueResourceListener.name()).thenReturn(QueueResourceListener.class.getSimpleName());
+ when(queueResourceListener.event()).thenReturn(TaskEvent.COMPLETE);
+ taskListenerFactory.setQueueResourceListener(queueResourceListener);
SinkResourceListener sinkResourceListener = mock(SinkResourceListener.class);
when(sinkResourceListener.listen(any(WorkflowContext.class))).thenReturn(ListenerResult.success());
diff --git a/inlong-manager/manager-workflow/src/main/java/org/apache/inlong/manager/workflow/event/task/QueueOperateListener.java b/inlong-manager/manager-workflow/src/main/java/org/apache/inlong/manager/workflow/event/task/QueueOperateListener.java
index 3e8f28475..fedd527df 100644
--- a/inlong-manager/manager-workflow/src/main/java/org/apache/inlong/manager/workflow/event/task/QueueOperateListener.java
+++ b/inlong-manager/manager-workflow/src/main/java/org/apache/inlong/manager/workflow/event/task/QueueOperateListener.java
@@ -17,6 +17,7 @@
package org.apache.inlong.manager.workflow.event.task;
+import org.apache.inlong.manager.common.pojo.workflow.form.process.GroupResourceProcessForm;
import org.apache.inlong.manager.workflow.WorkflowContext;
import org.apache.inlong.manager.workflow.event.ListenerResult;
@@ -39,4 +40,17 @@ public interface QueueOperateListener extends TaskEventListener {
};
+ /**
+ * Check whether the process form from the workflow context is {@link GroupResourceProcessForm}
+ *
+ * @param context workflow context
+ * @return true if the process form is instanceof GroupResourceProcessForm
+ */
+ default boolean isGroupProcessForm(WorkflowContext context) {
+ if (context == null) {
+ return false;
+ }
+ return context.getProcessForm() instanceof GroupResourceProcessForm;
+ }
+
}
diff --git a/inlong-manager/manager-workflow/src/main/java/org/apache/inlong/manager/workflow/event/task/SourceOperateListener.java b/inlong-manager/manager-workflow/src/main/java/org/apache/inlong/manager/workflow/event/task/SourceOperateListener.java
index 8abb11f5c..006e71df6 100644
--- a/inlong-manager/manager-workflow/src/main/java/org/apache/inlong/manager/workflow/event/task/SourceOperateListener.java
+++ b/inlong-manager/manager-workflow/src/main/java/org/apache/inlong/manager/workflow/event/task/SourceOperateListener.java
@@ -17,6 +17,7 @@
package org.apache.inlong.manager.workflow.event.task;
+import org.apache.inlong.manager.common.pojo.workflow.form.process.GroupResourceProcessForm;
import org.apache.inlong.manager.workflow.WorkflowContext;
import org.apache.inlong.manager.workflow.event.ListenerResult;
@@ -39,4 +40,17 @@ public interface SourceOperateListener extends TaskEventListener {
};
+ /**
+ * Check whether the process form from the workflow context is {@link GroupResourceProcessForm}
+ *
+ * @param context workflow context
+ * @return true if the process form is instanceof GroupResourceProcessForm
+ */
+ default boolean isGroupProcessForm(WorkflowContext context) {
+ if (context == null) {
+ return false;
+ }
+ return context.getProcessForm() instanceof GroupResourceProcessForm;
+ }
+
}