You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@inlong.apache.org by do...@apache.org on 2022/05/30 03:08:41 UTC
[incubator-inlong] branch master updated: [INLONG-4419][Manager] The manager-client module reuses the sink classes in manager-common (#4420)
This is an automated email from the ASF dual-hosted git repository.
dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 588d9ec86 [INLONG-4419][Manager] The manager-client module reuses the sink classes in manager-common (#4420)
588d9ec86 is described below
commit 588d9ec8667eb7c0639d72941bfe37e87b50d25f
Author: healzhou <he...@gmail.com>
AuthorDate: Mon May 30 11:08:36 2022 +0800
[INLONG-4419][Manager] The manager-client module reuses the sink classes in manager-common (#4420)
---
.../manager/client/AutoPush2HiveExample.java | 32 +-
.../apache/inlong/manager/client/BaseExample.java | 29 ++
.../inlong/manager/client/Binlog2KafkaExample.java | 13 +-
.../inlong/manager/client/File2HiveExample.java | 32 +-
.../inlong/manager/client/Kafka2HiveExample.java | 32 +-
.../inlong/manager/client/cli/CreateCommand.java | 2 +-
.../manager/client/cli/pojo/CreateGroupConf.java | 6 +-
.../inlong/manager/client/cli/util/GsonUtils.java | 4 +-
.../manager/client/cli/util/SeparatorAdapter.java | 2 +-
.../manager/client/cli/util/StreamSinkAdapter.java | 16 +-
.../client/cli/util/StreamSourceAdapter.java | 2 +-
.../manager/client/api/InlongGroupContext.java | 10 +-
.../inlong/manager/client/api/InlongStream.java | 45 +-
.../manager/client/api/InlongStreamBuilder.java | 8 +-
.../manager/client/api/InlongStreamConf.java | 4 +-
.../api/impl/DefaultInlongStreamBuilder.java | 35 +-
.../manager/client/api/impl/InlongClientImpl.java | 16 +-
.../manager/client/api/impl/InlongGroupImpl.java | 8 +-
.../manager/client/api/impl/InlongStreamImpl.java | 128 +++---
.../client/api/inner/InnerGroupContext.java | 2 +-
.../client/api/inner/InnerInlongManagerClient.java | 3 +-
.../client/api/inner/InnerStreamContext.java | 8 +-
.../manager/client/api/sink/ClickHouseSink.java | 91 ----
.../inlong/manager/client/api/sink/HbaseSink.java | 80 ----
.../inlong/manager/client/api/sink/HiveSink.java | 94 ----
.../inlong/manager/client/api/sink/KafkaSink.java | 63 ---
.../manager/client/api/sink/PostgresSink.java | 71 ---
.../manager/client/api/source/AgentFileSource.java | 2 +-
.../manager/client/api/source/AutoPushSource.java | 2 +-
.../manager/client/api/source/KafkaSource.java | 2 +-
.../client/api/source/MySQLBinlogSource.java | 2 +-
.../manager/client/api/source/MySQLSource.java | 2 +-
.../manager/client/api/source/PostgresSource.java | 2 +-
.../client/api/util/InlongStreamSinkTransfer.java | 511 ---------------------
.../api/util/InlongStreamSourceTransfer.java | 18 +-
.../client/api/util/InlongStreamTransfer.java | 11 +-
.../client/api/util/StreamSinkTransfer.java | 62 +++
.../client/api/impl/InlongStreamImplTest.java | 24 +-
.../inlong/manager/common/enums/FieldType.java | 15 +-
.../inlong/manager/common/enums/SourceStatus.java | 2 +-
.../common/pojo/{stream => sink}/SinkField.java | 19 +-
.../manager/common/pojo/sink/SinkFieldBase.java | 68 ---
.../manager/common/pojo/sink/SinkFieldRequest.java | 36 --
.../common/pojo/sink/SinkFieldResponse.java | 38 --
.../manager/common/pojo/sink/SinkPageRequest.java | 2 +-
.../manager/common/pojo/sink/SinkRequest.java | 34 +-
.../manager/common/pojo/sink/SinkResponse.java | 4 +-
.../common/pojo/{stream => sink}/StreamSink.java | 40 +-
.../common/pojo/sink/ck/ClickHouseSinkDTO.java | 2 +-
.../common/pojo/sink/ck/ClickHouseSinkRequest.java | 8 +-
.../pojo/sink/es/ElasticsearchSinkRequest.java | 4 +-
.../common/pojo/sink/hbase/HbaseSinkDTO.java | 32 +-
.../pojo/sink/hbase/HbaseSinkListResponse.java | 22 +-
.../common/pojo/sink/hbase/HbaseSinkRequest.java | 26 +-
.../common/pojo/sink/hbase/HbaseSinkResponse.java | 22 +-
.../common/pojo/sink/hive/HiveSinkRequest.java | 30 +-
.../common/pojo/sink/kafka/KafkaSinkRequest.java | 8 +-
.../manager/common/pojo/source/SourceRequest.java | 4 +-
.../manager/common/pojo/source/SourceResponse.java | 6 +-
.../pojo/{stream => source}/StreamSource.java | 50 +-
.../common/pojo/stream/InlongStreamFieldInfo.java | 67 ---
.../common/pojo/stream/InlongStreamInfo.java | 7 +-
.../common/pojo/stream/InlongStreamRequest.java | 2 +-
.../common/pojo/stream/InlongStreamResponse.java | 2 +-
.../manager/common/pojo/stream/StreamField.java | 29 +-
.../manager/common/pojo/stream/StreamNode.java | 4 -
.../manager/common/pojo/stream/StreamPipeline.java | 2 +-
.../manager/common/util/StreamParseUtils.java | 13 +-
.../common/pojo/stream/StreamPipelineTest.java | 10 +-
.../pojo/transform/TransformDefinitionTest.java | 18 +-
.../service/core/impl/InlongStreamServiceImpl.java | 22 +-
.../resource/hbase/HbaseResourceOperator.java | 4 +-
.../service/sink/ck/ClickHouseSinkOperation.java | 13 +-
.../sink/es/ElasticsearchSinkOperation.java | 13 +-
.../service/sink/hbase/HbaseSinkOperation.java | 13 +-
.../service/sink/hive/HiveSinkOperation.java | 13 +-
.../service/sink/iceberg/IcebergSinkOperation.java | 13 +-
.../service/sink/kafka/KafkaSinkOperation.java | 11 +-
.../sink/postgres/PostgresSinkOperation.java | 11 +-
.../service/sort/CreateSortConfigListenerV2.java | 28 +-
.../sort/CreateStreamSortConfigListener.java | 20 +-
.../service/sort/light/LightGroupSortListener.java | 79 ++--
.../service/sort/util/ExtractNodeUtils.java | 57 +--
.../manager/service/sort/util/FieldInfoUtils.java | 23 +-
.../service/sort/util/FieldRelationShipUtils.java | 51 +-
.../service/sort/util/FilterFunctionUtils.java | 4 +-
.../manager/service/sort/util/LoadNodeUtils.java | 117 ++---
.../service/sort/util/NodeRelationShipUtils.java | 55 ++-
.../manager/service/sort/util/SinkInfoUtils.java | 21 +-
.../service/sort/util/TransformNodeUtils.java | 15 +-
.../service/source/AbstractSourceOperation.java | 12 +-
.../transform/StreamTransformServiceImpl.java | 71 ++-
.../core/sink/HbaseStreamSinkServiceTest.java | 2 +-
.../manager/service/sort/DisableZkForSortTest.java | 52 +--
.../service/workflow/WorkflowServiceImplTest.java | 15 +-
95 files changed, 854 insertions(+), 1981 deletions(-)
diff --git a/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/AutoPush2HiveExample.java b/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/AutoPush2HiveExample.java
index da81ed118..eca40aded 100644
--- a/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/AutoPush2HiveExample.java
+++ b/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/AutoPush2HiveExample.java
@@ -19,25 +19,20 @@ package org.apache.inlong.manager.client;
import com.google.common.collect.Lists;
import org.apache.inlong.manager.client.api.ClientConfiguration;
-import org.apache.inlong.manager.common.enums.DataSeparator;
import org.apache.inlong.manager.client.api.InlongClient;
import org.apache.inlong.manager.client.api.InlongGroup;
import org.apache.inlong.manager.client.api.InlongGroupContext;
import org.apache.inlong.manager.client.api.InlongStreamBuilder;
import org.apache.inlong.manager.client.api.InlongStreamConf;
-import org.apache.inlong.manager.client.api.sink.HiveSink;
import org.apache.inlong.manager.client.api.source.AutoPushSource;
-import org.apache.inlong.manager.common.auth.DefaultAuthentication;
+import org.apache.inlong.manager.common.enums.DataSeparator;
import org.apache.inlong.manager.common.enums.FieldType;
-import org.apache.inlong.manager.common.enums.FileFormat;
import org.apache.inlong.manager.common.pojo.group.InlongGroupInfo;
-import org.apache.inlong.manager.common.pojo.stream.SinkField;
import org.apache.inlong.manager.common.pojo.stream.StreamField;
import org.apache.shiro.util.Assert;
import org.junit.Test;
import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
@@ -109,31 +104,10 @@ public class AutoPush2HiveExample extends BaseExample {
return autoPushSource;
}
- private HiveSink createHiveSink() {
- HiveSink hiveSink = new HiveSink();
- hiveSink.setDbName("{db.name}");
- hiveSink.setJdbcUrl("jdbc:hive2://{ip:port}");
- hiveSink.setAuthentication(new DefaultAuthentication("hive", "hive"));
- hiveSink.setCharset(StandardCharsets.UTF_8);
- hiveSink.setFileFormat(FileFormat.TextFile);
- hiveSink.setDataSeparator(DataSeparator.VERTICAL_BAR);
- hiveSink.setDataPath("hdfs://{ip:port}/usr/hive/warehouse/{db.name}");
-
- List<SinkField> fields = new ArrayList<>();
- SinkField field1 = new SinkField(0, FieldType.INT, "age", FieldType.INT, "age");
- SinkField field2 = new SinkField(1, FieldType.STRING, "name", FieldType.STRING, "name");
- fields.add(field1);
- fields.add(field2);
- hiveSink.setSinkFields(fields);
- hiveSink.setTableName("{table.name}");
- hiveSink.setSinkName("{hive.sink.name}");
- return hiveSink;
- }
-
private List<StreamField> createStreamFields() {
List<StreamField> streamFieldList = Lists.newArrayList();
- streamFieldList.add(new StreamField(0, FieldType.STRING, "name", null, null));
- streamFieldList.add(new StreamField(1, FieldType.INT, "age", null, null));
+ streamFieldList.add(new StreamField(0, FieldType.STRING.toString(), "name", null, null));
+ streamFieldList.add(new StreamField(1, FieldType.INT.toString(), "age", null, null));
return streamFieldList;
}
}
diff --git a/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/BaseExample.java b/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/BaseExample.java
index b6359b202..07135cadd 100644
--- a/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/BaseExample.java
+++ b/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/BaseExample.java
@@ -19,11 +19,19 @@ package org.apache.inlong.manager.client;
import lombok.Data;
import org.apache.inlong.manager.common.auth.DefaultAuthentication;
+import org.apache.inlong.manager.common.enums.DataSeparator;
+import org.apache.inlong.manager.common.enums.FieldType;
+import org.apache.inlong.manager.common.enums.FileFormat;
import org.apache.inlong.manager.common.pojo.group.InlongGroupInfo;
import org.apache.inlong.manager.common.pojo.group.pulsar.InlongPulsarInfo;
+import org.apache.inlong.manager.common.pojo.sink.SinkField;
+import org.apache.inlong.manager.common.pojo.sink.hive.HiveSinkRequest;
import org.apache.inlong.manager.common.pojo.sort.FlinkSortConf;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
/**
@@ -84,4 +92,25 @@ public class BaseExample {
return pulsarInfo;
}
+ public HiveSinkRequest createHiveSink() {
+ HiveSinkRequest hiveSink = new HiveSinkRequest();
+ hiveSink.setDbName("{db.name}");
+ hiveSink.setJdbcUrl("jdbc:hive2://{ip:port}");
+ hiveSink.setAuthentication(new DefaultAuthentication("hive", "hive"));
+ hiveSink.setDataEncoding(StandardCharsets.UTF_8.toString());
+ hiveSink.setFileFormat(FileFormat.TextFile.name());
+ hiveSink.setDataSeparator(DataSeparator.VERTICAL_BAR.getSeparator());
+ hiveSink.setDataPath("hdfs://{ip:port}/usr/hive/warehouse/{db.name}");
+
+ List<SinkField> fields = new ArrayList<>();
+ SinkField field1 = new SinkField(0, FieldType.INT.toString(), "age", FieldType.INT.toString(), "age");
+ SinkField field2 = new SinkField(1, FieldType.STRING.toString(), "name", FieldType.STRING.toString(), "name");
+ fields.add(field1);
+ fields.add(field2);
+ hiveSink.setFieldList(fields);
+ hiveSink.setTableName("{table.name}");
+ hiveSink.setSinkName("{hive.sink.name}");
+ return hiveSink;
+ }
+
}
\ No newline at end of file
diff --git a/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/Binlog2KafkaExample.java b/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/Binlog2KafkaExample.java
index 63eeb3c38..1f5060186 100644
--- a/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/Binlog2KafkaExample.java
+++ b/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/Binlog2KafkaExample.java
@@ -19,17 +19,16 @@ package org.apache.inlong.manager.client;
import lombok.extern.slf4j.Slf4j;
import org.apache.inlong.manager.client.api.ClientConfiguration;
-import org.apache.inlong.manager.common.enums.DataSeparator;
import org.apache.inlong.manager.client.api.InlongClient;
import org.apache.inlong.manager.client.api.InlongGroup;
import org.apache.inlong.manager.client.api.InlongGroupContext;
import org.apache.inlong.manager.client.api.InlongStreamBuilder;
import org.apache.inlong.manager.client.api.InlongStreamConf;
-import org.apache.inlong.manager.client.api.sink.KafkaSink;
import org.apache.inlong.manager.client.api.source.MySQLBinlogSource;
import org.apache.inlong.manager.common.auth.DefaultAuthentication;
-import org.apache.inlong.manager.common.enums.DataFormat;
+import org.apache.inlong.manager.common.enums.DataSeparator;
import org.apache.inlong.manager.common.pojo.group.InlongGroupInfo;
+import org.apache.inlong.manager.common.pojo.sink.kafka.KafkaSinkRequest;
import org.apache.shiro.util.Assert;
import org.junit.Test;
@@ -149,17 +148,17 @@ public class Binlog2KafkaExample extends BaseExample {
return mySQLBinlogSource;
}
- private KafkaSink createKafkaSink() {
- KafkaSink kafkaSink = new KafkaSink();
- kafkaSink.setDataFormat(DataFormat.CANAL);
+ private KafkaSinkRequest createKafkaSink() {
+ KafkaSinkRequest kafkaSink = new KafkaSinkRequest();
kafkaSink.setBootstrapServers("{kafka.bootstrap}");
kafkaSink.setTopicName("{kafka.topic}");
- kafkaSink.setNeedCreated(false);
+ kafkaSink.setEnableCreateResource(0);
kafkaSink.setSinkName("{kafka.sink.name}");
Map<String, Object> properties = new HashMap<>();
// Not needed if kafka cluster is not set
properties.put("transaction.timeout.ms", 9000000);
kafkaSink.setProperties(properties);
+
return kafkaSink;
}
}
diff --git a/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/File2HiveExample.java b/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/File2HiveExample.java
index b0449d333..4d11b9392 100644
--- a/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/File2HiveExample.java
+++ b/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/File2HiveExample.java
@@ -20,25 +20,20 @@ package org.apache.inlong.manager.client;
import com.google.common.collect.Lists;
import lombok.extern.slf4j.Slf4j;
import org.apache.inlong.manager.client.api.ClientConfiguration;
-import org.apache.inlong.manager.common.enums.DataSeparator;
import org.apache.inlong.manager.client.api.InlongClient;
import org.apache.inlong.manager.client.api.InlongGroup;
import org.apache.inlong.manager.client.api.InlongGroupContext;
import org.apache.inlong.manager.client.api.InlongStreamBuilder;
import org.apache.inlong.manager.client.api.InlongStreamConf;
-import org.apache.inlong.manager.client.api.sink.HiveSink;
import org.apache.inlong.manager.client.api.source.AgentFileSource;
-import org.apache.inlong.manager.common.auth.DefaultAuthentication;
+import org.apache.inlong.manager.common.enums.DataSeparator;
import org.apache.inlong.manager.common.enums.FieldType;
-import org.apache.inlong.manager.common.enums.FileFormat;
import org.apache.inlong.manager.common.pojo.group.InlongGroupInfo;
-import org.apache.inlong.manager.common.pojo.stream.SinkField;
import org.apache.inlong.manager.common.pojo.stream.StreamField;
import org.apache.shiro.util.Assert;
import org.junit.Test;
import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
@@ -113,31 +108,10 @@ public class File2HiveExample extends BaseExample {
return agentFileSource;
}
- private HiveSink createHiveSink() {
- HiveSink hiveSink = new HiveSink();
- hiveSink.setDbName("{db.name}");
- hiveSink.setJdbcUrl("jdbc:hive2://{ip:port}");
- hiveSink.setAuthentication(new DefaultAuthentication("hive", "hive"));
- hiveSink.setCharset(StandardCharsets.UTF_8);
- hiveSink.setFileFormat(FileFormat.TextFile);
- hiveSink.setDataSeparator(DataSeparator.VERTICAL_BAR);
- hiveSink.setDataPath("hdfs://{ip:port}/usr/hive/warehouse/{db.name}");
-
- List<SinkField> fields = new ArrayList<>();
- SinkField field1 = new SinkField(0, FieldType.INT, "age", FieldType.INT, "age");
- SinkField field2 = new SinkField(1, FieldType.STRING, "name", FieldType.STRING, "name");
- fields.add(field1);
- fields.add(field2);
- hiveSink.setSinkFields(fields);
- hiveSink.setTableName("{table.name}");
- hiveSink.setSinkName("{hive.sink.name}");
- return hiveSink;
- }
-
private List<StreamField> createStreamFields() {
List<StreamField> streamFieldList = Lists.newArrayList();
- streamFieldList.add(new StreamField(0, FieldType.STRING, "name", null, null));
- streamFieldList.add(new StreamField(1, FieldType.INT, "age", null, null));
+ streamFieldList.add(new StreamField(0, FieldType.STRING.toString(), "name", null, null));
+ streamFieldList.add(new StreamField(1, FieldType.INT.toString(), "age", null, null));
return streamFieldList;
}
}
diff --git a/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/Kafka2HiveExample.java b/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/Kafka2HiveExample.java
index 9ca34ac1b..acb7759ed 100644
--- a/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/Kafka2HiveExample.java
+++ b/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/Kafka2HiveExample.java
@@ -20,26 +20,21 @@ package org.apache.inlong.manager.client;
import com.google.common.collect.Lists;
import lombok.extern.slf4j.Slf4j;
import org.apache.inlong.manager.client.api.ClientConfiguration;
-import org.apache.inlong.manager.common.enums.DataSeparator;
import org.apache.inlong.manager.client.api.InlongClient;
import org.apache.inlong.manager.client.api.InlongGroup;
import org.apache.inlong.manager.client.api.InlongGroupContext;
import org.apache.inlong.manager.client.api.InlongStreamBuilder;
import org.apache.inlong.manager.client.api.InlongStreamConf;
-import org.apache.inlong.manager.client.api.sink.HiveSink;
import org.apache.inlong.manager.client.api.source.KafkaSource;
-import org.apache.inlong.manager.common.auth.DefaultAuthentication;
import org.apache.inlong.manager.common.enums.DataFormat;
+import org.apache.inlong.manager.common.enums.DataSeparator;
import org.apache.inlong.manager.common.enums.FieldType;
-import org.apache.inlong.manager.common.enums.FileFormat;
import org.apache.inlong.manager.common.pojo.group.InlongGroupInfo;
-import org.apache.inlong.manager.common.pojo.stream.SinkField;
import org.apache.inlong.manager.common.pojo.stream.StreamField;
import org.apache.shiro.util.Assert;
import org.junit.Test;
import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
@@ -115,31 +110,10 @@ public class Kafka2HiveExample extends BaseExample {
return kafkaSource;
}
- private HiveSink createHiveSink() {
- HiveSink hiveSink = new HiveSink();
- hiveSink.setDbName("{db.name}");
- hiveSink.setJdbcUrl("jdbc:hive2://{ip:port}");
- hiveSink.setAuthentication(new DefaultAuthentication("hive", "hive"));
- hiveSink.setCharset(StandardCharsets.UTF_8);
- hiveSink.setFileFormat(FileFormat.TextFile);
- hiveSink.setDataSeparator(DataSeparator.VERTICAL_BAR);
- hiveSink.setDataPath("hdfs://{ip:port}/usr/hive/warehouse/{db.name}");
-
- List<SinkField> fields = new ArrayList<>();
- SinkField field1 = new SinkField(0, FieldType.INT, "age", FieldType.INT, "age");
- SinkField field2 = new SinkField(1, FieldType.STRING, "name", FieldType.STRING, "name");
- fields.add(field1);
- fields.add(field2);
- hiveSink.setSinkFields(fields);
- hiveSink.setTableName("{table.name}");
- hiveSink.setSinkName("{hive.sink.name}");
- return hiveSink;
- }
-
private List<StreamField> createStreamFields() {
List<StreamField> streamFieldList = Lists.newArrayList();
- streamFieldList.add(new StreamField(0, FieldType.STRING, "name", null, null));
- streamFieldList.add(new StreamField(1, FieldType.INT, "age", null, null));
+ streamFieldList.add(new StreamField(0, FieldType.STRING.toString(), "name", null, null));
+ streamFieldList.add(new StreamField(1, FieldType.INT.toString(), "age", null, null));
return streamFieldList;
}
}
diff --git a/inlong-manager/manager-client-tools/src/main/java/org/apache/inlong/manager/client/cli/CreateCommand.java b/inlong-manager/manager-client-tools/src/main/java/org/apache/inlong/manager/client/cli/CreateCommand.java
index f5e42798a..9a2c1e38c 100644
--- a/inlong-manager/manager-client-tools/src/main/java/org/apache/inlong/manager/client/cli/CreateCommand.java
+++ b/inlong-manager/manager-client-tools/src/main/java/org/apache/inlong/manager/client/cli/CreateCommand.java
@@ -71,7 +71,7 @@ public class CreateCommand extends AbstractCommand {
InlongStreamBuilder streamBuilder = group.createStream(groupConf.getStreamConf());
streamBuilder.fields(groupConf.getStreamFieldList());
streamBuilder.source(groupConf.getStreamSource());
- streamBuilder.sink(groupConf.getStreamSink());
+ streamBuilder.sink(groupConf.getSinkRequest());
streamBuilder.initOrUpdate();
group.init();
System.out.println("Create group success!");
diff --git a/inlong-manager/manager-client-tools/src/main/java/org/apache/inlong/manager/client/cli/pojo/CreateGroupConf.java b/inlong-manager/manager-client-tools/src/main/java/org/apache/inlong/manager/client/cli/pojo/CreateGroupConf.java
index b8cd8515c..9fa74a285 100644
--- a/inlong-manager/manager-client-tools/src/main/java/org/apache/inlong/manager/client/cli/pojo/CreateGroupConf.java
+++ b/inlong-manager/manager-client-tools/src/main/java/org/apache/inlong/manager/client/cli/pojo/CreateGroupConf.java
@@ -20,9 +20,9 @@ package org.apache.inlong.manager.client.cli.pojo;
import lombok.Data;
import org.apache.inlong.manager.client.api.InlongStreamConf;
import org.apache.inlong.manager.common.pojo.group.InlongGroupInfo;
+import org.apache.inlong.manager.common.pojo.sink.SinkRequest;
import org.apache.inlong.manager.common.pojo.stream.StreamField;
-import org.apache.inlong.manager.common.pojo.stream.StreamSink;
-import org.apache.inlong.manager.common.pojo.stream.StreamSource;
+import org.apache.inlong.manager.common.pojo.source.StreamSource;
import java.util.List;
@@ -36,5 +36,5 @@ public class CreateGroupConf {
private InlongStreamConf streamConf;
private List<StreamField> streamFieldList;
private StreamSource streamSource;
- private StreamSink streamSink;
+ private SinkRequest sinkRequest;
}
diff --git a/inlong-manager/manager-client-tools/src/main/java/org/apache/inlong/manager/client/cli/util/GsonUtils.java b/inlong-manager/manager-client-tools/src/main/java/org/apache/inlong/manager/client/cli/util/GsonUtils.java
index e89b53ecf..19a88548c 100644
--- a/inlong-manager/manager-client-tools/src/main/java/org/apache/inlong/manager/client/cli/util/GsonUtils.java
+++ b/inlong-manager/manager-client-tools/src/main/java/org/apache/inlong/manager/client/cli/util/GsonUtils.java
@@ -23,8 +23,8 @@ import org.apache.inlong.manager.common.enums.DataFormat;
import org.apache.inlong.manager.common.enums.DataSeparator;
import org.apache.inlong.manager.common.pojo.group.InlongGroupInfo;
import org.apache.inlong.manager.common.pojo.sort.BaseSortConf;
-import org.apache.inlong.manager.common.pojo.stream.StreamSink;
-import org.apache.inlong.manager.common.pojo.stream.StreamSource;
+import org.apache.inlong.manager.common.pojo.sink.StreamSink;
+import org.apache.inlong.manager.common.pojo.source.StreamSource;
import java.nio.charset.Charset;
diff --git a/inlong-manager/manager-client-tools/src/main/java/org/apache/inlong/manager/client/cli/util/SeparatorAdapter.java b/inlong-manager/manager-client-tools/src/main/java/org/apache/inlong/manager/client/cli/util/SeparatorAdapter.java
index 2f0948a08..12d19f311 100644
--- a/inlong-manager/manager-client-tools/src/main/java/org/apache/inlong/manager/client/cli/util/SeparatorAdapter.java
+++ b/inlong-manager/manager-client-tools/src/main/java/org/apache/inlong/manager/client/cli/util/SeparatorAdapter.java
@@ -33,6 +33,6 @@ public class SeparatorAdapter implements JsonDeserializer<DataSeparator> {
@Override
public DataSeparator deserialize(JsonElement jsonElement, Type type, JsonDeserializationContext context)
throws JsonParseException {
- return DataSeparator.forAscii(jsonElement.getAsCharacter());
+ return DataSeparator.forAscii(jsonElement.getAsInt());
}
}
diff --git a/inlong-manager/manager-client-tools/src/main/java/org/apache/inlong/manager/client/cli/util/StreamSinkAdapter.java b/inlong-manager/manager-client-tools/src/main/java/org/apache/inlong/manager/client/cli/util/StreamSinkAdapter.java
index 477ef6b1d..c05c8e8cf 100644
--- a/inlong-manager/manager-client-tools/src/main/java/org/apache/inlong/manager/client/cli/util/StreamSinkAdapter.java
+++ b/inlong-manager/manager-client-tools/src/main/java/org/apache/inlong/manager/client/cli/util/StreamSinkAdapter.java
@@ -23,11 +23,7 @@ import com.google.gson.JsonDeserializer;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParseException;
-import org.apache.inlong.manager.client.api.sink.ClickHouseSink;
-import org.apache.inlong.manager.client.api.sink.HiveSink;
-import org.apache.inlong.manager.client.api.sink.KafkaSink;
-import org.apache.inlong.manager.common.enums.SinkType;
-import org.apache.inlong.manager.common.pojo.stream.StreamSink;
+import org.apache.inlong.manager.common.pojo.sink.StreamSink;
import java.lang.reflect.Type;
@@ -36,6 +32,9 @@ import java.lang.reflect.Type;
*/
public class StreamSinkAdapter implements JsonDeserializer<StreamSink> {
+ /**
+ * TODO refactor by {@link org.apache.inlong.manager.common.util.JsonTypeDefine}
+ */
@Override
public StreamSink deserialize(JsonElement jsonElement, Type type, JsonDeserializationContext context)
throws JsonParseException {
@@ -43,7 +42,7 @@ public class StreamSinkAdapter implements JsonDeserializer<StreamSink> {
String sinkType = jsonObject.get("sinkType").getAsString();
Gson gson = GsonUtils.GSON;
try {
- switch (sinkType) {
+ /*switch (sinkType) {
case SinkType.SINK_HIVE:
return gson.fromJson(jsonElement, (Type) Class.forName((HiveSink.class).getName()));
case SinkType.SINK_KAFKA:
@@ -52,8 +51,9 @@ public class StreamSinkAdapter implements JsonDeserializer<StreamSink> {
return gson.fromJson(jsonElement, (Type) Class.forName((ClickHouseSink.class).getName()));
default:
throw new IllegalArgumentException(String.format("Unsupported sink type=%s for Inlong", sinkType));
- }
- } catch (Exception e) {
+ }*/
+ } catch (
+ Exception e) {
e.printStackTrace();
}
return null;
diff --git a/inlong-manager/manager-client-tools/src/main/java/org/apache/inlong/manager/client/cli/util/StreamSourceAdapter.java b/inlong-manager/manager-client-tools/src/main/java/org/apache/inlong/manager/client/cli/util/StreamSourceAdapter.java
index 38387ce2d..2fb1c0173 100644
--- a/inlong-manager/manager-client-tools/src/main/java/org/apache/inlong/manager/client/cli/util/StreamSourceAdapter.java
+++ b/inlong-manager/manager-client-tools/src/main/java/org/apache/inlong/manager/client/cli/util/StreamSourceAdapter.java
@@ -27,7 +27,7 @@ import org.apache.inlong.manager.client.api.source.AgentFileSource;
import org.apache.inlong.manager.client.api.source.KafkaSource;
import org.apache.inlong.manager.client.api.source.MySQLBinlogSource;
import org.apache.inlong.manager.common.enums.SourceType;
-import org.apache.inlong.manager.common.pojo.stream.StreamSource;
+import org.apache.inlong.manager.common.pojo.source.StreamSource;
import java.lang.reflect.Type;
diff --git a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/InlongGroupContext.java b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/InlongGroupContext.java
index d98648db8..28e053d65 100644
--- a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/InlongGroupContext.java
+++ b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/InlongGroupContext.java
@@ -28,8 +28,8 @@ import org.apache.inlong.manager.client.api.util.GsonUtils;
import org.apache.inlong.manager.common.enums.GroupStatus;
import org.apache.inlong.manager.common.pojo.group.InlongGroupExtInfo;
import org.apache.inlong.manager.common.pojo.group.InlongGroupInfo;
-import org.apache.inlong.manager.common.pojo.stream.StreamSource;
-import org.apache.inlong.manager.common.pojo.stream.StreamSource.State;
+import org.apache.inlong.manager.common.pojo.source.StreamSource;
+import org.apache.inlong.manager.common.pojo.source.StreamSource.Status;
import org.apache.inlong.manager.common.util.AssertUtils;
import java.io.Serializable;
@@ -107,7 +107,7 @@ public class InlongGroupContext implements Serializable {
StreamSource source = entry.getValue();
if (source != null) {
sourcesInGroup.add(source);
- if (source.getState() == State.FAILED) {
+ if (source.getStatus() == Status.FAILED) {
failedSources.add(source);
}
}
@@ -127,7 +127,7 @@ public class InlongGroupContext implements Serializable {
switch (this.status) {
case STARTED:
for (StreamSource source : sourcesInGroup) {
- if (source.getState() != State.NORMAL) {
+ if (source.getStatus() != Status.NORMAL) {
log.warn("StreamSource:{} is not started", source);
this.status = InlongGroupStatus.INITIALIZING;
break;
@@ -136,7 +136,7 @@ public class InlongGroupContext implements Serializable {
return;
case STOPPED:
for (StreamSource source : sourcesInGroup) {
- if (source.getState() != State.FROZEN) {
+ if (source.getStatus() != Status.FROZEN) {
log.warn("StreamSource:{} is not stopped", source);
this.status = InlongGroupStatus.OPERATING;
break;
diff --git a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/InlongStream.java b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/InlongStream.java
index 2c0a01f76..7adef1bcc 100644
--- a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/InlongStream.java
+++ b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/InlongStream.java
@@ -17,10 +17,10 @@
package org.apache.inlong.manager.client.api;
+import org.apache.inlong.manager.common.pojo.sink.SinkRequest;
+import org.apache.inlong.manager.common.pojo.source.StreamSource;
import org.apache.inlong.manager.common.pojo.stream.StreamField;
import org.apache.inlong.manager.common.pojo.stream.StreamPipeline;
-import org.apache.inlong.manager.common.pojo.stream.StreamSink;
-import org.apache.inlong.manager.common.pojo.stream.StreamSource;
import org.apache.inlong.manager.common.pojo.stream.StreamTransform;
import java.util.List;
@@ -29,86 +29,87 @@ import java.util.Map;
/**
* Inlong stream.
*/
-public abstract class InlongStream {
+public interface InlongStream {
/**
* Return name of stream.
*/
- public abstract String getStreamId();
+ String getInlongStreamId();
/**
* Return field definitions of stream.
*/
- public abstract List<StreamField> listFields();
+ List<StreamField> getStreamFields();
/**
* Return data sources defined in stream, key is source name which must be unique within one stream scope.
*/
- public abstract Map<String, StreamSource> getSources();
+ Map<String, StreamSource> getSources();
/**
- * Return data sinks defined in stream, key is sink name which must be unique within one stream scope.
+ * Return sinks defined in stream, key is sink name which must be unique within one stream scope.
*/
- public abstract Map<String, StreamSink> getSinks();
+ Map<String, SinkRequest> getSinks();
/**
* Return data transform node defined in stream(split,string replace etc)
* key is transform name which must be unique within one stream scope.
*/
- public abstract Map<String, StreamTransform> getTransforms();
+ Map<String, StreamTransform> getTransforms();
/**
* Add data source to stream, this method will throw exception when source name already exists in stream.
*/
- public abstract InlongStream addSource(StreamSource source);
+ InlongStream addSource(StreamSource source);
/**
- * Add data sink to stream, this method will throw exception when sink name already exists in stream.
+ * Add sink to stream, this method will throw exception when sink name already exists in stream.
*/
- public abstract InlongStream addSink(StreamSink sink);
+ InlongStream addSink(SinkRequest sinkRequest);
/**
* Add data transform node to stream, this method will throw exception when transform name already exists in stream.
*/
- public abstract InlongStream addTransform(StreamTransform transform);
+ InlongStream addTransform(StreamTransform transform);
/**
* Delete data source by source name.
*/
- public abstract InlongStream deleteSource(String sourceName);
+ InlongStream deleteSource(String sourceName);
/**
* Delete data sink by sink name.
*/
- public abstract InlongStream deleteSink(String sinkName);
+ InlongStream deleteSink(String sinkName);
/**
* Delete data transform node by transform name.
*/
- public abstract InlongStream deleteTransform(String transformName);
+ InlongStream deleteTransform(String transformName);
/**
* Update data source by source name, add new one if source name not exists.
*/
- public abstract InlongStream updateSource(StreamSource source);
+ InlongStream updateSource(StreamSource source);
/**
- * Update data sink by sink name, add new one if sink name not exists.
+ * Update sink by sink name, add new one if sink name not exists.
*/
- public abstract InlongStream updateSink(StreamSink sink);
+ InlongStream updateSink(SinkRequest sink);
/**
* Update data transform node by transform name, add new one if transform name not exists.
*/
- public abstract InlongStream updateTransform(StreamTransform transform);
+ InlongStream updateTransform(StreamTransform transform);
/**
* Create stream pipeline by sources, transforms, sinks defined in stream.
*/
- public abstract StreamPipeline createPipeline();
+ StreamPipeline createPipeline();
/**
* Update stream definition in manager service, which must be invoked after add/delete/update source/sink/transform.
*/
- public abstract InlongStream update();
+ InlongStream update();
+
}
diff --git a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/InlongStreamBuilder.java b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/InlongStreamBuilder.java
index 1c7bd0d07..b8bbe15ac 100644
--- a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/InlongStreamBuilder.java
+++ b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/InlongStreamBuilder.java
@@ -17,9 +17,9 @@
package org.apache.inlong.manager.client.api;
+import org.apache.inlong.manager.common.pojo.sink.SinkRequest;
+import org.apache.inlong.manager.common.pojo.source.StreamSource;
import org.apache.inlong.manager.common.pojo.stream.StreamField;
-import org.apache.inlong.manager.common.pojo.stream.StreamSink;
-import org.apache.inlong.manager.common.pojo.stream.StreamSource;
import org.apache.inlong.manager.common.pojo.stream.StreamTransform;
import java.util.List;
@@ -37,12 +37,12 @@ public abstract class InlongStreamBuilder {
public abstract InlongStreamBuilder source(StreamSource source);
/**
- * create sink in stream.
+ * create sinkRequest in stream.
* *
*
* @return inlong stream builder
*/
- public abstract InlongStreamBuilder sink(StreamSink sink);
+ public abstract InlongStreamBuilder sink(SinkRequest sinkRequest);
/**
* Create or update stream fields.
diff --git a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/InlongStreamConf.java b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/InlongStreamConf.java
index e9a98f41f..61f1855e8 100644
--- a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/InlongStreamConf.java
+++ b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/InlongStreamConf.java
@@ -39,12 +39,12 @@ import java.util.List;
public class InlongStreamConf {
@ApiModelProperty(value = "Inlong stream ID", required = true)
- private String streamId;
+ private String inlongStreamId;
@ApiModelProperty(value = "Inlong stream name")
private String name;
- @ApiModelProperty(value = "MQ resource for stream. Default: ${streamId}")
+ @ApiModelProperty(value = "MQ resource for stream. Default: ${inlongStreamId}")
private String mqResource;
@ApiModelProperty("Inlong stream description")
diff --git a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/impl/DefaultInlongStreamBuilder.java b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/impl/DefaultInlongStreamBuilder.java
index 94f1390dd..ae9e3086c 100644
--- a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/impl/DefaultInlongStreamBuilder.java
+++ b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/impl/DefaultInlongStreamBuilder.java
@@ -29,20 +29,18 @@ import org.apache.inlong.manager.client.api.inner.InnerGroupContext;
import org.apache.inlong.manager.client.api.inner.InnerInlongManagerClient;
import org.apache.inlong.manager.client.api.inner.InnerStreamContext;
import org.apache.inlong.manager.client.api.util.GsonUtils;
-import org.apache.inlong.manager.client.api.util.InlongStreamSinkTransfer;
import org.apache.inlong.manager.client.api.util.InlongStreamSourceTransfer;
import org.apache.inlong.manager.client.api.util.InlongStreamTransfer;
import org.apache.inlong.manager.client.api.util.InlongStreamTransformTransfer;
+import org.apache.inlong.manager.common.pojo.group.InlongGroupInfo;
import org.apache.inlong.manager.common.pojo.sink.SinkListResponse;
import org.apache.inlong.manager.common.pojo.sink.SinkRequest;
import org.apache.inlong.manager.common.pojo.source.SourceListResponse;
import org.apache.inlong.manager.common.pojo.source.SourceRequest;
-import org.apache.inlong.manager.common.pojo.stream.InlongStreamFieldInfo;
+import org.apache.inlong.manager.common.pojo.source.StreamSource;
import org.apache.inlong.manager.common.pojo.stream.InlongStreamInfo;
import org.apache.inlong.manager.common.pojo.stream.StreamField;
import org.apache.inlong.manager.common.pojo.stream.StreamPipeline;
-import org.apache.inlong.manager.common.pojo.stream.StreamSink;
-import org.apache.inlong.manager.common.pojo.stream.StreamSource;
import org.apache.inlong.manager.common.pojo.stream.StreamTransform;
import org.apache.inlong.manager.common.pojo.transform.TransformRequest;
import org.apache.inlong.manager.common.pojo.transform.TransformResponse;
@@ -55,26 +53,25 @@ import java.util.Map;
*/
public class DefaultInlongStreamBuilder extends InlongStreamBuilder {
- private InlongStreamImpl inlongStream;
+ private final InlongStreamImpl inlongStream;
- private InnerStreamContext streamContext;
+ private final InnerStreamContext streamContext;
- private InnerInlongManagerClient managerClient;
+ private final InnerInlongManagerClient managerClient;
- public DefaultInlongStreamBuilder(
- InlongStreamConf streamConf,
- InnerGroupContext groupContext,
+ public DefaultInlongStreamBuilder(InlongStreamConf streamConf, InnerGroupContext groupContext,
InnerInlongManagerClient managerClient) {
this.managerClient = managerClient;
if (MapUtils.isEmpty(groupContext.getStreamContextMap())) {
groupContext.setStreamContextMap(Maps.newHashMap());
}
- InlongStreamInfo stream = InlongStreamTransfer.createStreamInfo(streamConf, groupContext.getGroupInfo());
+
+ InlongGroupInfo groupInfo = groupContext.getGroupInfo();
+ InlongStreamInfo stream = InlongStreamTransfer.createStreamInfo(streamConf, groupInfo);
InnerStreamContext streamContext = new InnerStreamContext(stream);
groupContext.setStreamContext(streamContext);
this.streamContext = streamContext;
- this.inlongStream = new InlongStreamImpl(groupContext.getGroupInfo().getName(), stream.getName(),
- managerClient);
+ this.inlongStream = new InlongStreamImpl(groupInfo.getName(), stream.getName(), managerClient);
if (CollectionUtils.isNotEmpty(streamConf.getStreamFields())) {
this.inlongStream.setStreamFields(streamConf.getStreamFields());
}
@@ -91,9 +88,11 @@ public class DefaultInlongStreamBuilder extends InlongStreamBuilder {
}
@Override
- public InlongStreamBuilder sink(StreamSink sink) {
- inlongStream.addSink(sink);
- SinkRequest sinkRequest = InlongStreamSinkTransfer.createSinkRequest(sink, streamContext.getStreamInfo());
+ public InlongStreamBuilder sink(SinkRequest sinkRequest) {
+ InlongStreamInfo streamInfo = streamContext.getStreamInfo();
+ sinkRequest.setInlongGroupId(streamInfo.getInlongGroupId());
+ sinkRequest.setInlongStreamId(streamInfo.getInlongStreamId());
+ inlongStream.addSink(sinkRequest);
streamContext.setSinkRequest(sinkRequest);
return this;
}
@@ -101,9 +100,9 @@ public class DefaultInlongStreamBuilder extends InlongStreamBuilder {
@Override
public InlongStreamBuilder fields(List<StreamField> fieldList) {
inlongStream.setStreamFields(fieldList);
- List<InlongStreamFieldInfo> streamFieldInfos = InlongStreamTransfer.createStreamFields(fieldList,
+ List<StreamField> streamFields = InlongStreamTransfer.createStreamFields(fieldList,
streamContext.getStreamInfo());
- streamContext.updateStreamFields(streamFieldInfos);
+ streamContext.updateStreamFields(streamFields);
return this;
}
diff --git a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/impl/InlongClientImpl.java b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/impl/InlongClientImpl.java
index 25b053337..0dff892a5 100644
--- a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/impl/InlongClientImpl.java
+++ b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/impl/InlongClientImpl.java
@@ -35,7 +35,7 @@ import org.apache.inlong.manager.common.pojo.group.InlongGroupInfo;
import org.apache.inlong.manager.common.pojo.group.InlongGroupListResponse;
import org.apache.inlong.manager.common.pojo.group.InlongGroupPageRequest;
import org.apache.inlong.manager.common.pojo.source.SourceListResponse;
-import org.apache.inlong.manager.common.pojo.stream.StreamSource.State;
+import org.apache.inlong.manager.common.pojo.source.StreamSource.Status;
import org.apache.inlong.manager.common.util.HttpUtils;
import java.util.List;
@@ -137,29 +137,29 @@ public class InlongClientImpl implements InlongClient {
private InlongGroupStatus recheckGroupState(InlongGroupStatus groupState,
List<SourceListResponse> sourceListResponses) {
- Map<State, List<SourceListResponse>> stateListMap = Maps.newHashMap();
+ Map<Status, List<SourceListResponse>> stateListMap = Maps.newHashMap();
sourceListResponses.forEach(sourceListResponse -> {
- State state = State.parseByStatus(sourceListResponse.getStatus());
- stateListMap.computeIfAbsent(state, k -> Lists.newArrayList()).add(sourceListResponse);
+ Status status = Status.parseByStatus(sourceListResponse.getStatus());
+ stateListMap.computeIfAbsent(status, k -> Lists.newArrayList()).add(sourceListResponse);
});
- if (CollectionUtils.isNotEmpty(stateListMap.get(State.FAILED))) {
+ if (CollectionUtils.isNotEmpty(stateListMap.get(Status.FAILED))) {
return InlongGroupStatus.FAILED;
}
switch (groupState) {
case STARTED:
- if (CollectionUtils.isNotEmpty(stateListMap.get(State.INIT))) {
+ if (CollectionUtils.isNotEmpty(stateListMap.get(Status.INIT))) {
return InlongGroupStatus.INITIALIZING;
} else {
return groupState;
}
case STOPPED:
- if (CollectionUtils.isNotEmpty(stateListMap.get(State.FROZING))) {
+ if (CollectionUtils.isNotEmpty(stateListMap.get(Status.FREEZING))) {
return InlongGroupStatus.OPERATING;
} else {
return groupState;
}
case DELETED:
- if (CollectionUtils.isNotEmpty(stateListMap.get(State.DELETING))) {
+ if (CollectionUtils.isNotEmpty(stateListMap.get(Status.DELETING))) {
return InlongGroupStatus.OPERATING;
} else {
return groupState;
diff --git a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/impl/InlongGroupImpl.java b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/impl/InlongGroupImpl.java
index 61474236f..0efd08c8b 100644
--- a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/impl/InlongGroupImpl.java
+++ b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/impl/InlongGroupImpl.java
@@ -222,7 +222,7 @@ public class InlongGroupImpl implements InlongGroup {
@Override
public List<InlongStream> listStreams() throws Exception {
String inlongGroupId = this.groupContext.getGroupId();
- return fetchDataStreams(inlongGroupId);
+ return fetchInlongStreams(inlongGroupId);
}
private InlongGroupContext generateSnapshot() {
@@ -231,7 +231,7 @@ public class InlongGroupImpl implements InlongGroup {
groupContext.setGroupInfo(groupInfo);
String inlongGroupId = groupInfo.getInlongGroupId();
// fetch stream in group
- List<InlongStream> dataStreams = fetchDataStreams(inlongGroupId);
+ List<InlongStream> dataStreams = fetchInlongStreams(inlongGroupId);
if (CollectionUtils.isNotEmpty(dataStreams)) {
dataStreams.forEach(groupContext::setStream);
}
@@ -276,13 +276,13 @@ public class InlongGroupImpl implements InlongGroup {
return inlongGroupContext;
}
- private List<InlongStream> fetchDataStreams(String groupId) {
+ private List<InlongStream> fetchInlongStreams(String groupId) {
List<FullStreamResponse> streamResponses = managerClient.listStreamInfo(groupId);
if (CollectionUtils.isEmpty(streamResponses)) {
return null;
}
return streamResponses.stream()
- .map(fullStreamResponse -> new InlongStreamImpl(fullStreamResponse, managerClient))
+ .map(response -> new InlongStreamImpl(response, managerClient))
.collect(Collectors.toList());
}
}
diff --git a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/impl/InlongStreamImpl.java b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/impl/InlongStreamImpl.java
index 7345ff2b8..0900269a8 100644
--- a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/impl/InlongStreamImpl.java
+++ b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/impl/InlongStreamImpl.java
@@ -21,32 +21,29 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import lombok.Data;
-import lombok.EqualsAndHashCode;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.inlong.manager.client.api.InlongStream;
import org.apache.inlong.manager.client.api.inner.InnerInlongManagerClient;
import org.apache.inlong.manager.client.api.util.GsonUtils;
-import org.apache.inlong.manager.client.api.util.InlongStreamSinkTransfer;
import org.apache.inlong.manager.client.api.util.InlongStreamSourceTransfer;
import org.apache.inlong.manager.client.api.util.InlongStreamTransfer;
import org.apache.inlong.manager.client.api.util.InlongStreamTransformTransfer;
-import org.apache.inlong.manager.common.enums.FieldType;
+import org.apache.inlong.manager.client.api.util.StreamSinkTransfer;
import org.apache.inlong.manager.common.pojo.sink.SinkListResponse;
import org.apache.inlong.manager.common.pojo.sink.SinkRequest;
import org.apache.inlong.manager.common.pojo.sink.SinkResponse;
+import org.apache.inlong.manager.common.pojo.sink.StreamSink;
import org.apache.inlong.manager.common.pojo.source.SourceListResponse;
import org.apache.inlong.manager.common.pojo.source.SourceRequest;
import org.apache.inlong.manager.common.pojo.source.SourceResponse;
+import org.apache.inlong.manager.common.pojo.source.StreamSource;
import org.apache.inlong.manager.common.pojo.stream.FullStreamResponse;
-import org.apache.inlong.manager.common.pojo.stream.InlongStreamFieldInfo;
import org.apache.inlong.manager.common.pojo.stream.InlongStreamInfo;
import org.apache.inlong.manager.common.pojo.stream.StreamField;
import org.apache.inlong.manager.common.pojo.stream.StreamNodeRelationship;
import org.apache.inlong.manager.common.pojo.stream.StreamPipeline;
-import org.apache.inlong.manager.common.pojo.stream.StreamSink;
-import org.apache.inlong.manager.common.pojo.stream.StreamSource;
import org.apache.inlong.manager.common.pojo.stream.StreamTransform;
import org.apache.inlong.manager.common.pojo.transform.TransformRequest;
import org.apache.inlong.manager.common.pojo.transform.TransformResponse;
@@ -61,18 +58,17 @@ import java.util.stream.Collectors;
* Inlong stream service implementation.
*/
@Data
-@EqualsAndHashCode(callSuper = true)
-public class InlongStreamImpl extends InlongStream {
+public class InlongStreamImpl implements InlongStream {
private InnerInlongManagerClient managerClient;
- private String groupId;
+ private String inlongGroupId;
- private String streamId;
+ private String inlongStreamId;
private Map<String, StreamSource> streamSources = Maps.newHashMap();
- private Map<String, StreamSink> streamSinks = Maps.newHashMap();
+ private Map<String, SinkRequest> sinkRequests = Maps.newHashMap();
private Map<String, StreamTransform> streamTransforms = Maps.newHashMap();
@@ -81,17 +77,17 @@ public class InlongStreamImpl extends InlongStream {
/**
* Constructor of InlongStreamImpl.
*/
- public InlongStreamImpl(FullStreamResponse fullStreamResponse, InnerInlongManagerClient managerClient) {
- InlongStreamInfo streamInfo = fullStreamResponse.getStreamInfo();
+ public InlongStreamImpl(FullStreamResponse streamResponse, InnerInlongManagerClient managerClient) {
+ InlongStreamInfo streamInfo = streamResponse.getStreamInfo();
this.managerClient = managerClient;
- this.streamId = streamInfo.getInlongStreamId();
- this.groupId = streamInfo.getInlongGroupId();
- List<InlongStreamFieldInfo> streamFieldInfos = streamInfo.getFieldList();
- if (CollectionUtils.isNotEmpty(streamFieldInfos)) {
- this.streamFields = streamFieldInfos.stream()
+ this.inlongGroupId = streamInfo.getInlongGroupId();
+ this.inlongStreamId = streamInfo.getInlongStreamId();
+ List<StreamField> streamFields = streamInfo.getFieldList();
+ if (CollectionUtils.isNotEmpty(streamFields)) {
+ this.streamFields = streamFields.stream()
.map(fieldInfo -> new StreamField(
fieldInfo.getId(),
- FieldType.forName(fieldInfo.getFieldType()),
+ fieldInfo.getFieldType(),
fieldInfo.getFieldName(),
fieldInfo.getFieldComment(),
fieldInfo.getFieldValue(),
@@ -100,40 +96,37 @@ public class InlongStreamImpl extends InlongStream {
)
).collect(Collectors.toList());
}
- List<SinkResponse> sinkList = fullStreamResponse.getSinkInfo();
- if (CollectionUtils.isNotEmpty(sinkList)) {
- this.streamSinks = sinkList.stream()
- .map(sinkResponse -> InlongStreamSinkTransfer.parseStreamSink(sinkResponse, null))
- .collect(Collectors.toMap(StreamSink::getSinkName, streamSink -> streamSink,
+ List<SinkResponse> responseList = streamResponse.getSinkInfo();
+ if (CollectionUtils.isNotEmpty(responseList)) {
+ this.sinkRequests = responseList.stream()
+ .map(StreamSinkTransfer::parseSinkRequest)
+ .collect(Collectors.toMap(SinkRequest::getSinkName, sinkRequest -> sinkRequest,
(sink1, sink2) -> {
- throw new RuntimeException(
- String.format("duplicate sinkName:%s in stream:%s", sink1.getSinkName(),
- this.streamId));
+ throw new RuntimeException(String.format("duplicate sinkName:%s in stream:%s",
+ sink1.getSinkName(), this.inlongStreamId));
}));
}
- List<SourceResponse> sourceList = fullStreamResponse.getSourceInfo();
+ List<SourceResponse> sourceList = streamResponse.getSourceInfo();
if (CollectionUtils.isNotEmpty(sourceList)) {
this.streamSources = sourceList.stream()
.map(InlongStreamSourceTransfer::parseStreamSource)
.collect(Collectors.toMap(StreamSource::getSourceName, streamSource -> streamSource,
(source1, source2) -> {
- throw new RuntimeException(
- String.format("duplicate sourceName:%s in stream:%s",
- source1.getSourceName(), this.streamId));
+ throw new RuntimeException(String.format("duplicate sourceName:%s in stream:%s",
+ source1.getSourceName(), this.inlongStreamId));
}
));
}
-
}
- public InlongStreamImpl(String group, String streamId, InnerInlongManagerClient managerClient) {
+ public InlongStreamImpl(String groupId, String streamId, InnerInlongManagerClient managerClient) {
this.managerClient = managerClient;
- this.groupId = group;
- this.streamId = streamId;
+ this.inlongGroupId = groupId;
+ this.inlongStreamId = streamId;
}
@Override
- public List<StreamField> listFields() {
+ public List<StreamField> getStreamFields() {
return this.streamFields;
}
@@ -143,8 +136,8 @@ public class InlongStreamImpl extends InlongStream {
}
@Override
- public Map<String, StreamSink> getSinks() {
- return this.streamSinks;
+ public Map<String, SinkRequest> getSinks() {
+ return this.sinkRequests;
}
@Override
@@ -164,13 +157,13 @@ public class InlongStreamImpl extends InlongStream {
}
@Override
- public InlongStream addSink(StreamSink sink) {
- AssertUtils.notNull(sink.getSinkName(), "Sink name should not be empty");
- String sinkName = sink.getSinkName();
- if (streamSinks.get(sinkName) != null) {
- throw new IllegalArgumentException(String.format("StreamSink=%s has already be set", sink));
+ public InlongStream addSink(SinkRequest sinkRequest) {
+ AssertUtils.notNull(sinkRequest.getSinkName(), "Sink name should not be empty");
+ String sinkName = sinkRequest.getSinkName();
+ if (sinkRequests.get(sinkName) != null) {
+ throw new IllegalArgumentException(String.format("StreamSink=%s has already be set", sinkRequest));
}
- streamSinks.put(sinkName, sink);
+ sinkRequests.put(sinkName, sinkRequest);
return this;
}
@@ -193,7 +186,7 @@ public class InlongStreamImpl extends InlongStream {
@Override
public InlongStream deleteSink(String sinkName) {
- streamSinks.remove(sinkName);
+ sinkRequests.remove(sinkName);
return this;
}
@@ -211,9 +204,9 @@ public class InlongStreamImpl extends InlongStream {
}
@Override
- public InlongStream updateSink(StreamSink sink) {
+ public InlongStream updateSink(SinkRequest sink) {
AssertUtils.notNull(sink.getSinkName(), "Sink name should not be empty");
- streamSinks.put(sink.getSinkName(), sink);
+ sinkRequests.put(sink.getSinkName(), sink);
return this;
}
@@ -230,7 +223,7 @@ public class InlongStreamImpl extends InlongStream {
if (MapUtils.isEmpty(streamTransforms)) {
StreamNodeRelationship relationship = new StreamNodeRelationship();
relationship.setInputNodes(streamSources.keySet());
- relationship.setOutputNodes(streamSinks.keySet());
+ relationship.setOutputNodes(sinkRequests.keySet());
streamPipeline.setPipeline(Lists.newArrayList(relationship));
return streamPipeline;
}
@@ -257,7 +250,7 @@ public class InlongStreamImpl extends InlongStream {
Set<String> postNodes = streamTransform.getPostNodes();
Set<String> sinkSet = Sets.newHashSet();
for (String postNode : postNodes) {
- StreamSink sink = streamSinks.get(postNode);
+ StreamSink sink = sinkRequests.get(postNode);
if (sink != null) {
sinkSet.add(sink.getSinkName());
}
@@ -299,12 +292,12 @@ public class InlongStreamImpl extends InlongStream {
@Override
public InlongStream update() {
InlongStreamInfo streamInfo = new InlongStreamInfo();
- streamInfo.setInlongStreamId(streamId);
- streamInfo.setInlongGroupId(groupId);
+ streamInfo.setInlongStreamId(inlongStreamId);
+ streamInfo.setInlongGroupId(inlongGroupId);
streamInfo = managerClient.getStreamInfo(streamInfo);
if (streamInfo == null) {
throw new IllegalArgumentException(
- String.format("Stream is not exists for group=%s and stream=%s", groupId, streamId));
+ String.format("Stream is not exists for group=%s and stream=%s", inlongGroupId, inlongStreamId));
}
streamInfo.setFieldList(InlongStreamTransfer.createStreamFields(this.streamFields, streamInfo));
StreamPipeline streamPipeline = createPipeline();
@@ -320,7 +313,7 @@ public class InlongStreamImpl extends InlongStream {
}
private void initOrUpdateTransform(InlongStreamInfo streamInfo) {
- List<TransformResponse> transformResponses = managerClient.listTransform(groupId, streamId);
+ List<TransformResponse> transformResponses = managerClient.listTransform(inlongGroupId, inlongStreamId);
List<String> updateTransformNames = Lists.newArrayList();
for (TransformResponse transformResponse : transformResponses) {
StreamTransform transform = InlongStreamTransformTransfer.parseStreamTransform(transformResponse);
@@ -359,7 +352,7 @@ public class InlongStreamImpl extends InlongStream {
}
private void initOrUpdateSource(InlongStreamInfo streamInfo) {
- List<SourceListResponse> sourceListResponses = managerClient.listSources(groupId, streamId);
+ List<SourceListResponse> sourceListResponses = managerClient.listSources(inlongGroupId, inlongStreamId);
List<String> updateSourceNames = Lists.newArrayList();
for (SourceListResponse sourceListResponse : sourceListResponses) {
final String sourceName = sourceListResponse.getSourceName();
@@ -393,20 +386,22 @@ public class InlongStreamImpl extends InlongStream {
}
private void initOrUpdateSink(InlongStreamInfo streamInfo) {
- List<SinkListResponse> sinkListResponses = managerClient.listSinks(groupId, streamId);
+ List<SinkListResponse> sinkListResponses = managerClient.listSinks(inlongGroupId, inlongStreamId);
+ // delete or update the sink info
List<String> updateSinkNames = Lists.newArrayList();
- for (SinkListResponse sinkListResponse : sinkListResponses) {
- final String sinkName = sinkListResponse.getSinkName();
- final int id = sinkListResponse.getId();
- if (this.streamSinks.get(sinkName) == null) {
+ for (SinkListResponse sinkResponse : sinkListResponses) {
+ final String sinkName = sinkResponse.getSinkName();
+ final int id = sinkResponse.getId();
+ if (this.sinkRequests.get(sinkName) == null) {
boolean isDelete = managerClient.deleteSink(id);
if (!isDelete) {
- throw new RuntimeException(String.format("Delete sink=%s failed", sinkListResponse));
+ throw new RuntimeException(String.format("Delete sink=%s failed", sinkResponse));
}
} else {
- StreamSink sink = this.streamSinks.get(sinkName);
- SinkRequest sinkRequest = InlongStreamSinkTransfer.createSinkRequest(sink, streamInfo);
+ SinkRequest sinkRequest = this.sinkRequests.get(sinkName);
sinkRequest.setId(id);
+ sinkRequest.setInlongGroupId(streamInfo.getInlongGroupId());
+ sinkRequest.setInlongStreamId(streamInfo.getInlongStreamId());
Pair<Boolean, String> updateState = managerClient.updateSink(sinkRequest);
if (!updateState.getKey()) {
throw new RuntimeException(String.format("Update sink=%s failed with err=%s", sinkRequest,
@@ -415,13 +410,16 @@ public class InlongStreamImpl extends InlongStream {
updateSinkNames.add(sinkName);
}
}
- for (Map.Entry<String, StreamSink> requestEntry : streamSinks.entrySet()) {
+
+ // create sink info after deleting or updating
+ for (Map.Entry<String, SinkRequest> requestEntry : sinkRequests.entrySet()) {
String sinkName = requestEntry.getKey();
if (updateSinkNames.contains(sinkName)) {
continue;
}
- StreamSink streamSink = requestEntry.getValue();
- SinkRequest sinkRequest = InlongStreamSinkTransfer.createSinkRequest(streamSink, streamInfo);
+ SinkRequest sinkRequest = requestEntry.getValue();
+ sinkRequest.setInlongGroupId(streamInfo.getInlongGroupId());
+ sinkRequest.setInlongStreamId(streamInfo.getInlongStreamId());
managerClient.createSink(sinkRequest);
}
}
diff --git a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/inner/InnerGroupContext.java b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/inner/InnerGroupContext.java
index 8b5f48d2b..dacd660a1 100644
--- a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/inner/InnerGroupContext.java
+++ b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/inner/InnerGroupContext.java
@@ -66,7 +66,7 @@ public class InnerGroupContext {
if (MapUtils.isEmpty(streamMap)) {
streamMap = Maps.newHashMap();
}
- streamMap.put(stream.getStreamId(), stream);
+ streamMap.put(stream.getInlongStreamId(), stream);
}
public InlongStream getStream(String streamId) {
diff --git a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/inner/InnerInlongManagerClient.java b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/inner/InnerInlongManagerClient.java
index 4515d0289..f3c503300 100644
--- a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/inner/InnerInlongManagerClient.java
+++ b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/inner/InnerInlongManagerClient.java
@@ -810,8 +810,7 @@ public class InnerInlongManagerClient {
String.format("Inlong request failed: %s", responseBody.getErrMsg()));
return responseBody.getData();
} catch (Exception e) {
- throw new RuntimeException(String.format("Inlong group init failed: %s", e.getMessage()),
- e);
+ throw new RuntimeException(String.format("Inlong group init failed: %s", e.getMessage()), e);
}
}
diff --git a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/inner/InnerStreamContext.java b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/inner/InnerStreamContext.java
index dd385f841..c47c0be17 100644
--- a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/inner/InnerStreamContext.java
+++ b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/inner/InnerStreamContext.java
@@ -22,15 +22,15 @@ import lombok.Data;
import lombok.NoArgsConstructor;
import org.apache.inlong.manager.common.pojo.sink.SinkRequest;
import org.apache.inlong.manager.common.pojo.source.SourceRequest;
-import org.apache.inlong.manager.common.pojo.stream.InlongStreamFieldInfo;
import org.apache.inlong.manager.common.pojo.stream.InlongStreamInfo;
+import org.apache.inlong.manager.common.pojo.stream.StreamField;
import org.apache.inlong.manager.common.pojo.transform.TransformRequest;
import java.util.List;
import java.util.Map;
/**
- * InnerStreamContext.
+ * Inner stream context.
*/
@Data
@NoArgsConstructor
@@ -48,8 +48,8 @@ public class InnerStreamContext {
this.streamInfo = streamInfo;
}
- public void updateStreamFields(List<InlongStreamFieldInfo> fieldInfoList) {
- streamInfo.setFieldList(fieldInfoList);
+ public void updateStreamFields(List<StreamField> fieldList) {
+ streamInfo.setFieldList(fieldList);
}
public void setSourceRequest(SourceRequest sourceRequest) {
diff --git a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/sink/ClickHouseSink.java b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/sink/ClickHouseSink.java
deleted file mode 100644
index d6697cbd1..000000000
--- a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/sink/ClickHouseSink.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.manager.client.api.sink;
-
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.NoArgsConstructor;
-import org.apache.inlong.manager.common.enums.DataFormat;
-import org.apache.inlong.manager.common.pojo.stream.SinkField;
-import org.apache.inlong.manager.common.pojo.stream.StreamSink;
-import org.apache.inlong.manager.common.auth.DefaultAuthentication;
-import org.apache.inlong.manager.common.enums.SinkType;
-
-import java.util.List;
-
-/**
- * ClickHouse sink.
- */
-@Data
-@EqualsAndHashCode(callSuper = true)
-@NoArgsConstructor
-@AllArgsConstructor
-@ApiModel("Clickhouse sink configuration")
-public class ClickHouseSink extends StreamSink {
-
- @ApiModelProperty(value = "Sink type", required = true)
- private SinkType sinkType = SinkType.CLICKHOUSE;
-
- @ApiModelProperty("ClickHouse JDBC URL")
- private String jdbcUrl;
-
- @ApiModelProperty("Target database name")
- private String dbName;
-
- @ApiModelProperty("Target table name")
- private String tableName;
-
- @ApiModelProperty("Authentication for clickhouse")
- private DefaultAuthentication authentication;
-
- @ApiModelProperty("Flush interval, unit: second, default is 1s")
- private Integer flushInterval;
-
- @ApiModelProperty("Flush when record number reaches flushRecord")
- private Integer flushRecord;
-
- @ApiModelProperty("Write max retry times, default is 3")
- private Integer retryTimes;
-
- @ApiModelProperty("Whether distributed table? 0: no, 1: yes")
- private Integer isDistributed;
-
- @ApiModelProperty("Partition strategy,support: BALANCE, RANDOM, HASH")
- private String partitionStrategy;
-
- @ApiModelProperty(value = "Partition files, separate with commas",
- notes = "Necessary when partitionStrategy is HASH, must be one of the field list")
- private String partitionFields;
-
- @ApiModelProperty("Key field names, separate with commas")
- private String keyFieldNames;
-
- @ApiModelProperty("Create topic or not")
- private boolean needCreated;
-
- @ApiModelProperty("Field definitions for clickhouse")
- private List<SinkField> sinkFields;
-
- @Override
- public DataFormat getDataFormat() {
- return DataFormat.NONE;
- }
-}
diff --git a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/sink/HbaseSink.java b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/sink/HbaseSink.java
deleted file mode 100644
index 1628bc8d8..000000000
--- a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/sink/HbaseSink.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.manager.client.api.sink;
-
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.NoArgsConstructor;
-import org.apache.inlong.manager.common.enums.DataFormat;
-import org.apache.inlong.manager.common.enums.SinkType;
-import org.apache.inlong.manager.common.pojo.stream.SinkField;
-import org.apache.inlong.manager.common.pojo.stream.StreamSink;
-
-import java.util.List;
-
-/**
- * Hbase sink.
- */
-@Data
-@EqualsAndHashCode(callSuper = true)
-@NoArgsConstructor
-@AllArgsConstructor
-@ApiModel("Hbase sink configuration")
-public class HbaseSink extends StreamSink {
-
- @ApiModelProperty(value = "Sink type", required = true)
- private SinkType sinkType = SinkType.HBASE;
-
- @ApiModelProperty("Target table name")
- private String tableName;
-
- @ApiModelProperty("Namespace")
- private String namespace;
-
- @ApiModelProperty("Row key")
- private String rowKey;
-
- @ApiModelProperty("Zookeeper quorm")
- private String zookeeperQuorum;
-
- @ApiModelProperty("Sink buffer flush maxsize")
- private String sinkBufferFlushMaxSize;
-
- @ApiModelProperty("Zookeeper znode parent")
- private String zookeeperZnodeParent;
-
- @ApiModelProperty("Sink buffer flush max rows")
- private String sinkBufferFlushMaxRows;
-
- @ApiModelProperty("Sink buffer flush interval")
- private String sinkBufferFlushInterval;
-
- @ApiModelProperty("Create table or not")
- private boolean needCreated;
-
- @ApiModelProperty("Field definitions for hbase")
- private List<SinkField> sinkFields;
-
- @Override
- public DataFormat getDataFormat() {
- return DataFormat.NONE;
- }
-}
diff --git a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/sink/HiveSink.java b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/sink/HiveSink.java
deleted file mode 100644
index 33514347b..000000000
--- a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/sink/HiveSink.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.manager.client.api.sink;
-
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.NoArgsConstructor;
-import org.apache.inlong.manager.common.enums.DataSeparator;
-import org.apache.inlong.manager.common.auth.DefaultAuthentication;
-import org.apache.inlong.manager.common.enums.DataFormat;
-import org.apache.inlong.manager.common.enums.FileFormat;
-import org.apache.inlong.manager.common.enums.SinkType;
-import org.apache.inlong.manager.common.pojo.sink.hive.HivePartitionField;
-import org.apache.inlong.manager.common.pojo.stream.SinkField;
-import org.apache.inlong.manager.common.pojo.stream.StreamSink;
-
-import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
-import java.util.List;
-
-/**
- * Hive sink.
- */
-@Data
-@EqualsAndHashCode(callSuper = true)
-@NoArgsConstructor
-@AllArgsConstructor
-@ApiModel("Hive sink configuration")
-public class HiveSink extends StreamSink {
-
- @ApiModelProperty(value = "Sink type", required = true)
- private SinkType sinkType = SinkType.HIVE;
-
- @ApiModelProperty("Hive meta db URL, etc jdbc:hive2://${ip}:${port}")
- private String jdbcUrl;
-
- @ApiModelProperty("Authentication for hive")
- private DefaultAuthentication authentication;
-
- @ApiModelProperty("Target database name")
- private String dbName;
-
- @ApiModelProperty("Target table name")
- private String tableName;
-
- @ApiModelProperty("Data path, such as: hdfs://ip:port/usr/hive/warehouse/test.db")
- private String dataPath;
-
- @ApiModelProperty("Data encoding format: UTF-8, GBK")
- private Charset charset = StandardCharsets.UTF_8;
-
- @ApiModelProperty("Data separator, stored as ASCII code")
- private DataSeparator dataSeparator = DataSeparator.SOH;
-
- @ApiModelProperty("File format, support: TextFile, ORCFile, SequenceFile, Parquet")
- private FileFormat fileFormat;
-
- @ApiModelProperty("Create table or not")
- private boolean needCreated;
-
- @ApiModelProperty("Partition field list")
- private List<HivePartitionField> partitionFieldList;
-
- @ApiModelProperty("Field definitions for hive")
- private List<SinkField> sinkFields;
-
- @ApiModelProperty("Data format type for stream sink")
- private DataFormat dataFormat;
-
- @ApiModelProperty("Version for hive")
- private String hiveVersion;
-
- @ApiModelProperty("Config directory of hive, needed by sort in light mode")
- private String hiveConfDir;
-}
-
diff --git a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/sink/KafkaSink.java b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/sink/KafkaSink.java
deleted file mode 100644
index 6ae542b49..000000000
--- a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/sink/KafkaSink.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.manager.client.api.sink;
-
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.NoArgsConstructor;
-import org.apache.inlong.manager.common.enums.DataFormat;
-import org.apache.inlong.manager.common.pojo.stream.SinkField;
-import org.apache.inlong.manager.common.pojo.stream.StreamSink;
-import org.apache.inlong.manager.common.enums.SinkType;
-
-import java.util.List;
-
-/**
- * Kafka sink.
- */
-@Data
-@EqualsAndHashCode(callSuper = true)
-@NoArgsConstructor
-@AllArgsConstructor
-@ApiModel("Kafka sink configuration")
-public class KafkaSink extends StreamSink {
-
- @ApiModelProperty(value = "Sink type", required = true)
- private SinkType sinkType = SinkType.KAFKA;
-
- @ApiModelProperty("Kafka bootstrap servers")
- private String bootstrapServers;
-
- @ApiModelProperty("Kafka topicName")
- private String topicName;
-
- @ApiModelProperty("Data format type for stream sink")
- private DataFormat dataFormat;
-
- @ApiModelProperty("Create topic or not")
- private boolean needCreated;
-
- @ApiModelProperty("Field definitions for kafka")
- private List<SinkField> sinkFields;
-
- @ApiModelProperty("Primary key is required when dataFormat is json, avro")
- private String primaryKey;
-}
diff --git a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/sink/PostgresSink.java b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/sink/PostgresSink.java
deleted file mode 100644
index 90e1b8574..000000000
--- a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/sink/PostgresSink.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.manager.client.api.sink;
-
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-import java.util.List;
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.NoArgsConstructor;
-import org.apache.inlong.manager.common.auth.DefaultAuthentication;
-import org.apache.inlong.manager.common.enums.DataFormat;
-import org.apache.inlong.manager.common.enums.SinkType;
-import org.apache.inlong.manager.common.pojo.stream.SinkField;
-import org.apache.inlong.manager.common.pojo.stream.StreamSink;
-
-/**
- * Postgres sink.
- */
-@Data
-@EqualsAndHashCode(callSuper = true)
-@NoArgsConstructor
-@AllArgsConstructor
-@ApiModel("Postgres sink configuration")
-public class PostgresSink extends StreamSink {
-
- @ApiModelProperty(value = "Sink type", required = true)
- private SinkType sinkType = SinkType.POSTGRES;
-
- @ApiModelProperty("Postgres JDBC URL")
- private String jdbcUrl;
-
- @ApiModelProperty("Target database name")
- private String dbName;
-
- @ApiModelProperty("Authentication for postgres")
- private DefaultAuthentication authentication;
-
- @ApiModelProperty("Target table name")
- private String tableName;
-
- @ApiModelProperty("Field definitions for postgres")
- private List<SinkField> sinkFields;
-
- @ApiModelProperty("Primary key")
- private String primaryKey;
-
- /**
- * get data format
- * @return just NONE
- */
- public DataFormat getDataFormat() {
- return DataFormat.NONE;
- }
-}
diff --git a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/source/AgentFileSource.java b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/source/AgentFileSource.java
index 236be6016..61d29df02 100644
--- a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/source/AgentFileSource.java
+++ b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/source/AgentFileSource.java
@@ -24,7 +24,7 @@ import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import org.apache.inlong.manager.common.enums.DataFormat;
-import org.apache.inlong.manager.common.pojo.stream.StreamSource;
+import org.apache.inlong.manager.common.pojo.source.StreamSource;
import org.apache.inlong.manager.common.enums.SourceType;
/**
diff --git a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/source/AutoPushSource.java b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/source/AutoPushSource.java
index 62f63c1ce..a5d1df42b 100644
--- a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/source/AutoPushSource.java
+++ b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/source/AutoPushSource.java
@@ -24,7 +24,7 @@ import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import org.apache.inlong.manager.common.enums.DataFormat;
-import org.apache.inlong.manager.common.pojo.stream.StreamSource;
+import org.apache.inlong.manager.common.pojo.source.StreamSource;
import org.apache.inlong.manager.common.enums.SourceType;
/**
diff --git a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/source/KafkaSource.java b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/source/KafkaSource.java
index e68bc8a89..29aa57163 100644
--- a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/source/KafkaSource.java
+++ b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/source/KafkaSource.java
@@ -25,7 +25,7 @@ import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import org.apache.inlong.manager.common.enums.DataFormat;
import org.apache.inlong.manager.common.pojo.source.kafka.KafkaOffset;
-import org.apache.inlong.manager.common.pojo.stream.StreamSource;
+import org.apache.inlong.manager.common.pojo.source.StreamSource;
import org.apache.inlong.manager.common.enums.SourceType;
/**
diff --git a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/source/MySQLBinlogSource.java b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/source/MySQLBinlogSource.java
index c84f72820..daf661046 100644
--- a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/source/MySQLBinlogSource.java
+++ b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/source/MySQLBinlogSource.java
@@ -26,7 +26,7 @@ import lombok.NoArgsConstructor;
import org.apache.inlong.manager.common.auth.DefaultAuthentication;
import org.apache.inlong.manager.common.enums.DataFormat;
import org.apache.inlong.manager.common.enums.SourceType;
-import org.apache.inlong.manager.common.pojo.stream.StreamSource;
+import org.apache.inlong.manager.common.pojo.source.StreamSource;
import java.util.List;
diff --git a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/source/MySQLSource.java b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/source/MySQLSource.java
index d1e19de04..735e8aa33 100644
--- a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/source/MySQLSource.java
+++ b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/source/MySQLSource.java
@@ -24,7 +24,7 @@ import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import org.apache.inlong.manager.common.enums.DataFormat;
-import org.apache.inlong.manager.common.pojo.stream.StreamSource;
+import org.apache.inlong.manager.common.pojo.source.StreamSource;
import org.apache.inlong.manager.common.enums.SourceType;
/**
diff --git a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/source/PostgresSource.java b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/source/PostgresSource.java
index f7b2bb0a6..3f8cb76c6 100644
--- a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/source/PostgresSource.java
+++ b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/source/PostgresSource.java
@@ -26,7 +26,7 @@ import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import org.apache.inlong.manager.common.enums.DataFormat;
import org.apache.inlong.manager.common.enums.SourceType;
-import org.apache.inlong.manager.common.pojo.stream.StreamSource;
+import org.apache.inlong.manager.common.pojo.source.StreamSource;
/**
* Postgres source.
diff --git a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/util/InlongStreamSinkTransfer.java b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/util/InlongStreamSinkTransfer.java
deleted file mode 100644
index a7e47b9f1..000000000
--- a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/util/InlongStreamSinkTransfer.java
+++ /dev/null
@@ -1,511 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.manager.client.api.util;
-
-import com.google.common.collect.Lists;
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.inlong.manager.common.enums.DataSeparator;
-import org.apache.inlong.manager.client.api.sink.ClickHouseSink;
-import org.apache.inlong.manager.client.api.sink.HbaseSink;
-import org.apache.inlong.manager.client.api.sink.HiveSink;
-import org.apache.inlong.manager.client.api.sink.KafkaSink;
-import org.apache.inlong.manager.common.auth.DefaultAuthentication;
-import org.apache.inlong.manager.client.api.sink.PostgresSink;
-import org.apache.inlong.manager.common.enums.DataFormat;
-import org.apache.inlong.manager.common.enums.FieldType;
-import org.apache.inlong.manager.common.enums.FileFormat;
-import org.apache.inlong.manager.common.enums.GlobalConstants;
-import org.apache.inlong.manager.common.enums.SinkType;
-import org.apache.inlong.manager.common.pojo.sink.SinkFieldRequest;
-import org.apache.inlong.manager.common.pojo.sink.SinkFieldResponse;
-import org.apache.inlong.manager.common.pojo.sink.SinkRequest;
-import org.apache.inlong.manager.common.pojo.sink.SinkResponse;
-import org.apache.inlong.manager.common.pojo.sink.ck.ClickHouseSinkRequest;
-import org.apache.inlong.manager.common.pojo.sink.ck.ClickHouseSinkResponse;
-import org.apache.inlong.manager.common.pojo.sink.hbase.HbaseSinkRequest;
-import org.apache.inlong.manager.common.pojo.sink.hbase.HbaseSinkResponse;
-import org.apache.inlong.manager.common.pojo.sink.hive.HiveSinkRequest;
-import org.apache.inlong.manager.common.pojo.sink.hive.HiveSinkResponse;
-import org.apache.inlong.manager.common.pojo.sink.kafka.KafkaSinkRequest;
-import org.apache.inlong.manager.common.pojo.sink.kafka.KafkaSinkResponse;
-import org.apache.inlong.manager.common.pojo.sink.postgres.PostgresSinkRequest;
-import org.apache.inlong.manager.common.pojo.sink.postgres.PostgresSinkResponse;
-import org.apache.inlong.manager.common.pojo.stream.InlongStreamInfo;
-import org.apache.inlong.manager.common.pojo.stream.SinkField;
-import org.apache.inlong.manager.common.pojo.stream.StreamSink;
-import org.apache.inlong.manager.common.util.AssertUtils;
-import org.apache.inlong.manager.common.util.CommonBeanUtils;
-
-import java.nio.charset.Charset;
-import java.util.List;
-import java.util.stream.Collectors;
-
-public class InlongStreamSinkTransfer {
-
- /**
- * Create sink request
- */
- public static SinkRequest createSinkRequest(StreamSink streamSink, InlongStreamInfo streamInfo) {
- SinkType sinkType = streamSink.getSinkType();
- switch (sinkType) {
- case KAFKA:
- return createKafkaRequest(streamSink, streamInfo);
- case HIVE:
- return createHiveRequest(streamSink, streamInfo);
- case CLICKHOUSE:
- return createClickHouseRequest(streamSink, streamInfo);
- case HBASE:
- return createHbaseRequest(streamSink, streamInfo);
- case POSTGRES:
- return createPostgresRequest(streamSink, streamInfo);
- default:
- throw new IllegalArgumentException(String.format("Unsupported sink type : %s for Inlong", sinkType));
- }
- }
-
- public static StreamSink parseStreamSink(SinkResponse sinkResponse) {
- return parseStreamSink(sinkResponse, null);
- }
-
- /**
- * Parse stream sink
- */
- public static StreamSink parseStreamSink(SinkResponse sinkResponse, StreamSink streamSink) {
- String type = sinkResponse.getSinkType();
- SinkType sinkType = SinkType.forType(type);
- switch (sinkType) {
- case KAFKA:
- return parseKafkaSink((KafkaSinkResponse) sinkResponse, streamSink);
- case HIVE:
- return parseHiveSink((HiveSinkResponse) sinkResponse, streamSink);
- case CLICKHOUSE:
- return parseClickHouseSink((ClickHouseSinkResponse) sinkResponse, streamSink);
- case HBASE:
- return parseHbaseSink((HbaseSinkResponse) sinkResponse, streamSink);
- case POSTGRES:
- return parsePostgresSink((PostgresSinkResponse) sinkResponse, streamSink);
- default:
- throw new IllegalArgumentException(String.format("Unsupported sink type : %s for Inlong", sinkType));
- }
- }
-
- /**
- * Create clickHouse request.
- *
- * @param streamSink streamSink object
- * @param streamInfo inlongStreamInfo object
- * @return clickHouseSinkRequest object
- */
- private static SinkRequest createClickHouseRequest(StreamSink streamSink, InlongStreamInfo streamInfo) {
- ClickHouseSinkRequest clickHouseSinkRequest = new ClickHouseSinkRequest();
- ClickHouseSink clickHouseSink = (ClickHouseSink) streamSink;
- clickHouseSinkRequest.setSinkName(clickHouseSink.getSinkName());
- clickHouseSinkRequest.setDbName(clickHouseSink.getDbName());
- clickHouseSinkRequest.setSinkType(clickHouseSink.getSinkType().name());
- clickHouseSinkRequest.setJdbcUrl(clickHouseSink.getJdbcUrl());
- DefaultAuthentication defaultAuthentication = clickHouseSink.getAuthentication();
- AssertUtils.notNull(defaultAuthentication,
- String.format("Clickhouse storage:%s must be authenticated", clickHouseSink.getDbName()));
- clickHouseSinkRequest.setUsername(defaultAuthentication.getUserName());
- clickHouseSinkRequest.setPassword(defaultAuthentication.getPassword());
- clickHouseSinkRequest.setTableName(clickHouseSink.getTableName());
- clickHouseSinkRequest.setIsDistributed(clickHouseSink.getIsDistributed());
- clickHouseSinkRequest.setFlushInterval(clickHouseSink.getFlushInterval());
- clickHouseSinkRequest.setFlushRecord(clickHouseSink.getFlushRecord());
- clickHouseSinkRequest.setKeyFieldNames(clickHouseSink.getKeyFieldNames());
- clickHouseSinkRequest.setPartitionFields(clickHouseSink.getPartitionFields());
- clickHouseSinkRequest.setPartitionStrategy(clickHouseSink.getPartitionStrategy());
- clickHouseSinkRequest.setRetryTimes(clickHouseSink.getRetryTimes());
- clickHouseSinkRequest.setInlongGroupId(streamInfo.getInlongGroupId());
- clickHouseSinkRequest.setInlongStreamId(streamInfo.getInlongStreamId());
- clickHouseSinkRequest.setProperties(clickHouseSink.getProperties());
- clickHouseSinkRequest.setEnableCreateResource(clickHouseSink.isNeedCreated() ? 1 : 0);
- if (CollectionUtils.isNotEmpty(clickHouseSink.getSinkFields())) {
- List<SinkFieldRequest> fieldRequests = createSinkFieldRequests(streamSink.getSinkFields());
- clickHouseSinkRequest.setFieldList(fieldRequests);
- }
- return clickHouseSinkRequest;
- }
-
- /**
- * Parse clickHouse sink.
- *
- * @param sinkResponse kafkaSinkResponse object
- * @param streamSink streamSink object
- * @return clickHouseSink object
- */
- private static StreamSink parseClickHouseSink(ClickHouseSinkResponse sinkResponse,
- StreamSink streamSink) {
- ClickHouseSink clickHouseSink = new ClickHouseSink();
- if (streamSink != null) {
- AssertUtils.isTrue(sinkResponse.getSinkName().equals(streamSink.getSinkName()),
- String.format("SinkName is not equal: %s != %s", sinkResponse, streamSink));
- ClickHouseSink snapshot = (ClickHouseSink) streamSink;
- clickHouseSink = CommonBeanUtils.copyProperties(snapshot, ClickHouseSink::new);
- } else {
- clickHouseSink.setIsDistributed(sinkResponse.getIsDistributed());
- clickHouseSink.setSinkName(sinkResponse.getSinkName());
- clickHouseSink.setFlushInterval(sinkResponse.getFlushInterval());
- clickHouseSink.setAuthentication(new DefaultAuthentication(sinkResponse.getSinkName(),
- sinkResponse.getPassword()));
- clickHouseSink.setDbName(sinkResponse.getDbName());
- clickHouseSink.setFlushRecord(sinkResponse.getFlushRecord());
- clickHouseSink.setJdbcUrl(sinkResponse.getJdbcUrl());
- clickHouseSink.setPartitionFields(sinkResponse.getPartitionFields());
- clickHouseSink.setKeyFieldNames(sinkResponse.getKeyFieldNames());
- clickHouseSink.setPartitionStrategy(sinkResponse.getPartitionStrategy());
- clickHouseSink.setRetryTimes(sinkResponse.getRetryTimes());
- clickHouseSink.setIsDistributed(sinkResponse.getIsDistributed());
- }
- clickHouseSink.setProperties(sinkResponse.getProperties());
- clickHouseSink.setNeedCreated(
- GlobalConstants.ENABLE_CREATE_RESOURCE.equals(sinkResponse.getEnableCreateResource()));
- if (CollectionUtils.isNotEmpty(sinkResponse.getFieldList())) {
- clickHouseSink.setSinkFields(convertToSinkFields(sinkResponse.getFieldList()));
- }
- return clickHouseSink;
- }
-
- /**
- * Convert sinkFieldResponse to sinkField
- */
- private static List<SinkField> convertToSinkFields(List<SinkFieldResponse> sinkFieldResponses) {
- return sinkFieldResponses.stream().map(sinkFieldResponse -> new SinkField(sinkFieldResponse.getId(),
- FieldType.forName(sinkFieldResponse.getFieldType()),
- sinkFieldResponse.getFieldName(),
- sinkFieldResponse.getFieldComment(),
- null, sinkFieldResponse.getSourceFieldName(),
- StringUtils.isBlank(sinkFieldResponse.getSourceFieldType()) ? null :
- FieldType.forName(sinkFieldResponse.getSourceFieldType()),
- sinkFieldResponse.getIsMetaField(),
- sinkFieldResponse.getFieldFormat())).collect(Collectors.toList());
- }
-
- /**
- * Create kafka request.
- *
- * @param streamSink streamSink object
- * @param streamInfo inlongStreamInfo object
- * @return kafkaSinkRequest object
- */
- private static SinkRequest createKafkaRequest(StreamSink streamSink, InlongStreamInfo streamInfo) {
- KafkaSinkRequest kafkaSinkRequest = new KafkaSinkRequest();
- KafkaSink kafkaSink = (KafkaSink) streamSink;
- kafkaSinkRequest.setSinkName(streamSink.getSinkName());
- kafkaSinkRequest.setBootstrapServers(kafkaSink.getBootstrapServers());
- kafkaSinkRequest.setTopicName(kafkaSink.getTopicName());
- kafkaSinkRequest.setSinkType(kafkaSink.getSinkType().name());
- kafkaSinkRequest.setInlongGroupId(streamInfo.getInlongGroupId());
- kafkaSinkRequest.setInlongStreamId(streamInfo.getInlongStreamId());
- kafkaSinkRequest.setSerializationType(kafkaSink.getDataFormat().name());
- kafkaSinkRequest.setEnableCreateResource(kafkaSink.isNeedCreated() ? 1 : 0);
- kafkaSinkRequest.setProperties(kafkaSink.getProperties());
- kafkaSinkRequest.setPrimaryKey(kafkaSink.getPrimaryKey());
- if (CollectionUtils.isNotEmpty(kafkaSink.getSinkFields())) {
- List<SinkFieldRequest> fieldRequests = createSinkFieldRequests(kafkaSink.getSinkFields());
- kafkaSinkRequest.setFieldList(fieldRequests);
- }
- return kafkaSinkRequest;
- }
-
- /**
- * Parse kafka sink.
- *
- * @param sinkResponse kafkaSinkResponse object
- * @param sink streamSink object
- * @return kafkaSink object
- */
- private static StreamSink parseKafkaSink(KafkaSinkResponse sinkResponse, StreamSink sink) {
- KafkaSink kafkaSink = new KafkaSink();
- if (sink != null) {
- AssertUtils.isTrue(sinkResponse.getSinkName().equals(sink.getSinkName()),
- String.format("SinkName is not equal: %s != %s", sinkResponse, sink));
- KafkaSink snapshot = (KafkaSink) sink;
- kafkaSink.setSinkName(snapshot.getSinkName());
- kafkaSink.setBootstrapServers(snapshot.getBootstrapServers());
- kafkaSink.setTopicName(snapshot.getTopicName());
- kafkaSink.setDataFormat(snapshot.getDataFormat());
- } else {
- kafkaSink.setSinkName(sinkResponse.getSinkName());
- kafkaSink.setBootstrapServers(sinkResponse.getBootstrapServers());
- kafkaSink.setTopicName(sinkResponse.getTopicName());
- kafkaSink.setDataFormat(DataFormat.forName(sinkResponse.getSerializationType()));
- }
- kafkaSink.setPrimaryKey(sinkResponse.getPrimaryKey());
- kafkaSink.setProperties(sinkResponse.getProperties());
- kafkaSink.setNeedCreated(GlobalConstants.ENABLE_CREATE_RESOURCE.equals(sinkResponse.getEnableCreateResource()));
- if (CollectionUtils.isNotEmpty(sinkResponse.getFieldList())) {
- kafkaSink.setSinkFields(convertToSinkFields(sinkResponse.getFieldList()));
- }
- return kafkaSink;
- }
-
- /**
- * Create habse request.
- *
- * @param streamSink streamSink object
- * @param streamInfo inlongStreamInfo object
- * @return hbaseSinkRequest object
- */
- private static SinkRequest createHbaseRequest(StreamSink streamSink, InlongStreamInfo streamInfo) {
- HbaseSinkRequest hbaseSinkRequest = new HbaseSinkRequest();
- HbaseSink hbaseSink = (HbaseSink) streamSink;
- hbaseSinkRequest.setSinkName(streamSink.getSinkName());
- hbaseSinkRequest.setInlongGroupId(streamInfo.getInlongGroupId());
- hbaseSinkRequest.setInlongStreamId(streamInfo.getInlongStreamId());
- hbaseSinkRequest.setNamespace(hbaseSink.getNamespace());
- hbaseSinkRequest.setTableName(hbaseSink.getTableName());
- hbaseSinkRequest.setRowKey(hbaseSink.getRowKey());
- hbaseSinkRequest.setZookeeperQuorum(hbaseSink.getZookeeperQuorum());
- hbaseSinkRequest.setZookeeperZnodeParent(hbaseSink.getZookeeperZnodeParent());
- hbaseSinkRequest.setSinkBufferFlushInterval(hbaseSink.getSinkBufferFlushInterval());
- hbaseSinkRequest.setSinkBufferFlushMaxRows(hbaseSink.getSinkBufferFlushMaxRows());
- hbaseSinkRequest.setSinkBufferFlushMaxSize(hbaseSink.getSinkBufferFlushMaxSize());
- hbaseSinkRequest.setSinkType(hbaseSink.getSinkType().name());
- hbaseSinkRequest.setEnableCreateResource(hbaseSink.isNeedCreated() ? 1 : 0);
- hbaseSinkRequest.setProperties(hbaseSink.getProperties());
- if (CollectionUtils.isNotEmpty(hbaseSink.getSinkFields())) {
- List<SinkFieldRequest> fieldRequests = createSinkFieldRequests(hbaseSink.getSinkFields());
- hbaseSinkRequest.setFieldList(fieldRequests);
- }
- return hbaseSinkRequest;
-
- }
-
- /**
- * Parse hbase sink.
- *
- * @param sinkResponse hbaseSinkResponse object
- * @param sink streamSink object
- * @return hbaseSink object
- */
- private static StreamSink parseHbaseSink(HbaseSinkResponse sinkResponse, StreamSink sink) {
- HbaseSink hbaseSink = new HbaseSink();
- if (sink != null) {
- AssertUtils.isTrue(sinkResponse.getSinkName().equals(sink.getSinkName()),
- String.format("SinkName is not equal: %s != %s", sinkResponse, sink));
- HbaseSink snapshot = (HbaseSink) sink;
- hbaseSink.setSinkName(snapshot.getSinkName());
- hbaseSink.setNamespace(snapshot.getNamespace());
- hbaseSink.setTableName(snapshot.getTableName());
- hbaseSink.setRowKey(snapshot.getRowKey());
- hbaseSink.setZookeeperQuorum(snapshot.getZookeeperQuorum());
- hbaseSink.setZookeeperZnodeParent(snapshot.getZookeeperZnodeParent());
- hbaseSink.setSinkBufferFlushInterval(snapshot.getSinkBufferFlushInterval());
- hbaseSink.setSinkBufferFlushMaxRows(snapshot.getSinkBufferFlushMaxRows());
- hbaseSink.setSinkBufferFlushMaxSize(snapshot.getSinkBufferFlushMaxSize());
- } else {
- hbaseSink.setSinkName(sinkResponse.getSinkName());
- hbaseSink.setNamespace(sinkResponse.getNamespace());
- hbaseSink.setTableName(sinkResponse.getTableName());
- hbaseSink.setRowKey(sinkResponse.getRowKey());
- hbaseSink.setZookeeperQuorum(sinkResponse.getZookeeperQuorum());
- hbaseSink.setZookeeperZnodeParent(sinkResponse.getZookeeperZnodeParent());
- hbaseSink.setSinkBufferFlushInterval(sinkResponse.getSinkBufferFlushInterval());
- hbaseSink.setSinkBufferFlushMaxRows(sinkResponse.getSinkBufferFlushMaxRows());
- hbaseSink.setSinkBufferFlushMaxSize(sinkResponse.getSinkBufferFlushMaxSize());
- }
- hbaseSink.setProperties(sinkResponse.getProperties());
- hbaseSink.setSinkType(SinkType.HBASE);
- hbaseSink.setNeedCreated(GlobalConstants.ENABLE_CREATE_RESOURCE.equals(sinkResponse.getEnableCreateResource()));
- if (CollectionUtils.isNotEmpty(sinkResponse.getFieldList())) {
- hbaseSink.setSinkFields(convertToSinkFields(sinkResponse.getFieldList()));
- }
- return hbaseSink;
- }
-
- /**
- * Create hive request.
- *
- * @param streamSink streamSink object
- * @param streamInfo inlongStreamInfo object
- * @return hiveSinkRequest object
- */
- private static HiveSinkRequest createHiveRequest(StreamSink streamSink, InlongStreamInfo streamInfo) {
- HiveSinkRequest hiveSinkRequest = new HiveSinkRequest();
- HiveSink hiveSink = (HiveSink) streamSink;
- hiveSinkRequest.setSinkName(streamSink.getSinkName());
- hiveSinkRequest.setInlongGroupId(streamInfo.getInlongGroupId());
- hiveSinkRequest.setInlongStreamId(streamInfo.getInlongStreamId());
- hiveSinkRequest.setDataEncoding(hiveSink.getCharset().name());
- hiveSinkRequest.setEnableCreateTable(hiveSink.isNeedCreated() ? 1 : 0);
- hiveSinkRequest.setDataSeparator(String.valueOf(hiveSink.getDataSeparator().getAsciiCode()));
- hiveSinkRequest.setDbName(hiveSink.getDbName());
- hiveSinkRequest.setTableName(hiveSink.getTableName());
- hiveSinkRequest.setDataPath(hiveSink.getDataPath());
- hiveSinkRequest.setJdbcUrl(hiveSink.getJdbcUrl());
- hiveSinkRequest.setFileFormat(hiveSink.getFileFormat().name());
- hiveSinkRequest.setSinkType(hiveSink.getSinkType().name());
- hiveSinkRequest.setPartitionFieldList(hiveSink.getPartitionFieldList());
- hiveSinkRequest.setHiveConfDir(hiveSink.getHiveConfDir());
- hiveSinkRequest.setHiveVersion(hiveSink.getHiveVersion());
- DefaultAuthentication defaultAuthentication = hiveSink.getAuthentication();
- AssertUtils.notNull(defaultAuthentication,
- String.format("Hive storage:%s must be authenticated", hiveSink.getDbName()));
- hiveSinkRequest.setUsername(defaultAuthentication.getUserName());
- hiveSinkRequest.setPassword(defaultAuthentication.getPassword());
- hiveSinkRequest.setProperties(hiveSink.getProperties());
- if (CollectionUtils.isNotEmpty(hiveSink.getSinkFields())) {
- List<SinkFieldRequest> fieldRequests = createSinkFieldRequests(streamSink.getSinkFields());
- hiveSinkRequest.setFieldList(fieldRequests);
- }
- return hiveSinkRequest;
- }
-
- /**
- * Create sink field requests
- */
- private static List<SinkFieldRequest> createSinkFieldRequests(List<SinkField> sinkFields) {
- List<SinkFieldRequest> fieldRequestList = Lists.newArrayList();
- for (SinkField sinkField : sinkFields) {
- SinkFieldRequest request = new SinkFieldRequest();
- request.setFieldName(sinkField.getFieldName());
- request.setFieldType(sinkField.getFieldType().toString());
- request.setFieldComment(sinkField.getFieldComment());
- request.setSourceFieldName(sinkField.getSourceFieldName());
- request.setSourceFieldType(
- sinkField.getSourceFieldType() == null ? null : sinkField.getSourceFieldType().toString());
- request.setIsMetaField(sinkField.getIsMetaField());
- request.setFieldFormat(sinkField.getFieldFormat());
- fieldRequestList.add(request);
- }
- return fieldRequestList;
- }
-
- /**
- * Parse hive sink.
- *
- * @param sinkResponse hiveSinkResponse object
- * @param sink streamSink object
- * @return hiveSink object
- */
- private static HiveSink parseHiveSink(HiveSinkResponse sinkResponse, StreamSink sink) {
- HiveSink hiveSink = new HiveSink();
- if (sink != null) {
- AssertUtils.isTrue(sinkResponse.getSinkName().equals(sink.getSinkName()),
- String.format("SinkName is not equal: %s != %s", sinkResponse, sink));
- HiveSink snapshot = (HiveSink) sink;
- hiveSink.setSinkName(snapshot.getSinkName());
- hiveSink.setDataSeparator(snapshot.getDataSeparator());
- hiveSink.setCharset(snapshot.getCharset());
- hiveSink.setAuthentication(snapshot.getAuthentication());
- hiveSink.setFileFormat(snapshot.getFileFormat());
- hiveSink.setJdbcUrl(snapshot.getJdbcUrl());
- hiveSink.setTableName(snapshot.getTableName());
- hiveSink.setDbName(snapshot.getDbName());
- hiveSink.setDataPath(snapshot.getDataPath());
- hiveSink.setHiveVersion(snapshot.getHiveVersion());
- hiveSink.setHiveConfDir(snapshot.getHiveConfDir());
- hiveSink.setPartitionFieldList(snapshot.getPartitionFieldList());
- } else {
- hiveSink.setSinkName(sinkResponse.getSinkName());
- hiveSink.setDataSeparator(DataSeparator.forAscii(Integer.parseInt(sinkResponse.getDataSeparator())));
- hiveSink.setCharset(Charset.forName(sinkResponse.getDataEncoding()));
- String password = sinkResponse.getPassword();
- String uname = sinkResponse.getUsername();
- hiveSink.setAuthentication(new DefaultAuthentication(uname, password));
- hiveSink.setFileFormat(FileFormat.forName(sinkResponse.getFileFormat()));
- hiveSink.setJdbcUrl(sinkResponse.getJdbcUrl());
- hiveSink.setTableName(sinkResponse.getTableName());
- hiveSink.setDbName(sinkResponse.getDbName());
- hiveSink.setDataPath(sinkResponse.getDataPath());
- hiveSink.setHiveConfDir(sinkResponse.getHiveConfDir());
- hiveSink.setHiveVersion(sinkResponse.getHiveVersion());
- hiveSink.setPartitionFieldList(sinkResponse.getPartitionFieldList());
- }
-
- hiveSink.setProperties(sinkResponse.getProperties());
- hiveSink.setSinkType(SinkType.HIVE);
- hiveSink.setNeedCreated(GlobalConstants.ENABLE_CREATE_RESOURCE.equals(sinkResponse.getEnableCreateResource()));
- if (CollectionUtils.isNotEmpty(sinkResponse.getFieldList())) {
- hiveSink.setSinkFields(convertToSinkFields(sinkResponse.getFieldList()));
- }
- return hiveSink;
- }
-
- /**
- * Create postgres request
- *
- * @param streamSink streamSink
- * @param streamInfo streamInfo
- * @return postgres sinkRequest
- */
- private static SinkRequest createPostgresRequest(StreamSink streamSink, InlongStreamInfo streamInfo) {
- PostgresSinkRequest postgresSinkRequest = new PostgresSinkRequest();
- PostgresSink postgresSink = (PostgresSink) streamSink;
- postgresSinkRequest.setJdbcUrl(postgresSink.getJdbcUrl());
- postgresSinkRequest.setUsername(postgresSink.getAuthentication().getUserName());
- postgresSinkRequest.setPassword(postgresSink.getAuthentication().getPassword());
- postgresSinkRequest.setDbName(postgresSink.getDbName());
- postgresSinkRequest.setTableName(postgresSink.getTableName());
-
- postgresSinkRequest.setSinkName(postgresSink.getSinkName());
- postgresSinkRequest.setSinkType(postgresSink.getSinkType().name());
- postgresSinkRequest.setInlongGroupId(streamInfo.getInlongGroupId());
- postgresSinkRequest.setInlongStreamId(streamInfo.getInlongStreamId());
-
- postgresSinkRequest.setProperties(postgresSink.getProperties());
- postgresSinkRequest.setPrimaryKey(postgresSink.getPrimaryKey());
-
- if (CollectionUtils.isNotEmpty(postgresSink.getSinkFields())) {
- List<SinkFieldRequest> fieldRequests = createSinkFieldRequests(postgresSink.getSinkFields());
- postgresSinkRequest.setFieldList(fieldRequests);
- }
- return postgresSinkRequest;
- }
-
- /**
- * parse postgres sink
- *
- * @param sinkResponse sinkResponse
- * @param sink sink
- * @return postgres streamSink
- */
- private static StreamSink parsePostgresSink(PostgresSinkResponse sinkResponse, StreamSink sink) {
- PostgresSink postgresSink = new PostgresSink();
- if (sink != null) {
- AssertUtils.isTrue(sinkResponse.getSinkName().equals(sink.getSinkName()),
- String.format("SinkName is not equal: %s != %s", sinkResponse, sink));
- PostgresSink snapshot = (PostgresSink) sink;
-
- postgresSink.setSinkName(snapshot.getSinkName());
- postgresSink.setAuthentication(snapshot.getAuthentication());
-
- postgresSink.setJdbcUrl(snapshot.getJdbcUrl());
- postgresSink.setTableName(snapshot.getTableName());
- postgresSink.setDbName(snapshot.getDbName());
- } else {
- postgresSink.setSinkName(sinkResponse.getSinkName());
- String password = sinkResponse.getPassword();
- String uname = sinkResponse.getUsername();
- postgresSink.setAuthentication(new DefaultAuthentication(uname, password));
-
- postgresSink.setJdbcUrl(sinkResponse.getJdbcUrl());
- postgresSink.setTableName(sinkResponse.getTableName());
- postgresSink.setDbName(sinkResponse.getDbName());
- }
- postgresSink.setPrimaryKey(sinkResponse.getPrimaryKey());
- postgresSink.setProperties(sinkResponse.getProperties());
- postgresSink.setSinkType(SinkType.POSTGRES);
- if (CollectionUtils.isNotEmpty(sinkResponse.getFieldList())) {
- postgresSink.setSinkFields(convertToSinkFields(sinkResponse.getFieldList()));
- }
- return postgresSink;
- }
-}
diff --git a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/util/InlongStreamSourceTransfer.java b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/util/InlongStreamSourceTransfer.java
index c85f19235..2a94fef5b 100644
--- a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/util/InlongStreamSourceTransfer.java
+++ b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/util/InlongStreamSourceTransfer.java
@@ -30,6 +30,9 @@ import org.apache.inlong.manager.common.enums.DataFormat;
import org.apache.inlong.manager.common.enums.SourceType;
import org.apache.inlong.manager.common.pojo.source.SourceRequest;
import org.apache.inlong.manager.common.pojo.source.SourceResponse;
+import org.apache.inlong.manager.common.pojo.source.StreamSource;
+import org.apache.inlong.manager.common.pojo.source.StreamSource.Status;
+import org.apache.inlong.manager.common.pojo.source.StreamSource.SyncType;
import org.apache.inlong.manager.common.pojo.source.autopush.AutoPushSourceRequest;
import org.apache.inlong.manager.common.pojo.source.autopush.AutoPushSourceResponse;
import org.apache.inlong.manager.common.pojo.source.binlog.BinlogSourceRequest;
@@ -41,9 +44,6 @@ import org.apache.inlong.manager.common.pojo.source.kafka.KafkaSourceResponse;
import org.apache.inlong.manager.common.pojo.source.postgres.PostgresSourceRequest;
import org.apache.inlong.manager.common.pojo.source.postgres.PostgresSourceResponse;
import org.apache.inlong.manager.common.pojo.stream.InlongStreamInfo;
-import org.apache.inlong.manager.common.pojo.stream.StreamSource;
-import org.apache.inlong.manager.common.pojo.stream.StreamSource.State;
-import org.apache.inlong.manager.common.pojo.stream.StreamSource.SyncType;
import java.util.Arrays;
@@ -97,7 +97,7 @@ public class InlongStreamSourceTransfer {
kafkaSource.setConsumerGroup(response.getGroupId());
DataFormat dataFormat = DataFormat.forName(response.getSerializationType());
kafkaSource.setDataFormat(dataFormat);
- kafkaSource.setState(State.parseByStatus(response.getStatus()));
+ kafkaSource.setStatus(Status.parseByStatus(response.getStatus()));
kafkaSource.setAgentIp(response.getAgentIp());
kafkaSource.setTopic(response.getTopic());
kafkaSource.setBootstrapServers(response.getBootstrapServers());
@@ -122,9 +122,8 @@ public class InlongStreamSourceTransfer {
binlogSource.setDataFormat(dataFormat);
binlogSource.setPort(response.getPort());
binlogSource.setAgentIp(response.getAgentIp());
- binlogSource.setState(State.parseByStatus(response.getStatus()));
+ binlogSource.setStatus(Status.parseByStatus(response.getStatus()));
binlogSource.setServerId(response.getServerId());
- binlogSource.setDataNodeName(response.getDataNodeName());
DefaultAuthentication defaultAuthentication = new DefaultAuthentication(
response.getUser(),
response.getPassword());
@@ -148,7 +147,7 @@ public class InlongStreamSourceTransfer {
private static AgentFileSource parseAgentFileSource(FileSourceResponse response) {
AgentFileSource fileSource = new AgentFileSource();
fileSource.setSourceName(response.getSourceName());
- fileSource.setState(State.parseByStatus(response.getStatus()));
+ fileSource.setStatus(Status.parseByStatus(response.getStatus()));
DataFormat dataFormat = DataFormat.forName(response.getSerializationType());
fileSource.setDataFormat(dataFormat);
fileSource.setPattern(response.getPattern());
@@ -161,7 +160,7 @@ public class InlongStreamSourceTransfer {
private static AutoPushSource parseAutoPushSource(AutoPushSourceResponse response) {
AutoPushSource autoPushSource = new AutoPushSource();
autoPushSource.setSourceName(response.getSourceName());
- autoPushSource.setState(State.parseByStatus(response.getStatus()));
+ autoPushSource.setStatus(Status.parseByStatus(response.getStatus()));
DataFormat dataFormat = DataFormat.forName(response.getSerializationType());
autoPushSource.setDataFormat(dataFormat);
autoPushSource.setDataProxyGroup(response.getDataProxyGroup());
@@ -172,7 +171,7 @@ public class InlongStreamSourceTransfer {
private static PostgresSource parsePostgresSource(PostgresSourceResponse response) {
PostgresSource postgresSource = new PostgresSource();
postgresSource.setSourceName(response.getSourceName());
- postgresSource.setState(State.parseByStatus(response.getStatus()));
+ postgresSource.setStatus(Status.parseByStatus(response.getStatus()));
DataFormat dataFormat = DataFormat.forName(response.getSerializationType());
postgresSource.setDataFormat(dataFormat);
postgresSource.setFields(InlongStreamTransfer.parseStreamFields(response.getFieldList()));
@@ -226,7 +225,6 @@ public class InlongStreamSourceTransfer {
sourceRequest.setHostname(binlogSource.getHostname());
sourceRequest.setPort(binlogSource.getPort());
sourceRequest.setServerId(binlogSource.getServerId());
- sourceRequest.setDataNodeName(binlogSource.getDataNodeName());
sourceRequest.setIncludeSchema(binlogSource.getIncludeSchema());
sourceRequest.setServerTimezone(binlogSource.getServerTimezone());
sourceRequest.setMonitoredDdl(binlogSource.getMonitoredDdl());
diff --git a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/util/InlongStreamTransfer.java b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/util/InlongStreamTransfer.java
index c875c3cb9..d88ecc0d6 100644
--- a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/util/InlongStreamTransfer.java
+++ b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/util/InlongStreamTransfer.java
@@ -22,7 +22,6 @@ import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.inlong.manager.client.api.InlongStreamConf;
import org.apache.inlong.manager.common.pojo.group.InlongGroupInfo;
-import org.apache.inlong.manager.common.pojo.stream.InlongStreamFieldInfo;
import org.apache.inlong.manager.common.pojo.stream.InlongStreamInfo;
import org.apache.inlong.manager.common.pojo.stream.StreamField;
import org.apache.inlong.manager.common.util.CommonBeanUtils;
@@ -38,11 +37,11 @@ public class InlongStreamTransfer {
public static InlongStreamInfo createStreamInfo(InlongStreamConf streamConf, InlongGroupInfo groupInfo) {
InlongStreamInfo dataStreamInfo = new InlongStreamInfo();
dataStreamInfo.setInlongGroupId(groupInfo.getInlongGroupId());
- dataStreamInfo.setInlongStreamId(streamConf.getStreamId());
+ dataStreamInfo.setInlongStreamId(streamConf.getInlongStreamId());
dataStreamInfo.setName(streamConf.getName());
dataStreamInfo.setDataEncoding(streamConf.getCharset().name());
if (StringUtils.isEmpty(streamConf.getMqResource())) {
- dataStreamInfo.setMqResource(streamConf.getStreamId());
+ dataStreamInfo.setMqResource(streamConf.getInlongStreamId());
} else {
dataStreamInfo.setMqResource(streamConf.getMqResource());
}
@@ -62,13 +61,13 @@ public class InlongStreamTransfer {
/**
* Create inlong stream fields.
*/
- public static List<InlongStreamFieldInfo> createStreamFields(
+ public static List<StreamField> createStreamFields(
List<StreamField> fieldList, InlongStreamInfo streamInfo) {
if (CollectionUtils.isEmpty(fieldList)) {
return Lists.newArrayList();
}
return fieldList.stream().map(field -> {
- InlongStreamFieldInfo fieldInfo = new InlongStreamFieldInfo();
+ StreamField fieldInfo = new StreamField();
fieldInfo.setInlongStreamId(streamInfo.getInlongStreamId());
fieldInfo.setInlongGroupId(streamInfo.getInlongGroupId());
fieldInfo.setFieldName(field.getFieldName());
@@ -84,7 +83,7 @@ public class InlongStreamTransfer {
/**
* Parse information of stream fields.
*/
- public static List<StreamField> parseStreamFields(List<InlongStreamFieldInfo> fields) {
+ public static List<StreamField> parseStreamFields(List<StreamField> fields) {
if (CollectionUtils.isEmpty(fields)) {
return null;
}
diff --git a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/util/StreamSinkTransfer.java b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/util/StreamSinkTransfer.java
new file mode 100644
index 000000000..39d7fe777
--- /dev/null
+++ b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/util/StreamSinkTransfer.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.manager.client.api.util;
+
+import org.apache.inlong.manager.common.enums.SinkType;
+import org.apache.inlong.manager.common.pojo.sink.SinkRequest;
+import org.apache.inlong.manager.common.pojo.sink.SinkResponse;
+import org.apache.inlong.manager.common.pojo.sink.ck.ClickHouseSinkRequest;
+import org.apache.inlong.manager.common.pojo.sink.ck.ClickHouseSinkResponse;
+import org.apache.inlong.manager.common.pojo.sink.hbase.HbaseSinkRequest;
+import org.apache.inlong.manager.common.pojo.sink.hbase.HbaseSinkResponse;
+import org.apache.inlong.manager.common.pojo.sink.hive.HiveSinkRequest;
+import org.apache.inlong.manager.common.pojo.sink.hive.HiveSinkResponse;
+import org.apache.inlong.manager.common.pojo.sink.kafka.KafkaSinkRequest;
+import org.apache.inlong.manager.common.pojo.sink.kafka.KafkaSinkResponse;
+import org.apache.inlong.manager.common.pojo.sink.postgres.PostgresSinkRequest;
+import org.apache.inlong.manager.common.pojo.sink.postgres.PostgresSinkResponse;
+import org.apache.inlong.manager.common.util.CommonBeanUtils;
+
+/**
+ * Transfer util for stream sink.
+ */
+public class StreamSinkTransfer {
+
+ /**
+ * Parse stream sink
+ */
+ public static SinkRequest parseSinkRequest(SinkResponse sinkResponse) {
+ SinkType sinkType = SinkType.forType(sinkResponse.getSinkType());
+ switch (sinkType) {
+ case KAFKA:
+ return CommonBeanUtils.copyProperties((KafkaSinkResponse) sinkResponse, KafkaSinkRequest::new);
+ case HIVE:
+ return CommonBeanUtils.copyProperties((HiveSinkResponse) sinkResponse, HiveSinkRequest::new);
+ case CLICKHOUSE:
+ return CommonBeanUtils.copyProperties((ClickHouseSinkResponse) sinkResponse,
+ ClickHouseSinkRequest::new);
+ case HBASE:
+ return CommonBeanUtils.copyProperties((HbaseSinkResponse) sinkResponse, HbaseSinkRequest::new);
+ case POSTGRES:
+ return CommonBeanUtils.copyProperties((PostgresSinkResponse) sinkResponse, PostgresSinkRequest::new);
+ default:
+ throw new IllegalArgumentException(String.format("Unsupported sink type : %s for Inlong", sinkType));
+ }
+ }
+
+}
diff --git a/inlong-manager/manager-client/src/test/java/org/apache/inlong/manager/client/api/impl/InlongStreamImplTest.java b/inlong-manager/manager-client/src/test/java/org/apache/inlong/manager/client/api/impl/InlongStreamImplTest.java
index 4b964698f..5f87fa8e6 100644
--- a/inlong-manager/manager-client/src/test/java/org/apache/inlong/manager/client/api/impl/InlongStreamImplTest.java
+++ b/inlong-manager/manager-client/src/test/java/org/apache/inlong/manager/client/api/impl/InlongStreamImplTest.java
@@ -18,14 +18,14 @@
package org.apache.inlong.manager.client.api.impl;
import org.apache.inlong.manager.client.api.InlongStream;
-import org.apache.inlong.manager.client.api.sink.ClickHouseSink;
-import org.apache.inlong.manager.client.api.sink.HiveSink;
-import org.apache.inlong.manager.client.api.sink.KafkaSink;
import org.apache.inlong.manager.client.api.source.KafkaSource;
import org.apache.inlong.manager.client.api.source.MySQLBinlogSource;
import org.apache.inlong.manager.client.api.transform.MultiDependencyTransform;
import org.apache.inlong.manager.client.api.transform.SingleDependencyTransform;
import org.apache.inlong.manager.client.api.util.GsonUtils;
+import org.apache.inlong.manager.common.pojo.sink.ck.ClickHouseSinkRequest;
+import org.apache.inlong.manager.common.pojo.sink.hive.HiveSinkRequest;
+import org.apache.inlong.manager.common.pojo.sink.kafka.KafkaSinkRequest;
import org.apache.inlong.manager.common.pojo.stream.StreamPipeline;
import org.apache.inlong.manager.common.pojo.stream.StreamTransform;
import org.apache.inlong.manager.common.pojo.transform.filter.FilterDefinition;
@@ -53,18 +53,22 @@ public class InlongStreamImplTest {
inlongStream.addSource(kafkaSource);
inlongStream.addSource(mySQLBinlogSource);
// add stream sink
- ClickHouseSink clickHouseSink = new ClickHouseSink();
+ ClickHouseSinkRequest clickHouseSink = new ClickHouseSinkRequest();
clickHouseSink.setSinkName("E");
- HiveSink hiveSink = new HiveSink();
- hiveSink.setSinkName("F");
- KafkaSink kafkaSink1 = new KafkaSink();
- kafkaSink1.setSinkName("I");
- KafkaSink kafkaSink2 = new KafkaSink();
- kafkaSink2.setSinkName("M");
inlongStream.addSink(clickHouseSink);
+
+ HiveSinkRequest hiveSink = new HiveSinkRequest();
+ hiveSink.setSinkName("F");
inlongStream.addSink(hiveSink);
+
+ KafkaSinkRequest kafkaSink1 = new KafkaSinkRequest();
+ kafkaSink1.setSinkName("I");
inlongStream.addSink(kafkaSink1);
+
+ KafkaSinkRequest kafkaSink2 = new KafkaSinkRequest();
+ kafkaSink2.setSinkName("M");
inlongStream.addSink(kafkaSink2);
+
// add stream transform
StreamTransform multiDependencyTransform = new MultiDependencyTransform(
"C",
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/FieldType.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/FieldType.java
index 735cf6b62..901dec247 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/FieldType.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/FieldType.java
@@ -19,9 +19,8 @@ package org.apache.inlong.manager.common.enums;
import org.apache.inlong.manager.common.util.Preconditions;
-import java.util.Locale;
-
public enum FieldType {
+
INT,
TINYINT,
SMALLINT,
@@ -40,18 +39,18 @@ public enum FieldType {
TIME,
TIMESTAMP;
- @Override
- public String toString() {
- return name();
- }
-
public static FieldType forName(String name) {
Preconditions.checkNotNull(name, "FieldType should not be null");
for (FieldType value : values()) {
- if (value.toString().equals(name) || value.toString().equals(name.toUpperCase(Locale.ROOT))) {
+ if (value.toString().equals(name)) {
return value;
}
}
throw new IllegalArgumentException(String.format("Unsupported FieldType : %s", name));
}
+
+ @Override
+ public String toString() {
+ return name();
+ }
}
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/SourceStatus.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/SourceStatus.java
index 4ea363906..1561af0ec 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/SourceStatus.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/SourceStatus.java
@@ -25,7 +25,7 @@ import java.util.Map;
import java.util.Set;
/**
- * State of stream source
+ * Status of stream source
*/
public enum SourceStatus {
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/stream/SinkField.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/SinkField.java
similarity index 75%
rename from inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/stream/SinkField.java
rename to inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/SinkField.java
index 97f40e6d7..df6db86e2 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/stream/SinkField.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/SinkField.java
@@ -15,21 +15,19 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.common.pojo.stream;
+package org.apache.inlong.manager.common.pojo.sink;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import lombok.EqualsAndHashCode;
-import lombok.NoArgsConstructor;
-import org.apache.inlong.manager.common.enums.FieldType;
+import org.apache.inlong.manager.common.pojo.stream.StreamField;
/**
* Sink field info.
*/
@Data
@EqualsAndHashCode(callSuper = true)
-@NoArgsConstructor
@ApiModel("Sink field configuration")
public class SinkField extends StreamField {
@@ -37,15 +35,18 @@ public class SinkField extends StreamField {
private String sourceFieldName;
@ApiModelProperty("Source field type")
- private FieldType sourceFieldType;
+ private String sourceFieldType;
- public SinkField(int index, FieldType fieldType, String fieldName, FieldType sourceFieldType,
- String sourceFieldName) {
+ public SinkField() {
+
+ }
+
+ public SinkField(int index, String fieldType, String fieldName, String sourceFieldType, String sourceFieldName) {
this(index, fieldType, fieldName, null, null, sourceFieldName, sourceFieldType, 0, null);
}
- public SinkField(int index, FieldType fieldType, String fieldName, String fieldComment,
- String fieldValue, String sourceFieldName, FieldType sourceFieldType,
+ public SinkField(int index, String fieldType, String fieldName, String fieldComment,
+ String fieldValue, String sourceFieldName, String sourceFieldType,
Integer isMetaField, String fieldFormat) {
super(index, fieldType, fieldName, fieldComment, fieldValue, isMetaField, fieldFormat);
this.sourceFieldName = sourceFieldName;
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/SinkFieldBase.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/SinkFieldBase.java
deleted file mode 100644
index 555109f84..000000000
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/SinkFieldBase.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.manager.common.pojo.sink;
-
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-import lombok.Data;
-
-/**
- * Sink field base info
- */
-@Data
-@ApiModel("Sink field base info")
-public class SinkFieldBase {
-
- @ApiModelProperty("Field name")
- private String fieldName;
-
- @ApiModelProperty("Field type")
- private String fieldType;
-
- @ApiModelProperty("Field comment")
- private String fieldComment;
-
- @ApiModelProperty("Source field name")
- private String sourceFieldName;
-
- @ApiModelProperty("Source field type")
- private String sourceFieldType;
-
- @ApiModelProperty("Length of fixed type")
- private Integer fieldLength;
-
- @ApiModelProperty("Precision of decimal type, that is, field length. precision >= scale")
- private Integer fieldPrecision;
-
- @ApiModelProperty("Range of decimal type, that is, the number of decimal places. precision >= scale")
- private Integer fieldScale;
-
- @ApiModelProperty("Partition strategy, including: None, Identity, Year, Month, Day, Hour, Bucket, Truncate")
- private String partitionStrategy;
-
- @ApiModelProperty("Extra Param in JSON style")
- private String extParams;
-
- @ApiModelProperty("Field format, including: MICROSECONDS, MILLISECONDS, SECONDS, SQL, ISO_8601"
- + " and custom such as 'yyyy-MM-dd HH:mm:ss'. This is mainly used for time format")
- private String fieldFormat;
-
- @ApiModelProperty("Field order")
- private Short rankNum;
-
-}
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/SinkFieldRequest.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/SinkFieldRequest.java
deleted file mode 100644
index a0a511949..000000000
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/SinkFieldRequest.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.manager.common.pojo.sink;
-
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-
-/**
- * Sink field request.
- */
-@Data
-@EqualsAndHashCode(callSuper = true)
-@ApiModel("Sink field request")
-public class SinkFieldRequest extends SinkFieldBase {
-
- @ApiModelProperty("Is this field a meta field, 0: no, 1: yes")
- private Integer isMetaField = 0;
-
-}
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/SinkFieldResponse.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/SinkFieldResponse.java
deleted file mode 100644
index 7ad5eec4f..000000000
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/SinkFieldResponse.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.manager.common.pojo.sink;
-
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-
-/**
- * Sink field response
- */
-@Data
-@EqualsAndHashCode(callSuper = true)
-@ApiModel("Sink field response")
-public class SinkFieldResponse extends SinkFieldBase {
-
- private Integer id;
-
- @ApiModelProperty("Is this field a meta field, 0: no, 1: yes")
- private Integer isMetaField;
-
-}
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/SinkPageRequest.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/SinkPageRequest.java
index 6ea62c618..4bc733ef2 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/SinkPageRequest.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/SinkPageRequest.java
@@ -34,7 +34,7 @@ import javax.validation.constraints.NotNull;
public class SinkPageRequest extends PageRequest {
@ApiModelProperty(value = "Inlong group id", required = true)
- @NotNull
+ @NotNull(message = "inlongGroupId cannot be null")
private String inlongGroupId;
@ApiModelProperty(value = "Inlong stream id")
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/SinkRequest.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/SinkRequest.java
index 6c5d44358..733b5636d 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/SinkRequest.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/SinkRequest.java
@@ -19,40 +19,19 @@ package org.apache.inlong.manager.common.pojo.sink;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
import com.fasterxml.jackson.annotation.JsonTypeInfo.Id;
-import com.google.common.collect.Maps;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
-
-import javax.validation.constraints.NotNull;
-import java.util.List;
-import java.util.Map;
+import lombok.EqualsAndHashCode;
/**
* Request of sink
*/
@Data
+@EqualsAndHashCode(callSuper = true)
@ApiModel("Request of sink")
@JsonTypeInfo(use = Id.NAME, visible = true, property = "sinkType")
-public class SinkRequest {
-
- private Integer id;
-
- @ApiModelProperty("Inlong group id")
- @NotNull
- private String inlongGroupId;
-
- @ApiModelProperty("Inlong stream id")
- @NotNull
- private String inlongStreamId;
-
- @ApiModelProperty("Sink type, including: HIVE, ES, etc.")
- @NotNull
- private String sinkType;
-
- @ApiModelProperty("Sink name, unique in one stream.")
- @NotNull
- private String sinkName;
+public abstract class SinkRequest extends StreamSink {
@ApiModelProperty("Sink description")
private String description;
@@ -69,12 +48,7 @@ public class SinkRequest {
@ApiModelProperty("Sort consumer group")
private String sortConsumerGroup;
- @ApiModelProperty(value = "Whether to enable create sink resource? 0: disable, 1: enable. default is 1")
+ @ApiModelProperty(value = "Whether to enable create sink resource? 0: disable, 1: enable. Default is 1")
private Integer enableCreateResource = 1;
- @ApiModelProperty("Sink field list")
- private List<SinkFieldRequest> fieldList;
-
- @ApiModelProperty("Properties for sink")
- private Map<String, Object> properties = Maps.newHashMap();
}
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/SinkResponse.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/SinkResponse.java
index 30f249237..eb13dcb80 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/SinkResponse.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/SinkResponse.java
@@ -75,7 +75,7 @@ public class SinkResponse {
@ApiModelProperty("Status")
private Integer status;
- @ApiModelProperty("Previous State")
+ @ApiModelProperty("Previous Status")
private Integer previousStatus;
@ApiModelProperty("Creator")
@@ -91,7 +91,7 @@ public class SinkResponse {
private Date modifyTime;
@ApiModelProperty("Sink field list")
- private List<SinkFieldResponse> fieldList;
+ private List<SinkField> fieldList;
@ApiModelProperty("Properties for sink")
private Map<String, Object> properties;
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/stream/StreamSink.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/StreamSink.java
similarity index 53%
rename from inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/stream/StreamSink.java
rename to inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/StreamSink.java
index a2165f1db..6b1f7b05a 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/stream/StreamSink.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/StreamSink.java
@@ -15,14 +15,17 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.common.pojo.stream;
+package org.apache.inlong.manager.common.pojo.sink;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.google.common.collect.Maps;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import lombok.EqualsAndHashCode;
+import org.apache.inlong.manager.common.auth.DefaultAuthentication;
import org.apache.inlong.manager.common.enums.DataFormat;
-import org.apache.inlong.manager.common.enums.SinkType;
+import org.apache.inlong.manager.common.pojo.stream.StreamNode;
import java.util.List;
import java.util.Map;
@@ -35,16 +38,37 @@ import java.util.Map;
@ApiModel("Stream sink configuration")
public abstract class StreamSink extends StreamNode {
- @ApiModelProperty(value = "DataSink name", required = true)
+ @ApiModelProperty("Sink id")
+ private Integer id;
+
+ @ApiModelProperty("Inlong group id")
+ private String inlongGroupId;
+
+ @ApiModelProperty("Inlong stream id")
+ private String inlongStreamId;
+
+ @ApiModelProperty("Sink type, including: HIVE, ES, etc.")
+ private String sinkType;
+
+ @ApiModelProperty("Sink name, unique in one stream")
private String sinkName;
- @ApiModelProperty("Other properties if need")
- private Map<String, Object> properties;
+ @JsonIgnore
+ @ApiModelProperty("Data format type for stream sink")
+ private DataFormat dataFormat;
+
+ @JsonIgnore
+ @ApiModelProperty("Authentication info if needed")
+ private DefaultAuthentication authentication;
- public abstract SinkType getSinkType();
+ @ApiModelProperty("Sink field list")
+ private List<SinkField> fieldList;
- public abstract List<SinkField> getSinkFields();
+ @ApiModelProperty("Other properties if needed")
+ private Map<String, Object> properties = Maps.newHashMap();
- public abstract DataFormat getDataFormat();
+ public DataFormat getDataFormat() {
+ return DataFormat.NONE;
+ }
}
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/ck/ClickHouseSinkDTO.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/ck/ClickHouseSinkDTO.java
index e18b8a92b..e63a19ae7 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/ck/ClickHouseSinkDTO.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/ck/ClickHouseSinkDTO.java
@@ -82,7 +82,7 @@ public class ClickHouseSinkDTO {
@ApiModelProperty("Table engine, support MergeTree Mem and so on")
private String engine;
- @ApiModelProperty("Table Partiion information")
+ @ApiModelProperty("Table partition information")
private String partitionBy;
@ApiModelProperty("Table order information")
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/ck/ClickHouseSinkRequest.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/ck/ClickHouseSinkRequest.java
index f4ebdc31d..afece3cd2 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/ck/ClickHouseSinkRequest.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/ck/ClickHouseSinkRequest.java
@@ -52,13 +52,13 @@ public class ClickHouseSinkRequest extends SinkRequest {
private String tableName;
@ApiModelProperty("Flush interval, unit: second, default is 1s")
- private Integer flushInterval;
+ private Integer flushInterval = 1;
@ApiModelProperty("Flush when record number reaches flushRecord")
private Integer flushRecord;
@ApiModelProperty("Write max retry times, default is 3")
- private Integer retryTimes;
+ private Integer retryTimes = 3;
@ApiModelProperty("Whether distributed table? 0: no, 1: yes")
private Integer isDistributed;
@@ -73,10 +73,10 @@ public class ClickHouseSinkRequest extends SinkRequest {
@ApiModelProperty("Key field names, separate with commas")
private String keyFieldNames;
- @ApiModelProperty("table engine, support MergeTree Mem and so on")
+ @ApiModelProperty("Table engine, support MergeTree Mem and so on")
private String engine;
- @ApiModelProperty("Table Partiion information")
+ @ApiModelProperty("Table partition information")
private String partitionBy;
@ApiModelProperty("Table order information")
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/es/ElasticsearchSinkRequest.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/es/ElasticsearchSinkRequest.java
index 0661833b2..c64f4e151 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/es/ElasticsearchSinkRequest.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/es/ElasticsearchSinkRequest.java
@@ -23,7 +23,7 @@ import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.ToString;
import org.apache.inlong.manager.common.enums.SinkType;
-import org.apache.inlong.manager.common.pojo.sink.SinkFieldRequest;
+import org.apache.inlong.manager.common.pojo.sink.SinkField;
import org.apache.inlong.manager.common.pojo.sink.SinkRequest;
import org.apache.inlong.manager.common.util.JsonTypeDefine;
@@ -64,6 +64,6 @@ public class ElasticsearchSinkRequest extends SinkRequest {
private Integer retryTimes;
@ApiModelProperty("Sink filed info")
- private List<SinkFieldRequest> fieldList;
+ private List<SinkField> fieldList;
}
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/hbase/HbaseSinkDTO.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/hbase/HbaseSinkDTO.java
index 8b2499bbc..1936dbfdf 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/hbase/HbaseSinkDTO.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/hbase/HbaseSinkDTO.java
@@ -42,29 +42,29 @@ public class HbaseSinkDTO {
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
- @ApiModelProperty("Target table name")
- private String tableName;
-
@ApiModelProperty("Namespace")
private String namespace;
+ @ApiModelProperty("Target table name")
+ private String tableName;
+
@ApiModelProperty("Row key")
private String rowKey;
- @ApiModelProperty("Zookeeper quorm")
- private String zookeeperQuorum;
+ @ApiModelProperty("ZooKeeper quorum")
+ private String zkQuorum;
- @ApiModelProperty("Sink buffer flush maxsize")
- private String sinkBufferFlushMaxSize;
+ @ApiModelProperty("ZooKeeper node parent")
+ private String zkNodeParent;
- @ApiModelProperty("Zookeeper znode parent")
- private String zookeeperZnodeParent;
+ @ApiModelProperty("Sink buffer flush maxsize")
+ private String bufferFlushMaxSize;
@ApiModelProperty("Sink buffer flush max rows")
- private String sinkBufferFlushMaxRows;
+ private String bufferFlushMaxRows;
@ApiModelProperty("Sink buffer flush interval")
- private String sinkBufferFlushInterval;
+ private String bufferFlushInterval;
@ApiModelProperty("Properties for hbase")
private Map<String, Object> properties;
@@ -77,11 +77,11 @@ public class HbaseSinkDTO {
.tableName(request.getTableName())
.namespace(request.getNamespace())
.rowKey(request.getRowKey())
- .zookeeperQuorum(request.getZookeeperQuorum())
- .sinkBufferFlushMaxSize(request.getSinkBufferFlushMaxSize())
- .zookeeperZnodeParent(request.getZookeeperZnodeParent())
- .sinkBufferFlushMaxRows(request.getSinkBufferFlushMaxRows())
- .sinkBufferFlushInterval(request.getSinkBufferFlushInterval())
+ .zkQuorum(request.getZkQuorum())
+ .bufferFlushMaxSize(request.getBufferFlushMaxSize())
+ .zkNodeParent(request.getZkNodeParent())
+ .bufferFlushMaxRows(request.getBufferFlushMaxRows())
+ .bufferFlushInterval(request.getBufferFlushInterval())
.properties(request.getProperties())
.build();
}
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/hbase/HbaseSinkListResponse.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/hbase/HbaseSinkListResponse.java
index e0183ee88..39e233d82 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/hbase/HbaseSinkListResponse.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/hbase/HbaseSinkListResponse.java
@@ -31,28 +31,28 @@ import org.apache.inlong.manager.common.pojo.sink.SinkListResponse;
@ApiModel("Response of Hbase sink paging list")
public class HbaseSinkListResponse extends SinkListResponse {
- @ApiModelProperty("Target table name")
- private String tableName;
-
@ApiModelProperty("Namespace")
private String namespace;
+ @ApiModelProperty("Target table name")
+ private String tableName;
+
@ApiModelProperty("Row key")
private String rowKey;
- @ApiModelProperty("Zookeeper quorm")
- private String zookeeperQuorum;
+ @ApiModelProperty("ZooKeeper quorum")
+ private String zkQuorum;
- @ApiModelProperty("Sink buffer flush maxsize")
- private String sinkBufferFlushMaxSize;
+ @ApiModelProperty("ZooKeeper node parent")
+ private String zkNodeParent;
- @ApiModelProperty("Zookeeper znode parent")
- private String zookeeperZnodeParent;
+ @ApiModelProperty("Sink buffer flush maxsize")
+ private String bufferFlushMaxSize;
@ApiModelProperty("Sink buffer flush max rows")
- private String sinkBufferFlushMaxRows;
+ private String bufferFlushMaxRows;
@ApiModelProperty("Sink buffer flush interval")
- private String sinkBufferFlushInterval;
+ private String bufferFlushInterval;
}
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/hbase/HbaseSinkRequest.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/hbase/HbaseSinkRequest.java
index 1fae9067b..99eb4bf4b 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/hbase/HbaseSinkRequest.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/hbase/HbaseSinkRequest.java
@@ -27,37 +27,37 @@ import org.apache.inlong.manager.common.pojo.sink.SinkRequest;
import org.apache.inlong.manager.common.util.JsonTypeDefine;
/**
- * Request of the Hbase sink
+ * Request of the HBase sink
*/
@Data
@ToString(callSuper = true)
@EqualsAndHashCode(callSuper = true)
-@ApiModel(value = "Request of the Hbase sink")
+@ApiModel(value = "Request of the HBase sink")
@JsonTypeDefine(value = SinkType.SINK_HBASE)
public class HbaseSinkRequest extends SinkRequest {
- @ApiModelProperty("Target table name")
- private String tableName;
-
@ApiModelProperty("Namespace")
private String namespace;
+ @ApiModelProperty("Target table name")
+ private String tableName;
+
@ApiModelProperty("Row key")
private String rowKey;
- @ApiModelProperty("Zookeeper quorm")
- private String zookeeperQuorum;
+ @ApiModelProperty("ZooKeeper quorum")
+ private String zkQuorum;
- @ApiModelProperty("Sink buffer flush maxsize")
- private String sinkBufferFlushMaxSize;
+ @ApiModelProperty("ZooKeeper node parent")
+ private String zkNodeParent;
- @ApiModelProperty("Zookeeper znode parent")
- private String zookeeperZnodeParent;
+ @ApiModelProperty("Sink buffer flush maxsize")
+ private String bufferFlushMaxSize;
@ApiModelProperty("Sink buffer flush max rows")
- private String sinkBufferFlushMaxRows;
+ private String bufferFlushMaxRows;
@ApiModelProperty("Sink buffer flush interval")
- private String sinkBufferFlushInterval;
+ private String bufferFlushInterval;
}
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/hbase/HbaseSinkResponse.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/hbase/HbaseSinkResponse.java
index 4bb09d480..09ea5bf1a 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/hbase/HbaseSinkResponse.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/hbase/HbaseSinkResponse.java
@@ -34,29 +34,29 @@ import org.apache.inlong.manager.common.pojo.sink.SinkResponse;
@ApiModel(value = "Response of the Hbase sink")
public class HbaseSinkResponse extends SinkResponse {
- @ApiModelProperty("Target table name")
- private String tableName;
-
@ApiModelProperty("Namespace")
private String namespace;
+ @ApiModelProperty("Target table name")
+ private String tableName;
+
@ApiModelProperty("Row key")
private String rowKey;
- @ApiModelProperty("Zookeeper quorm")
- private String zookeeperQuorum;
+ @ApiModelProperty("ZooKeeper quorum")
+ private String zkQuorum;
- @ApiModelProperty("Sink buffer flush maxsize")
- private String sinkBufferFlushMaxSize;
+ @ApiModelProperty("ZooKeeper node parent")
+ private String zkNodeParent;
- @ApiModelProperty("Zookeeper znode parent")
- private String zookeeperZnodeParent;
+ @ApiModelProperty("Sink buffer flush maxsize")
+ private String bufferFlushMaxSize;
@ApiModelProperty("Sink buffer flush max rows")
- private String sinkBufferFlushMaxRows;
+ private String bufferFlushMaxRows;
@ApiModelProperty("Sink buffer flush interval")
- private String sinkBufferFlushInterval;
+ private String bufferFlushInterval;
public HbaseSinkResponse() {
this.sinkType = SinkType.SINK_HBASE;
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/hive/HiveSinkRequest.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/hive/HiveSinkRequest.java
index c098bee96..40ff6dd25 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/hive/HiveSinkRequest.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/hive/HiveSinkRequest.java
@@ -22,10 +22,14 @@ import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.ToString;
+import org.apache.inlong.manager.common.enums.DataSeparator;
+import org.apache.inlong.manager.common.enums.FileFormat;
import org.apache.inlong.manager.common.enums.SinkType;
import org.apache.inlong.manager.common.pojo.sink.SinkRequest;
import org.apache.inlong.manager.common.util.JsonTypeDefine;
+import javax.validation.constraints.NotNull;
+import java.nio.charset.StandardCharsets;
import java.util.List;
/**
@@ -38,10 +42,8 @@ import java.util.List;
@JsonTypeDefine(value = SinkType.SINK_HIVE)
public class HiveSinkRequest extends SinkRequest {
- @ApiModelProperty("Whether to enable create table, 1: enable, 0: disable, default is 1")
- private Integer enableCreateTable = 1;
-
- @ApiModelProperty("Hive JDBC URL")
+ @NotNull(message = "jdbcUrl cannot be null")
+ @ApiModelProperty("Hive JDBC URL, such as jdbc:hive2://${ip}:${port}")
private String jdbcUrl;
@ApiModelProperty("Username for JDBC URL")
@@ -50,12 +52,15 @@ public class HiveSinkRequest extends SinkRequest {
@ApiModelProperty("User password")
private String password;
+ @NotNull(message = "dbName cannot be null")
@ApiModelProperty("Target database name")
private String dbName;
+ @NotNull(message = "tableName cannot be null")
@ApiModelProperty("Target table name")
private String tableName;
+ @NotNull(message = "dataPath cannot be null")
@ApiModelProperty("Data path, such as: hdfs://ip:port/user/hive/warehouse/test.db")
private String dataPath;
@@ -68,19 +73,20 @@ public class HiveSinkRequest extends SinkRequest {
@ApiModelProperty("Partition creation strategy, partition start, partition close")
private String partitionCreationStrategy;
- @ApiModelProperty("File format, support: TextFile, RCFile, SequenceFile, Avro")
- private String fileFormat;
+ @ApiModelProperty("File format, support: TextFile, ORCFile, RCFile, SequenceFile, Avro, Parquet, etc")
+ private String fileFormat = FileFormat.TextFile.name();
- @ApiModelProperty("Data encoding type")
- private String dataEncoding;
+ @ApiModelProperty("Data encoding format: UTF-8, GBK")
+ private String dataEncoding = StandardCharsets.UTF_8.toString();
- @ApiModelProperty("Data field separator")
- private String dataSeparator;
+ @ApiModelProperty("Data separator, stored as ASCII code")
+ private String dataSeparator = DataSeparator.SOH.getSeparator();
- @ApiModelProperty("Version for hive")
+ @ApiModelProperty("Version for Hive, such as: 3.2.1")
private String hiveVersion;
- @ApiModelProperty("Config directory of hive, needed by sort in light mode")
+ @NotNull(message = "hiveConfDir cannot be null")
+ @ApiModelProperty("Config directory of Hive on HDFS, must include hive-site.xml")
private String hiveConfDir;
}
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/kafka/KafkaSinkRequest.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/kafka/KafkaSinkRequest.java
index 31dc27717..33721d2fd 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/kafka/KafkaSinkRequest.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/kafka/KafkaSinkRequest.java
@@ -22,6 +22,7 @@ import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.ToString;
+import org.apache.inlong.manager.common.enums.DataFormat;
import org.apache.inlong.manager.common.enums.SinkType;
import org.apache.inlong.manager.common.pojo.sink.SinkRequest;
import org.apache.inlong.manager.common.util.JsonTypeDefine;
@@ -46,7 +47,7 @@ public class KafkaSinkRequest extends SinkRequest {
private String partitionNum;
@ApiModelProperty("Data Serialization, support: json, canal, avro")
- private String serializationType;
+ private String serializationType = DataFormat.CANAL.getName();
@ApiModelProperty(value = "The strategy of auto offset reset",
notes = "including earliest, latest (the default), none")
@@ -55,4 +56,9 @@ public class KafkaSinkRequest extends SinkRequest {
@ApiModelProperty("Primary key is required when serializationType is json, avro")
private String primaryKey;
+ @Override
+ public DataFormat getDataFormat() {
+ return DataFormat.CANAL;
+ }
+
}
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/source/SourceRequest.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/source/SourceRequest.java
index 19ccca4a7..1b3f89969 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/source/SourceRequest.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/source/SourceRequest.java
@@ -22,7 +22,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo.Id;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
-import org.apache.inlong.manager.common.pojo.stream.InlongStreamFieldInfo;
+import org.apache.inlong.manager.common.pojo.stream.StreamField;
import javax.validation.constraints.NotNull;
import java.util.List;
@@ -72,6 +72,6 @@ public class SourceRequest {
private Integer version;
@ApiModelProperty("Field list, only support when inlong group in light weight mode")
- private List<InlongStreamFieldInfo> fieldList;
+ private List<StreamField> fieldList;
}
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/source/SourceResponse.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/source/SourceResponse.java
index 8a5be5858..a3387c9a7 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/source/SourceResponse.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/source/SourceResponse.java
@@ -21,7 +21,7 @@ import com.fasterxml.jackson.annotation.JsonFormat;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
-import org.apache.inlong.manager.common.pojo.stream.InlongStreamFieldInfo;
+import org.apache.inlong.manager.common.pojo.stream.StreamField;
import java.util.Date;
import java.util.List;
@@ -71,7 +71,7 @@ public class SourceResponse {
@ApiModelProperty("Status")
private Integer status;
- @ApiModelProperty("Previous State")
+ @ApiModelProperty("Previous Status")
private Integer previousStatus;
@ApiModelProperty("Creator")
@@ -87,6 +87,6 @@ public class SourceResponse {
private Date modifyTime;
@ApiModelProperty(value = "Field list")
- private List<InlongStreamFieldInfo> fieldList;
+ private List<StreamField> fieldList;
}
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/stream/StreamSource.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/source/StreamSource.java
similarity index 87%
rename from inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/stream/StreamSource.java
rename to inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/source/StreamSource.java
index 8961f5be8..275205507 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/stream/StreamSource.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/source/StreamSource.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.common.pojo.stream;
+package org.apache.inlong.manager.common.pojo.source;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
@@ -24,6 +24,7 @@ import lombok.EqualsAndHashCode;
import org.apache.inlong.manager.common.enums.DataFormat;
import org.apache.inlong.manager.common.enums.SourceStatus;
import org.apache.inlong.manager.common.enums.SourceType;
+import org.apache.inlong.manager.common.pojo.stream.StreamNode;
/**
* Stream source configuration, including source name, agent ip, etc.
@@ -33,13 +34,32 @@ import org.apache.inlong.manager.common.enums.SourceType;
@ApiModel("Stream source configuration")
public abstract class StreamSource extends StreamNode {
- public enum State {
- INIT, NORMAL, FROZING, FROZEN, FAILED, DELETING, DELETE;
+ @ApiModelProperty(value = "DataSource name", required = true)
+ private String sourceName;
+
+ @ApiModelProperty("Ip of the agent running the task")
+ private String agentIp;
+
+ @ApiModelProperty("Mac uuid of the agent running the task")
+ private String uuid = "";
+
+ @ApiModelProperty("Status of stream source")
+ private Status status;
+
+ public abstract SourceType getSourceType();
+
+ public abstract SyncType getSyncType();
+
+ public abstract DataFormat getDataFormat();
+
+ public enum Status {
+
+ INIT, NORMAL, FREEZING, FROZEN, FAILED, DELETING, DELETE;
/**
* Parse status of stream source.
*/
- public static State parseByStatus(int status) {
+ public static Status parseByStatus(int status) {
SourceStatus sourceStatus = SourceStatus.forCode(status);
switch (sourceStatus) {
case SOURCE_NEW:
@@ -52,7 +72,7 @@ public abstract class StreamSource extends StreamNode {
return NORMAL;
case TO_BE_ISSUED_FROZEN:
case BEEN_ISSUED_FROZEN:
- return FROZING;
+ return FREEZING;
case SOURCE_FROZEN:
return FROZEN;
case SOURCE_FAILED:
@@ -64,7 +84,7 @@ public abstract class StreamSource extends StreamNode {
return DELETE;
default:
throw new IllegalStateException(
- String.format("Unsupported source state=%s for Inlong", sourceStatus));
+ String.format("Unsupported source status=%s for Inlong", sourceStatus));
}
}
}
@@ -72,22 +92,4 @@ public abstract class StreamSource extends StreamNode {
public enum SyncType {
FULL, INCREMENT
}
-
- @ApiModelProperty(value = "DataSource name", required = true)
- private String sourceName;
-
- @ApiModelProperty("Ip of the agent running the task")
- private String agentIp;
-
- @ApiModelProperty("Mac uuid of the agent running the task")
- private String uuid = "";
-
- @ApiModelProperty("State of stream source")
- private State state;
-
- public abstract SourceType getSourceType();
-
- public abstract SyncType getSyncType();
-
- public abstract DataFormat getDataFormat();
}
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/stream/InlongStreamFieldInfo.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/stream/InlongStreamFieldInfo.java
deleted file mode 100644
index d991e0b9c..000000000
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/stream/InlongStreamFieldInfo.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.manager.common.pojo.stream;
-
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-import lombok.Data;
-
-/**
- * Inlong stream field info.
- */
-@Data
-@ApiModel("Inlong stream field info")
-public class InlongStreamFieldInfo {
-
- private Integer id;
-
- @ApiModelProperty(value = "inlong group id", required = true)
- private String inlongGroupId;
-
- @ApiModelProperty(value = "inlong stream id", required = true)
- private String inlongStreamId;
-
- @ApiModelProperty(value = "is predefined field, 1: ye, 0: no")
- private Integer isPredefinedField;
-
- @ApiModelProperty(value = "Field name")
- private String fieldName;
-
- @ApiModelProperty(value = "Field value")
- private String fieldValue;
-
- @ApiModelProperty(value = "value expression of predefined field")
- private String preExpression;
-
- @ApiModelProperty("Field type")
- private String fieldType;
-
- @ApiModelProperty("Field comment")
- private String fieldComment;
-
- @ApiModelProperty("Is this field a meta field, 0: no, 1: yes")
- private Integer isMetaField = 0;
-
- @ApiModelProperty("Field format, including: MICROSECONDS, MILLISECONDS, SECONDS, SQL, ISO_8601"
- + " and custom such as 'yyyy-MM-dd HH:mm:ss'. This is mainly used for time format")
- private String fieldFormat;
-
- @ApiModelProperty(value = "field rank num")
- private Short rankNum;
-
-}
\ No newline at end of file
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/stream/InlongStreamInfo.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/stream/InlongStreamInfo.java
index 9826f8d52..c908a2aca 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/stream/InlongStreamInfo.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/stream/InlongStreamInfo.java
@@ -48,8 +48,7 @@ public class InlongStreamInfo {
@ApiModelProperty(value = "Inlong stream description")
private String description;
- @ApiModelProperty(value = "MQ resource, in inlong group",
- notes = "Tube corresponds to Topic, Pulsar corresponds to Namespace")
+ @ApiModelProperty(value = "MQ resource for inlong stream. Default: ${inlongStreamId}")
private String mqResource;
@ApiModelProperty(value = "Data type, including: TEXT, KV, etc.")
@@ -67,7 +66,7 @@ public class InlongStreamInfo {
@ApiModelProperty(value = "Whether to send synchronously, 0: no, 1: yes",
notes = "Each task under this stream sends data synchronously, "
+ "which will affect the throughput of data collection, please choose carefully")
- private Integer syncSend;
+ private Integer syncSend = 0;
@ApiModelProperty(value = "Number of access items per day, unit: 10,000 items per day")
private Integer dailyRecords;
@@ -107,7 +106,7 @@ public class InlongStreamInfo {
private Date modifyTime;
@ApiModelProperty(value = "Field list")
- private List<InlongStreamFieldInfo> fieldList;
+ private List<StreamField> fieldList;
@ApiModelProperty(value = "Inlong stream Extension properties")
private List<InlongStreamExtInfo> extList;
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/stream/InlongStreamRequest.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/stream/InlongStreamRequest.java
index a317d6b69..55a4c91b9 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/stream/InlongStreamRequest.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/stream/InlongStreamRequest.java
@@ -87,7 +87,7 @@ public class InlongStreamRequest {
private String extParams;
@ApiModelProperty(value = "Field list")
- private List<InlongStreamFieldInfo> fieldList;
+ private List<StreamField> fieldList;
@ApiModelProperty(value = "Inlong stream Extension properties")
private List<InlongStreamExtInfo> extList;
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/stream/InlongStreamResponse.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/stream/InlongStreamResponse.java
index fc875b98a..65fcd8d96 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/stream/InlongStreamResponse.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/stream/InlongStreamResponse.java
@@ -113,7 +113,7 @@ public class InlongStreamResponse {
private Date modifyTime;
@ApiModelProperty(value = "Field list")
- private List<InlongStreamFieldInfo> fieldList;
+ private List<StreamField> fieldList;
@ApiModelProperty(value = "Inlong stream Extension properties")
private List<InlongStreamExtInfo> extList;
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/stream/StreamField.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/stream/StreamField.java
index 6f1e31895..02a5e9358 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/stream/StreamField.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/stream/StreamField.java
@@ -36,25 +36,37 @@ public class StreamField {
public static final StreamField PROCESSING_TIME = new StreamField(
100,
- FieldType.BIGINT,
+ FieldType.BIGINT.toString(),
MetaFieldType.PROCESSING_TIME.getName(),
null, null, 1);
@ApiModelProperty("Field index")
private Integer id;
- @ApiModelProperty(value = "Field type", required = true)
- private FieldType fieldType;
+ @ApiModelProperty(value = "inlong group id", required = true)
+ private String inlongGroupId;
+
+ @ApiModelProperty(value = "inlong stream id", required = true)
+ private String inlongStreamId;
@ApiModelProperty(value = "Field name", required = true)
private String fieldName;
+ @ApiModelProperty(value = "Field type", required = true)
+ private String fieldType;
+
@ApiModelProperty(value = "Field comment")
private String fieldComment;
@ApiModelProperty(value = "Field value for constants")
private String fieldValue;
+ @ApiModelProperty(value = "Is predefined field, 1: yes, 0: no")
+ private Integer isPredefinedField;
+
+ @ApiModelProperty(value = "Value expression of predefined field")
+ private String preExpression;
+
@ApiModelProperty("Is this field a meta field, 0: no, 1: yes")
private Integer isMetaField = 0;
@@ -68,7 +80,10 @@ public class StreamField {
@ApiModelProperty("Origin field name before transform operation")
private String originFieldName;
- public StreamField(int index, FieldType fieldType, String fieldName, String fieldComment, String fieldValue) {
+ @ApiModelProperty("Extra Param in JSON style")
+ private String extParams;
+
+ public StreamField(int index, String fieldType, String fieldName, String fieldComment, String fieldValue) {
this.id = index;
this.fieldType = fieldType;
this.fieldName = fieldName;
@@ -76,20 +91,20 @@ public class StreamField {
this.fieldValue = fieldValue;
}
- public StreamField(int index, FieldType fieldType, String fieldName, String fieldComment, String fieldValue,
+ public StreamField(int index, String fieldType, String fieldName, String fieldComment, String fieldValue,
Integer isMetaField) {
this(index, fieldType, fieldName, fieldComment, fieldValue);
this.isMetaField = isMetaField;
}
- public StreamField(int index, FieldType fieldType, String fieldName, String fieldComment, String fieldValue,
+ public StreamField(int index, String fieldType, String fieldName, String fieldComment, String fieldValue,
Integer isMetaField, String originNodeName) {
this(index, fieldType, fieldName, fieldComment, fieldValue);
this.isMetaField = isMetaField;
this.originNodeName = originNodeName;
}
- public StreamField(int index, FieldType fieldType, String fieldName, String fieldComment, String fieldValue,
+ public StreamField(int index, String fieldType, String fieldName, String fieldComment, String fieldValue,
Integer isMetaField, String originNodeName, String originFieldName) {
this(index, fieldType, fieldName, fieldComment, fieldValue);
this.isMetaField = isMetaField;
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/stream/StreamNode.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/stream/StreamNode.java
index 39ac2a0b7..532739922 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/stream/StreamNode.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/stream/StreamNode.java
@@ -18,7 +18,6 @@
package org.apache.inlong.manager.common.pojo.stream;
import com.google.common.collect.Sets;
-import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import org.apache.inlong.manager.common.util.Preconditions;
@@ -37,9 +36,6 @@ public class StreamNode {
protected List<StreamField> fields;
- @ApiModelProperty("Data node name")
- protected String dataNodeName;
-
public void addPre(String pre) {
Preconditions.checkNotEmpty(pre, "Pre node should not be empty");
if (preNodes == null) {
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/stream/StreamPipeline.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/stream/StreamPipeline.java
index dd50cbcbd..6efd61ec5 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/stream/StreamPipeline.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/stream/StreamPipeline.java
@@ -49,7 +49,7 @@ public class StreamPipeline {
this.pipeline = pipeline;
}
- public void addRelationShip(StreamNodeRelationship relationship) {
+ public void addRelationship(StreamNodeRelationship relationship) {
pipeline.add(relationship);
}
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/util/StreamParseUtils.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/util/StreamParseUtils.java
index 8b76adc59..9e8b82eb1 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/util/StreamParseUtils.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/util/StreamParseUtils.java
@@ -20,14 +20,13 @@ package org.apache.inlong.manager.common.util;
import com.google.gson.Gson;
import com.google.gson.JsonObject;
import org.apache.inlong.manager.common.enums.DataFormat;
-import org.apache.inlong.manager.common.enums.SinkType;
import org.apache.inlong.manager.common.enums.SourceType;
import org.apache.inlong.manager.common.enums.TransformType;
-import org.apache.inlong.manager.common.pojo.stream.SinkField;
+import org.apache.inlong.manager.common.pojo.sink.SinkField;
+import org.apache.inlong.manager.common.pojo.sink.StreamSink;
+import org.apache.inlong.manager.common.pojo.source.StreamSource;
import org.apache.inlong.manager.common.pojo.stream.StreamNode;
import org.apache.inlong.manager.common.pojo.stream.StreamPipeline;
-import org.apache.inlong.manager.common.pojo.stream.StreamSink;
-import org.apache.inlong.manager.common.pojo.stream.StreamSource;
import org.apache.inlong.manager.common.pojo.stream.StreamTransform;
import org.apache.inlong.manager.common.pojo.transform.TransformDefinition;
import org.apache.inlong.manager.common.pojo.transform.deduplication.DeDuplicationDefinition;
@@ -108,13 +107,9 @@ public class StreamParseUtils {
} else if (jsonObject.has(SINK_TYPE)) {
String sinkName = jsonObject.get(SINK_NAME).getAsString();
StreamSink sink = new StreamSink() {
- @Override
- public SinkType getSinkType() {
- return null;
- }
@Override
- public List<SinkField> getSinkFields() {
+ public List<SinkField> getFieldList() {
return null;
}
diff --git a/inlong-manager/manager-common/src/test/java/org/apache/inlong/manager/common/pojo/stream/StreamPipelineTest.java b/inlong-manager/manager-common/src/test/java/org/apache/inlong/manager/common/pojo/stream/StreamPipelineTest.java
index 1fb6bc785..af29276b5 100644
--- a/inlong-manager/manager-common/src/test/java/org/apache/inlong/manager/common/pojo/stream/StreamPipelineTest.java
+++ b/inlong-manager/manager-common/src/test/java/org/apache/inlong/manager/common/pojo/stream/StreamPipelineTest.java
@@ -30,11 +30,11 @@ public class StreamPipelineTest {
@Test
public void testCheckHasCircle() {
StreamPipeline streamPipeline = new StreamPipeline();
- streamPipeline.addRelationShip(new StreamNodeRelationship(Sets.newHashSet("A", "B"), Sets.newHashSet("C")));
- streamPipeline.addRelationShip(new StreamNodeRelationship(Sets.newHashSet("C"), Sets.newHashSet("D")));
- streamPipeline.addRelationShip(new StreamNodeRelationship(Sets.newHashSet("D"), Sets.newHashSet("E", "F")));
- streamPipeline.addRelationShip(new StreamNodeRelationship(Sets.newHashSet("F"), Sets.newHashSet("G")));
- streamPipeline.addRelationShip(new StreamNodeRelationship(Sets.newHashSet("E"), Sets.newHashSet("H", "C")));
+ streamPipeline.addRelationship(new StreamNodeRelationship(Sets.newHashSet("A", "B"), Sets.newHashSet("C")));
+ streamPipeline.addRelationship(new StreamNodeRelationship(Sets.newHashSet("C"), Sets.newHashSet("D")));
+ streamPipeline.addRelationship(new StreamNodeRelationship(Sets.newHashSet("D"), Sets.newHashSet("E", "F")));
+ streamPipeline.addRelationship(new StreamNodeRelationship(Sets.newHashSet("F"), Sets.newHashSet("G")));
+ streamPipeline.addRelationship(new StreamNodeRelationship(Sets.newHashSet("E"), Sets.newHashSet("H", "C")));
Pair<Boolean, Pair<String, String>> circleState = streamPipeline.hasCircle();
Assert.assertTrue(circleState.getLeft());
Assert.assertTrue(Sets.newHashSet("E","C").contains(circleState.getRight().getLeft()));
diff --git a/inlong-manager/manager-common/src/test/java/org/apache/inlong/manager/common/pojo/transform/TransformDefinitionTest.java b/inlong-manager/manager-common/src/test/java/org/apache/inlong/manager/common/pojo/transform/TransformDefinitionTest.java
index d7c40b9df..1285aaf54 100644
--- a/inlong-manager/manager-common/src/test/java/org/apache/inlong/manager/common/pojo/transform/TransformDefinitionTest.java
+++ b/inlong-manager/manager-common/src/test/java/org/apache/inlong/manager/common/pojo/transform/TransformDefinitionTest.java
@@ -43,16 +43,12 @@ import java.util.concurrent.TimeUnit;
*/
public class TransformDefinitionTest {
- public class BlankStreamNode extends StreamNode {
-
- }
-
public static Gson gson = new Gson();
@Test
public void testParseDeDuplicationDefinition() {
List<StreamField> streamFields = createStreamFields();
- StreamField timingField = new StreamField(2, FieldType.TIMESTAMP, "event_time", null, null);
+ StreamField timingField = new StreamField(2, FieldType.TIMESTAMP.toString(), "event_time", null, null);
DeDuplicationDefinition deDuplicationDefinition = new DeDuplicationDefinition(streamFields, timingField, 100,
TimeUnit.MICROSECONDS, DeDuplicationStrategy.RESERVE_FIRST);
String definitionJson = gson.toJson(deDuplicationDefinition);
@@ -85,18 +81,22 @@ public class TransformDefinitionTest {
private List<StreamField> createStreamFields() {
List<StreamField> streamFieldList = Lists.newArrayList();
- streamFieldList.add(new StreamField(0, FieldType.STRING, "name", null, null));
- streamFieldList.add(new StreamField(1, FieldType.INT, "age", null, null));
+ streamFieldList.add(new StreamField(0, FieldType.STRING.toString(), "name", null, null));
+ streamFieldList.add(new StreamField(1, FieldType.INT.toString(), "age", null, null));
return streamFieldList;
}
private List<FilterRule> createFilterRule() {
List<FilterRule> filterRules = Lists.newArrayList();
- filterRules.add(new FilterRule(new StreamField(0, FieldType.STRING, "name", null, null),
+ filterRules.add(new FilterRule(new StreamField(0, FieldType.STRING.toString(), "name", null, null),
OperationType.not_null, null, RuleRelation.OR));
- filterRules.add(new FilterRule(new StreamField(1, FieldType.INT, "age", null, null),
+ filterRules.add(new FilterRule(new StreamField(1, FieldType.INT.toString(), "age", null, null),
OperationType.gt, new TargetValue(true, null, "50"), null));
return filterRules;
}
+ public static class BlankStreamNode extends StreamNode {
+
+ }
+
}
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/InlongStreamServiceImpl.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/InlongStreamServiceImpl.java
index 564350292..2e1ccee15 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/InlongStreamServiceImpl.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/InlongStreamServiceImpl.java
@@ -36,13 +36,13 @@ import org.apache.inlong.manager.common.pojo.stream.FullStreamRequest;
import org.apache.inlong.manager.common.pojo.stream.FullStreamResponse;
import org.apache.inlong.manager.common.pojo.stream.InlongStreamApproveRequest;
import org.apache.inlong.manager.common.pojo.stream.InlongStreamExtInfo;
-import org.apache.inlong.manager.common.pojo.stream.InlongStreamFieldInfo;
import org.apache.inlong.manager.common.pojo.stream.InlongStreamInfo;
import org.apache.inlong.manager.common.pojo.stream.InlongStreamListResponse;
import org.apache.inlong.manager.common.pojo.stream.InlongStreamPageRequest;
import org.apache.inlong.manager.common.pojo.stream.InlongStreamRequest;
import org.apache.inlong.manager.common.pojo.stream.InlongStreamTopicInfo;
import org.apache.inlong.manager.common.pojo.stream.StreamBriefResponse;
+import org.apache.inlong.manager.common.pojo.stream.StreamField;
import org.apache.inlong.manager.common.util.CommonBeanUtils;
import org.apache.inlong.manager.common.util.Preconditions;
import org.apache.inlong.manager.dao.entity.InlongGroupEntity;
@@ -144,7 +144,7 @@ public class InlongStreamServiceImpl implements InlongStreamService {
}
InlongStreamInfo streamInfo = CommonBeanUtils.copyProperties(streamEntity, InlongStreamInfo::new);
- List<InlongStreamFieldInfo> streamFields = getStreamFields(groupId, streamId);
+ List<StreamField> streamFields = getStreamFields(groupId, streamId);
streamInfo.setFieldList(streamFields);
List<InlongStreamExtEntity> extEntities = streamExtMapper.selectByRelatedId(groupId, streamId);
List<InlongStreamExtInfo> exts = CommonBeanUtils.copyListProperties(extEntities, InlongStreamExtInfo::new);
@@ -159,9 +159,9 @@ public class InlongStreamServiceImpl implements InlongStreamService {
List<InlongStreamEntity> inlongStreamEntityList = streamMapper.selectByGroupId(groupId);
List<InlongStreamInfo> streamList = CommonBeanUtils.copyListProperties(inlongStreamEntityList,
InlongStreamInfo::new);
- List<InlongStreamFieldInfo> streamFields = getStreamFields(groupId, null);
- Map<String, List<InlongStreamFieldInfo>> streamFieldMap = streamFields.stream().collect(
- Collectors.groupingBy(InlongStreamFieldInfo::getInlongStreamId,
+ List<StreamField> streamFields = getStreamFields(groupId, null);
+ Map<String, List<StreamField>> streamFieldMap = streamFields.stream().collect(
+ Collectors.groupingBy(StreamField::getInlongStreamId,
HashMap::new,
Collectors.toCollection(ArrayList::new)));
List<InlongStreamExtEntity> extEntities = streamExtMapper.selectByRelatedId(groupId, null);
@@ -172,7 +172,7 @@ public class InlongStreamServiceImpl implements InlongStreamService {
Collectors.toCollection(ArrayList::new)));
streamList.stream().forEach(streamInfo -> {
String streamId = streamInfo.getInlongStreamId();
- List<InlongStreamFieldInfo> fieldInfos = streamFieldMap.get(streamId);
+ List<StreamField> fieldInfos = streamFieldMap.get(streamId);
streamInfo.setFieldList(fieldInfos);
List<InlongStreamExtInfo> extInfos = extInfoMap.get(streamId);
streamInfo.setExtList(extInfos);
@@ -191,12 +191,12 @@ public class InlongStreamServiceImpl implements InlongStreamService {
/**
* Query and set the extended information and data source fields of the inlong stream
*/
- private List<InlongStreamFieldInfo> getStreamFields(String groupId, String streamId) {
+ private List<StreamField> getStreamFields(String groupId, String streamId) {
List<InlongStreamFieldEntity> fieldEntityList = streamFieldMapper.selectByIdentifier(groupId, streamId);
if (CollectionUtils.isEmpty(fieldEntityList)) {
return Collections.emptyList();
}
- return CommonBeanUtils.copyListProperties(fieldEntityList, InlongStreamFieldInfo::new);
+ return CommonBeanUtils.copyListProperties(fieldEntityList, StreamField::new);
}
@Override
@@ -464,7 +464,7 @@ public class InlongStreamServiceImpl implements InlongStreamService {
for (InlongStreamInfo streamInfo : streamInfoList) {
// Set the field information of the inlong stream
String streamId = streamInfo.getInlongStreamId();
- List<InlongStreamFieldInfo> streamFields = getStreamFields(groupId, streamId);
+ List<StreamField> streamFields = getStreamFields(groupId, streamId);
streamInfo.setFieldList(streamFields);
List<InlongStreamExtInfo> streamExtInfos = CommonBeanUtils.copyListProperties(
streamExtMapper.selectByRelatedId(groupId, streamId), InlongStreamExtInfo::new);
@@ -590,7 +590,7 @@ public class InlongStreamServiceImpl implements InlongStreamService {
* <p/>First physically delete the existing field information, and then add the field information of this batch
*/
@Transactional(rollbackFor = Throwable.class)
- void updateField(String groupId, String streamId, List<InlongStreamFieldInfo> fieldList) {
+ void updateField(String groupId, String streamId, List<StreamField> fieldList) {
LOGGER.debug("begin to update inlong stream field, groupId={}, streamId={}, field={}", groupId, streamId,
fieldList);
try {
@@ -604,7 +604,7 @@ public class InlongStreamServiceImpl implements InlongStreamService {
}
@Transactional(rollbackFor = Throwable.class)
- void saveField(String groupId, String streamId, List<InlongStreamFieldInfo> infoList) {
+ void saveField(String groupId, String streamId, List<StreamField> infoList) {
if (CollectionUtils.isEmpty(infoList)) {
return;
}
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/hbase/HbaseResourceOperator.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/hbase/HbaseResourceOperator.java
index 1be7412a8..131e78a23 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/hbase/HbaseResourceOperator.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/hbase/HbaseResourceOperator.java
@@ -92,8 +92,8 @@ public class HbaseResourceOperator implements SinkResourceOperator {
}
HbaseTableInfo tableInfo = HbaseSinkDTO.getHbaseTableInfo(hbaseInfo, columnFamilies);
- String zkAddress = hbaseInfo.getZookeeperQuorum();
- String zkNode = hbaseInfo.getZookeeperZnodeParent();
+ String zkAddress = hbaseInfo.getZkQuorum();
+ String zkNode = hbaseInfo.getZkNodeParent();
String namespace = hbaseInfo.getNamespace();
String tableName = hbaseInfo.getTableName();
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sink/ck/ClickHouseSinkOperation.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sink/ck/ClickHouseSinkOperation.java
index f5f618ce4..bfa00dc8c 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sink/ck/ClickHouseSinkOperation.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sink/ck/ClickHouseSinkOperation.java
@@ -27,8 +27,7 @@ import org.apache.inlong.manager.common.enums.GlobalConstants;
import org.apache.inlong.manager.common.enums.SinkStatus;
import org.apache.inlong.manager.common.enums.SinkType;
import org.apache.inlong.manager.common.exceptions.BusinessException;
-import org.apache.inlong.manager.common.pojo.sink.SinkFieldRequest;
-import org.apache.inlong.manager.common.pojo.sink.SinkFieldResponse;
+import org.apache.inlong.manager.common.pojo.sink.SinkField;
import org.apache.inlong.manager.common.pojo.sink.SinkListResponse;
import org.apache.inlong.manager.common.pojo.sink.SinkRequest;
import org.apache.inlong.manager.common.pojo.sink.SinkResponse;
@@ -108,7 +107,7 @@ public class ClickHouseSinkOperation implements StreamSinkOperation {
@Override
public void saveFieldOpt(SinkRequest request) {
- List<SinkFieldRequest> fieldList = request.getFieldList();
+ List<SinkField> fieldList = request.getFieldList();
LOGGER.info("begin to save field={}", fieldList);
if (CollectionUtils.isEmpty(fieldList)) {
return;
@@ -120,7 +119,7 @@ public class ClickHouseSinkOperation implements StreamSinkOperation {
String streamId = request.getInlongStreamId();
String sinkType = request.getSinkType();
Integer sinkId = request.getId();
- for (SinkFieldRequest fieldInfo : fieldList) {
+ for (SinkField fieldInfo : fieldList) {
StreamSinkFieldEntity fieldEntity = CommonBeanUtils.copyProperties(fieldInfo, StreamSinkFieldEntity::new);
if (StringUtils.isEmpty(fieldEntity.getFieldComment())) {
fieldEntity.setFieldComment(fieldEntity.getFieldName());
@@ -146,8 +145,8 @@ public class ClickHouseSinkOperation implements StreamSinkOperation {
SinkResponse response = this.getFromEntity(entity, ClickHouseSinkResponse::new);
List<StreamSinkFieldEntity> entities = sinkFieldMapper.selectBySinkId(entity.getId());
- List<SinkFieldResponse> infos = CommonBeanUtils.copyListProperties(entities,
- SinkFieldResponse::new);
+ List<SinkField> infos = CommonBeanUtils.copyListProperties(entities,
+ SinkField::new);
response.setFieldList(infos);
return response;
@@ -211,7 +210,7 @@ public class ClickHouseSinkOperation implements StreamSinkOperation {
@Override
public void updateFieldOpt(Boolean onlyAdd, SinkRequest request) {
Integer sinkId = request.getId();
- List<SinkFieldRequest> fieldRequestList = request.getFieldList();
+ List<SinkField> fieldRequestList = request.getFieldList();
if (CollectionUtils.isEmpty(fieldRequestList)) {
return;
}
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sink/es/ElasticsearchSinkOperation.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sink/es/ElasticsearchSinkOperation.java
index 36733e601..2e5a80d2a 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sink/es/ElasticsearchSinkOperation.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sink/es/ElasticsearchSinkOperation.java
@@ -27,8 +27,7 @@ import org.apache.inlong.manager.common.enums.GlobalConstants;
import org.apache.inlong.manager.common.enums.SinkStatus;
import org.apache.inlong.manager.common.enums.SinkType;
import org.apache.inlong.manager.common.exceptions.BusinessException;
-import org.apache.inlong.manager.common.pojo.sink.SinkFieldRequest;
-import org.apache.inlong.manager.common.pojo.sink.SinkFieldResponse;
+import org.apache.inlong.manager.common.pojo.sink.SinkField;
import org.apache.inlong.manager.common.pojo.sink.SinkListResponse;
import org.apache.inlong.manager.common.pojo.sink.SinkRequest;
import org.apache.inlong.manager.common.pojo.sink.SinkResponse;
@@ -108,7 +107,7 @@ public class ElasticsearchSinkOperation implements StreamSinkOperation {
@Override
public void saveFieldOpt(SinkRequest request) {
- List<SinkFieldRequest> fieldList = request.getFieldList();
+ List<SinkField> fieldList = request.getFieldList();
LOGGER.info("begin to save field={}", fieldList);
if (CollectionUtils.isEmpty(fieldList)) {
return;
@@ -120,7 +119,7 @@ public class ElasticsearchSinkOperation implements StreamSinkOperation {
String streamId = request.getInlongStreamId();
String sinkType = request.getSinkType();
Integer sinkId = request.getId();
- for (SinkFieldRequest fieldInfo : fieldList) {
+ for (SinkField fieldInfo : fieldList) {
StreamSinkFieldEntity fieldEntity = CommonBeanUtils.copyProperties(fieldInfo, StreamSinkFieldEntity::new);
if (StringUtils.isEmpty(fieldEntity.getFieldComment())) {
fieldEntity.setFieldComment(fieldEntity.getFieldName());
@@ -146,8 +145,8 @@ public class ElasticsearchSinkOperation implements StreamSinkOperation {
SinkResponse response = this.getFromEntity(entity, ElasticsearchSinkResponse::new);
List<StreamSinkFieldEntity> entities = sinkFieldMapper.selectBySinkId(entity.getId());
- List<SinkFieldResponse> infos = CommonBeanUtils.copyListProperties(entities,
- SinkFieldResponse::new);
+ List<SinkField> infos = CommonBeanUtils.copyListProperties(entities,
+ SinkField::new);
response.setFieldList(infos);
return response;
@@ -211,7 +210,7 @@ public class ElasticsearchSinkOperation implements StreamSinkOperation {
@Override
public void updateFieldOpt(Boolean onlyAdd, SinkRequest request) {
Integer sinkId = request.getId();
- List<SinkFieldRequest> fieldRequestList = request.getFieldList();
+ List<SinkField> fieldRequestList = request.getFieldList();
if (CollectionUtils.isEmpty(fieldRequestList)) {
return;
}
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sink/hbase/HbaseSinkOperation.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sink/hbase/HbaseSinkOperation.java
index 8d1eafdcf..93e7f9f0b 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sink/hbase/HbaseSinkOperation.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sink/hbase/HbaseSinkOperation.java
@@ -27,8 +27,7 @@ import org.apache.inlong.manager.common.enums.GlobalConstants;
import org.apache.inlong.manager.common.enums.SinkStatus;
import org.apache.inlong.manager.common.enums.SinkType;
import org.apache.inlong.manager.common.exceptions.BusinessException;
-import org.apache.inlong.manager.common.pojo.sink.SinkFieldRequest;
-import org.apache.inlong.manager.common.pojo.sink.SinkFieldResponse;
+import org.apache.inlong.manager.common.pojo.sink.SinkField;
import org.apache.inlong.manager.common.pojo.sink.SinkListResponse;
import org.apache.inlong.manager.common.pojo.sink.SinkRequest;
import org.apache.inlong.manager.common.pojo.sink.SinkResponse;
@@ -105,7 +104,7 @@ public class HbaseSinkOperation implements StreamSinkOperation {
@Override
public void saveFieldOpt(SinkRequest request) {
- List<SinkFieldRequest> fieldList = request.getFieldList();
+ List<SinkField> fieldList = request.getFieldList();
LOGGER.info("begin to save hbase field={}", fieldList);
if (CollectionUtils.isEmpty(fieldList)) {
return;
@@ -116,7 +115,7 @@ public class HbaseSinkOperation implements StreamSinkOperation {
String streamId = request.getInlongStreamId();
String sinkType = request.getSinkType();
Integer sinkId = request.getId();
- for (SinkFieldRequest fieldInfo : fieldList) {
+ for (SinkField fieldInfo : fieldList) {
StreamSinkFieldEntity fieldEntity = CommonBeanUtils.copyProperties(fieldInfo, StreamSinkFieldEntity::new);
if (StringUtils.isEmpty(fieldEntity.getFieldComment())) {
fieldEntity.setFieldComment(fieldEntity.getFieldName());
@@ -142,8 +141,8 @@ public class HbaseSinkOperation implements StreamSinkOperation {
SinkResponse response = this.getFromEntity(entity, HbaseSinkResponse::new);
List<StreamSinkFieldEntity> entities = sinkFieldMapper.selectBySinkId(entity.getId());
- List<SinkFieldResponse> infos = CommonBeanUtils.copyListProperties(entities,
- SinkFieldResponse::new);
+ List<SinkField> infos = CommonBeanUtils.copyListProperties(entities,
+ SinkField::new);
response.setFieldList(infos);
return response;
@@ -207,7 +206,7 @@ public class HbaseSinkOperation implements StreamSinkOperation {
@Override
public void updateFieldOpt(Boolean onlyAdd, SinkRequest request) {
Integer sinkId = request.getId();
- List<SinkFieldRequest> fieldRequestList = request.getFieldList();
+ List<SinkField> fieldRequestList = request.getFieldList();
if (CollectionUtils.isEmpty(fieldRequestList)) {
return;
}
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sink/hive/HiveSinkOperation.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sink/hive/HiveSinkOperation.java
index 5da3a1322..149af2b97 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sink/hive/HiveSinkOperation.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sink/hive/HiveSinkOperation.java
@@ -27,8 +27,7 @@ import org.apache.inlong.manager.common.enums.GlobalConstants;
import org.apache.inlong.manager.common.enums.SinkStatus;
import org.apache.inlong.manager.common.enums.SinkType;
import org.apache.inlong.manager.common.exceptions.BusinessException;
-import org.apache.inlong.manager.common.pojo.sink.SinkFieldRequest;
-import org.apache.inlong.manager.common.pojo.sink.SinkFieldResponse;
+import org.apache.inlong.manager.common.pojo.sink.SinkField;
import org.apache.inlong.manager.common.pojo.sink.SinkListResponse;
import org.apache.inlong.manager.common.pojo.sink.SinkRequest;
import org.apache.inlong.manager.common.pojo.sink.SinkResponse;
@@ -108,7 +107,7 @@ public class HiveSinkOperation implements StreamSinkOperation {
@Override
public void saveFieldOpt(SinkRequest request) {
- List<SinkFieldRequest> fieldList = request.getFieldList();
+ List<SinkField> fieldList = request.getFieldList();
LOGGER.info("begin to save hive field={}", fieldList);
if (CollectionUtils.isEmpty(fieldList)) {
return;
@@ -120,7 +119,7 @@ public class HiveSinkOperation implements StreamSinkOperation {
String streamId = request.getInlongStreamId();
String sinkType = request.getSinkType();
Integer sinkId = request.getId();
- for (SinkFieldRequest fieldInfo : fieldList) {
+ for (SinkField fieldInfo : fieldList) {
StreamSinkFieldEntity fieldEntity = CommonBeanUtils.copyProperties(fieldInfo, StreamSinkFieldEntity::new);
if (StringUtils.isEmpty(fieldEntity.getFieldComment())) {
fieldEntity.setFieldComment(fieldEntity.getFieldName());
@@ -146,8 +145,8 @@ public class HiveSinkOperation implements StreamSinkOperation {
SinkResponse response = this.getFromEntity(entity, HiveSinkResponse::new);
List<StreamSinkFieldEntity> entities = sinkFieldMapper.selectBySinkId(entity.getId());
- List<SinkFieldResponse> infos = CommonBeanUtils.copyListProperties(entities,
- SinkFieldResponse::new);
+ List<SinkField> infos = CommonBeanUtils.copyListProperties(entities,
+ SinkField::new);
response.setFieldList(infos);
return response;
@@ -212,7 +211,7 @@ public class HiveSinkOperation implements StreamSinkOperation {
@Override
public void updateFieldOpt(Boolean onlyAdd, SinkRequest request) {
Integer sinkId = request.getId();
- List<SinkFieldRequest> fieldRequestList = request.getFieldList();
+ List<SinkField> fieldRequestList = request.getFieldList();
if (CollectionUtils.isEmpty(fieldRequestList)) {
return;
}
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sink/iceberg/IcebergSinkOperation.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sink/iceberg/IcebergSinkOperation.java
index d0c01423c..e231203e6 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sink/iceberg/IcebergSinkOperation.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sink/iceberg/IcebergSinkOperation.java
@@ -27,8 +27,7 @@ import org.apache.inlong.manager.common.enums.GlobalConstants;
import org.apache.inlong.manager.common.enums.SinkStatus;
import org.apache.inlong.manager.common.enums.SinkType;
import org.apache.inlong.manager.common.exceptions.BusinessException;
-import org.apache.inlong.manager.common.pojo.sink.SinkFieldRequest;
-import org.apache.inlong.manager.common.pojo.sink.SinkFieldResponse;
+import org.apache.inlong.manager.common.pojo.sink.SinkField;
import org.apache.inlong.manager.common.pojo.sink.SinkListResponse;
import org.apache.inlong.manager.common.pojo.sink.SinkRequest;
import org.apache.inlong.manager.common.pojo.sink.SinkResponse;
@@ -105,7 +104,7 @@ public class IcebergSinkOperation implements StreamSinkOperation {
@Override
public void saveFieldOpt(SinkRequest request) {
- List<SinkFieldRequest> fieldList = request.getFieldList();
+ List<SinkField> fieldList = request.getFieldList();
LOGGER.info("begin to save iceberg field={}", fieldList);
if (CollectionUtils.isEmpty(fieldList)) {
return;
@@ -117,7 +116,7 @@ public class IcebergSinkOperation implements StreamSinkOperation {
String streamId = request.getInlongStreamId();
String sinkType = request.getSinkType();
Integer sinkId = request.getId();
- for (SinkFieldRequest fieldInfo : fieldList) {
+ for (SinkField fieldInfo : fieldList) {
StreamSinkFieldEntity fieldEntity = CommonBeanUtils.copyProperties(fieldInfo, StreamSinkFieldEntity::new);
if (StringUtils.isEmpty(fieldEntity.getFieldComment())) {
fieldEntity.setFieldComment(fieldEntity.getFieldName());
@@ -143,8 +142,8 @@ public class IcebergSinkOperation implements StreamSinkOperation {
SinkResponse response = this.getFromEntity(entity, IcebergSinkResponse::new);
List<StreamSinkFieldEntity> entities = sinkFieldMapper.selectBySinkId(entity.getId());
- List<SinkFieldResponse> infos = CommonBeanUtils.copyListProperties(entities,
- SinkFieldResponse::new);
+ List<SinkField> infos = CommonBeanUtils.copyListProperties(entities,
+ SinkField::new);
response.setFieldList(infos);
return response;
@@ -208,7 +207,7 @@ public class IcebergSinkOperation implements StreamSinkOperation {
@Override
public void updateFieldOpt(Boolean onlyAdd, SinkRequest request) {
Integer sinkId = request.getId();
- List<SinkFieldRequest> fieldRequestList = request.getFieldList();
+ List<SinkField> fieldRequestList = request.getFieldList();
if (CollectionUtils.isEmpty(fieldRequestList)) {
return;
}
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sink/kafka/KafkaSinkOperation.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sink/kafka/KafkaSinkOperation.java
index 0e9e041ff..1ab6c0e98 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sink/kafka/KafkaSinkOperation.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sink/kafka/KafkaSinkOperation.java
@@ -27,8 +27,7 @@ import org.apache.inlong.manager.common.enums.GlobalConstants;
import org.apache.inlong.manager.common.enums.SinkStatus;
import org.apache.inlong.manager.common.enums.SinkType;
import org.apache.inlong.manager.common.exceptions.BusinessException;
-import org.apache.inlong.manager.common.pojo.sink.SinkFieldRequest;
-import org.apache.inlong.manager.common.pojo.sink.SinkFieldResponse;
+import org.apache.inlong.manager.common.pojo.sink.SinkField;
import org.apache.inlong.manager.common.pojo.sink.SinkListResponse;
import org.apache.inlong.manager.common.pojo.sink.SinkRequest;
import org.apache.inlong.manager.common.pojo.sink.SinkResponse;
@@ -106,7 +105,7 @@ public class KafkaSinkOperation implements StreamSinkOperation {
@Override
public void saveFieldOpt(SinkRequest request) {
- List<SinkFieldRequest> fieldList = request.getFieldList();
+ List<SinkField> fieldList = request.getFieldList();
LOGGER.info("begin to save field={}", fieldList);
if (CollectionUtils.isEmpty(fieldList)) {
return;
@@ -118,7 +117,7 @@ public class KafkaSinkOperation implements StreamSinkOperation {
String streamId = request.getInlongStreamId();
String sinkType = request.getSinkType();
Integer sinkId = request.getId();
- for (SinkFieldRequest fieldInfo : fieldList) {
+ for (SinkField fieldInfo : fieldList) {
StreamSinkFieldEntity fieldEntity = CommonBeanUtils.copyProperties(fieldInfo, StreamSinkFieldEntity::new);
if (StringUtils.isEmpty(fieldEntity.getFieldComment())) {
fieldEntity.setFieldComment(fieldEntity.getFieldName());
@@ -143,7 +142,7 @@ public class KafkaSinkOperation implements StreamSinkOperation {
String.format(ErrorCodeEnum.SINK_TYPE_NOT_SAME.getMessage(), SinkType.SINK_KAFKA, existType));
SinkResponse response = this.getFromEntity(entity, KafkaSinkResponse::new);
List<StreamSinkFieldEntity> entities = sinkFieldMapper.selectBySinkId(entity.getId());
- List<SinkFieldResponse> infos = CommonBeanUtils.copyListProperties(entities, SinkFieldResponse::new);
+ List<SinkField> infos = CommonBeanUtils.copyListProperties(entities, SinkField::new);
response.setFieldList(infos);
return response;
}
@@ -205,7 +204,7 @@ public class KafkaSinkOperation implements StreamSinkOperation {
@Override
public void updateFieldOpt(Boolean onlyAdd, SinkRequest request) {
Integer sinkId = request.getId();
- List<SinkFieldRequest> fieldRequestList = request.getFieldList();
+ List<SinkField> fieldRequestList = request.getFieldList();
if (CollectionUtils.isEmpty(fieldRequestList)) {
return;
}
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sink/postgres/PostgresSinkOperation.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sink/postgres/PostgresSinkOperation.java
index 60c4e7d85..fc68cf914 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sink/postgres/PostgresSinkOperation.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sink/postgres/PostgresSinkOperation.java
@@ -32,8 +32,7 @@ import org.apache.inlong.manager.common.enums.GlobalConstants;
import org.apache.inlong.manager.common.enums.SinkStatus;
import org.apache.inlong.manager.common.enums.SinkType;
import org.apache.inlong.manager.common.exceptions.BusinessException;
-import org.apache.inlong.manager.common.pojo.sink.SinkFieldRequest;
-import org.apache.inlong.manager.common.pojo.sink.SinkFieldResponse;
+import org.apache.inlong.manager.common.pojo.sink.SinkField;
import org.apache.inlong.manager.common.pojo.sink.SinkListResponse;
import org.apache.inlong.manager.common.pojo.sink.SinkRequest;
import org.apache.inlong.manager.common.pojo.sink.SinkResponse;
@@ -105,7 +104,7 @@ public class PostgresSinkOperation implements StreamSinkOperation {
@Override
public void saveFieldOpt(SinkRequest request) {
- List<SinkFieldRequest> fieldList = request.getFieldList();
+ List<SinkField> fieldList = request.getFieldList();
LOGGER.info("begin to save field={}", fieldList);
if (CollectionUtils.isEmpty(fieldList)) {
return;
@@ -117,7 +116,7 @@ public class PostgresSinkOperation implements StreamSinkOperation {
String streamId = request.getInlongStreamId();
String sinkType = request.getSinkType();
Integer sinkId = request.getId();
- for (SinkFieldRequest fieldInfo : fieldList) {
+ for (SinkField fieldInfo : fieldList) {
StreamSinkFieldEntity fieldEntity = CommonBeanUtils.copyProperties(fieldInfo, StreamSinkFieldEntity::new);
if (StringUtils.isEmpty(fieldEntity.getFieldComment())) {
fieldEntity.setFieldComment(fieldEntity.getFieldName());
@@ -142,7 +141,7 @@ public class PostgresSinkOperation implements StreamSinkOperation {
String.format(ErrorCodeEnum.SINK_TYPE_NOT_SAME.getMessage(), SinkType.SINK_POSTGRES, existType));
SinkResponse response = this.getFromEntity(entity, PostgresSinkResponse::new);
List<StreamSinkFieldEntity> entities = sinkFieldMapper.selectBySinkId(entity.getId());
- List<SinkFieldResponse> infos = CommonBeanUtils.copyListProperties(entities, SinkFieldResponse::new);
+ List<SinkField> infos = CommonBeanUtils.copyListProperties(entities, SinkField::new);
response.setFieldList(infos);
return response;
}
@@ -204,7 +203,7 @@ public class PostgresSinkOperation implements StreamSinkOperation {
@Override
public void updateFieldOpt(Boolean onlyAdd, SinkRequest request) {
Integer sinkId = request.getId();
- List<SinkFieldRequest> fieldRequestList = request.getFieldList();
+ List<SinkField> fieldRequestList = request.getFieldList();
if (CollectionUtils.isEmpty(fieldRequestList)) {
return;
}
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/CreateSortConfigListenerV2.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/CreateSortConfigListenerV2.java
index fb443eced..1a00a22f8 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/CreateSortConfigListenerV2.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/CreateSortConfigListenerV2.java
@@ -107,7 +107,7 @@ public class CreateSortConfigListenerV2 implements SortOperateListener {
String groupId = groupInfo.getInlongGroupId();
List<SinkResponse> sinkResponses = sinkService.listSink(groupId, null);
Map<String, List<SinkResponse>> sinkResponseMap = sinkResponses.stream()
- .collect(Collectors.groupingBy(sinkResponse -> sinkResponse.getInlongStreamId(), HashMap::new,
+ .collect(Collectors.groupingBy(SinkResponse::getInlongStreamId, HashMap::new,
Collectors.toCollection(ArrayList::new)));
Map<String, List<SourceResponse>> sourceResponseMap = createPulsarSources(groupInfo, streamInfoList);
List<StreamInfo> streamInfos = streamInfoList.stream()
@@ -115,7 +115,7 @@ public class CreateSortConfigListenerV2 implements SortOperateListener {
createNodesForStream(
sourceResponseMap.get(inlongStreamInfo.getInlongStreamId()),
sinkResponseMap.get(inlongStreamInfo.getInlongStreamId())),
- createNodeRelationShipsForStream(
+ createNodeRelationshipsForStream(
sourceResponseMap.get(inlongStreamInfo.getInlongStreamId()),
sinkResponseMap.get(inlongStreamInfo.getInlongStreamId())))
).collect(Collectors.toList());
@@ -126,13 +126,13 @@ public class CreateSortConfigListenerV2 implements SortOperateListener {
List<InlongStreamInfo> streamInfoList) {
MQType mqType = MQType.forType(groupInfo.getMqType());
if (mqType != MQType.PULSAR) {
- String errMsg = String.format("Unsupported MqType={} for Inlong", mqType);
+ String errMsg = String.format("Unsupported MqType={%s} for Inlong", mqType);
log.error(errMsg);
throw new WorkflowListenerException(errMsg);
}
- Map<String, List<SourceResponse>> sourceReponses = Maps.newHashMap();
+ Map<String, List<SourceResponse>> sourceResponseMap = Maps.newHashMap();
PulsarClusterInfo pulsarCluster = commonOperateService.getPulsarClusterInfo(groupInfo.getMqType());
- streamInfoList.stream().forEach(streamInfo -> {
+ streamInfoList.forEach(streamInfo -> {
PulsarSourceResponse pulsarSourceResponse = new PulsarSourceResponse();
pulsarSourceResponse.setSourceName(streamInfo.getInlongStreamId());
pulsarSourceResponse.setNamespace(groupInfo.getMqResource());
@@ -153,10 +153,10 @@ public class CreateSortConfigListenerV2 implements SortOperateListener {
}
pulsarSourceResponse.setScanStartupMode("earliest");
pulsarSourceResponse.setFieldList(streamInfo.getFieldList());
- sourceReponses.computeIfAbsent(streamInfo.getInlongStreamId(), key -> Lists.newArrayList())
+ sourceResponseMap.computeIfAbsent(streamInfo.getInlongStreamId(), key -> Lists.newArrayList())
.add(pulsarSourceResponse);
});
- return sourceReponses;
+ return sourceResponseMap;
}
private List<Node> createNodesForStream(
@@ -168,17 +168,17 @@ public class CreateSortConfigListenerV2 implements SortOperateListener {
return nodes;
}
- private List<NodeRelationShip> createNodeRelationShipsForStream(
+ private List<NodeRelationShip> createNodeRelationshipsForStream(
List<SourceResponse> sourceResponses,
List<SinkResponse> sinkResponses) {
- NodeRelationShip relationShip = new NodeRelationShip();
- List<String> inputs = sourceResponses.stream().map(sourceResponse -> sourceResponse.getSourceName())
+ NodeRelationShip relationship = new NodeRelationShip();
+ List<String> inputs = sourceResponses.stream().map(SourceResponse::getSourceName)
.collect(Collectors.toList());
- List<String> outputs = sinkResponses.stream().map(sinkResponse -> sinkResponse.getSinkName())
+ List<String> outputs = sinkResponses.stream().map(SinkResponse::getSinkName)
.collect(Collectors.toList());
- relationShip.setInputs(inputs);
- relationShip.setOutputs(outputs);
- return Lists.newArrayList(relationShip);
+ relationship.setInputs(inputs);
+ relationship.setOutputs(outputs);
+ return Lists.newArrayList(relationship);
}
private void upsertDataFlow(InlongGroupInfo groupInfo, InlongGroupExtInfo extInfo) {
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/CreateStreamSortConfigListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/CreateStreamSortConfigListener.java
index 12c21d0f6..bf03da23a 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/CreateStreamSortConfigListener.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/CreateStreamSortConfigListener.java
@@ -95,8 +95,8 @@ public class CreateStreamSortConfigListener implements SortOperateListener {
try {
List<SourceResponse> sourceResponses = createPulsarSources(groupInfo, streamInfo);
List<Node> nodes = createNodesForStream(sourceResponses, sinkResponses);
- List<NodeRelationShip> nodeRelationShips = createNodeRelationShipsForStream(sourceResponses, sinkResponses);
- StreamInfo sortStreamInfo = new StreamInfo(streamId, nodes, nodeRelationShips);
+ List<NodeRelationShip> nodeRelationships = createNodeRelationshipsForStream(sourceResponses, sinkResponses);
+ StreamInfo sortStreamInfo = new StreamInfo(streamId, nodes, nodeRelationships);
GroupInfo sortGroupInfo = new GroupInfo(groupId, Lists.newArrayList(sortStreamInfo));
String dataFlows = OBJECT_MAPPER.writeValueAsString(sortGroupInfo);
InlongStreamExtInfo extInfo = new InlongStreamExtInfo();
@@ -120,7 +120,7 @@ public class CreateStreamSortConfigListener implements SortOperateListener {
private List<SourceResponse> createPulsarSources(InlongGroupInfo groupInfo, InlongStreamInfo streamInfo) {
MQType mqType = MQType.forType(groupInfo.getMqType());
if (mqType != MQType.PULSAR) {
- String errMsg = String.format("Unsupported MqType={} for Inlong", mqType);
+ String errMsg = String.format("Unsupported MqType={%s} for Inlong", mqType);
log.error(errMsg);
throw new WorkflowListenerException(errMsg);
}
@@ -157,17 +157,17 @@ public class CreateStreamSortConfigListener implements SortOperateListener {
return nodes;
}
- private List<NodeRelationShip> createNodeRelationShipsForStream(
+ private List<NodeRelationShip> createNodeRelationshipsForStream(
List<SourceResponse> sourceResponses,
List<SinkResponse> sinkResponses) {
- NodeRelationShip relationShip = new NodeRelationShip();
- List<String> inputs = sourceResponses.stream().map(sourceResponse -> sourceResponse.getSourceName())
+ NodeRelationShip relationship = new NodeRelationShip();
+ List<String> inputs = sourceResponses.stream().map(SourceResponse::getSourceName)
.collect(Collectors.toList());
- List<String> outputs = sinkResponses.stream().map(sinkResponse -> sinkResponse.getSinkName())
+ List<String> outputs = sinkResponses.stream().map(SinkResponse::getSinkName)
.collect(Collectors.toList());
- relationShip.setInputs(inputs);
- relationShip.setOutputs(outputs);
- return Lists.newArrayList(relationShip);
+ relationship.setInputs(inputs);
+ relationship.setOutputs(outputs);
+ return Lists.newArrayList(relationship);
}
private void upsertDataFlow(InlongStreamInfo streamInfo, InlongStreamExtInfo extInfo, String keyName) {
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/light/LightGroupSortListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/light/LightGroupSortListener.java
index 73fb97f1a..640398a9a 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/light/LightGroupSortListener.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/light/LightGroupSortListener.java
@@ -19,6 +19,7 @@ package org.apache.inlong.manager.service.sort.light;
import com.google.common.collect.Lists;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.inlong.manager.common.exceptions.WorkflowListenerException;
import org.apache.inlong.manager.common.pojo.group.InlongGroupExtInfo;
import org.apache.inlong.manager.common.pojo.group.InlongGroupInfo;
import org.apache.inlong.manager.common.pojo.sink.SinkResponse;
@@ -74,53 +75,61 @@ public class LightGroupSortListener implements SortOperateListener {
}
@Override
- public ListenerResult listen(WorkflowContext context) throws Exception {
- LOGGER.info("Create light group sort config for context={}", context);
- LightGroupResourceProcessForm processForm = (LightGroupResourceProcessForm) context.getProcessForm();
- InlongGroupInfo groupInfo = processForm.getGroupInfo();
- List<InlongStreamInfo> streamInfos = processForm.getStreamInfos();
-
- final String groupId = groupInfo.getInlongGroupId();
- GroupInfo configInfo = createGroupInfo(groupInfo, streamInfos);
- String dataFlows = OBJECT_MAPPER.writeValueAsString(configInfo);
- InlongGroupExtInfo extInfo = new InlongGroupExtInfo();
- extInfo.setInlongGroupId(groupId);
- extInfo.setKeyName(InlongGroupSettings.DATA_FLOW);
- extInfo.setKeyValue(dataFlows);
- if (groupInfo.getExtList() == null) {
- groupInfo.setExtList(Lists.newArrayList());
+ public ListenerResult listen(WorkflowContext context) throws WorkflowListenerException {
+ LOGGER.info("create sort config for light group, context={}", context);
+ try {
+ LightGroupResourceProcessForm processForm = (LightGroupResourceProcessForm) context.getProcessForm();
+ InlongGroupInfo groupInfo = processForm.getGroupInfo();
+ List<InlongStreamInfo> streamInfos = processForm.getStreamInfos();
+ final String groupId = groupInfo.getInlongGroupId();
+ GroupInfo configInfo = this.createGroupInfo(groupInfo, streamInfos);
+ String dataFlows = OBJECT_MAPPER.writeValueAsString(configInfo);
+ InlongGroupExtInfo extInfo = new InlongGroupExtInfo();
+ extInfo.setInlongGroupId(groupId);
+ extInfo.setKeyName(InlongGroupSettings.DATA_FLOW);
+ extInfo.setKeyValue(dataFlows);
+ if (groupInfo.getExtList() == null) {
+ groupInfo.setExtList(Lists.newArrayList());
+ }
+ upsertDataFlow(groupInfo, extInfo);
+ return ListenerResult.success();
+ } catch (Throwable t) {
+ LOGGER.error("create sort config error: ", t);
+ throw new WorkflowListenerException("create sort config error: " + t.getMessage());
}
- upsertDataFlow(groupInfo, extInfo);
- return ListenerResult.success();
}
private GroupInfo createGroupInfo(InlongGroupInfo groupInfo, List<InlongStreamInfo> streamInfoList) {
final String groupId = groupInfo.getInlongGroupId();
List<SourceResponse> sourceResponses = sourceService.listSource(groupId, null);
- Map<String, List<SourceResponse>> sourceResponseMap = sourceResponses.stream()
- .collect(Collectors.groupingBy(sourceResponse -> sourceResponse.getInlongStreamId(), HashMap::new,
+ Map<String, List<SourceResponse>> sourceMap = sourceResponses.stream()
+ .collect(Collectors.groupingBy(SourceResponse::getInlongStreamId, HashMap::new,
Collectors.toCollection(ArrayList::new)));
+
List<SinkResponse> sinkResponses = sinkService.listSink(groupId, null);
- Map<String, List<SinkResponse>> sinkResponseMap = sinkResponses.stream()
- .collect(Collectors.groupingBy(sinkResponse -> sinkResponse.getInlongStreamId(), HashMap::new,
+ Map<String, List<SinkResponse>> sinkMap = sinkResponses.stream()
+ .collect(Collectors.groupingBy(SinkResponse::getInlongStreamId, HashMap::new,
Collectors.toCollection(ArrayList::new)));
+
List<TransformResponse> transformResponses = transformService.listTransform(groupId, null);
- Map<String, List<TransformResponse>> transformResponseMap = transformResponses.stream()
- .collect(Collectors.groupingBy(transformResponse -> transformResponse.getInlongStreamId(), HashMap::new,
+ Map<String, List<TransformResponse>> transformMap = transformResponses.stream()
+ .collect(Collectors.groupingBy(TransformResponse::getInlongStreamId, HashMap::new,
Collectors.toCollection(ArrayList::new)));
- List<StreamInfo> streamInfos = streamInfoList.stream()
- .map(inlongStreamInfo -> new StreamInfo(inlongStreamInfo.getInlongStreamId(),
- createNodesForStream(
- sourceResponseMap.get(inlongStreamInfo.getInlongStreamId()),
- transformResponseMap.get(inlongStreamInfo.getInlongStreamId()),
- sinkResponseMap.get(inlongStreamInfo.getInlongStreamId())),
- NodeRelationShipUtils.createNodeRelationShipsForStream(inlongStreamInfo)))
- .collect(Collectors.toList());
- // Rebuild joinerNode relationship
- streamInfos.stream().forEach(streamInfo -> {
- List<TransformResponse> transformResponseList = transformResponseMap.get(streamInfo.getStreamId());
+
+ List<StreamInfo> streamInfos = new ArrayList<>();
+ for (InlongStreamInfo stream : streamInfoList) {
+ String streamId = stream.getInlongStreamId();
+ List<Node> nodes = this.createNodesForStream(sourceMap.get(streamId),
+ transformMap.get(streamId), sinkMap.get(streamId));
+ StreamInfo streamInfo = new StreamInfo(streamId, nodes,
+ NodeRelationShipUtils.createNodeRelationShipsForStream(stream));
+ streamInfos.add(streamInfo);
+
+ // Rebuild joinerNode relationship
+ List<TransformResponse> transformResponseList = transformMap.get(streamId);
NodeRelationShipUtils.optimizeNodeRelationShips(streamInfo, transformResponseList);
- });
+ }
+
return new GroupInfo(groupInfo.getInlongGroupId(), streamInfos);
}
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/ExtractNodeUtils.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/ExtractNodeUtils.java
index 93868e601..24f23ef2b 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/ExtractNodeUtils.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/ExtractNodeUtils.java
@@ -31,7 +31,7 @@ import org.apache.inlong.manager.common.pojo.source.kafka.KafkaOffset;
import org.apache.inlong.manager.common.pojo.source.kafka.KafkaSourceResponse;
import org.apache.inlong.manager.common.pojo.source.postgres.PostgresSourceResponse;
import org.apache.inlong.manager.common.pojo.source.pulsar.PulsarSourceResponse;
-import org.apache.inlong.manager.common.pojo.stream.InlongStreamFieldInfo;
+import org.apache.inlong.manager.common.pojo.stream.StreamField;
import org.apache.inlong.sort.protocol.FieldInfo;
import org.apache.inlong.sort.protocol.enums.KafkaScanStartupMode;
import org.apache.inlong.sort.protocol.enums.PulsarScanStartupMode;
@@ -58,11 +58,14 @@ import java.util.stream.Collectors;
@Slf4j
public class ExtractNodeUtils {
+ /**
+ * Create extract nodes from the source responses.
+ */
public static List<ExtractNode> createExtractNodes(List<SourceResponse> sourceResponses) {
if (CollectionUtils.isEmpty(sourceResponses)) {
return Lists.newArrayList();
}
- return sourceResponses.stream().map(sourceResponse -> createExtractNode(sourceResponse))
+ return sourceResponses.stream().map(ExtractNodeUtils::createExtractNode)
.collect(Collectors.toList());
}
@@ -86,34 +89,34 @@ public class ExtractNodeUtils {
/**
* Create MySqlExtractNode based on BinlogSourceResponse
*
- * @param binlogSourceResponse binlog source response info
+ * @param sourceResponse binlog source response info
* @return MySql extract node info
*/
- public static MySqlExtractNode createExtractNode(BinlogSourceResponse binlogSourceResponse) {
- final String id = binlogSourceResponse.getSourceName();
- final String name = binlogSourceResponse.getSourceName();
- final String database = binlogSourceResponse.getDatabaseWhiteList();
- final String primaryKey = binlogSourceResponse.getPrimaryKey();
- final String hostName = binlogSourceResponse.getHostname();
- final String userName = binlogSourceResponse.getUser();
- final String password = binlogSourceResponse.getPassword();
- final Integer port = binlogSourceResponse.getPort();
+ public static MySqlExtractNode createExtractNode(BinlogSourceResponse sourceResponse) {
+ final String id = sourceResponse.getSourceName();
+ final String name = sourceResponse.getSourceName();
+ final String database = sourceResponse.getDatabaseWhiteList();
+ final String primaryKey = sourceResponse.getPrimaryKey();
+ final String hostName = sourceResponse.getHostname();
+ final String userName = sourceResponse.getUser();
+ final String password = sourceResponse.getPassword();
+ final Integer port = sourceResponse.getPort();
Integer serverId = null;
- if (binlogSourceResponse.getServerId() != null && binlogSourceResponse.getServerId() > 0) {
- serverId = binlogSourceResponse.getServerId();
+ if (sourceResponse.getServerId() != null && sourceResponse.getServerId() > 0) {
+ serverId = sourceResponse.getServerId();
}
- String tables = binlogSourceResponse.getTableWhiteList();
+ String tables = sourceResponse.getTableWhiteList();
final List<String> tableNames = Splitter.on(",").splitToList(tables);
- final List<InlongStreamFieldInfo> streamFieldInfos = binlogSourceResponse.getFieldList();
- final List<FieldInfo> fieldInfos = streamFieldInfos.stream()
- .map(streamFieldInfo -> FieldInfoUtils.parseStreamFieldInfo(streamFieldInfo, name))
+ final List<StreamField> streamFields = sourceResponse.getFieldList();
+ final List<FieldInfo> fieldInfos = streamFields.stream()
+ .map(streamField -> FieldInfoUtils.parseStreamFieldInfo(streamField, name))
.collect(Collectors.toList());
- final String serverTimeZone = binlogSourceResponse.getServerTimezone();
+ final String serverTimeZone = sourceResponse.getServerTimezone();
boolean incrementalSnapshotEnabled = true;
// TODO Needs to be configurable for those parameters
Map<String, String> properties = Maps.newHashMap();
- if (binlogSourceResponse.isAllMigration()) {
+ if (sourceResponse.isAllMigration()) {
// Unique properties when migrate all tables in database
incrementalSnapshotEnabled = false;
properties.put("migrate-all", "true");
@@ -149,13 +152,13 @@ public class ExtractNodeUtils {
public static KafkaExtractNode createExtractNode(KafkaSourceResponse kafkaSourceResponse) {
String id = kafkaSourceResponse.getSourceName();
String name = kafkaSourceResponse.getSourceName();
- List<InlongStreamFieldInfo> streamFieldInfos = kafkaSourceResponse.getFieldList();
- List<FieldInfo> fieldInfos = streamFieldInfos.stream()
+ List<StreamField> streamFields = kafkaSourceResponse.getFieldList();
+ List<FieldInfo> fieldInfos = streamFields.stream()
.map(streamFieldInfo -> FieldInfoUtils.parseStreamFieldInfo(streamFieldInfo, name))
.collect(Collectors.toList());
String topic = kafkaSourceResponse.getTopic();
String bootstrapServers = kafkaSourceResponse.getBootstrapServers();
- Format format = null;
+ Format format;
DataTypeEnum dataType = DataTypeEnum.forName(kafkaSourceResponse.getSerializationType());
switch (dataType) {
case CSV:
@@ -211,8 +214,8 @@ public class ExtractNodeUtils {
public static PulsarExtractNode createExtractNode(PulsarSourceResponse pulsarSourceResponse) {
String id = pulsarSourceResponse.getSourceName();
String name = pulsarSourceResponse.getSourceName();
- List<InlongStreamFieldInfo> streamFieldInfos = pulsarSourceResponse.getFieldList();
- List<FieldInfo> fieldInfos = streamFieldInfos.stream()
+ List<StreamField> streamFields = pulsarSourceResponse.getFieldList();
+ List<FieldInfo> fieldInfos = streamFields.stream()
.map(streamFieldInfo -> FieldInfoUtils.parseStreamFieldInfo(streamFieldInfo, name))
.collect(Collectors.toList());
String topic = pulsarSourceResponse.getTopic();
@@ -267,10 +270,10 @@ public class ExtractNodeUtils {
* @return postgres extract node info
*/
public static PostgresExtractNode createExtractNode(PostgresSourceResponse postgresSourceResponse) {
- List<InlongStreamFieldInfo> streamFieldInfos = postgresSourceResponse.getFieldList();
+ List<StreamField> streamFields = postgresSourceResponse.getFieldList();
String id = postgresSourceResponse.getSourceName();
String name = postgresSourceResponse.getSourceName();
- List<FieldInfo> fields = streamFieldInfos.stream()
+ List<FieldInfo> fields = streamFields.stream()
.map(streamFieldInfo -> FieldInfoUtils.parseStreamFieldInfo(streamFieldInfo, name))
.collect(Collectors.toList());
return new PostgresExtractNode(id, name, fields, null, null,
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/FieldInfoUtils.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/FieldInfoUtils.java
index f0a19492d..3f28cadb8 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/FieldInfoUtils.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/FieldInfoUtils.java
@@ -21,8 +21,7 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.inlong.manager.common.enums.FieldType;
import org.apache.inlong.manager.common.enums.MetaFieldType;
-import org.apache.inlong.manager.common.pojo.sink.SinkFieldResponse;
-import org.apache.inlong.manager.common.pojo.stream.InlongStreamFieldInfo;
+import org.apache.inlong.manager.common.pojo.sink.SinkField;
import org.apache.inlong.manager.common.pojo.stream.StreamField;
import org.apache.inlong.sort.formats.common.ArrayFormatInfo;
import org.apache.inlong.sort.formats.common.BooleanFormatInfo;
@@ -77,15 +76,16 @@ public class FieldInfoUtils {
BUILT_IN_FIELD_MAP.put(MetaFieldType.MYSQL_DATA.getName(), BuiltInField.MYSQL_METADATA_DATA);
}
- public static FieldInfo parseSinkFieldInfo(SinkFieldResponse sinkFieldResponse, String nodeId) {
- boolean isBuiltIn = sinkFieldResponse.getIsMetaField() == 1;
- FieldInfo fieldInfo = getFieldInfo(sinkFieldResponse.getFieldName(), sinkFieldResponse.getFieldType(),
- isBuiltIn, sinkFieldResponse.getFieldFormat());
+ public static FieldInfo parseSinkFieldInfo(SinkField sinkField, String nodeId) {
+ boolean isBuiltIn = sinkField.getIsMetaField() == 1;
+ FieldInfo fieldInfo = getFieldInfo(sinkField.getFieldName(),
+ sinkField.getFieldType(),
+ isBuiltIn, sinkField.getFieldFormat());
fieldInfo.setNodeId(nodeId);
return fieldInfo;
}
- public static FieldInfo parseStreamFieldInfo(InlongStreamFieldInfo streamField, String nodeId) {
+ public static FieldInfo parseStreamFieldInfo(StreamField streamField, String nodeId) {
boolean isBuiltIn = streamField.getIsMetaField() == 1;
FieldInfo fieldInfo = getFieldInfo(streamField.getFieldName(), streamField.getFieldType(), isBuiltIn,
streamField.getFieldFormat());
@@ -95,7 +95,7 @@ public class FieldInfoUtils {
public static FieldInfo parseStreamField(StreamField streamField) {
boolean isBuiltIn = streamField.getIsMetaField() == 1;
- FieldInfo fieldInfo = getFieldInfo(streamField.getFieldName(), streamField.getFieldType().name(), isBuiltIn,
+ FieldInfo fieldInfo = getFieldInfo(streamField.getFieldName(), streamField.getFieldType(), isBuiltIn,
streamField.getFieldFormat());
fieldInfo.setNodeId(streamField.getOriginNodeName());
return fieldInfo;
@@ -106,11 +106,11 @@ public class FieldInfoUtils {
* TODO 1. Support partition field(not need to add index at 0), 2. Add is_metadata field in StreamSinkFieldEntity
*/
public static List<FieldMappingUnit> createFieldInfo(
- List<InlongStreamFieldInfo> streamFieldList, List<SinkFieldResponse> fieldList,
+ List<StreamField> streamFieldList, List<SinkField> fieldList,
List<FieldInfo> sourceFields, List<FieldInfo> sinkFields) {
// Set source field info list.
- for (InlongStreamFieldInfo field : streamFieldList) {
+ for (StreamField field : streamFieldList) {
FieldInfo sourceField = getFieldInfo(field.getFieldName(), field.getFieldType(),
field.getIsMetaField() == 1, field.getFieldFormat());
sourceFields.add(sourceField);
@@ -118,7 +118,7 @@ public class FieldInfoUtils {
List<FieldMappingUnit> mappingUnitList = new ArrayList<>();
// Get sink field info list, if the field name equals to build-in field, new a build-in field info
- for (SinkFieldResponse field : fieldList) {
+ for (SinkField field : fieldList) {
FieldInfo sinkField = getFieldInfo(field.getFieldName(), field.getFieldType(),
field.getIsMetaField() == 1, field.getFieldFormat());
sinkFields.add(sinkField);
@@ -138,7 +138,6 @@ public class FieldInfoUtils {
* @apiNote If the field name equals to build-in field, new a build-in field info
*/
private static FieldInfo getFieldInfo(String fieldName, String fieldType, boolean isBuiltin, String format) {
- FieldInfo fieldInfo;
BuiltInField builtInField = BUILT_IN_FIELD_MAP.get(fieldName);
FormatInfo formatInfo = convertFieldFormat(fieldType.toLowerCase(), format);
if (isBuiltin && builtInField != null) {
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/FieldRelationShipUtils.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/FieldRelationShipUtils.java
index 2db7ae56d..46216cc62 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/FieldRelationShipUtils.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/FieldRelationShipUtils.java
@@ -49,7 +49,7 @@ import java.util.Set;
import java.util.stream.Collectors;
/**
- * Util for creat field relation ship.
+ * Util for creat field relationship.
*/
public class FieldRelationShipUtils {
@@ -86,7 +86,7 @@ public class FieldRelationShipUtils {
*/
private static List<FieldRelationShip> createFieldRelationShips(List<StreamField> fieldList, String transformName) {
return fieldList.stream()
- .map(streamFieldInfo -> FieldInfoUtils.parseStreamField(streamFieldInfo))
+ .map(FieldInfoUtils::parseStreamField)
.map(fieldInfo -> {
FieldInfo inputField = new FieldInfo(fieldInfo.getName(), fieldInfo.getNodeId(),
fieldInfo.getFormatInfo());
@@ -102,13 +102,12 @@ public class FieldRelationShipUtils {
private static List<FieldRelationShip> createJoinerFieldRelationShips(List<StreamField> fieldList,
String transformName) {
return fieldList.stream()
- .map(streamFieldInfo -> {
+ .map(streamField -> {
FormatInfo formatInfo = FieldInfoUtils.convertFieldFormat(
- streamFieldInfo.getFieldType().name(),
- streamFieldInfo.getFieldFormat());
- FieldInfo inputField = new FieldInfo(streamFieldInfo.getOriginFieldName(),
- streamFieldInfo.getOriginNodeName(), formatInfo);
- FieldInfo outputField = new FieldInfo(streamFieldInfo.getFieldName(),
+ streamField.getFieldType(), streamField.getFieldFormat());
+ FieldInfo inputField = new FieldInfo(streamField.getOriginFieldName(),
+ streamField.getOriginNodeName(), formatInfo);
+ FieldInfo outputField = new FieldInfo(streamField.getFieldName(),
transformName, formatInfo);
return new FieldRelationShip(inputField, outputField);
}).collect(Collectors.toList());
@@ -123,7 +122,7 @@ public class FieldRelationShipUtils {
String preNode = preNodes.split(",")[0];
List<SplitRule> splitRules = splitterDefinition.getSplitRules();
Set<String> splitFields = Sets.newHashSet();
- List<FieldRelationShip> fieldRelationShips = splitRules.stream()
+ List<FieldRelationShip> fieldRelationships = splitRules.stream()
.map(splitRule -> parseSplitRule(splitRule, splitFields, transformName, preNode))
.reduce(Lists.newArrayList(), (list1, list2) -> {
list1.addAll(list2);
@@ -132,8 +131,8 @@ public class FieldRelationShipUtils {
List<StreamField> filteredFieldList = fieldList.stream()
.filter(streamFieldInfo -> !splitFields.contains(streamFieldInfo.getFieldName()))
.collect(Collectors.toList());
- fieldRelationShips.addAll(createFieldRelationShips(filteredFieldList, transformName));
- return fieldRelationShips;
+ fieldRelationships.addAll(createFieldRelationShips(filteredFieldList, transformName));
+ return fieldRelationships;
}
/**
@@ -145,37 +144,37 @@ public class FieldRelationShipUtils {
String preNode = preNodes.split(",")[0];
List<ReplaceRule> replaceRules = replacerDefinition.getReplaceRules();
Set<String> replaceFields = Sets.newHashSet();
- List<FieldRelationShip> fieldRelationShips = replaceRules.stream()
+ List<FieldRelationShip> fieldRelationships = replaceRules.stream()
.map(replaceRule -> parseReplaceRule(replaceRule, replaceFields, transformName, preNode))
.collect(Collectors.toList());
- fieldRelationShips = cascadeFunctionRelationShips(fieldRelationShips);
+ fieldRelationships = cascadeFunctionRelationships(fieldRelationships);
List<StreamField> filteredFieldList = fieldList.stream()
.filter(streamFieldInfo -> !replaceFields.contains(streamFieldInfo.getFieldName()))
.collect(Collectors.toList());
- fieldRelationShips.addAll(createFieldRelationShips(filteredFieldList, transformName));
- return fieldRelationShips;
+ fieldRelationships.addAll(createFieldRelationShips(filteredFieldList, transformName));
+ return fieldRelationships;
}
/**
* Create relationship of fields in cascade function.
*/
- private static List<FieldRelationShip> cascadeFunctionRelationShips(List<FieldRelationShip> fieldRelationShips) {
+ private static List<FieldRelationShip> cascadeFunctionRelationships(List<FieldRelationShip> fieldRelationships) {
Map<String, List<CascadeFunction>> cascadeFunctions = Maps.newHashMap();
Map<String, FieldInfo> targetFields = Maps.newHashMap();
- for (FieldRelationShip fieldRelationShip : fieldRelationShips) {
- CascadeFunction cascadeFunction = (CascadeFunction) fieldRelationShip.getInputField();
- String targetField = fieldRelationShip.getOutputField().getName();
+ for (FieldRelationShip fieldRelationship : fieldRelationships) {
+ CascadeFunction cascadeFunction = (CascadeFunction) fieldRelationship.getInputField();
+ String targetField = fieldRelationship.getOutputField().getName();
cascadeFunctions.computeIfAbsent(targetField, k -> Lists.newArrayList()).add(cascadeFunction);
- targetFields.put(targetField, fieldRelationShip.getOutputField());
+ targetFields.put(targetField, fieldRelationship.getOutputField());
}
- List<FieldRelationShip> cascadeRelationShips = Lists.newArrayList();
+ List<FieldRelationShip> cascadeRelationships = Lists.newArrayList();
for (Map.Entry<String, List<CascadeFunction>> entry : cascadeFunctions.entrySet()) {
String targetField = entry.getKey();
CascadeFunctionWrapper functionWrapper = new CascadeFunctionWrapper(entry.getValue());
FieldInfo targetFieldInfo = targetFields.get(targetField);
- cascadeRelationShips.add(new FieldRelationShip(functionWrapper, targetFieldInfo));
+ cascadeRelationships.add(new FieldRelationShip(functionWrapper, targetFieldInfo));
}
- return cascadeRelationShips;
+ return cascadeRelationships;
}
/**
@@ -214,7 +213,7 @@ public class FieldRelationShipUtils {
fieldInfo.setNodeId(preNode);
String seperator = splitRule.getSeperator();
List<String> targetSources = splitRule.getTargetFields();
- List<FieldRelationShip> splitRelationShips = Lists.newArrayList();
+ List<FieldRelationShip> splitRelationships = Lists.newArrayList();
for (int index = 0; index < targetSources.size(); index++) {
SplitIndexFunction splitIndexFunction = new SplitIndexFunction(
fieldInfo, new StringConstantParam(seperator), new ConstantParam(index));
@@ -222,8 +221,8 @@ public class FieldRelationShipUtils {
targetSources.get(index), transformName, FieldInfoUtils.convertFieldFormat(FieldType.STRING.name())
);
splitFields.add(targetSources.get(index));
- splitRelationShips.add(new FieldRelationShip(splitIndexFunction, targetFieldInfo));
+ splitRelationships.add(new FieldRelationShip(splitIndexFunction, targetFieldInfo));
}
- return splitRelationShips;
+ return splitRelationships;
}
}
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/FilterFunctionUtils.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/FilterFunctionUtils.java
index 82f72f19d..2bc91662e 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/FilterFunctionUtils.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/FilterFunctionUtils.java
@@ -135,7 +135,7 @@ public class FilterFunctionUtils {
private static FilterFunction createFilterFunction(FilterRule filterRule, String transformName) {
StreamField streamField = filterRule.getSourceField();
- String fieldType = streamField.getFieldType().name();
+ String fieldType = streamField.getFieldType();
String fieldFormat = streamField.getFieldFormat();
String fieldName = streamField.getFieldName();
FieldInfo sourceFieldInfo = new FieldInfo(fieldName, transformName,
@@ -173,7 +173,7 @@ public class FilterFunctionUtils {
return new ConstantParam(constant);
} else {
StreamField targetField = value.getTargetField();
- String fieldType = targetField.getFieldType().name();
+ String fieldType = targetField.getFieldType();
String fieldFormat = targetField.getFieldFormat();
String fieldName = targetField.getFieldName();
return new FieldInfo(fieldName, transformName,
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/LoadNodeUtils.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/LoadNodeUtils.java
index 1d3ef57db..cd677e911 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/LoadNodeUtils.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/LoadNodeUtils.java
@@ -22,7 +22,7 @@ import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.inlong.common.enums.DataTypeEnum;
import org.apache.inlong.manager.common.enums.SinkType;
-import org.apache.inlong.manager.common.pojo.sink.SinkFieldResponse;
+import org.apache.inlong.manager.common.pojo.sink.SinkField;
import org.apache.inlong.manager.common.pojo.sink.SinkResponse;
import org.apache.inlong.manager.common.pojo.sink.ck.ClickHouseSinkResponse;
import org.apache.inlong.manager.common.pojo.sink.hbase.HbaseSinkResponse;
@@ -49,7 +49,7 @@ import java.util.Map;
import java.util.stream.Collectors;
/**
- * Util for loead node info.
+ * Util for load node info.
*/
public class LoadNodeUtils {
@@ -60,8 +60,7 @@ public class LoadNodeUtils {
if (CollectionUtils.isEmpty(sinkResponses)) {
return Lists.newArrayList();
}
- return sinkResponses.stream().map(sourceResponse -> createLoadNode(sourceResponse))
- .collect(Collectors.toList());
+ return sinkResponses.stream().map(LoadNodeUtils::createLoadNode).collect(Collectors.toList());
}
/**
@@ -90,16 +89,15 @@ public class LoadNodeUtils {
* Create node of data load about kafka.
*/
public static KafkaLoadNode createLoadNode(KafkaSinkResponse kafkaSinkResponse) {
-
String id = kafkaSinkResponse.getSinkName();
String name = kafkaSinkResponse.getSinkName();
String topicName = kafkaSinkResponse.getTopicName();
String bootstrapServers = kafkaSinkResponse.getBootstrapServers();
- List<SinkFieldResponse> sinkFieldResponses = kafkaSinkResponse.getFieldList();
- List<FieldInfo> fieldInfos = sinkFieldResponses.stream()
- .map(sinkFieldResponse -> FieldInfoUtils.parseSinkFieldInfo(sinkFieldResponse, name))
+ List<SinkField> fieldList = kafkaSinkResponse.getFieldList();
+ List<FieldInfo> fieldInfos = fieldList.stream()
+ .map(field -> FieldInfoUtils.parseSinkFieldInfo(field, name))
.collect(Collectors.toList());
- List<FieldRelationShip> fieldRelationShips = parseSinkFields(sinkFieldResponses, name);
+ List<FieldRelationShip> fieldRelationships = parseSinkFields(fieldList, name);
Map<String, String> properties = kafkaSinkResponse.getProperties().entrySet().stream()
.collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().toString()));
Integer sinkParallelism = null;
@@ -131,7 +129,7 @@ public class LoadNodeUtils {
return new KafkaLoadNode(id,
name,
fieldInfos,
- fieldRelationShips,
+ fieldRelationships,
Lists.newArrayList(),
null,
topicName,
@@ -152,11 +150,11 @@ public class LoadNodeUtils {
String tableName = hiveSinkResponse.getTableName();
String hiveConfDir = hiveSinkResponse.getHiveConfDir();
String hiveVersion = hiveSinkResponse.getHiveVersion();
- List<SinkFieldResponse> sinkFieldResponses = hiveSinkResponse.getFieldList();
- List<FieldInfo> fields = sinkFieldResponses.stream()
- .map(sinkFieldResponse -> FieldInfoUtils.parseSinkFieldInfo(sinkFieldResponse, name))
+ List<SinkField> fieldList = hiveSinkResponse.getFieldList();
+ List<FieldInfo> fields = fieldList.stream()
+ .map(sinkField -> FieldInfoUtils.parseSinkFieldInfo(sinkField, name))
.collect(Collectors.toList());
- List<FieldRelationShip> fieldRelationShips = parseSinkFields(sinkFieldResponses, name);
+ List<FieldRelationShip> fieldRelationships = parseSinkFields(fieldList, name);
Map<String, String> properties = hiveSinkResponse.getProperties().entrySet().stream()
.collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().toString()));
List<FieldInfo> partitionFields = Lists.newArrayList();
@@ -170,7 +168,7 @@ public class LoadNodeUtils {
id,
name,
fields,
- fieldRelationShips,
+ fieldRelationships,
Lists.newArrayList(),
null,
null,
@@ -187,66 +185,50 @@ public class LoadNodeUtils {
/**
* Create hbase load node from response.
- *
- * @param hbaseSinkResponse hbaseSinkResponse
- * @return hbaseLoadNode
*/
- public static HbaseLoadNode createLoadNode(HbaseSinkResponse hbaseSinkResponse) {
- String id = hbaseSinkResponse.getSinkName();
- String name = hbaseSinkResponse.getSinkName();
- String tableName = hbaseSinkResponse.getTableName();
- String nameSpace = hbaseSinkResponse.getNamespace();
- String rowKey = hbaseSinkResponse.getRowKey();
- String zookeeperQuorum = hbaseSinkResponse.getZookeeperQuorum();
- String sinkBufferFlushMaxSize = hbaseSinkResponse.getSinkBufferFlushMaxSize();
- String zookeeperZnodeParent = hbaseSinkResponse.getZookeeperZnodeParent();
- String sinkBufferFlushMaxRows = hbaseSinkResponse.getSinkBufferFlushMaxRows();
- String sinkBufferFlushInterval = hbaseSinkResponse.getSinkBufferFlushInterval();
- List<SinkFieldResponse> sinkFieldResponses = hbaseSinkResponse.getFieldList();
- List<FieldInfo> fields = sinkFieldResponses.stream()
- .map(sinkFieldResponse -> FieldInfoUtils.parseSinkFieldInfo(sinkFieldResponse, name))
+ public static HbaseLoadNode createLoadNode(HbaseSinkResponse sinkResponse) {
+ String id = sinkResponse.getSinkName();
+ String name = sinkResponse.getSinkName();
+ List<SinkField> fieldList = sinkResponse.getFieldList();
+ List<FieldInfo> fields = fieldList.stream()
+ .map(sinkField -> FieldInfoUtils.parseSinkFieldInfo(sinkField, name))
.collect(Collectors.toList());
- List<FieldRelationShip> fieldRelationShips = parseSinkFields(sinkFieldResponses, name);
- Map<String, String> properties = hbaseSinkResponse.getProperties().entrySet().stream()
+ List<FieldRelationShip> fieldRelationships = parseSinkFields(fieldList, name);
+ Map<String, String> properties = sinkResponse.getProperties().entrySet().stream()
.collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().toString()));
return new HbaseLoadNode(
id,
name,
fields,
- fieldRelationShips,
+ fieldRelationships,
Lists.newArrayList(),
null,
null,
properties,
- tableName,
- nameSpace,
- zookeeperQuorum,
- rowKey,
- sinkBufferFlushMaxSize,
- zookeeperZnodeParent,
- sinkBufferFlushMaxRows,
- sinkBufferFlushInterval
+ sinkResponse.getTableName(),
+ sinkResponse.getNamespace(),
+ sinkResponse.getZkQuorum(),
+ sinkResponse.getRowKey(),
+ sinkResponse.getBufferFlushMaxSize(),
+ sinkResponse.getZkNodeParent(),
+ sinkResponse.getBufferFlushMaxRows(),
+ sinkResponse.getBufferFlushInterval()
);
}
/**
- * create postgres load node
- *
- * @param postgresSinkResponse postgresSinkResponse
- * @return postgres load node
+ * Create postgres load node
*/
public static PostgresLoadNode createLoadNode(PostgresSinkResponse postgresSinkResponse) {
- List<SinkFieldResponse> sinkFieldResponses = postgresSinkResponse.getFieldList();
-
+ List<SinkField> fieldList = postgresSinkResponse.getFieldList();
String name = postgresSinkResponse.getSinkName();
- List<FieldInfo> fields = sinkFieldResponses.stream()
- .map(sinkFieldResponse -> FieldInfoUtils.parseSinkFieldInfo(sinkFieldResponse,
- name))
+ List<FieldInfo> fields = fieldList.stream()
+ .map(sinkField -> FieldInfoUtils.parseSinkFieldInfo(sinkField, name))
.collect(Collectors.toList());
- List<FieldRelationShip> fieldRelationShips = parseSinkFields(sinkFieldResponses, name);
+ List<FieldRelationShip> fieldRelationships = parseSinkFields(fieldList, name);
return new PostgresLoadNode(postgresSinkResponse.getSinkName(),
postgresSinkResponse.getSinkName(),
- fields, fieldRelationShips, null, null, 1,
+ fields, fieldRelationships, null, null, 1,
null, postgresSinkResponse.getJdbcUrl(), postgresSinkResponse.getUsername(),
postgresSinkResponse.getPassword(),
postgresSinkResponse.getDbName() + "." + postgresSinkResponse.getTableName(),
@@ -254,13 +236,10 @@ public class LoadNodeUtils {
}
/**
- * create clickHouse load node
- *
- * @param clickHouseSinkResponse clickHouseSinkResponse
- * @return
+ * Create ClickHouse load node
*/
public static ClickHouseLoadNode createLoadNode(ClickHouseSinkResponse clickHouseSinkResponse) {
- List<SinkFieldResponse> sinkFieldResponses = clickHouseSinkResponse.getFieldList();
+ List<SinkField> sinkFieldResponses = clickHouseSinkResponse.getFieldList();
String name = clickHouseSinkResponse.getSinkName();
List<FieldInfo> fields = sinkFieldResponses.stream()
.map(sinkFieldResponse -> FieldInfoUtils.parseSinkFieldInfo(sinkFieldResponse,
@@ -279,20 +258,20 @@ public class LoadNodeUtils {
/**
* Parse information field of data sink.
*/
- public static List<FieldRelationShip> parseSinkFields(List<SinkFieldResponse> sinkFieldResponses, String sinkName) {
- if (CollectionUtils.isEmpty(sinkFieldResponses)) {
+ public static List<FieldRelationShip> parseSinkFields(List<SinkField> fieldList, String sinkName) {
+ if (CollectionUtils.isEmpty(fieldList)) {
return Lists.newArrayList();
}
- return sinkFieldResponses.stream()
- .filter(sinkFieldResponse -> StringUtils.isNotEmpty(sinkFieldResponse.getSourceFieldName()))
- .map(sinkFieldResponse -> {
- String fieldName = sinkFieldResponse.getFieldName();
- String fieldType = sinkFieldResponse.getFieldType();
- String fieldFormat = sinkFieldResponse.getFieldFormat();
+ return fieldList.stream()
+ .filter(sinkField -> StringUtils.isNotEmpty(sinkField.getSourceFieldName()))
+ .map(field -> {
+ String fieldName = field.getFieldName();
+ String fieldType = field.getFieldType();
+ String fieldFormat = field.getFieldFormat();
FieldInfo sinkField = new FieldInfo(fieldName, sinkName,
FieldInfoUtils.convertFieldFormat(fieldType, fieldFormat));
- String sourceFieldName = sinkFieldResponse.getSourceFieldName();
- String sourceFieldType = sinkFieldResponse.getSourceFieldType();
+ String sourceFieldName = field.getSourceFieldName();
+ String sourceFieldType = field.getSourceFieldType();
FieldInfo sourceField = new FieldInfo(sourceFieldName, sinkName,
FieldInfoUtils.convertFieldFormat(sourceFieldType));
return new FieldRelationShip(sourceField, sinkField);
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/NodeRelationShipUtils.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/NodeRelationShipUtils.java
index 4255b983a..fb8078384 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/NodeRelationShipUtils.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/NodeRelationShipUtils.java
@@ -23,12 +23,12 @@ import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.curator.shaded.com.google.common.collect.Maps;
import org.apache.inlong.manager.common.enums.TransformType;
+import org.apache.inlong.manager.common.pojo.sink.StreamSink;
+import org.apache.inlong.manager.common.pojo.source.StreamSource;
import org.apache.inlong.manager.common.pojo.stream.InlongStreamInfo;
import org.apache.inlong.manager.common.pojo.stream.StreamField;
import org.apache.inlong.manager.common.pojo.stream.StreamNode;
import org.apache.inlong.manager.common.pojo.stream.StreamPipeline;
-import org.apache.inlong.manager.common.pojo.stream.StreamSink;
-import org.apache.inlong.manager.common.pojo.stream.StreamSource;
import org.apache.inlong.manager.common.pojo.stream.StreamTransform;
import org.apache.inlong.manager.common.pojo.transform.TransformDefinition;
import org.apache.inlong.manager.common.pojo.transform.TransformResponse;
@@ -56,11 +56,14 @@ import java.util.Map;
import java.util.stream.Collectors;
/**
- * Util for creat node relation ship.
+ * Util for creat node relationship.
*/
@Slf4j
public class NodeRelationShipUtils {
+ /**
+ * Create node relationship for the given stream
+ */
public static List<NodeRelationShip> createNodeRelationShipsForStream(InlongStreamInfo streamInfo) {
String tempView = streamInfo.getExtParams();
if (StringUtils.isEmpty(tempView)) {
@@ -70,15 +73,14 @@ public class NodeRelationShipUtils {
StreamPipeline pipeline = StreamParseUtils.parseStreamPipeline(streamInfo.getExtParams(),
streamInfo.getInlongStreamId());
return pipeline.getPipeline().stream()
- .map(streamNodeRelationShip -> new NodeRelationShip(
- Lists.newArrayList(streamNodeRelationShip.getInputNodes()),
- Lists.newArrayList(streamNodeRelationShip.getOutputNodes())))
+ .map(nodeRelationship -> new NodeRelationShip(
+ Lists.newArrayList(nodeRelationship.getInputNodes()),
+ Lists.newArrayList(nodeRelationship.getOutputNodes())))
.collect(Collectors.toList());
}
/**
- * Optimize relationship of node.
- * JoinerRelationship must be rebuild.
+ * Optimize relationship of node, JoinerRelationship must be rebuilt.
*/
public static void optimizeNodeRelationShips(StreamInfo streamInfo, List<TransformResponse> transformResponses) {
if (CollectionUtils.isEmpty(transformResponses)) {
@@ -97,29 +99,28 @@ public class NodeRelationShipUtils {
TransformDefinition transformDefinition = transformTypeMap.get(transformNode.getName());
return transformDefinition.getTransformType() == TransformType.JOINER;
}).collect(Collectors.toMap(TransformNode::getName, transformNode -> transformNode));
- List<NodeRelationShip> relationShips = streamInfo.getRelations();
- Iterator<NodeRelationShip> shipIterator = relationShips.listIterator();
- List<NodeRelationShip> joinRelationShips = Lists.newArrayList();
+
+ List<NodeRelationShip> relationships = streamInfo.getRelations();
+ Iterator<NodeRelationShip> shipIterator = relationships.listIterator();
+ List<NodeRelationShip> joinRelationships = Lists.newArrayList();
while (shipIterator.hasNext()) {
- NodeRelationShip relationShip = shipIterator.next();
- List<String> outputs = relationShip.getOutputs();
+ NodeRelationShip relationship = shipIterator.next();
+ List<String> outputs = relationship.getOutputs();
if (outputs.size() == 1) {
String nodeName = outputs.get(0);
if (joinNodes.get(nodeName) != null) {
TransformDefinition transformDefinition = transformTypeMap.get(nodeName);
TransformNode transformNode = joinNodes.get(nodeName);
- joinRelationShips.add(createNodeRelationShip((JoinerDefinition) transformDefinition,
- relationShip,
- transformNode.getId()));
+ joinRelationships.add(createNodeRelationShip((JoinerDefinition) transformDefinition, relationship));
shipIterator.remove();
}
}
}
- relationShips.addAll(joinRelationShips);
+ relationships.addAll(joinRelationships);
}
private static NodeRelationShip createNodeRelationShip(JoinerDefinition joinerDefinition,
- NodeRelationShip nodeRelationShip, String nodeId) {
+ NodeRelationShip nodeRelationship) {
JoinMode joinMode = joinerDefinition.getJoinMode();
String leftNode = getNodeName(joinerDefinition.getLeftNode());
String rightNode = getNodeName(joinerDefinition.getRightNode());
@@ -130,27 +131,23 @@ public class NodeRelationShipUtils {
for (int index = 0; index < leftJoinFields.size(); index++) {
StreamField leftField = leftJoinFields.get(index);
StreamField rightField = rightJoinFields.get(index);
- LogicOperator operator = null;
+ LogicOperator operator;
if (index != leftJoinFields.size() - 1) {
operator = AndOperator.getInstance();
} else {
operator = EmptyOperator.getInstance();
}
- filterFunctions.add(
- createFilterFunction(leftField, rightField, operator));
+ filterFunctions.add(createFilterFunction(leftField, rightField, operator));
}
Map<String, List<FilterFunction>> joinConditions = Maps.newHashMap();
joinConditions.put(rightNode, filterFunctions);
switch (joinMode) {
case LEFT_JOIN:
- return new LeftOuterJoinNodeRelationShip(preNodes, nodeRelationShip.getOutputs(),
- joinConditions);
+ return new LeftOuterJoinNodeRelationShip(preNodes, nodeRelationship.getOutputs(), joinConditions);
case INNER_JOIN:
- return new InnerJoinNodeRelationShip(preNodes, nodeRelationShip.getOutputs(),
- joinConditions);
+ return new InnerJoinNodeRelationShip(preNodes, nodeRelationship.getOutputs(), joinConditions);
case RIGHT_JOIN:
- return new RightOuterJoinNodeRelationShip(preNodes, nodeRelationShip.getOutputs(),
- joinConditions);
+ return new RightOuterJoinNodeRelationShip(preNodes, nodeRelationship.getOutputs(), joinConditions);
default:
throw new IllegalArgumentException(String.format("Unsupported join mode=%s for inlong", joinMode));
}
@@ -159,9 +156,9 @@ public class NodeRelationShipUtils {
private static SingleValueFilterFunction createFilterFunction(StreamField leftField, StreamField rightField,
LogicOperator operator) {
FieldInfo sourceField = new FieldInfo(leftField.getOriginFieldName(), leftField.getOriginNodeName(),
- FieldInfoUtils.convertFieldFormat(leftField.getFieldType().name(), leftField.getFieldFormat()));
+ FieldInfoUtils.convertFieldFormat(leftField.getFieldType(), leftField.getFieldFormat()));
FieldInfo targetField = new FieldInfo(rightField.getOriginFieldName(), rightField.getOriginNodeName(),
- FieldInfoUtils.convertFieldFormat(rightField.getFieldType().name(), rightField.getFieldFormat()));
+ FieldInfoUtils.convertFieldFormat(rightField.getFieldType(), rightField.getFieldFormat()));
return new SingleValueFilterFunction(operator, sourceField, EqualOperator.getInstance(), targetField);
}
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/SinkInfoUtils.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/SinkInfoUtils.java
index 2d17d8bd4..6842d0a7f 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/SinkInfoUtils.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/SinkInfoUtils.java
@@ -24,7 +24,7 @@ import org.apache.inlong.manager.common.enums.FieldType;
import org.apache.inlong.manager.common.enums.FileFormat;
import org.apache.inlong.manager.common.enums.SinkType;
import org.apache.inlong.manager.common.exceptions.BusinessException;
-import org.apache.inlong.manager.common.pojo.sink.SinkFieldBase;
+import org.apache.inlong.manager.common.pojo.sink.SinkField;
import org.apache.inlong.manager.common.pojo.sink.SinkResponse;
import org.apache.inlong.manager.common.pojo.sink.ck.ClickHouseSinkResponse;
import org.apache.inlong.manager.common.pojo.sink.es.ElasticsearchSinkResponse;
@@ -216,8 +216,7 @@ public class SinkInfoUtils {
/**
* Check the validation of Hive partition field.
*/
- public static void checkPartitionField(List<? extends SinkFieldBase> fieldList,
- List<HivePartitionField> partitionList) {
+ public static void checkPartitionField(List<SinkField> fieldList, List<HivePartitionField> partitionList) {
if (CollectionUtils.isEmpty(partitionList)) {
return;
}
@@ -226,7 +225,7 @@ public class SinkInfoUtils {
throw new BusinessException(ErrorCodeEnum.SINK_FIELD_LIST_IS_EMPTY);
}
- Map<String, SinkFieldBase> sinkFieldMap = new HashMap<>(fieldList.size());
+ Map<String, SinkField> sinkFieldMap = new HashMap<>(fieldList.size());
fieldList.forEach(field -> sinkFieldMap.put(field.getFieldName(), field));
for (HivePartitionField partitionField : partitionList) {
@@ -235,7 +234,7 @@ public class SinkInfoUtils {
throw new BusinessException(ErrorCodeEnum.PARTITION_FIELD_NAME_IS_EMPTY);
}
- SinkFieldBase sinkField = sinkFieldMap.get(fieldName);
+ SinkField sinkField = sinkFieldMap.get(fieldName);
if (sinkField == null) {
throw new BusinessException(
String.format(ErrorCodeEnum.PARTITION_FIELD_NOT_FOUND.getMessage(), fieldName));
@@ -252,18 +251,18 @@ public class SinkInfoUtils {
* Creat HBase sink info.
*/
private static HbaseSinkInfo createHbaseSinkInfo(HbaseSinkResponse sinkResponse, List<FieldInfo> sinkFields) {
- if (StringUtils.isEmpty(sinkResponse.getZookeeperQuorum())) {
+ if (StringUtils.isEmpty(sinkResponse.getZkQuorum())) {
throw new BusinessException(String.format("HBase={%s} zookeeper quorum url cannot be empty", sinkResponse));
- } else if (StringUtils.isEmpty(sinkResponse.getZookeeperZnodeParent())) {
+ } else if (StringUtils.isEmpty(sinkResponse.getZkNodeParent())) {
throw new BusinessException(String.format("HBase={%s} zookeeper node cannot be empty", sinkResponse));
} else if (StringUtils.isEmpty(sinkResponse.getTableName())) {
throw new BusinessException(String.format("HBase={%s} table name cannot be empty", sinkResponse));
}
- return new HbaseSinkInfo(sinkFields.toArray(new FieldInfo[0]), sinkResponse.getZookeeperQuorum(),
- sinkResponse.getZookeeperZnodeParent(), sinkResponse.getNamespace(), sinkResponse.getTableName(),
- sinkResponse.getSinkBufferFlushMaxSize(), sinkResponse.getSinkBufferFlushMaxSize(),
- sinkResponse.getSinkBufferFlushInterval());
+ return new HbaseSinkInfo(sinkFields.toArray(new FieldInfo[0]), sinkResponse.getZkQuorum(),
+ sinkResponse.getZkNodeParent(), sinkResponse.getNamespace(), sinkResponse.getTableName(),
+ sinkResponse.getBufferFlushMaxSize(), sinkResponse.getBufferFlushMaxSize(),
+ sinkResponse.getBufferFlushInterval());
}
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/TransformNodeUtils.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/TransformNodeUtils.java
index 5d44a62dd..12d211996 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/TransformNodeUtils.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/TransformNodeUtils.java
@@ -45,9 +45,7 @@ public class TransformNodeUtils {
if (CollectionUtils.isEmpty(transformResponses)) {
return Lists.newArrayList();
}
- List<TransformNode> transformNodes = transformResponses.stream()
- .map(transformResponse -> createTransformNode(transformResponse)).collect(Collectors.toList());
- return transformNodes;
+ return transformResponses.stream().map(TransformNodeUtils::createTransformNode).collect(Collectors.toList());
}
public static TransformNode createTransformNode(TransformResponse transformResponse) {
@@ -63,16 +61,12 @@ public class TransformNodeUtils {
/**
* Create distinct node based on deDuplicationDefinition
- *
- * @param deDuplicationDefinition
- * @param transformResponse
- * @return
*/
public static DistinctNode createDistinctNode(DeDuplicationDefinition deDuplicationDefinition,
TransformResponse transformResponse) {
List<StreamField> streamFields = deDuplicationDefinition.getDupFields();
List<FieldInfo> distinctFields = streamFields.stream()
- .map(streamField -> FieldInfoUtils.parseStreamField(streamField))
+ .map(FieldInfoUtils::parseStreamField)
.collect(Collectors.toList());
StreamField timingField = deDuplicationDefinition.getTimingField();
FieldInfo orderField = FieldInfoUtils.parseStreamField(timingField);
@@ -104,16 +98,13 @@ public class TransformNodeUtils {
/**
* Create transform node based on transformResponse
- *
- * @param transformResponse
- * @return
*/
public static TransformNode createNormalTransformNode(TransformResponse transformResponse) {
TransformNode transformNode = new TransformNode();
transformNode.setId(transformResponse.getTransformName());
transformNode.setName(transformResponse.getTransformName());
List<FieldInfo> fieldInfos = transformResponse.getFieldList().stream()
- .map(streamFieldInfo -> FieldInfoUtils.parseStreamField(streamFieldInfo)).collect(Collectors.toList());
+ .map(FieldInfoUtils::parseStreamField).collect(Collectors.toList());
transformNode.setFields(fieldInfos);
transformNode.setFieldRelationShips(FieldRelationShipUtils.createFieldRelationShips(transformResponse));
transformNode.setFilters(
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/AbstractSourceOperation.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/AbstractSourceOperation.java
index 1e229425b..0bbf80474 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/AbstractSourceOperation.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/AbstractSourceOperation.java
@@ -26,7 +26,7 @@ import org.apache.inlong.manager.common.enums.SourceStatus;
import org.apache.inlong.manager.common.exceptions.BusinessException;
import org.apache.inlong.manager.common.pojo.source.SourceRequest;
import org.apache.inlong.manager.common.pojo.source.SourceResponse;
-import org.apache.inlong.manager.common.pojo.stream.InlongStreamFieldInfo;
+import org.apache.inlong.manager.common.pojo.stream.StreamField;
import org.apache.inlong.manager.common.util.CommonBeanUtils;
import org.apache.inlong.manager.common.util.Preconditions;
import org.apache.inlong.manager.dao.entity.StreamSourceEntity;
@@ -121,8 +121,8 @@ public abstract class AbstractSourceOperation implements StreamSourceOperation {
SourceResponse sourceResponse = this.getFromEntity(entity, this::getResponse);
List<StreamSourceFieldEntity> sourceFieldEntities = sourceFieldMapper.selectBySourceId(entity.getId());
- List<InlongStreamFieldInfo> fieldInfos = CommonBeanUtils.copyListProperties(sourceFieldEntities,
- InlongStreamFieldInfo::new);
+ List<StreamField> fieldInfos = CommonBeanUtils.copyListProperties(sourceFieldEntities,
+ StreamField::new);
sourceResponse.setFieldList(fieldInfos);
return sourceResponse;
}
@@ -200,7 +200,7 @@ public abstract class AbstractSourceOperation implements StreamSourceOperation {
sourceMapper.updateByPrimaryKeySelective(curEntity);
}
- private void updateFieldOpt(StreamSourceEntity entity, List<InlongStreamFieldInfo> fieldInfos) {
+ private void updateFieldOpt(StreamSourceEntity entity, List<StreamField> fieldInfos) {
Integer sourceId = entity.getId();
if (CollectionUtils.isEmpty(fieldInfos)) {
return;
@@ -214,7 +214,7 @@ public abstract class AbstractSourceOperation implements StreamSourceOperation {
LOGGER.info("success to update field");
}
- private void saveFieldOpt(StreamSourceEntity entity, List<InlongStreamFieldInfo> fieldInfos) {
+ private void saveFieldOpt(StreamSourceEntity entity, List<StreamField> fieldInfos) {
LOGGER.info("begin to save source field={}", fieldInfos);
if (CollectionUtils.isEmpty(fieldInfos)) {
return;
@@ -226,7 +226,7 @@ public abstract class AbstractSourceOperation implements StreamSourceOperation {
String streamId = entity.getInlongStreamId();
String sourceType = entity.getSourceType();
Integer sourceId = entity.getId();
- for (InlongStreamFieldInfo fieldInfo : fieldInfos) {
+ for (StreamField fieldInfo : fieldInfos) {
StreamSourceFieldEntity fieldEntity = CommonBeanUtils.copyProperties(fieldInfo,
StreamSourceFieldEntity::new);
if (StringUtils.isEmpty(fieldEntity.getFieldComment())) {
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/transform/StreamTransformServiceImpl.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/transform/StreamTransformServiceImpl.java
index 1c4c78050..7aacda685 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/transform/StreamTransformServiceImpl.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/transform/StreamTransformServiceImpl.java
@@ -22,7 +22,6 @@ import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.inlong.manager.common.enums.ErrorCodeEnum;
-import org.apache.inlong.manager.common.enums.FieldType;
import org.apache.inlong.manager.common.enums.GlobalConstants;
import org.apache.inlong.manager.common.exceptions.BusinessException;
import org.apache.inlong.manager.common.pojo.stream.StreamField;
@@ -55,9 +54,9 @@ import java.util.stream.Collectors;
public class StreamTransformServiceImpl implements StreamTransformService {
@Autowired
- protected StreamTransformEntityMapper transformEntityMapper;
+ protected StreamTransformEntityMapper transformMapper;
@Autowired
- protected StreamTransformFieldEntityMapper transformFieldEntityMapper;
+ protected StreamTransformFieldEntityMapper transformFieldMapper;
@Autowired
protected CommonOperateService commonOperateService;
@@ -73,7 +72,7 @@ public class StreamTransformServiceImpl implements StreamTransformService {
final String transformName = transformRequest.getTransformName();
commonOperateService.checkGroupStatus(groupId, operator);
- List<StreamTransformEntity> transformEntities = transformEntityMapper.selectByRelatedId(groupId,
+ List<StreamTransformEntity> transformEntities = transformMapper.selectByRelatedId(groupId,
streamId, transformName);
if (CollectionUtils.isNotEmpty(transformEntities)) {
String err = "stream transform already exists with groupId=%s, streamId=%s, transformName=%s";
@@ -88,7 +87,7 @@ public class StreamTransformServiceImpl implements StreamTransformService {
transformEntity.setCreateTime(now);
transformEntity.setModifyTime(now);
transformEntity.setIsDeleted(GlobalConstants.UN_DELETED);
- transformEntityMapper.insert(transformEntity);
+ transformMapper.insert(transformEntity);
saveFieldOpt(transformEntity, transformRequest.getFieldList());
return transformEntity.getId();
}
@@ -97,28 +96,25 @@ public class StreamTransformServiceImpl implements StreamTransformService {
public List<TransformResponse> listTransform(String groupId, String streamId) {
log.info("begin to fetch transform info by groupId={} and streamId={} ", groupId, streamId);
Preconditions.checkNotNull(groupId, ErrorCodeEnum.GROUP_ID_IS_EMPTY.getMessage());
- List<StreamTransformEntity> transformEntities = transformEntityMapper.selectByRelatedId(groupId, streamId,
- null);
- if (CollectionUtils.isEmpty(transformEntities)) {
+ List<StreamTransformEntity> entityList = transformMapper.selectByRelatedId(groupId, streamId, null);
+ if (CollectionUtils.isEmpty(entityList)) {
return Collections.emptyList();
}
- List<Integer> transformIds = transformEntities.stream().map(transformEntity -> transformEntity.getId())
- .collect(Collectors.toList());
- List<StreamTransformFieldEntity> transformFieldEntities = transformFieldEntityMapper.selectByTransformIds(
- transformIds);
- Map<Integer, List<StreamField>> fieldInfoMap = transformFieldEntities.stream()
+
+ List<Integer> transformIds = entityList.stream().map(StreamTransformEntity::getId).collect(Collectors.toList());
+ List<StreamTransformFieldEntity> fieldEntities = transformFieldMapper.selectByTransformIds(transformIds);
+ Map<Integer, List<StreamField>> fieldInfoMap = fieldEntities.stream()
.map(transformFieldEntity -> {
- StreamField fieldInfo = CommonBeanUtils.copyProperties(transformFieldEntity,
- StreamField::new);
- fieldInfo.setFieldType(FieldType.forName(transformFieldEntity.getFieldType()));
- fieldInfo.setId(Integer.valueOf(transformFieldEntity.getRankNum()));
+ StreamField fieldInfo = CommonBeanUtils.copyProperties(transformFieldEntity, StreamField::new);
+ fieldInfo.setFieldType(transformFieldEntity.getFieldType());
+ fieldInfo.setId(transformFieldEntity.getRankNum());
return Pair.of(transformFieldEntity.getTransformId(), fieldInfo);
}).collect(Collectors.groupingBy(Pair::getLeft,
Collectors.mapping(Pair::getRight, Collectors.toList())));
- List<TransformResponse> transformResponses = transformEntities.stream()
+ List<TransformResponse> transformResponses = entityList.stream()
.map(entity -> CommonBeanUtils.copyProperties(entity, TransformResponse::new))
.collect(Collectors.toList());
- transformResponses.stream().forEach(transformResponse -> {
+ transformResponses.forEach(transformResponse -> {
int transformId = transformResponse.getId();
List<StreamField> fieldInfos = fieldInfoMap.get(transformId);
if (CollectionUtils.isNotEmpty(fieldInfos)) {
@@ -141,9 +137,9 @@ public class StreamTransformServiceImpl implements StreamTransformService {
StreamTransformEntity::new);
transformEntity.setModifier(operator);
transformEntity.setVersion(transformEntity.getVersion() + 1);
- Date now = new Date();
- transformEntity.setModifyTime(now);
- transformEntityMapper.updateByIdSelective(transformEntity);
+ transformEntity.setModifyTime(new Date());
+
+ transformMapper.updateByIdSelective(transformEntity);
updateFieldOpt(transformEntity, transformRequest.getFieldList());
return true;
}
@@ -151,14 +147,13 @@ public class StreamTransformServiceImpl implements StreamTransformService {
@Override
@Transactional(rollbackFor = Throwable.class, propagation = Propagation.REQUIRES_NEW)
public boolean delete(String groupId, String streamId, String transformName, String operator) {
- log.info("begin to delete transform by groupId={} streamId={}, transformName={}", groupId, streamId,
+ log.info("begin to logic delete transform by groupId={} streamId={}, transformName={}", groupId, streamId,
transformName);
Preconditions.checkNotNull(groupId, ErrorCodeEnum.GROUP_ID_IS_EMPTY.getMessage());
Preconditions.checkNotNull(streamId, ErrorCodeEnum.STREAM_ID_IS_EMPTY.getMessage());
commonOperateService.checkGroupStatus(groupId, operator);
Date now = new Date();
- List<StreamTransformEntity> entityList = transformEntityMapper.selectByRelatedId(groupId, streamId,
- transformName);
+ List<StreamTransformEntity> entityList = transformMapper.selectByRelatedId(groupId, streamId, transformName);
if (CollectionUtils.isNotEmpty(entityList)) {
for (StreamTransformEntity entity : entityList) {
Integer id = entity.getId();
@@ -166,8 +161,8 @@ public class StreamTransformServiceImpl implements StreamTransformService {
entity.setIsDeleted(id);
entity.setModifier(operator);
entity.setModifyTime(now);
- transformEntityMapper.updateByIdSelective(entity);
- transformFieldEntityMapper.deleteAll(id);
+ transformMapper.updateByIdSelective(entity);
+ transformFieldMapper.deleteAll(id);
}
}
log.info("success to logic delete transform by groupId={}, streamId={}, transformName={}", groupId, streamId,
@@ -187,33 +182,33 @@ public class StreamTransformServiceImpl implements StreamTransformService {
Preconditions.checkNotNull(transformName, ErrorCodeEnum.TRANSFORM_NAME_IS_NULL.getMessage());
}
- private void updateFieldOpt(StreamTransformEntity entity, List<StreamField> fieldInfos) {
+ private void updateFieldOpt(StreamTransformEntity entity, List<StreamField> fieldList) {
Integer transformId = entity.getId();
- if (CollectionUtils.isEmpty(fieldInfos)) {
+ if (CollectionUtils.isEmpty(fieldList)) {
return;
}
// First physically delete the existing fields
- transformFieldEntityMapper.deleteAll(transformId);
+ transformFieldMapper.deleteAll(transformId);
// Then batch save the source fields
- this.saveFieldOpt(entity, fieldInfos);
+ this.saveFieldOpt(entity, fieldList);
log.info("success to update transform field");
}
- private void saveFieldOpt(StreamTransformEntity entity, List<StreamField> fieldInfos) {
- log.info("begin to save transform field={}", fieldInfos);
- if (CollectionUtils.isEmpty(fieldInfos)) {
+ private void saveFieldOpt(StreamTransformEntity entity, List<StreamField> fieldList) {
+ log.info("begin to save transform field={}", fieldList);
+ if (CollectionUtils.isEmpty(fieldList)) {
return;
}
- int size = fieldInfos.size();
+ int size = fieldList.size();
List<StreamTransformFieldEntity> entityList = new ArrayList<>(size);
String groupId = entity.getInlongGroupId();
String streamId = entity.getInlongStreamId();
String transformType = entity.getTransformType();
Integer transformId = entity.getId();
- for (StreamField fieldInfo : fieldInfos) {
+ for (StreamField fieldInfo : fieldList) {
StreamTransformFieldEntity fieldEntity = CommonBeanUtils.copyProperties(fieldInfo,
StreamTransformFieldEntity::new);
if (StringUtils.isEmpty(fieldEntity.getFieldComment())) {
@@ -222,7 +217,7 @@ public class StreamTransformServiceImpl implements StreamTransformService {
fieldEntity.setId(null);
fieldEntity.setInlongGroupId(groupId);
fieldEntity.setInlongStreamId(streamId);
- fieldEntity.setFieldType(fieldInfo.getFieldType().name());
+ fieldEntity.setFieldType(fieldInfo.getFieldType());
fieldEntity.setRankNum(fieldInfo.getId());
fieldEntity.setTransformId(transformId);
fieldEntity.setTransformType(transformType);
@@ -230,7 +225,7 @@ public class StreamTransformServiceImpl implements StreamTransformService {
entityList.add(fieldEntity);
}
- transformFieldEntityMapper.insertAll(entityList);
+ transformFieldMapper.insertAll(entityList);
log.info("success to save transform fields");
}
}
diff --git a/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/core/sink/HbaseStreamSinkServiceTest.java b/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/core/sink/HbaseStreamSinkServiceTest.java
index 52d5eaded..52cdfc862 100644
--- a/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/core/sink/HbaseStreamSinkServiceTest.java
+++ b/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/core/sink/HbaseStreamSinkServiceTest.java
@@ -60,7 +60,7 @@ public class HbaseStreamSinkServiceTest extends ServiceBaseTest {
sinkInfo.setTableName(tableName);
sinkInfo.setNamespace(nameSpace);
sinkInfo.setRowKey(rowkey);
- sinkInfo.setZookeeperQuorum(zookeeperQuorum);
+ sinkInfo.setZkQuorum(zookeeperQuorum);
return sinkService.save(sinkInfo, globalOperator);
}
diff --git a/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/sort/DisableZkForSortTest.java b/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/sort/DisableZkForSortTest.java
index 115267879..70e809e10 100644
--- a/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/sort/DisableZkForSortTest.java
+++ b/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/sort/DisableZkForSortTest.java
@@ -22,7 +22,7 @@ import org.apache.inlong.manager.common.enums.GroupOperateType;
import org.apache.inlong.manager.common.enums.GroupStatus;
import org.apache.inlong.manager.common.enums.ProcessStatus;
import org.apache.inlong.manager.common.pojo.group.InlongGroupInfo;
-import org.apache.inlong.manager.common.pojo.sink.SinkFieldRequest;
+import org.apache.inlong.manager.common.pojo.sink.SinkField;
import org.apache.inlong.manager.common.pojo.sink.hive.HiveSinkRequest;
import org.apache.inlong.manager.common.pojo.source.kafka.KafkaSourceRequest;
import org.apache.inlong.manager.common.pojo.stream.InlongStreamInfo;
@@ -71,38 +71,38 @@ public class DisableZkForSortTest extends WorkflowServiceImplTest {
}
/**
- * Creat hvie sink by inlong stream info.
+ * Create Hive sink by inlong stream info.
*/
public HiveSinkRequest createHiveSink(InlongStreamInfo streamInfo) {
- HiveSinkRequest hiveSinkRequest = new HiveSinkRequest();
- hiveSinkRequest.setInlongGroupId(streamInfo.getInlongGroupId());
- hiveSinkRequest.setSinkType("HIVE");
- hiveSinkRequest.setSinkName("HIVE");
- hiveSinkRequest.setInlongStreamId(streamInfo.getInlongStreamId());
- List<SinkFieldRequest> sinkFieldRequests = createStreamFields(streamInfo.getInlongGroupId(),
+ HiveSinkRequest sinkRequest = new HiveSinkRequest();
+ sinkRequest.setInlongGroupId(streamInfo.getInlongGroupId());
+ sinkRequest.setSinkType("HIVE");
+ sinkRequest.setSinkName("HIVE");
+ sinkRequest.setInlongStreamId(streamInfo.getInlongStreamId());
+ List<SinkField> sinkFields = createStreamFields(streamInfo.getInlongGroupId(),
streamInfo.getInlongStreamId())
.stream()
- .map(streamFieldInfo -> {
- SinkFieldRequest fieldInfo = new SinkFieldRequest();
- fieldInfo.setFieldName(streamFieldInfo.getFieldName());
- fieldInfo.setFieldType(streamFieldInfo.getFieldType());
- fieldInfo.setFieldComment(streamFieldInfo.getFieldComment());
+ .map(streamField -> {
+ SinkField fieldInfo = new SinkField();
+ fieldInfo.setFieldName(streamField.getFieldName());
+ fieldInfo.setFieldType(streamField.getFieldType());
+ fieldInfo.setFieldComment(streamField.getFieldComment());
return fieldInfo;
})
.collect(Collectors.toList());
- hiveSinkRequest.setFieldList(sinkFieldRequests);
- hiveSinkRequest.setEnableCreateTable(0);
- hiveSinkRequest.setUsername(OPERATOR);
- hiveSinkRequest.setPassword("password");
- hiveSinkRequest.setDbName("default");
- hiveSinkRequest.setTableName("kip_test");
- hiveSinkRequest.setJdbcUrl("jdbc:hive2://localhost:7001");
- hiveSinkRequest.setFileFormat("TextFile");
- hiveSinkRequest.setDataPath("hdfs://localhost:4007/user/hive/warehouse/default");
- hiveSinkRequest.setFileFormat(StandardCharsets.UTF_8.name());
- hiveSinkRequest.setDataSeparator("124");
- streamSinkService.save(hiveSinkRequest, OPERATOR);
- return hiveSinkRequest;
+ sinkRequest.setFieldList(sinkFields);
+ sinkRequest.setEnableCreateResource(0);
+ sinkRequest.setUsername(OPERATOR);
+ sinkRequest.setPassword("password");
+ sinkRequest.setDbName("default");
+ sinkRequest.setTableName("kip_test");
+ sinkRequest.setJdbcUrl("jdbc:hive2://localhost:7001");
+ sinkRequest.setFileFormat("TextFile");
+ sinkRequest.setDataPath("hdfs://localhost:4007/user/hive/warehouse/default");
+ sinkRequest.setFileFormat(StandardCharsets.UTF_8.name());
+ sinkRequest.setDataSeparator("124");
+ streamSinkService.save(sinkRequest, OPERATOR);
+ return sinkRequest;
}
/**
diff --git a/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/workflow/WorkflowServiceImplTest.java b/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/workflow/WorkflowServiceImplTest.java
index 269d73fed..ba4b4a087 100644
--- a/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/workflow/WorkflowServiceImplTest.java
+++ b/inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/workflow/WorkflowServiceImplTest.java
@@ -19,6 +19,7 @@ package org.apache.inlong.manager.service.workflow;
import com.github.pagehelper.PageInfo;
import com.google.common.collect.Lists;
+import org.apache.inlong.manager.common.enums.FieldType;
import org.apache.inlong.manager.common.enums.GroupOperateType;
import org.apache.inlong.manager.common.enums.GroupStatus;
import org.apache.inlong.manager.common.enums.MQType;
@@ -26,9 +27,9 @@ import org.apache.inlong.manager.common.enums.ProcessStatus;
import org.apache.inlong.manager.common.pojo.group.InlongGroupInfo;
import org.apache.inlong.manager.common.pojo.group.none.InlongNoneMqInfo;
import org.apache.inlong.manager.common.pojo.group.pulsar.InlongPulsarInfo;
-import org.apache.inlong.manager.common.pojo.stream.InlongStreamFieldInfo;
import org.apache.inlong.manager.common.pojo.stream.InlongStreamInfo;
import org.apache.inlong.manager.common.pojo.stream.InlongStreamRequest;
+import org.apache.inlong.manager.common.pojo.stream.StreamField;
import org.apache.inlong.manager.common.pojo.workflow.ProcessResponse;
import org.apache.inlong.manager.common.pojo.workflow.TaskExecuteLogQuery;
import org.apache.inlong.manager.common.pojo.workflow.WorkflowResult;
@@ -189,16 +190,16 @@ public class WorkflowServiceImplTest extends ServiceBaseTest {
return streamService.get(request.getInlongGroupId(), request.getInlongStreamId());
}
- public List<InlongStreamFieldInfo> createStreamFields(String groupId, String streamId) {
- final List<InlongStreamFieldInfo> streamFieldInfos = new ArrayList<>();
- InlongStreamFieldInfo fieldInfo = new InlongStreamFieldInfo();
+ public List<StreamField> createStreamFields(String groupId, String streamId) {
+ final List<StreamField> streamFields = new ArrayList<>();
+ StreamField fieldInfo = new StreamField();
fieldInfo.setInlongGroupId(groupId);
fieldInfo.setInlongStreamId(streamId);
fieldInfo.setFieldName("id");
- fieldInfo.setFieldType("int");
+ fieldInfo.setFieldType(FieldType.INT.toString());
fieldInfo.setFieldComment("idx");
- streamFieldInfos.add(fieldInfo);
- return streamFieldInfos;
+ streamFields.add(fieldInfo);
+ return streamFields;
}
/**