You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@inlong.apache.org by he...@apache.org on 2022/04/26 00:37:16 UTC
[incubator-inlong] branch master updated: [INLONG-3921][Manager] Add primary key in Kafka source (#3941)
This is an automated email from the ASF dual-hosted git repository.
healchow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 5884d69f9 [INLONG-3921][Manager] Add primary key in Kafka source (#3941)
5884d69f9 is described below
commit 5884d69f9519d41da4edbbacb370bc0a665e0d18
Author: kipshi <48...@users.noreply.github.com>
AuthorDate: Tue Apr 26 08:37:10 2022 +0800
[INLONG-3921][Manager] Add primary key in Kafka source (#3941)
---
.../inlong/manager/client/api/sink/KafkaSink.java | 3 ++
.../client/api/util/InlongStreamSinkTransfer.java | 2 +
.../common/pojo/sink/kafka/KafkaSinkDTO.java | 4 ++
.../pojo/sink/kafka/KafkaSinkListResponse.java | 3 ++
.../common/pojo/sink/kafka/KafkaSinkRequest.java | 3 ++
.../common/pojo/sink/kafka/KafkaSinkResponse.java | 3 ++
.../manager/service/sort/util/LoadNodeUtils.java | 60 +++++++++++++++++++++-
7 files changed, 76 insertions(+), 2 deletions(-)
diff --git a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/sink/KafkaSink.java b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/sink/KafkaSink.java
index a84df575b..b54bb14c8 100644
--- a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/sink/KafkaSink.java
+++ b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/sink/KafkaSink.java
@@ -54,4 +54,7 @@ public class KafkaSink extends StreamSink {
@ApiModelProperty("Field definitions for kafka")
private List<SinkField> sinkFields;
+
+ @ApiModelProperty("Primary key is required when dataFormat is json, avro")
+ private String primaryKey;
}
diff --git a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/util/InlongStreamSinkTransfer.java b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/util/InlongStreamSinkTransfer.java
index c9c12341f..fe052febc 100644
--- a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/util/InlongStreamSinkTransfer.java
+++ b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/util/InlongStreamSinkTransfer.java
@@ -174,6 +174,7 @@ public class InlongStreamSinkTransfer {
kafkaSinkRequest.setSerializationType(kafkaSink.getDataFormat().name());
kafkaSinkRequest.setEnableCreateResource(kafkaSink.isNeedCreated() ? 1 : 0);
kafkaSinkRequest.setProperties(kafkaSink.getProperties());
+ kafkaSinkRequest.setPrimaryKey(kafkaSink.getPrimaryKey());
if (CollectionUtils.isNotEmpty(kafkaSink.getSinkFields())) {
List<SinkFieldRequest> fieldRequests = createSinkFieldRequests(kafkaSink.getSinkFields());
kafkaSinkRequest.setFieldList(fieldRequests);
@@ -197,6 +198,7 @@ public class InlongStreamSinkTransfer {
kafkaSink.setTopicName(sinkResponse.getTopicName());
kafkaSink.setDataFormat(DataFormat.forName(sinkResponse.getSerializationType()));
}
+ kafkaSink.setPrimaryKey(sinkResponse.getPrimaryKey());
kafkaSink.setProperties(sinkResponse.getProperties());
kafkaSink.setNeedCreated(GlobalConstants.ENABLE_CREATE_RESOURCE.equals(sinkResponse.getEnableCreateResource()));
if (CollectionUtils.isNotEmpty(sinkResponse.getFieldList())) {
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/kafka/KafkaSinkDTO.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/kafka/KafkaSinkDTO.java
index 1f7f853d5..30aaa0e6c 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/kafka/KafkaSinkDTO.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/kafka/KafkaSinkDTO.java
@@ -57,6 +57,9 @@ public class KafkaSinkDTO {
notes = "including earliest, latest (the default), none")
private String autoOffsetReset;
+ @ApiModelProperty("Primary key is required when serializationType is json, avro")
+ private String primaryKey;
+
@ApiModelProperty("Properties for kafka")
private Map<String, Object> properties;
@@ -68,6 +71,7 @@ public class KafkaSinkDTO {
.bootstrapServers(request.getBootstrapServers())
.topicName(request.getTopicName())
.serializationType(request.getSerializationType())
+ .primaryKey(request.getPrimaryKey())
.properties(request.getProperties())
.build();
}
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/kafka/KafkaSinkListResponse.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/kafka/KafkaSinkListResponse.java
index 85c4515ec..a65c76b24 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/kafka/KafkaSinkListResponse.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/kafka/KafkaSinkListResponse.java
@@ -47,4 +47,7 @@ public class KafkaSinkListResponse extends SinkListResponse {
notes = "including earliest, latest (the default), none")
private String autoOffsetReset;
+ @ApiModelProperty("Primary key is required when serializationType is json, avro")
+ private String primaryKey;
+
}
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/kafka/KafkaSinkRequest.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/kafka/KafkaSinkRequest.java
index 49fb9f81f..31dc27717 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/kafka/KafkaSinkRequest.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/kafka/KafkaSinkRequest.java
@@ -52,4 +52,7 @@ public class KafkaSinkRequest extends SinkRequest {
notes = "including earliest, latest (the default), none")
private String autoOffsetReset;
+ @ApiModelProperty("Primary key is required when serializationType is json, avro")
+ private String primaryKey;
+
}
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/kafka/KafkaSinkResponse.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/kafka/KafkaSinkResponse.java
index e6f62f4f6..ae33d1a36 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/kafka/KafkaSinkResponse.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/kafka/KafkaSinkResponse.java
@@ -50,6 +50,9 @@ public class KafkaSinkResponse extends SinkResponse {
notes = "including earliest, latest (the default), none")
private String autoOffsetReset;
+ @ApiModelProperty("Primary key is required when serializationType is json, avro")
+ private String primaryKey;
+
public KafkaSinkResponse() {
this.sinkType = SinkType.SINK_KAFKA;
}
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/LoadNodeUtils.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/LoadNodeUtils.java
index 6eed3bd34..155c46596 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/LoadNodeUtils.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/LoadNodeUtils.java
@@ -19,13 +19,24 @@ package org.apache.inlong.manager.service.sort.util;
import com.google.common.collect.Lists;
import org.apache.commons.collections.CollectionUtils;
+import org.apache.inlong.common.enums.DataTypeEnum;
import org.apache.inlong.manager.common.enums.SinkType;
+import org.apache.inlong.manager.common.pojo.sink.SinkFieldResponse;
import org.apache.inlong.manager.common.pojo.sink.SinkResponse;
import org.apache.inlong.manager.common.pojo.sink.kafka.KafkaSinkResponse;
+import org.apache.inlong.sort.protocol.FieldInfo;
import org.apache.inlong.sort.protocol.node.LoadNode;
+import org.apache.inlong.sort.protocol.node.format.AvroFormat;
+import org.apache.inlong.sort.protocol.node.format.CanalJsonFormat;
+import org.apache.inlong.sort.protocol.node.format.CsvFormat;
+import org.apache.inlong.sort.protocol.node.format.DebeziumJsonFormat;
+import org.apache.inlong.sort.protocol.node.format.Format;
+import org.apache.inlong.sort.protocol.node.format.JsonFormat;
import org.apache.inlong.sort.protocol.node.load.KafkaLoadNode;
+import org.apache.inlong.sort.protocol.transformation.FieldRelationShip;
import java.util.List;
+import java.util.Map;
import java.util.stream.Collectors;
public class LoadNodeUtils {
@@ -50,7 +61,52 @@ public class LoadNodeUtils {
}
public static KafkaLoadNode createLoadNode(KafkaSinkResponse kafkaSinkResponse) {
- //todo
- return null;
+ String id = kafkaSinkResponse.getSinkName();
+ String name = kafkaSinkResponse.getSinkName();
+ String topicName = kafkaSinkResponse.getTopicName();
+ String bootstrapServers = kafkaSinkResponse.getBootstrapServers();
+ List<SinkFieldResponse> sinkFieldResponses = kafkaSinkResponse.getFieldList();
+ List<FieldInfo> fieldInfos = sinkFieldResponses.stream()
+ .map(sinkFieldResponse -> new FieldInfo(sinkFieldResponse.getFieldName(), name,
+ FieldInfoUtils.convertFieldFormat(sinkFieldResponse.getFieldType(),
+ sinkFieldResponse.getFieldFormat()))).collect(Collectors.toList());
+ List<FieldRelationShip> fieldRelationShips = fieldInfos.stream()
+ .map(fieldInfo -> new FieldRelationShip(fieldInfo, fieldInfo)).collect(Collectors.toList());
+ Map<String, String> properties = kafkaSinkResponse.getProperties().entrySet().stream()
+ .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().toString()));
+ Integer sinkParallelism = Integer.parseInt(kafkaSinkResponse.getPartitionNum());
+ DataTypeEnum dataType = DataTypeEnum.forName(kafkaSinkResponse.getSerializationType());
+ Format format;
+ switch (dataType) {
+ case CSV:
+ format = new CsvFormat();
+ break;
+ case AVRO:
+ format = new AvroFormat();
+ break;
+ case JSON:
+ format = new JsonFormat();
+ break;
+ case CANAL:
+ format = new CanalJsonFormat();
+ break;
+ case DEBEZIUM_JSON:
+ format = new DebeziumJsonFormat();
+ break;
+ default:
+ throw new IllegalArgumentException(String.format("Unsupported dataType=%s for kafka source", dataType));
+ }
+ String primaryKey = kafkaSinkResponse.getPrimaryKey();
+ return new KafkaLoadNode(id,
+ name,
+ fieldInfos,
+ fieldRelationShips,
+ Lists.newArrayList(),
+ topicName,
+ bootstrapServers,
+ format,
+ sinkParallelism,
+ properties,
+ primaryKey);
}
}