You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@inlong.apache.org by he...@apache.org on 2022/04/26 00:37:16 UTC

[incubator-inlong] branch master updated: [INLONG-3921][Manager] Add primary key in Kafka source (#3941)

This is an automated email from the ASF dual-hosted git repository.

healchow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-inlong.git


The following commit(s) were added to refs/heads/master by this push:
     new 5884d69f9 [INLONG-3921][Manager] Add primary key in Kafka source (#3941)
5884d69f9 is described below

commit 5884d69f9519d41da4edbbacb370bc0a665e0d18
Author: kipshi <48...@users.noreply.github.com>
AuthorDate: Tue Apr 26 08:37:10 2022 +0800

    [INLONG-3921][Manager] Add primary key in Kafka source (#3941)
---
 .../inlong/manager/client/api/sink/KafkaSink.java  |  3 ++
 .../client/api/util/InlongStreamSinkTransfer.java  |  2 +
 .../common/pojo/sink/kafka/KafkaSinkDTO.java       |  4 ++
 .../pojo/sink/kafka/KafkaSinkListResponse.java     |  3 ++
 .../common/pojo/sink/kafka/KafkaSinkRequest.java   |  3 ++
 .../common/pojo/sink/kafka/KafkaSinkResponse.java  |  3 ++
 .../manager/service/sort/util/LoadNodeUtils.java   | 60 +++++++++++++++++++++-
 7 files changed, 76 insertions(+), 2 deletions(-)

diff --git a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/sink/KafkaSink.java b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/sink/KafkaSink.java
index a84df575b..b54bb14c8 100644
--- a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/sink/KafkaSink.java
+++ b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/sink/KafkaSink.java
@@ -54,4 +54,7 @@ public class KafkaSink extends StreamSink {
 
     @ApiModelProperty("Field definitions for kafka")
     private List<SinkField> sinkFields;
+
+    @ApiModelProperty("Primary key is required when dataFormat is json, avro")
+    private String primaryKey;
 }
diff --git a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/util/InlongStreamSinkTransfer.java b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/util/InlongStreamSinkTransfer.java
index c9c12341f..fe052febc 100644
--- a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/util/InlongStreamSinkTransfer.java
+++ b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/util/InlongStreamSinkTransfer.java
@@ -174,6 +174,7 @@ public class InlongStreamSinkTransfer {
         kafkaSinkRequest.setSerializationType(kafkaSink.getDataFormat().name());
         kafkaSinkRequest.setEnableCreateResource(kafkaSink.isNeedCreated() ? 1 : 0);
         kafkaSinkRequest.setProperties(kafkaSink.getProperties());
+        kafkaSinkRequest.setPrimaryKey(kafkaSink.getPrimaryKey());
         if (CollectionUtils.isNotEmpty(kafkaSink.getSinkFields())) {
             List<SinkFieldRequest> fieldRequests = createSinkFieldRequests(kafkaSink.getSinkFields());
             kafkaSinkRequest.setFieldList(fieldRequests);
@@ -197,6 +198,7 @@ public class InlongStreamSinkTransfer {
             kafkaSink.setTopicName(sinkResponse.getTopicName());
             kafkaSink.setDataFormat(DataFormat.forName(sinkResponse.getSerializationType()));
         }
+        kafkaSink.setPrimaryKey(sinkResponse.getPrimaryKey());
         kafkaSink.setProperties(sinkResponse.getProperties());
         kafkaSink.setNeedCreated(GlobalConstants.ENABLE_CREATE_RESOURCE.equals(sinkResponse.getEnableCreateResource()));
         if (CollectionUtils.isNotEmpty(sinkResponse.getFieldList())) {
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/kafka/KafkaSinkDTO.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/kafka/KafkaSinkDTO.java
index 1f7f853d5..30aaa0e6c 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/kafka/KafkaSinkDTO.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/kafka/KafkaSinkDTO.java
@@ -57,6 +57,9 @@ public class KafkaSinkDTO {
             notes = "including earliest, latest (the default), none")
     private String autoOffsetReset;
 
+    @ApiModelProperty("Primary key is required when serializationType is json, avro")
+    private String primaryKey;
+
     @ApiModelProperty("Properties for kafka")
     private Map<String, Object> properties;
 
@@ -68,6 +71,7 @@ public class KafkaSinkDTO {
                 .bootstrapServers(request.getBootstrapServers())
                 .topicName(request.getTopicName())
                 .serializationType(request.getSerializationType())
+                .primaryKey(request.getPrimaryKey())
                 .properties(request.getProperties())
                 .build();
     }
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/kafka/KafkaSinkListResponse.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/kafka/KafkaSinkListResponse.java
index 85c4515ec..a65c76b24 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/kafka/KafkaSinkListResponse.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/kafka/KafkaSinkListResponse.java
@@ -47,4 +47,7 @@ public class KafkaSinkListResponse extends SinkListResponse {
             notes = "including earliest, latest (the default), none")
     private String autoOffsetReset;
 
+    @ApiModelProperty("Primary key is required when serializationType is json, avro")
+    private String primaryKey;
+
 }
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/kafka/KafkaSinkRequest.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/kafka/KafkaSinkRequest.java
index 49fb9f81f..31dc27717 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/kafka/KafkaSinkRequest.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/kafka/KafkaSinkRequest.java
@@ -52,4 +52,7 @@ public class KafkaSinkRequest extends SinkRequest {
             notes = "including earliest, latest (the default), none")
     private String autoOffsetReset;
 
+    @ApiModelProperty("Primary key is required when serializationType is json, avro")
+    private String primaryKey;
+
 }
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/kafka/KafkaSinkResponse.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/kafka/KafkaSinkResponse.java
index e6f62f4f6..ae33d1a36 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/kafka/KafkaSinkResponse.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/kafka/KafkaSinkResponse.java
@@ -50,6 +50,9 @@ public class KafkaSinkResponse extends SinkResponse {
             notes = "including earliest, latest (the default), none")
     private String autoOffsetReset;
 
+    @ApiModelProperty("Primary key is required when serializationType is json, avro")
+    private String primaryKey;
+
     public KafkaSinkResponse() {
         this.sinkType = SinkType.SINK_KAFKA;
     }
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/LoadNodeUtils.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/LoadNodeUtils.java
index 6eed3bd34..155c46596 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/LoadNodeUtils.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/sort/util/LoadNodeUtils.java
@@ -19,13 +19,24 @@ package org.apache.inlong.manager.service.sort.util;
 
 import com.google.common.collect.Lists;
 import org.apache.commons.collections.CollectionUtils;
+import org.apache.inlong.common.enums.DataTypeEnum;
 import org.apache.inlong.manager.common.enums.SinkType;
+import org.apache.inlong.manager.common.pojo.sink.SinkFieldResponse;
 import org.apache.inlong.manager.common.pojo.sink.SinkResponse;
 import org.apache.inlong.manager.common.pojo.sink.kafka.KafkaSinkResponse;
+import org.apache.inlong.sort.protocol.FieldInfo;
 import org.apache.inlong.sort.protocol.node.LoadNode;
+import org.apache.inlong.sort.protocol.node.format.AvroFormat;
+import org.apache.inlong.sort.protocol.node.format.CanalJsonFormat;
+import org.apache.inlong.sort.protocol.node.format.CsvFormat;
+import org.apache.inlong.sort.protocol.node.format.DebeziumJsonFormat;
+import org.apache.inlong.sort.protocol.node.format.Format;
+import org.apache.inlong.sort.protocol.node.format.JsonFormat;
 import org.apache.inlong.sort.protocol.node.load.KafkaLoadNode;
+import org.apache.inlong.sort.protocol.transformation.FieldRelationShip;
 
 import java.util.List;
+import java.util.Map;
 import java.util.stream.Collectors;
 
 public class LoadNodeUtils {
@@ -50,7 +61,52 @@ public class LoadNodeUtils {
     }
 
     public static KafkaLoadNode createLoadNode(KafkaSinkResponse kafkaSinkResponse) {
-        //todo
-        return null;
+        String id = kafkaSinkResponse.getSinkName();
+        String name = kafkaSinkResponse.getSinkName();
+        String topicName = kafkaSinkResponse.getTopicName();
+        String bootstrapServers = kafkaSinkResponse.getBootstrapServers();
+        List<SinkFieldResponse> sinkFieldResponses = kafkaSinkResponse.getFieldList();
+        List<FieldInfo> fieldInfos = sinkFieldResponses.stream()
+                .map(sinkFieldResponse -> new FieldInfo(sinkFieldResponse.getFieldName(), name,
+                        FieldInfoUtils.convertFieldFormat(sinkFieldResponse.getFieldType(),
+                                sinkFieldResponse.getFieldFormat()))).collect(Collectors.toList());
+        List<FieldRelationShip> fieldRelationShips = fieldInfos.stream()
+                .map(fieldInfo -> new FieldRelationShip(fieldInfo, fieldInfo)).collect(Collectors.toList());
+        Map<String, String> properties = kafkaSinkResponse.getProperties().entrySet().stream()
+                .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().toString()));
+        Integer sinkParallelism = Integer.parseInt(kafkaSinkResponse.getPartitionNum());
+        DataTypeEnum dataType = DataTypeEnum.forName(kafkaSinkResponse.getSerializationType());
+        Format format;
+        switch (dataType) {
+            case CSV:
+                format = new CsvFormat();
+                break;
+            case AVRO:
+                format = new AvroFormat();
+                break;
+            case JSON:
+                format = new JsonFormat();
+                break;
+            case CANAL:
+                format = new CanalJsonFormat();
+                break;
+            case DEBEZIUM_JSON:
+                format = new DebeziumJsonFormat();
+                break;
+            default:
+                throw new IllegalArgumentException(String.format("Unsupported dataType=%s for kafka source", dataType));
+        }
+        String primaryKey = kafkaSinkResponse.getPrimaryKey();
+        return new KafkaLoadNode(id,
+                name,
+                fieldInfos,
+                fieldRelationShips,
+                Lists.newArrayList(),
+                topicName,
+                bootstrapServers,
+                format,
+                sinkParallelism,
+                properties,
+                primaryKey);
     }
 }