You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@inlong.apache.org by do...@apache.org on 2022/04/21 02:24:37 UTC
[incubator-inlong] branch master updated: [INLONG-3788][Sort] Add Node interface and derived interface definitions to support transform (#3857)
This is an automated email from the ASF dual-hosted git repository.
dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 02a530892 [INLONG-3788][Sort] Add Node interface and derived interface definitions to support transform (#3857)
02a530892 is described below
commit 02a5308929ac98e484f743fe5891eab83a6c71e7
Author: yunqingmoswu <44...@users.noreply.github.com>
AuthorDate: Thu Apr 21 10:24:31 2022 +0800
[INLONG-3788][Sort] Add Node interface and derived interface definitions to support transform (#3857)
Co-authored-by: yunqingmo <yu...@tencent.com>
---
.../inlong/sort/protocol/node/ExtractNode.java | 77 ++++++++++++
.../apache/inlong/sort/protocol/node/LoadNode.java | 90 ++++++++++++++
.../org/apache/inlong/sort/protocol/node/Node.java | 77 ++++++++++++
.../protocol/node/extract/MySqlExtractNode.java | 138 +++++++++++++++++++++
.../sort/protocol/node/load/KafkaLoadNode.java | 89 +++++++++++++
.../protocol/node/transform/TransformNode.java | 80 ++++++++++++
.../sort/protocol/node/KafkaLoadNodeTest.java | 49 ++++++++
.../sort/protocol/node/MySqlExtractNodeTest.java | 46 +++++++
.../inlong/sort/protocol/node/NodeBaseTest.java | 56 +++++++++
9 files changed, 702 insertions(+)
diff --git a/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/ExtractNode.java b/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/ExtractNode.java
new file mode 100644
index 000000000..e0ef2cf4d
--- /dev/null
+++ b/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/ExtractNode.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sort.protocol.node;
+
+import com.google.common.base.Preconditions;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonInclude;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonInclude.Include;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonSubTypes;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonTypeInfo;
+import org.apache.inlong.sort.protocol.FieldInfo;
+import org.apache.inlong.sort.protocol.node.extract.MySqlExtractNode;
+import org.apache.inlong.sort.protocol.transformation.WatermarkField;
+
+import javax.annotation.Nullable;
+import java.util.List;
+import java.util.Map;
+
+@JsonTypeInfo(
+ use = JsonTypeInfo.Id.NAME,
+ include = JsonTypeInfo.As.PROPERTY,
+ property = "type")
+@JsonSubTypes({
+ @JsonSubTypes.Type(value = MySqlExtractNode.class, name = "mysqlExtract")
+})
+@Data
+@NoArgsConstructor
+public abstract class ExtractNode implements Node {
+
+ @JsonProperty("id")
+ private String id;
+ @JsonInclude(Include.NON_NULL)
+ @JsonProperty("name")
+ private String name;
+ @JsonProperty("fields")
+ private List<FieldInfo> fields;
+ @Nullable
+ @JsonProperty("watermarkField")
+ @JsonInclude(Include.NON_NULL)
+ private WatermarkField watermarkField;
+ @Nullable
+ @JsonInclude(Include.NON_NULL)
+ @JsonProperty("properties")
+ private Map<String, String> properties;
+
+ @JsonCreator
+ public ExtractNode(@JsonProperty("id") String id,
+ @JsonProperty("name") String name,
+ @JsonProperty("fields") List<FieldInfo> fields,
+ @Nullable @JsonProperty("watermark_field") WatermarkField watermarkField,
+ @JsonProperty("properties") Map<String, String> properties) {
+ this.id = Preconditions.checkNotNull(id, "id is null");
+ this.name = name;
+ this.fields = Preconditions.checkNotNull(fields, "fields is null");
+ Preconditions.checkState(!fields.isEmpty(), "fields is empty");
+ this.watermarkField = watermarkField;
+ this.properties = properties;
+ }
+}
diff --git a/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/LoadNode.java b/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/LoadNode.java
new file mode 100644
index 000000000..18b7e669d
--- /dev/null
+++ b/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/LoadNode.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sort.protocol.node;
+
+import com.google.common.base.Preconditions;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonInclude;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonInclude.Include;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonSubTypes;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonTypeInfo;
+import org.apache.inlong.sort.protocol.FieldInfo;
+import org.apache.inlong.sort.protocol.node.load.KafkaLoadNode;
+import org.apache.inlong.sort.protocol.transformation.FieldRelationShip;
+import org.apache.inlong.sort.protocol.transformation.FilterFunction;
+
+import javax.annotation.Nullable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+@JsonTypeInfo(
+ use = JsonTypeInfo.Id.NAME,
+ include = JsonTypeInfo.As.PROPERTY,
+ property = "type")
+@JsonSubTypes({
+ @JsonSubTypes.Type(value = KafkaLoadNode.class, name = "kafkaLoad")
+})
+@NoArgsConstructor
+@Data
+public abstract class LoadNode implements Node {
+
+ @JsonProperty("id")
+ private String id;
+ @JsonInclude(Include.NON_NULL)
+ @JsonProperty("name")
+ private String name;
+ @JsonProperty("fields")
+ private List<FieldInfo> fields;
+ @JsonProperty("fieldRelationShips")
+ private List<FieldRelationShip> fieldRelationShips;
+ @Nullable
+ @JsonInclude(Include.NON_NULL)
+ @JsonProperty("sinkParallelism")
+ private Integer sinkParallelism;
+ @JsonProperty("filters")
+ private List<FilterFunction> filters = new ArrayList<>();
+ @Nullable
+ @JsonInclude(Include.NON_NULL)
+ @JsonProperty("properties")
+ private Map<String, String> properties;
+
+ @JsonCreator
+ public LoadNode(@JsonProperty("id") String id,
+ @JsonProperty("name") String name,
+ @JsonProperty("fields") List<FieldInfo> fields,
+ @JsonProperty("fieldRelationShips") List<FieldRelationShip> fieldRelationShips,
+ @JsonProperty("filters") List<FilterFunction> filters,
+ @Nullable @JsonProperty("sinkParallelism") Integer sinkParallelism,
+ @JsonProperty("properties") Map<String, String> properties) {
+ this.id = Preconditions.checkNotNull(id, "id is null");
+ this.name = name;
+ this.fields = Preconditions.checkNotNull(fields, "fields is null");
+ Preconditions.checkState(!fields.isEmpty(), "fields is empty");
+ this.fieldRelationShips = Preconditions.checkNotNull(fieldRelationShips,
+ "fieldRelationShips is null");
+ Preconditions.checkState(!fieldRelationShips.isEmpty(), "fieldRelationShips is empty");
+ this.filters = filters;
+ this.sinkParallelism = sinkParallelism;
+ this.properties = properties;
+ }
+
+}
diff --git a/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/Node.java b/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/Node.java
new file mode 100644
index 000000000..80c852861
--- /dev/null
+++ b/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/Node.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sort.protocol.node;
+
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonInclude;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonInclude.Include;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonSubTypes;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonTypeInfo;
+import org.apache.inlong.sort.protocol.FieldInfo;
+import org.apache.inlong.sort.protocol.node.extract.MySqlExtractNode;
+import org.apache.inlong.sort.protocol.node.load.KafkaLoadNode;
+import org.apache.inlong.sort.protocol.node.transform.TransformNode;
+
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+@JsonTypeInfo(
+ use = JsonTypeInfo.Id.NAME,
+ include = JsonTypeInfo.As.PROPERTY,
+ property = "type")
+@JsonSubTypes({
+ @JsonSubTypes.Type(value = MySqlExtractNode.class, name = "mysqlExtract"),
+ @JsonSubTypes.Type(value = TransformNode.class, name = "baseTransform"),
+ @JsonSubTypes.Type(value = KafkaLoadNode.class, name = "kafkaLoad")
+})
+public interface Node {
+
+ String getId();
+
+ @JsonInclude(Include.NON_NULL)
+ String getName();
+
+ List<FieldInfo> getFields();
+
+ @JsonInclude(Include.NON_NULL)
+ default Map<String, String> getProperties() {
+ return new TreeMap<>();
+ }
+
+ @JsonInclude(Include.NON_NULL)
+ default Map<String, String> tableOptions() {
+ Map<String, String> options = new TreeMap<>();
+ if (getProperties() != null && !getProperties().isEmpty()) {
+ options.putAll(getProperties());
+ }
+ return options;
+ }
+
+ String genTableName();
+
+ @JsonInclude(Include.NON_NULL)
+ default String getPrimaryKey() {
+ return null;
+ }
+
+ @JsonInclude(Include.NON_NULL)
+ default List<FieldInfo> getPartitionFields() {
+ return null;
+ }
+
+}
diff --git a/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/extract/MySqlExtractNode.java b/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/extract/MySqlExtractNode.java
new file mode 100644
index 000000000..b631b3c90
--- /dev/null
+++ b/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/extract/MySqlExtractNode.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sort.protocol.node.extract;
+
+import com.google.common.base.Preconditions;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonInclude;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonInclude.Include;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonTypeName;
+import org.apache.inlong.sort.protocol.FieldInfo;
+import org.apache.inlong.sort.protocol.node.ExtractNode;
+import org.apache.inlong.sort.protocol.transformation.WatermarkField;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+
+@EqualsAndHashCode(callSuper = true)
+@JsonTypeName("mysqlExtract")
+@Data
+public class MySqlExtractNode extends ExtractNode implements Serializable {
+
+ private static final long serialVersionUID = -5521981462461235277L;
+
+ @JsonInclude(Include.NON_NULL)
+ @JsonProperty("primaryKey")
+ private String primaryKey;
+ @JsonProperty("tableNames")
+ @Nonnull
+ private List<String> tableNames;
+ @JsonProperty("hostname")
+ private String hostname;
+ @JsonProperty("username")
+ private String username;
+ @JsonProperty("password")
+ private String password;
+ @JsonProperty("database")
+ private String database;
+ @JsonInclude(Include.NON_NULL)
+ @JsonProperty("port")
+ private Integer port;
+ @JsonInclude(Include.NON_NULL)
+ @JsonProperty("serverId")
+ private Integer serverId;
+ @JsonInclude(Include.NON_NULL)
+ @JsonProperty("incrementalSnapshotEnabled")
+ private Boolean incrementalSnapshotEnabled;
+ @JsonInclude(Include.NON_NULL)
+ @JsonProperty("serverTimeZone")
+ private String serverTimeZone;
+
+ @JsonCreator
+ public MySqlExtractNode(@JsonProperty("id") String id,
+ @JsonProperty("name") String name,
+ @JsonProperty("fields") List<FieldInfo> fields,
+ @Nullable @JsonProperty("watermarkField") WatermarkField waterMarkField,
+ @JsonProperty("properties") Map<String, String> properties,
+ @JsonProperty("primaryKey") String primaryKey,
+ @JsonProperty("tableNames") @Nonnull List<String> tableNames,
+ @JsonProperty("hostname") String hostname,
+ @JsonProperty("username") String username,
+ @JsonProperty("password") String password,
+ @JsonProperty("database") String database,
+ @JsonProperty("port") Integer port,
+ @JsonProperty("serverId") Integer serverId,
+ @JsonProperty("incrementalSnapshotEnabled") Boolean incrementalSnapshotEnabled,
+ @JsonProperty("serverTimeZone") String serverTimeZone) {
+ super(id, name, fields, waterMarkField, properties);
+ this.tableNames = Preconditions.checkNotNull(tableNames, "tableNames is null");
+ Preconditions.checkState(!tableNames.isEmpty(), "tableNames is empty");
+ this.hostname = Preconditions.checkNotNull(hostname, "hostname is null");
+ this.username = Preconditions.checkNotNull(username, "username is null");
+ this.password = Preconditions.checkNotNull(password, "password is null");
+ this.database = Preconditions.checkNotNull(database, "database is null");
+ this.primaryKey = primaryKey;
+ this.port = port;
+ this.serverId = serverId;
+ this.incrementalSnapshotEnabled = incrementalSnapshotEnabled;
+ this.serverTimeZone = serverTimeZone;
+ }
+
+ @Override
+ public String genTableName() {
+ return String.format("table_%s", super.getId());
+ }
+
+ @Override
+ public String getPrimaryKey() {
+ return primaryKey;
+ }
+
+ @Override
+ public Map<String, String> tableOptions() {
+ Map<String, String> options = super.tableOptions();
+ options.put("connector", "mysql-cdc");
+ options.put("hostname", hostname);
+ options.put("username", username);
+ options.put("password", password);
+ options.put("database-name", database);
+ if (port != null) {
+ options.put("port", port.toString());
+ }
+ if (serverId != null) {
+ options.put("server-id", serverId.toString());
+ }
+ if (incrementalSnapshotEnabled != null) {
+ options.put("scan.incremental.snapshot.enabled", incrementalSnapshotEnabled.toString());
+ }
+ if (serverTimeZone != null) {
+ options.put("scan.incremental.snapshot.enabled", serverTimeZone.toString());
+ }
+ String formatTable = tableNames.size() == 1 ? tableNames.get(0) :
+ String.format("(%s)", StringUtils.join(tableNames, "|"));
+ options.put("table-name", String.format("%s", formatTable));
+ return options;
+ }
+}
diff --git a/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/load/KafkaLoadNode.java b/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/load/KafkaLoadNode.java
new file mode 100644
index 000000000..500f82c31
--- /dev/null
+++ b/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/load/KafkaLoadNode.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sort.protocol.node.load;
+
+import com.google.common.base.Preconditions;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.NoArgsConstructor;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonTypeName;
+import org.apache.inlong.sort.protocol.FieldInfo;
+import org.apache.inlong.sort.protocol.node.LoadNode;
+import org.apache.inlong.sort.protocol.transformation.FieldRelationShip;
+import org.apache.inlong.sort.protocol.transformation.FilterFunction;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+
+@EqualsAndHashCode(callSuper = true)
+@JsonTypeName("kafkaLoad")
+@Data
+@NoArgsConstructor
+public class KafkaLoadNode extends LoadNode implements Serializable {
+
+
+ private static final long serialVersionUID = -558158965060708408L;
+
+ @Nonnull
+ @JsonProperty("topic")
+ private String topic;
+ @Nonnull
+ @JsonProperty("bootstrapServers")
+ private String bootstrapServers;
+ @Nonnull
+ @JsonProperty("format")
+ private String format;
+
+ public KafkaLoadNode(@JsonProperty("id") String id,
+ @JsonProperty("name") String name,
+ @JsonProperty("fields") List<FieldInfo> fields,
+ @JsonProperty("fieldRelationShips") List<FieldRelationShip> fieldRelationShips,
+ @JsonProperty("filters") List<FilterFunction> filters,
+ @Nonnull @JsonProperty("topic") String topic,
+ @Nonnull @JsonProperty("bootstrapServers") String bootstrapServers,
+ @Nonnull @JsonProperty("format") String format,
+ @Nullable @JsonProperty("sinkParallelism") Integer sinkParallelism,
+ @JsonProperty("properties") Map<String, String> properties) {
+ super(id, name, fields, fieldRelationShips, filters, sinkParallelism, properties);
+ this.topic = Preconditions.checkNotNull(topic, "topic is null");
+ this.bootstrapServers = Preconditions.checkNotNull(bootstrapServers, "bootstrapServers is null");
+ this.format = Preconditions.checkNotNull(format, "format is null");
+ }
+
+ @Override
+ public String genTableName() {
+ return "node_" + super.getId() + "_" + topic;
+ }
+
+ @Override
+ public Map<String, String> tableOptions() {
+ Map<String, String> options = super.tableOptions();
+ options.put("connector", "kafka");
+ options.put("topic", topic);
+ options.put("properties.bootstrap.servers", bootstrapServers);
+ options.put("format", format);
+ if (getSinkParallelism() != null) {
+ options.put("sink.parallelism", getSinkParallelism().toString());
+ }
+ return options;
+ }
+}
diff --git a/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/transform/TransformNode.java b/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/transform/TransformNode.java
new file mode 100644
index 000000000..e0a8e39d2
--- /dev/null
+++ b/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/transform/TransformNode.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sort.protocol.node.transform;
+
+import com.google.common.base.Preconditions;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonSubTypes;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonTypeInfo;
+import org.apache.inlong.sort.protocol.FieldInfo;
+import org.apache.inlong.sort.protocol.node.Node;
+import org.apache.inlong.sort.protocol.transformation.FieldRelationShip;
+import org.apache.inlong.sort.protocol.transformation.FilterFunction;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+@JsonTypeInfo(
+ use = JsonTypeInfo.Id.NAME,
+ include = JsonTypeInfo.As.PROPERTY,
+ property = "type")
+@JsonSubTypes({
+ @JsonSubTypes.Type(value = TransformNode.class, name = "baseTransform")
+})
+@Data
+@NoArgsConstructor
+public class TransformNode implements Node, Serializable {
+
+ private static final long serialVersionUID = -1202158328274891592L;
+
+ @JsonProperty("id")
+ private String id;
+ @JsonProperty("name")
+ private String name;
+ @JsonProperty("fields")
+ private List<FieldInfo> fields;
+ @JsonProperty("fieldRelationShips")
+ private List<FieldRelationShip> fieldRelationShips;
+ @JsonProperty("filters")
+ private List<FilterFunction> filters = new ArrayList<>();
+
+ @JsonCreator
+ public TransformNode(@JsonProperty("id") String id,
+ @JsonProperty("name") String name,
+ @JsonProperty("fields") List<FieldInfo> fields,
+ @JsonProperty("fieldRelationShips") List<FieldRelationShip> fieldRelationShips,
+ @JsonProperty("filters") List<FilterFunction> filters) {
+ this.id = Preconditions.checkNotNull(id, "id is null");
+ this.name = name;
+ this.fields = Preconditions.checkNotNull(fields, "fields is null");
+ Preconditions.checkState(!fields.isEmpty(), "fields is empty");
+ this.fieldRelationShips = Preconditions.checkNotNull(fieldRelationShips,
+ "fieldRelationShips is null");
+ Preconditions.checkState(!fieldRelationShips.isEmpty(), "fieldRelationShips is empty");
+ this.filters = filters;
+ }
+
+ @Override
+ public String genTableName() {
+ return "tansform_" + id;
+ }
+}
diff --git a/inlong-sort/sort-common/src/test/java/org/apache/inlong/sort/protocol/node/KafkaLoadNodeTest.java b/inlong-sort/sort-common/src/test/java/org/apache/inlong/sort/protocol/node/KafkaLoadNodeTest.java
new file mode 100644
index 000000000..dcd5975c2
--- /dev/null
+++ b/inlong-sort/sort-common/src/test/java/org/apache/inlong/sort/protocol/node/KafkaLoadNodeTest.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sort.protocol.node;
+
+import org.apache.inlong.sort.formats.common.StringFormatInfo;
+import org.apache.inlong.sort.protocol.FieldInfo;
+import org.apache.inlong.sort.protocol.node.load.KafkaLoadNode;
+import org.apache.inlong.sort.protocol.transformation.FieldRelationShip;
+
+import java.util.Arrays;
+import java.util.TreeMap;
+
+public class KafkaLoadNodeTest extends NodeBaseTest {
+
+ @Override
+ public Node getNode() {
+ return new KafkaLoadNode("1", null,
+ Arrays.asList(new FieldInfo("field", new StringFormatInfo())),
+ Arrays.asList(new FieldRelationShip(new FieldInfo("field", new StringFormatInfo()),
+ new FieldInfo("field", new StringFormatInfo()))), null,
+ "topic", "localhost:9092", "json",
+ 1, new TreeMap<>());
+ }
+
+ @Override
+ public String getExpectSerializeStr() {
+ return "{\"type\":\"kafkaLoad\",\"id\":\"1\",\"fields\":[{\"type\":\"base\",\"name\":\"field\","
+ + "\"formatInfo\":{\"type\":\"string\"}}],\"fieldRelationShips\":[{\"type\":\"fieldRelationShip\","
+ + "\"inputField\":{\"type\":\"base\",\"name\":\"field\",\"formatInfo\":{\"type\":\"string\"}},"
+ + "\"outputField\":{\"type\":\"base\",\"name\":\"field\",\"formatInfo\":{\"type\":\"string\"}}}],"
+ + "\"filters\":null,\"topic\":\"topic\",\"bootstrapServers\":\"localhost:9092\","
+ + "\"format\":\"json\",\"sinkParallelism\":1,\"properties\":{}}";
+ }
+}
diff --git a/inlong-sort/sort-common/src/test/java/org/apache/inlong/sort/protocol/node/MySqlExtractNodeTest.java b/inlong-sort/sort-common/src/test/java/org/apache/inlong/sort/protocol/node/MySqlExtractNodeTest.java
new file mode 100644
index 000000000..037102a9a
--- /dev/null
+++ b/inlong-sort/sort-common/src/test/java/org/apache/inlong/sort/protocol/node/MySqlExtractNodeTest.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sort.protocol.node;
+
+import org.apache.inlong.sort.formats.common.StringFormatInfo;
+import org.apache.inlong.sort.protocol.FieldInfo;
+import org.apache.inlong.sort.protocol.node.extract.MySqlExtractNode;
+
+import java.util.Arrays;
+
+public class MySqlExtractNodeTest extends NodeBaseTest {
+
+ @Override
+ public Node getNode() {
+ return new MySqlExtractNode("1", null,
+ Arrays.asList(new FieldInfo("field", new StringFormatInfo())), null, null,
+ "primary_key_field", Arrays.asList("table1", "table2"),
+ "localhost", "username",
+ "password", "dabasename", 3306, 123,
+ true, null);
+ }
+
+ @Override
+ public String getExpectSerializeStr() {
+ return "{\"type\":\"mysqlExtract\",\"id\":\"1\","
+ + "\"fields\":[{\"type\":\"base\",\"name\":\"field\",\"formatInfo\":{\"type\":\"string\"}}],"
+ + "\"primaryKey\":\"primary_key_field\",\"tableNames\":[\"table1\",\"table2\"],"
+ + "\"hostname\":\"localhost\",\"username\":\"username\",\"password\":\"password\","
+ + "\"database\":\"dabasename\",\"port\":3306,\"serverId\":123,\"incrementalSnapshotEnabled\":true}";
+ }
+}
diff --git a/inlong-sort/sort-common/src/test/java/org/apache/inlong/sort/protocol/node/NodeBaseTest.java b/inlong-sort/sort-common/src/test/java/org/apache/inlong/sort/protocol/node/NodeBaseTest.java
new file mode 100644
index 000000000..e4bd8c780
--- /dev/null
+++ b/inlong-sort/sort-common/src/test/java/org/apache/inlong/sort/protocol/node/NodeBaseTest.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sort.protocol.node;
+
+import com.google.common.base.Preconditions;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public abstract class NodeBaseTest {
+
+ private final ObjectMapper objectMapper = new ObjectMapper();
+ private String expectSerializeStr;
+ private Node node;
+
+ public abstract Node getNode();
+
+ public abstract String getExpectSerializeStr();
+
+ @Before
+ public void init() {
+ this.node = Preconditions.checkNotNull(getNode());
+ this.expectSerializeStr = Preconditions.checkNotNull(getExpectSerializeStr());
+ }
+
+ @Test
+ public void testSerialize() throws JsonProcessingException {
+ assertEquals(expectSerializeStr, objectMapper.writeValueAsString(node));
+ }
+
+ @Test
+ public void testDeserialize() throws JsonProcessingException {
+ ObjectMapper objectMapper = new ObjectMapper();
+ Node expected = objectMapper.readValue(expectSerializeStr, Node.class);
+ assertEquals(expected, node);
+ }
+
+}