You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@inlong.apache.org by do...@apache.org on 2022/04/21 02:24:37 UTC

[incubator-inlong] branch master updated: [INLONG-3788][Sort] Add Node interface and derived interface definitions to support transform (#3857)

This is an automated email from the ASF dual-hosted git repository.

dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-inlong.git


The following commit(s) were added to refs/heads/master by this push:
     new 02a530892 [INLONG-3788][Sort] Add Node interface and derived interface definitions to support transform (#3857)
02a530892 is described below

commit 02a5308929ac98e484f743fe5891eab83a6c71e7
Author: yunqingmoswu <44...@users.noreply.github.com>
AuthorDate: Thu Apr 21 10:24:31 2022 +0800

    [INLONG-3788][Sort] Add Node interface and derived interface definitions to support transform (#3857)
    
    Co-authored-by: yunqingmo <yu...@tencent.com>
---
 .../inlong/sort/protocol/node/ExtractNode.java     |  77 ++++++++++++
 .../apache/inlong/sort/protocol/node/LoadNode.java |  90 ++++++++++++++
 .../org/apache/inlong/sort/protocol/node/Node.java |  77 ++++++++++++
 .../protocol/node/extract/MySqlExtractNode.java    | 138 +++++++++++++++++++++
 .../sort/protocol/node/load/KafkaLoadNode.java     |  89 +++++++++++++
 .../protocol/node/transform/TransformNode.java     |  80 ++++++++++++
 .../sort/protocol/node/KafkaLoadNodeTest.java      |  49 ++++++++
 .../sort/protocol/node/MySqlExtractNodeTest.java   |  46 +++++++
 .../inlong/sort/protocol/node/NodeBaseTest.java    |  56 +++++++++
 9 files changed, 702 insertions(+)

diff --git a/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/ExtractNode.java b/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/ExtractNode.java
new file mode 100644
index 000000000..e0ef2cf4d
--- /dev/null
+++ b/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/ExtractNode.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sort.protocol.node;
+
+import com.google.common.base.Preconditions;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonInclude;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonInclude.Include;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonSubTypes;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonTypeInfo;
+import org.apache.inlong.sort.protocol.FieldInfo;
+import org.apache.inlong.sort.protocol.node.extract.MySqlExtractNode;
+import org.apache.inlong.sort.protocol.transformation.WatermarkField;
+
+import javax.annotation.Nullable;
+import java.util.List;
+import java.util.Map;
+
+@JsonTypeInfo(
+        use = JsonTypeInfo.Id.NAME,
+        include = JsonTypeInfo.As.PROPERTY,
+        property = "type")
+@JsonSubTypes({
+        @JsonSubTypes.Type(value = MySqlExtractNode.class, name = "mysqlExtract")
+})
+@Data
+@NoArgsConstructor
+public abstract class ExtractNode implements Node {
+
+    @JsonProperty("id")
+    private String id;
+    @JsonInclude(Include.NON_NULL)
+    @JsonProperty("name")
+    private String name;
+    @JsonProperty("fields")
+    private List<FieldInfo> fields;
+    @Nullable
+    @JsonProperty("watermarkField")
+    @JsonInclude(Include.NON_NULL)
+    private WatermarkField watermarkField;
+    @Nullable
+    @JsonInclude(Include.NON_NULL)
+    @JsonProperty("properties")
+    private Map<String, String> properties;
+
+    @JsonCreator
+    public ExtractNode(@JsonProperty("id") String id,
+            @JsonProperty("name") String name,
+            @JsonProperty("fields") List<FieldInfo> fields,
+            @Nullable @JsonProperty("watermark_field") WatermarkField watermarkField,
+            @JsonProperty("properties") Map<String, String> properties) {
+        this.id = Preconditions.checkNotNull(id, "id is null");
+        this.name = name;
+        this.fields = Preconditions.checkNotNull(fields, "fields is null");
+        Preconditions.checkState(!fields.isEmpty(), "fields is empty");
+        this.watermarkField = watermarkField;
+        this.properties = properties;
+    }
+}
diff --git a/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/LoadNode.java b/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/LoadNode.java
new file mode 100644
index 000000000..18b7e669d
--- /dev/null
+++ b/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/LoadNode.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sort.protocol.node;
+
+import com.google.common.base.Preconditions;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonInclude;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonInclude.Include;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonSubTypes;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonTypeInfo;
+import org.apache.inlong.sort.protocol.FieldInfo;
+import org.apache.inlong.sort.protocol.node.load.KafkaLoadNode;
+import org.apache.inlong.sort.protocol.transformation.FieldRelationShip;
+import org.apache.inlong.sort.protocol.transformation.FilterFunction;
+
+import javax.annotation.Nullable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+@JsonTypeInfo(
+        use = JsonTypeInfo.Id.NAME,
+        include = JsonTypeInfo.As.PROPERTY,
+        property = "type")
+@JsonSubTypes({
+        @JsonSubTypes.Type(value = KafkaLoadNode.class, name = "kafkaLoad")
+})
+@NoArgsConstructor
+@Data
+public abstract class LoadNode implements Node {
+
+    @JsonProperty("id")
+    private String id;
+    @JsonInclude(Include.NON_NULL)
+    @JsonProperty("name")
+    private String name;
+    @JsonProperty("fields")
+    private List<FieldInfo> fields;
+    @JsonProperty("fieldRelationShips")
+    private List<FieldRelationShip> fieldRelationShips;
+    @Nullable
+    @JsonInclude(Include.NON_NULL)
+    @JsonProperty("sinkParallelism")
+    private Integer sinkParallelism;
+    @JsonProperty("filters")
+    private List<FilterFunction> filters = new ArrayList<>();
+    @Nullable
+    @JsonInclude(Include.NON_NULL)
+    @JsonProperty("properties")
+    private Map<String, String> properties;
+
+    @JsonCreator
+    public LoadNode(@JsonProperty("id") String id,
+            @JsonProperty("name") String name,
+            @JsonProperty("fields") List<FieldInfo> fields,
+            @JsonProperty("fieldRelationShips") List<FieldRelationShip> fieldRelationShips,
+            @JsonProperty("filters") List<FilterFunction> filters,
+            @Nullable @JsonProperty("sinkParallelism") Integer sinkParallelism,
+            @JsonProperty("properties") Map<String, String> properties) {
+        this.id = Preconditions.checkNotNull(id, "id is null");
+        this.name = name;
+        this.fields = Preconditions.checkNotNull(fields, "fields is null");
+        Preconditions.checkState(!fields.isEmpty(), "fields is empty");
+        this.fieldRelationShips = Preconditions.checkNotNull(fieldRelationShips,
+                "fieldRelationShips is null");
+        Preconditions.checkState(!fieldRelationShips.isEmpty(), "fieldRelationShips is empty");
+        this.filters = filters;
+        this.sinkParallelism = sinkParallelism;
+        this.properties = properties;
+    }
+
+}
diff --git a/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/Node.java b/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/Node.java
new file mode 100644
index 000000000..80c852861
--- /dev/null
+++ b/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/Node.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sort.protocol.node;
+
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonInclude;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonInclude.Include;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonSubTypes;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonTypeInfo;
+import org.apache.inlong.sort.protocol.FieldInfo;
+import org.apache.inlong.sort.protocol.node.extract.MySqlExtractNode;
+import org.apache.inlong.sort.protocol.node.load.KafkaLoadNode;
+import org.apache.inlong.sort.protocol.node.transform.TransformNode;
+
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+@JsonTypeInfo(
+        use = JsonTypeInfo.Id.NAME,
+        include = JsonTypeInfo.As.PROPERTY,
+        property = "type")
+@JsonSubTypes({
+        @JsonSubTypes.Type(value = MySqlExtractNode.class, name = "mysqlExtract"),
+        @JsonSubTypes.Type(value = TransformNode.class, name = "baseTransform"),
+        @JsonSubTypes.Type(value = KafkaLoadNode.class, name = "kafkaLoad")
+})
+public interface Node {
+
+    String getId();
+
+    @JsonInclude(Include.NON_NULL)
+    String getName();
+
+    List<FieldInfo> getFields();
+
+    @JsonInclude(Include.NON_NULL)
+    default Map<String, String> getProperties() {
+        return new TreeMap<>();
+    }
+
+    @JsonInclude(Include.NON_NULL)
+    default Map<String, String> tableOptions() {
+        Map<String, String> options = new TreeMap<>();
+        if (getProperties() != null && !getProperties().isEmpty()) {
+            options.putAll(getProperties());
+        }
+        return options;
+    }
+
+    String genTableName();
+
+    @JsonInclude(Include.NON_NULL)
+    default String getPrimaryKey() {
+        return null;
+    }
+
+    @JsonInclude(Include.NON_NULL)
+    default List<FieldInfo> getPartitionFields() {
+        return null;
+    }
+
+}
diff --git a/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/extract/MySqlExtractNode.java b/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/extract/MySqlExtractNode.java
new file mode 100644
index 000000000..b631b3c90
--- /dev/null
+++ b/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/extract/MySqlExtractNode.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sort.protocol.node.extract;
+
+import com.google.common.base.Preconditions;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonInclude;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonInclude.Include;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonTypeName;
+import org.apache.inlong.sort.protocol.FieldInfo;
+import org.apache.inlong.sort.protocol.node.ExtractNode;
+import org.apache.inlong.sort.protocol.transformation.WatermarkField;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+
+@EqualsAndHashCode(callSuper = true)
+@JsonTypeName("mysqlExtract")
+@Data
+public class MySqlExtractNode extends ExtractNode implements Serializable {
+
+    private static final long serialVersionUID = -5521981462461235277L;
+
+    @JsonInclude(Include.NON_NULL)
+    @JsonProperty("primaryKey")
+    private String primaryKey;
+    @JsonProperty("tableNames")
+    @Nonnull
+    private List<String> tableNames;
+    @JsonProperty("hostname")
+    private String hostname;
+    @JsonProperty("username")
+    private String username;
+    @JsonProperty("password")
+    private String password;
+    @JsonProperty("database")
+    private String database;
+    @JsonInclude(Include.NON_NULL)
+    @JsonProperty("port")
+    private Integer port;
+    @JsonInclude(Include.NON_NULL)
+    @JsonProperty("serverId")
+    private Integer serverId;
+    @JsonInclude(Include.NON_NULL)
+    @JsonProperty("incrementalSnapshotEnabled")
+    private Boolean incrementalSnapshotEnabled;
+    @JsonInclude(Include.NON_NULL)
+    @JsonProperty("serverTimeZone")
+    private String serverTimeZone;
+
+    @JsonCreator
+    public MySqlExtractNode(@JsonProperty("id") String id,
+            @JsonProperty("name") String name,
+            @JsonProperty("fields") List<FieldInfo> fields,
+            @Nullable @JsonProperty("watermarkField") WatermarkField waterMarkField,
+            @JsonProperty("properties") Map<String, String> properties,
+            @JsonProperty("primaryKey") String primaryKey,
+            @JsonProperty("tableNames") @Nonnull List<String> tableNames,
+            @JsonProperty("hostname") String hostname,
+            @JsonProperty("username") String username,
+            @JsonProperty("password") String password,
+            @JsonProperty("database") String database,
+            @JsonProperty("port") Integer port,
+            @JsonProperty("serverId") Integer serverId,
+            @JsonProperty("incrementalSnapshotEnabled") Boolean incrementalSnapshotEnabled,
+            @JsonProperty("serverTimeZone") String serverTimeZone) {
+        super(id, name, fields, waterMarkField, properties);
+        this.tableNames = Preconditions.checkNotNull(tableNames, "tableNames is null");
+        Preconditions.checkState(!tableNames.isEmpty(), "tableNames is empty");
+        this.hostname = Preconditions.checkNotNull(hostname, "hostname is null");
+        this.username = Preconditions.checkNotNull(username, "username is null");
+        this.password = Preconditions.checkNotNull(password, "password is null");
+        this.database = Preconditions.checkNotNull(database, "database is null");
+        this.primaryKey = primaryKey;
+        this.port = port;
+        this.serverId = serverId;
+        this.incrementalSnapshotEnabled = incrementalSnapshotEnabled;
+        this.serverTimeZone = serverTimeZone;
+    }
+
+    @Override
+    public String genTableName() {
+        return String.format("table_%s", super.getId());
+    }
+
+    @Override
+    public String getPrimaryKey() {
+        return primaryKey;
+    }
+
+    @Override
+    public Map<String, String> tableOptions() {
+        Map<String, String> options = super.tableOptions();
+        options.put("connector", "mysql-cdc");
+        options.put("hostname", hostname);
+        options.put("username", username);
+        options.put("password", password);
+        options.put("database-name", database);
+        if (port != null) {
+            options.put("port", port.toString());
+        }
+        if (serverId != null) {
+            options.put("server-id", serverId.toString());
+        }
+        if (incrementalSnapshotEnabled != null) {
+            options.put("scan.incremental.snapshot.enabled", incrementalSnapshotEnabled.toString());
+        }
+        if (serverTimeZone != null) {
+            options.put("scan.incremental.snapshot.enabled", serverTimeZone.toString());
+        }
+        String formatTable = tableNames.size() == 1 ? tableNames.get(0) :
+                String.format("(%s)", StringUtils.join(tableNames, "|"));
+        options.put("table-name", String.format("%s", formatTable));
+        return options;
+    }
+}
diff --git a/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/load/KafkaLoadNode.java b/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/load/KafkaLoadNode.java
new file mode 100644
index 000000000..500f82c31
--- /dev/null
+++ b/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/load/KafkaLoadNode.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sort.protocol.node.load;
+
+import com.google.common.base.Preconditions;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.NoArgsConstructor;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonTypeName;
+import org.apache.inlong.sort.protocol.FieldInfo;
+import org.apache.inlong.sort.protocol.node.LoadNode;
+import org.apache.inlong.sort.protocol.transformation.FieldRelationShip;
+import org.apache.inlong.sort.protocol.transformation.FilterFunction;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+
+@EqualsAndHashCode(callSuper = true)
+@JsonTypeName("kafkaLoad")
+@Data
+@NoArgsConstructor
+public class KafkaLoadNode extends LoadNode implements Serializable {
+
+
+    private static final long serialVersionUID = -558158965060708408L;
+
+    @Nonnull
+    @JsonProperty("topic")
+    private String topic;
+    @Nonnull
+    @JsonProperty("bootstrapServers")
+    private String bootstrapServers;
+    @Nonnull
+    @JsonProperty("format")
+    private String format;
+
+    public KafkaLoadNode(@JsonProperty("id") String id,
+            @JsonProperty("name") String name,
+            @JsonProperty("fields") List<FieldInfo> fields,
+            @JsonProperty("fieldRelationShips") List<FieldRelationShip> fieldRelationShips,
+            @JsonProperty("filters") List<FilterFunction> filters,
+            @Nonnull @JsonProperty("topic") String topic,
+            @Nonnull @JsonProperty("bootstrapServers") String bootstrapServers,
+            @Nonnull @JsonProperty("format") String format,
+            @Nullable @JsonProperty("sinkParallelism") Integer sinkParallelism,
+            @JsonProperty("properties") Map<String, String> properties) {
+        super(id, name, fields, fieldRelationShips, filters, sinkParallelism, properties);
+        this.topic = Preconditions.checkNotNull(topic, "topic is null");
+        this.bootstrapServers = Preconditions.checkNotNull(bootstrapServers, "bootstrapServers is null");
+        this.format = Preconditions.checkNotNull(format, "format is null");
+    }
+
+    @Override
+    public String genTableName() {
+        return "node_" + super.getId() + "_" + topic;
+    }
+
+    @Override
+    public Map<String, String> tableOptions() {
+        Map<String, String> options = super.tableOptions();
+        options.put("connector", "kafka");
+        options.put("topic", topic);
+        options.put("properties.bootstrap.servers", bootstrapServers);
+        options.put("format", format);
+        if (getSinkParallelism() != null) {
+            options.put("sink.parallelism", getSinkParallelism().toString());
+        }
+        return options;
+    }
+}
diff --git a/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/transform/TransformNode.java b/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/transform/TransformNode.java
new file mode 100644
index 000000000..e0a8e39d2
--- /dev/null
+++ b/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/transform/TransformNode.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sort.protocol.node.transform;
+
+import com.google.common.base.Preconditions;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonSubTypes;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonTypeInfo;
+import org.apache.inlong.sort.protocol.FieldInfo;
+import org.apache.inlong.sort.protocol.node.Node;
+import org.apache.inlong.sort.protocol.transformation.FieldRelationShip;
+import org.apache.inlong.sort.protocol.transformation.FilterFunction;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+@JsonTypeInfo(
+        use = JsonTypeInfo.Id.NAME,
+        include = JsonTypeInfo.As.PROPERTY,
+        property = "type")
+@JsonSubTypes({
+        @JsonSubTypes.Type(value = TransformNode.class, name = "baseTransform")
+})
+@Data
+@NoArgsConstructor
+public class TransformNode implements Node, Serializable {
+
+    private static final long serialVersionUID = -1202158328274891592L;
+
+    @JsonProperty("id")
+    private String id;
+    @JsonProperty("name")
+    private String name;
+    @JsonProperty("fields")
+    private List<FieldInfo> fields;
+    @JsonProperty("fieldRelationShips")
+    private List<FieldRelationShip> fieldRelationShips;
+    @JsonProperty("filters")
+    private List<FilterFunction> filters = new ArrayList<>();
+
+    @JsonCreator
+    public TransformNode(@JsonProperty("id") String id,
+            @JsonProperty("name") String name,
+            @JsonProperty("fields") List<FieldInfo> fields,
+            @JsonProperty("fieldRelationShips") List<FieldRelationShip> fieldRelationShips,
+            @JsonProperty("filters") List<FilterFunction> filters) {
+        this.id = Preconditions.checkNotNull(id, "id is null");
+        this.name = name;
+        this.fields = Preconditions.checkNotNull(fields, "fields is null");
+        Preconditions.checkState(!fields.isEmpty(), "fields is empty");
+        this.fieldRelationShips = Preconditions.checkNotNull(fieldRelationShips,
+                "fieldRelationShips is null");
+        Preconditions.checkState(!fieldRelationShips.isEmpty(), "fieldRelationShips is empty");
+        this.filters = filters;
+    }
+
+    @Override
+    public String genTableName() {
+        return "tansform_" + id;
+    }
+}
diff --git a/inlong-sort/sort-common/src/test/java/org/apache/inlong/sort/protocol/node/KafkaLoadNodeTest.java b/inlong-sort/sort-common/src/test/java/org/apache/inlong/sort/protocol/node/KafkaLoadNodeTest.java
new file mode 100644
index 000000000..dcd5975c2
--- /dev/null
+++ b/inlong-sort/sort-common/src/test/java/org/apache/inlong/sort/protocol/node/KafkaLoadNodeTest.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sort.protocol.node;
+
+import org.apache.inlong.sort.formats.common.StringFormatInfo;
+import org.apache.inlong.sort.protocol.FieldInfo;
+import org.apache.inlong.sort.protocol.node.load.KafkaLoadNode;
+import org.apache.inlong.sort.protocol.transformation.FieldRelationShip;
+
+import java.util.Arrays;
+import java.util.TreeMap;
+
+public class KafkaLoadNodeTest extends NodeBaseTest {
+
+    @Override
+    public Node getNode() {
+        return new KafkaLoadNode("1", null,
+                Arrays.asList(new FieldInfo("field", new StringFormatInfo())),
+                Arrays.asList(new FieldRelationShip(new FieldInfo("field", new StringFormatInfo()),
+                        new FieldInfo("field", new StringFormatInfo()))), null,
+                "topic", "localhost:9092", "json",
+                1, new TreeMap<>());
+    }
+
+    @Override
+    public String getExpectSerializeStr() {
+        return "{\"type\":\"kafkaLoad\",\"id\":\"1\",\"fields\":[{\"type\":\"base\",\"name\":\"field\","
+                + "\"formatInfo\":{\"type\":\"string\"}}],\"fieldRelationShips\":[{\"type\":\"fieldRelationShip\","
+                + "\"inputField\":{\"type\":\"base\",\"name\":\"field\",\"formatInfo\":{\"type\":\"string\"}},"
+                + "\"outputField\":{\"type\":\"base\",\"name\":\"field\",\"formatInfo\":{\"type\":\"string\"}}}],"
+                + "\"filters\":null,\"topic\":\"topic\",\"bootstrapServers\":\"localhost:9092\","
+                + "\"format\":\"json\",\"sinkParallelism\":1,\"properties\":{}}";
+    }
+}
diff --git a/inlong-sort/sort-common/src/test/java/org/apache/inlong/sort/protocol/node/MySqlExtractNodeTest.java b/inlong-sort/sort-common/src/test/java/org/apache/inlong/sort/protocol/node/MySqlExtractNodeTest.java
new file mode 100644
index 000000000..037102a9a
--- /dev/null
+++ b/inlong-sort/sort-common/src/test/java/org/apache/inlong/sort/protocol/node/MySqlExtractNodeTest.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sort.protocol.node;
+
+import org.apache.inlong.sort.formats.common.StringFormatInfo;
+import org.apache.inlong.sort.protocol.FieldInfo;
+import org.apache.inlong.sort.protocol.node.extract.MySqlExtractNode;
+
+import java.util.Arrays;
+
+public class MySqlExtractNodeTest extends NodeBaseTest {
+
+    @Override
+    public Node getNode() {
+        return new MySqlExtractNode("1", null,
+                Arrays.asList(new FieldInfo("field", new StringFormatInfo())), null, null,
+                "primary_key_field", Arrays.asList("table1", "table2"),
+                "localhost", "username",
+                "password", "dabasename", 3306, 123,
+                true, null);
+    }
+
+    @Override
+    public String getExpectSerializeStr() {
+        return "{\"type\":\"mysqlExtract\",\"id\":\"1\","
+                + "\"fields\":[{\"type\":\"base\",\"name\":\"field\",\"formatInfo\":{\"type\":\"string\"}}],"
+                + "\"primaryKey\":\"primary_key_field\",\"tableNames\":[\"table1\",\"table2\"],"
+                + "\"hostname\":\"localhost\",\"username\":\"username\",\"password\":\"password\","
+                + "\"database\":\"dabasename\",\"port\":3306,\"serverId\":123,\"incrementalSnapshotEnabled\":true}";
+    }
+}
diff --git a/inlong-sort/sort-common/src/test/java/org/apache/inlong/sort/protocol/node/NodeBaseTest.java b/inlong-sort/sort-common/src/test/java/org/apache/inlong/sort/protocol/node/NodeBaseTest.java
new file mode 100644
index 000000000..e4bd8c780
--- /dev/null
+++ b/inlong-sort/sort-common/src/test/java/org/apache/inlong/sort/protocol/node/NodeBaseTest.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sort.protocol.node;
+
+import com.google.common.base.Preconditions;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public abstract class NodeBaseTest {
+
+    private final ObjectMapper objectMapper = new ObjectMapper();
+    private String expectSerializeStr;
+    private Node node;
+
+    public abstract Node getNode();
+
+    public abstract String getExpectSerializeStr();
+
+    @Before
+    public void init() {
+        this.node = Preconditions.checkNotNull(getNode());
+        this.expectSerializeStr = Preconditions.checkNotNull(getExpectSerializeStr());
+    }
+
+    @Test
+    public void testSerialize() throws JsonProcessingException {
+        assertEquals(expectSerializeStr, objectMapper.writeValueAsString(node));
+    }
+
+    @Test
+    public void testDeserialize() throws JsonProcessingException {
+        ObjectMapper objectMapper = new ObjectMapper();
+        Node expected = objectMapper.readValue(expectSerializeStr, Node.class);
+        assertEquals(expected, node);
+    }
+
+}