You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2022/02/11 15:35:11 UTC

[incubator-doris-flink-connector] 01/32: [Feature] Flink Doris Connector (#5372) (#5375)

This is an automated email from the ASF dual-hosted git repository.

morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris-flink-connector.git

commit d499ac5961214975afecb3d286831c7638a0b167
Author: wudi <67...@qq.com>
AuthorDate: Fri Apr 23 09:43:48 2021 +0800

    [Feature] Flink Doris Connector (#5372) (#5375)
---
 build.sh                                           |  60 +++
 pom.xml                                            | 307 +++++++++++
 .../apache/doris/flink/backend/BackendClient.java  | 217 ++++++++
 .../doris/flink/cfg/ConfigurationOptions.java      |  65 +++
 .../doris/flink/cfg/DorisConnectionOptions.java    |  80 +++
 .../doris/flink/cfg/DorisExecutionOptions.java     |  71 +++
 .../org/apache/doris/flink/cfg/DorisOptions.java   | 103 ++++
 .../apache/doris/flink/cfg/DorisReadOptions.java   | 185 +++++++
 .../apache/doris/flink/cfg/DorisStreamOptions.java |  74 +++
 .../flink/datastream/DorisSourceFunction.java      |  82 +++
 .../DorisDeserializationSchema.java                |  25 +
 .../SimpleListDeserializationSchema.java           |  30 ++
 .../flink/exception/ConnectedFailedException.java  |  28 +
 .../doris/flink/exception/DorisException.java      |  38 ++
 .../flink/exception/DorisInternalException.java    |  29 ++
 .../flink/exception/IllegalArgumentException.java  |  28 +
 .../exception/ShouldNeverHappenException.java      |  20 +
 .../doris/flink/exception/StreamLoadException.java |  38 ++
 .../doris/flink/rest/PartitionDefinition.java      | 150 ++++++
 .../org/apache/doris/flink/rest/RestService.java   | 573 +++++++++++++++++++++
 .../org/apache/doris/flink/rest/SchemaUtils.java   |  38 ++
 .../apache/doris/flink/rest/models/Backend.java    |  41 ++
 .../apache/doris/flink/rest/models/BackendRow.java |  67 +++
 .../org/apache/doris/flink/rest/models/Field.java  | 110 ++++
 .../apache/doris/flink/rest/models/QueryPlan.java  |  70 +++
 .../doris/flink/rest/models/RespContent.java       |  96 ++++
 .../org/apache/doris/flink/rest/models/Schema.java |  96 ++++
 .../org/apache/doris/flink/rest/models/Tablet.java |  80 +++
 .../apache/doris/flink/serialization/Routing.java  |  70 +++
 .../apache/doris/flink/serialization/RowBatch.java | 308 +++++++++++
 .../flink/table/DorisDynamicOutputFormat.java      | 200 +++++++
 .../flink/table/DorisDynamicTableFactory.java      | 250 +++++++++
 .../doris/flink/table/DorisDynamicTableSink.java   |  73 +++
 .../doris/flink/table/DorisDynamicTableSource.java | 101 ++++
 .../doris/flink/table/DorisRowDataInputFormat.java | 227 ++++++++
 .../apache/doris/flink/table/DorisStreamLoad.java  | 179 +++++++
 .../doris/flink/table/DorisTableInputSplit.java    |  44 ++
 .../org/apache/doris/flink/util/ErrorMessages.java |  27 +
 .../java/org/apache/doris/flink/util/IOUtils.java  |  49 ++
 .../org.apache.flink.table.factories.Factory       |  18 +
 src/main/resources/log4j.properties                |  23 +
 .../doris/flink/datastream/ScalaValueReader.scala  | 223 ++++++++
 src/main/thrift/doris/DorisExternalService.thrift  | 122 +++++
 src/main/thrift/doris/Status.thrift                |  66 +++
 src/main/thrift/doris/Types.thrift                 | 370 +++++++++++++
 .../org/apache/doris/flink/DorisSinkExample.java   |  56 ++
 .../apache/doris/flink/DorisSourceDataStream.java  |  46 ++
 .../org/apache/doris/flink/DorisSourceExample.java |  64 +++
 .../apache/doris/flink/DorisSourceSinkExample.java |  64 +++
 .../doris/flink/serialization/TestRowBatch.java    | 439 ++++++++++++++++
 50 files changed, 5820 insertions(+)

diff --git a/build.sh b/build.sh
new file mode 100644
index 0000000..d3e7ca3
--- /dev/null
+++ b/build.sh
@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+##############################################################
+# This script is used to compile Flink-Doris-Connector
+# Usage:
+#    sh build.sh
+#
+##############################################################
+
+set -eo pipefail
+
+ROOT=`dirname "$0"`
+ROOT=`cd "$ROOT"; pwd`
+
+export DORIS_HOME=${ROOT}/../../
+
+# include custom environment variables
+if [[ -f ${DORIS_HOME}/custom_env.sh ]]; then
+    . ${DORIS_HOME}/custom_env.sh
+fi
+
+# check maven
+MVN_CMD=mvn
+if [[ ! -z ${CUSTOM_MVN} ]]; then
+    MVN_CMD=${CUSTOM_MVN}
+fi
+if ! ${MVN_CMD} --version; then
+    echo "Error: mvn is not found"
+    exit 1
+fi
+export MVN_CMD
+
+${MVN_CMD} clean package
+
+
+mkdir -p output/
+cp target/doris-flink-1.0-SNAPSHOT.jar ./output/
+
+echo "*****************************************"
+echo "Successfully build Flink-Doris-Connector"
+echo "*****************************************"
+
+exit 0
+
diff --git a/pom.xml b/pom.xml
new file mode 100644
index 0000000..fad5bf9
--- /dev/null
+++ b/pom.xml
@@ -0,0 +1,307 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>org.apache</groupId>
+    <artifactId>doris-flink</artifactId>
+    <version>1.0-SNAPSHOT</version>
+
+    <properties>
+        <scala.version>2.12</scala.version>
+        <flink.version>1.11.2</flink.version>
+        <libthrift.version>0.9.3</libthrift.version>
+        <arrow.version>0.15.1</arrow.version>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <doris.home>${basedir}/../../</doris.home>
+        <doris.thirdparty>${basedir}/../../thirdparty</doris.thirdparty>
+    </properties>
+
+    <profiles>
+        <profile>
+            <id>thirdparty</id>
+            <activation>
+                <property>
+                    <name>env.DORIS_THIRDPARTY</name>
+                </property>
+            </activation>
+            <properties>
+                <doris.thirdparty>${env.DORIS_THIRDPARTY}</doris.thirdparty>
+            </properties>
+        </profile>
+        <!-- for custom internal repository -->
+        <profile>
+            <id>custom-env</id>
+            <activation>
+                <property>
+                    <name>env.CUSTOM_MAVEN_REPO</name>
+                </property>
+            </activation>
+
+            <repositories>
+                <repository>
+                    <id>custom-nexus</id>
+                    <url>${env.CUSTOM_MAVEN_REPO}</url>
+                </repository>
+            </repositories>
+
+            <pluginRepositories>
+                <pluginRepository>
+                    <id>custom-nexus</id>
+                    <url>${env.CUSTOM_MAVEN_REPO}</url>
+                </pluginRepository>
+            </pluginRepositories>
+        </profile>
+
+        <!-- for general repository -->
+        <profile>
+            <id>general-env</id>
+            <activation>
+                <property>
+                    <name>!env.CUSTOM_MAVEN_REPO</name>
+                </property>
+            </activation>
+
+            <repositories>
+                <repository>
+                    <id>central</id>
+                    <name>central maven repo https</name>
+                    <url>https://repo.maven.apache.org/maven2</url>
+                </repository>
+            </repositories>
+        </profile>
+    </profiles>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-java</artifactId>
+            <version>${flink.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-streaming-java_${scala.version}</artifactId>
+            <version>${flink.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-clients_${scala.version}</artifactId>
+            <version>${flink.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <!--	flink table	-->
+        <dependency>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-table-common</artifactId>
+            <version>${flink.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-table-api-java-bridge_${scala.version}</artifactId>
+            <version>${flink.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-table-planner-blink_${scala.version}</artifactId>
+            <version>${flink.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.thrift</groupId>
+            <artifactId>libthrift</artifactId>
+            <version>${libthrift.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.arrow</groupId>
+            <artifactId>arrow-vector</artifactId>
+            <version>${arrow.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+            <version>1.7.25</version>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-log4j12</artifactId>
+            <version>1.7.25</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>log4j</groupId>
+            <artifactId>log4j</artifactId>
+            <version>1.2.17</version>
+        </dependency>
+        <!--Test-->
+        <dependency>
+            <groupId>org.hamcrest</groupId>
+            <artifactId>hamcrest-core</artifactId>
+            <version>1.3</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-scala_${scala.version}</artifactId>
+            <version>1.4.7</version>
+            <exclusions>
+                <exclusion>
+                    <artifactId>hamcrest-core</artifactId>
+                    <groupId>org.hamcrest</groupId>
+                </exclusion>
+            </exclusions>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <version>4.11</version>
+            <exclusions>
+                <exclusion>
+                    <artifactId>hamcrest-core</artifactId>
+                    <groupId>org.hamcrest</groupId>
+                </exclusion>
+            </exclusions>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.thrift.tools</groupId>
+                <artifactId>maven-thrift-plugin</artifactId>
+                <version>0.1.11</version>
+                <configuration>
+                    <thriftExecutable>${doris.thirdparty}/installed/bin/thrift</thriftExecutable>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>thrift-sources</id>
+                        <phase>generate-sources</phase>
+                        <goals>
+                            <goal>compile</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>net.alchim31.maven</groupId>
+                <artifactId>scala-maven-plugin</artifactId>
+                <version>3.2.1</version>
+                <executions>
+                    <execution>
+                        <id>scala-compile-first</id>
+                        <phase>process-resources</phase>
+                        <goals>
+                            <goal>compile</goal>
+                        </goals>
+                    </execution>
+                    <execution>
+                        <id>scala-test-compile</id>
+                        <phase>process-test-resources</phase>
+                        <goals>
+                            <goal>testCompile</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <configuration>
+                    <args>
+                        <arg>-feature</arg>
+                    </args>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <version>3.2.1</version>
+                <configuration>
+                    <artifactSet>
+                        <excludes>
+                            <exclude>com.google.code.findbugs:*</exclude>
+                            <exclude>org.slf4j:*</exclude>
+                        </excludes>
+                    </artifactSet>
+                    <relocations>
+                        <relocation>
+                            <pattern>org.apache.arrow</pattern>
+                            <shadedPattern>org.apache.doris.shaded.org.apache.arrow</shadedPattern>
+                        </relocation>
+                        <relocation>
+                            <pattern>io.netty</pattern>
+                            <shadedPattern>org.apache.doris.shaded.io.netty</shadedPattern>
+                        </relocation>
+                        <relocation>
+                            <pattern>com.fasterxml.jackson</pattern>
+                            <shadedPattern>org.apache.doris.shaded.com.fasterxml.jackson</shadedPattern>
+                        </relocation>
+                        <relocation>
+                            <pattern>org.apache.commons.codec</pattern>
+                            <shadedPattern>org.apache.doris.shaded.org.apache.commons.codec</shadedPattern>
+                        </relocation>
+                        <relocation>
+                            <pattern>com.google.flatbuffers</pattern>
+                            <shadedPattern>org.apache.doris.shaded.com.google.flatbuffers</shadedPattern>
+                        </relocation>
+                        <relocation>
+                            <pattern>org.apache.thrift</pattern>
+                            <shadedPattern>org.apache.doris.shaded.org.apache.thrift</shadedPattern>
+                        </relocation>
+                    </relocations>
+                </configuration>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.jacoco</groupId>
+                <artifactId>jacoco-maven-plugin</artifactId>
+                <version>0.7.8</version>
+                <configuration>
+                    <excludes>
+                        <exclude>**/thrift/**</exclude></excludes>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>prepare-agent</id>
+                        <goals>
+                            <goal>prepare-agent</goal>
+                        </goals>
+                    </execution>
+                    <execution>
+                        <id>check</id>
+                        <goals>
+                            <goal>check</goal>
+                        </goals>
+                    </execution>
+                    <execution>
+                        <id>report</id>
+                        <phase>test</phase>
+                        <goals>
+                            <goal>report</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>3.8.1</version>
+                <configuration>
+                    <source>8</source>
+                    <target>8</target>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>
diff --git a/src/main/java/org/apache/doris/flink/backend/BackendClient.java b/src/main/java/org/apache/doris/flink/backend/BackendClient.java
new file mode 100644
index 0000000..93b353c
--- /dev/null
+++ b/src/main/java/org/apache/doris/flink/backend/BackendClient.java
@@ -0,0 +1,217 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.backend;
+
+import org.apache.doris.flink.cfg.ConfigurationOptions;
+import org.apache.doris.flink.cfg.DorisReadOptions;
+import org.apache.doris.flink.exception.ConnectedFailedException;
+import org.apache.doris.flink.exception.DorisException;
+import org.apache.doris.flink.exception.DorisInternalException;
+import org.apache.doris.flink.serialization.Routing;
+import org.apache.doris.flink.util.ErrorMessages;
+import org.apache.doris.thrift.TDorisExternalService;
+import org.apache.doris.thrift.TScanBatchResult;
+import org.apache.doris.thrift.TScanCloseParams;
+import org.apache.doris.thrift.TScanCloseResult;
+import org.apache.doris.thrift.TScanNextBatchParams;
+import org.apache.doris.thrift.TScanOpenParams;
+import org.apache.doris.thrift.TScanOpenResult;
+import org.apache.doris.thrift.TStatusCode;
+import org.apache.thrift.TException;
+import org.apache.thrift.protocol.TBinaryProtocol;
+import org.apache.thrift.protocol.TProtocol;
+import org.apache.thrift.transport.TSocket;
+import org.apache.thrift.transport.TTransport;
+import org.apache.thrift.transport.TTransportException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Client to request Doris BE
+ */
+public class BackendClient {
+    private static Logger logger = LoggerFactory.getLogger(BackendClient.class);
+
+    private Routing routing;
+
+    private TDorisExternalService.Client client;
+    private TTransport transport;
+
+    private boolean isConnected = false;
+    private final int retries;
+    private final int socketTimeout;
+    private final int connectTimeout;
+
+    public BackendClient(Routing routing, DorisReadOptions readOptions) throws ConnectedFailedException {
+        this.routing = routing;
+        this.connectTimeout = readOptions.getRequestConnectTimeoutMs() == null ? ConfigurationOptions.DORIS_REQUEST_CONNECT_TIMEOUT_MS_DEFAULT : readOptions.getRequestConnectTimeoutMs();
+        this.socketTimeout = readOptions.getRequestReadTimeoutMs() == null ? ConfigurationOptions.DORIS_REQUEST_READ_TIMEOUT_MS_DEFAULT : readOptions.getRequestReadTimeoutMs();
+        this.retries = readOptions.getRequestRetries() == null ? ConfigurationOptions.DORIS_REQUEST_RETRIES_DEFAULT : readOptions.getRequestRetries();
+        logger.trace("connect timeout set to '{}'. socket timeout set to '{}'. retries set to '{}'.",
+                this.connectTimeout, this.socketTimeout, this.retries);
+        open();
+    }
+
+    private void open() throws ConnectedFailedException {
+        logger.debug("Open client to Doris BE '{}'.", routing);
+        TException ex = null;
+        for (int attempt = 0; !isConnected && attempt < retries; ++attempt) {
+            logger.debug("Attempt {} to connect {}.", attempt, routing);
+            TBinaryProtocol.Factory factory = new TBinaryProtocol.Factory();
+            transport = new TSocket(routing.getHost(), routing.getPort(), socketTimeout, connectTimeout);
+            TProtocol protocol = factory.getProtocol(transport);
+            client = new TDorisExternalService.Client(protocol);
+            if (isConnected) {
+                logger.info("Success connect to {}.", routing);
+                return;
+            }
+            try {
+                logger.trace("Connect status before open transport to {} is '{}'.", routing, isConnected);
+                if (!transport.isOpen()) {
+                    transport.open();
+                    isConnected = true;
+                }
+            } catch (TTransportException e) {
+                logger.warn(ErrorMessages.CONNECT_FAILED_MESSAGE, routing, e);
+                ex = e;
+            }
+
+        }
+        if (!isConnected) {
+            logger.error(ErrorMessages.CONNECT_FAILED_MESSAGE, routing);
+            throw new ConnectedFailedException(routing.toString(), ex);
+        }
+    }
+
+    private void close() {
+        logger.trace("Connect status before close with '{}' is '{}'.", routing, isConnected);
+        isConnected = false;
+        if ((transport != null) && transport.isOpen()) {
+            transport.close();
+            logger.info("Closed a connection to {}.", routing);
+        }
+        if (null != client) {
+            client = null;
+        }
+    }
+
+    /**
+     * Open a scanner for reading Doris data.
+     * @param openParams thrift struct to required by request
+     * @return scan open result
+     * @throws ConnectedFailedException throw if cannot connect to Doris BE
+     */
+    public TScanOpenResult openScanner(TScanOpenParams openParams) throws ConnectedFailedException {
+        logger.debug("OpenScanner to '{}', parameter is '{}'.", routing, openParams);
+        if (!isConnected) {
+            open();
+        }
+        TException ex = null;
+        for (int attempt = 0; attempt < retries; ++attempt) {
+            logger.debug("Attempt {} to openScanner {}.", attempt, routing);
+            try {
+                TScanOpenResult result = client.open_scanner(openParams);
+                if (result == null) {
+                    logger.warn("Open scanner result from {} is null.", routing);
+                    continue;
+                }
+                if (!TStatusCode.OK.equals(result.getStatus().getStatus_code())) {
+                    logger.warn("The status of open scanner result from {} is '{}', error message is: {}.",
+                            routing, result.getStatus().getStatus_code(), result.getStatus().getError_msgs());
+                    continue;
+                }
+                return result;
+            } catch (TException e) {
+                logger.warn("Open scanner from {} failed.", routing, e);
+                ex = e;
+            }
+        }
+        logger.error(ErrorMessages.CONNECT_FAILED_MESSAGE, routing);
+        throw new ConnectedFailedException(routing.toString(), ex);
+    }
+
+    /**
+     * get next row batch from Doris BE
+     * @param nextBatchParams thrift struct to required by request
+     * @return scan batch result
+     * @throws ConnectedFailedException throw if cannot connect to Doris BE
+     */
+    public TScanBatchResult getNext(TScanNextBatchParams nextBatchParams) throws DorisException {
+        logger.debug("GetNext to '{}', parameter is '{}'.", routing, nextBatchParams);
+        if (!isConnected) {
+            open();
+        }
+        TException ex = null;
+        TScanBatchResult result = null;
+        for (int attempt = 0; attempt < retries; ++attempt) {
+            logger.debug("Attempt {} to getNext {}.", attempt, routing);
+            try {
+                result  = client.get_next(nextBatchParams);
+                if (result == null) {
+                    logger.warn("GetNext result from {} is null.", routing);
+                    continue;
+                }
+                if (!TStatusCode.OK.equals(result.getStatus().getStatus_code())) {
+                    logger.warn("The status of get next result from {} is '{}', error message is: {}.",
+                            routing, result.getStatus().getStatus_code(), result.getStatus().getError_msgs());
+                    continue;
+                }
+                return result;
+            } catch (TException e) {
+                logger.warn("Get next from {} failed.", routing, e);
+                ex = e;
+            }
+        }
+        if (result != null && (TStatusCode.OK != (result.getStatus().getStatus_code()))) {
+            logger.error(ErrorMessages.DORIS_INTERNAL_FAIL_MESSAGE, routing, result.getStatus().getStatus_code(),
+                    result.getStatus().getError_msgs());
+            throw new DorisInternalException(routing.toString(), result.getStatus().getStatus_code(),
+                    result.getStatus().getError_msgs());
+        }
+        logger.error(ErrorMessages.CONNECT_FAILED_MESSAGE, routing);
+        throw new ConnectedFailedException(routing.toString(), ex);
+    }
+
+    /**
+     * close an scanner.
+     * @param closeParams thrift struct to required by request
+     */
+    public void closeScanner(TScanCloseParams closeParams) {
+        logger.debug("CloseScanner to '{}', parameter is '{}'.", routing, closeParams);
+        for (int attempt = 0; attempt < retries; ++attempt) {
+            logger.debug("Attempt {} to closeScanner {}.", attempt, routing);
+            try {
+                TScanCloseResult result = client.close_scanner(closeParams);
+                if (result == null) {
+                    logger.warn("CloseScanner result from {} is null.", routing);
+                    continue;
+                }
+                if (!TStatusCode.OK.equals(result.getStatus().getStatus_code())) {
+                    logger.warn("The status of get next result from {} is '{}', error message is: {}.",
+                            routing, result.getStatus().getStatus_code(), result.getStatus().getError_msgs());
+                    continue;
+                }
+                break;
+            } catch (TException e) {
+                logger.warn("Close scanner from {} failed.", routing, e);
+            }
+        }
+        logger.info("CloseScanner to Doris BE '{}' success.", routing);
+        close();
+    }
+}
diff --git a/src/main/java/org/apache/doris/flink/cfg/ConfigurationOptions.java b/src/main/java/org/apache/doris/flink/cfg/ConfigurationOptions.java
new file mode 100644
index 0000000..47d07b8
--- /dev/null
+++ b/src/main/java/org/apache/doris/flink/cfg/ConfigurationOptions.java
@@ -0,0 +1,65 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.cfg;
+
+public interface ConfigurationOptions {
+    // doris fe node address
+    String DORIS_FENODES = "fenodes";
+
+    String DORIS_DEFAULT_CLUSTER = "default_cluster";
+
+    String TABLE_IDENTIFIER = "table.identifier";
+    String DORIS_TABLE_IDENTIFIER = "doris.table.identifier";
+    String DORIS_READ_FIELD = "doris.read.field";
+    String DORIS_FILTER_QUERY = "doris.filter.query";
+    String DORIS_FILTER_QUERY_IN_MAX_COUNT = "doris.filter.query.in.max.count";
+    Integer DORIS_FILTER_QUERY_IN_VALUE_UPPER_LIMIT = 10000;
+
+    String DORIS_USER = "username";
+    String DORIS_PASSWORD = "password";
+
+    String DORIS_REQUEST_AUTH_USER = "doris.request.auth.user";
+    String DORIS_REQUEST_AUTH_PASSWORD = "doris.request.auth.password";
+    String DORIS_REQUEST_RETRIES = "doris.request.retries";
+    String DORIS_REQUEST_CONNECT_TIMEOUT_MS = "doris.request.connect.timeout.ms";
+    String DORIS_REQUEST_READ_TIMEOUT_MS = "doris.request.read.timeout.ms";
+    String DORIS_REQUEST_QUERY_TIMEOUT_S = "doris.request.query.timeout.s";
+    Integer DORIS_REQUEST_RETRIES_DEFAULT = 3;
+    Integer DORIS_REQUEST_CONNECT_TIMEOUT_MS_DEFAULT = 30 * 1000;
+    Integer DORIS_REQUEST_READ_TIMEOUT_MS_DEFAULT = 30 * 1000;
+    Integer DORIS_REQUEST_QUERY_TIMEOUT_S_DEFAULT = 3600;
+
+    String DORIS_TABLET_SIZE = "doris.request.tablet.size";
+    Integer DORIS_TABLET_SIZE_DEFAULT = Integer.MAX_VALUE;
+    Integer DORIS_TABLET_SIZE_MIN = 1;
+
+    String DORIS_BATCH_SIZE = "doris.batch.size";
+    Integer DORIS_BATCH_SIZE_DEFAULT = 1024;
+
+    String DORIS_EXEC_MEM_LIMIT = "doris.exec.mem.limit";
+    Long DORIS_EXEC_MEM_LIMIT_DEFAULT = 2147483648L;
+
+    String DORIS_VALUE_READER_CLASS = "doris.value.reader.class";
+
+    String DORIS_DESERIALIZE_ARROW_ASYNC = "doris.deserialize.arrow.async";
+    Boolean DORIS_DESERIALIZE_ARROW_ASYNC_DEFAULT = false;
+
+    String DORIS_DESERIALIZE_QUEUE_SIZE = "doris.deserialize.queue.size";
+    Integer DORIS_DESERIALIZE_QUEUE_SIZE_DEFAULT = 64;
+
+}
diff --git a/src/main/java/org/apache/doris/flink/cfg/DorisConnectionOptions.java b/src/main/java/org/apache/doris/flink/cfg/DorisConnectionOptions.java
new file mode 100644
index 0000000..619ce74
--- /dev/null
+++ b/src/main/java/org/apache/doris/flink/cfg/DorisConnectionOptions.java
@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.doris.flink.cfg;
+
+import org.apache.flink.util.Preconditions;
+
+import java.io.Serializable;
+
+/**
+ *  Doris connection options.
+ */
+public class DorisConnectionOptions implements Serializable {
+
+    private static final long serialVersionUID = 1L;
+
+    protected final String fenodes;
+    protected final String username;
+    protected final String password;
+
+    public DorisConnectionOptions(String fenodes, String username, String password) {
+        this.fenodes = Preconditions.checkNotNull(fenodes, "fenodes  is empty");
+        this.username = username;
+        this.password = password;
+    }
+
+    public String getFenodes() {
+        return fenodes;
+    }
+
+    public String getUsername() {
+        return username;
+    }
+
+    public String getPassword() {
+        return password;
+    }
+
+    /**
+     * Builder for {@link DorisConnectionOptions}.
+     */
+    public static class DorisConnectionOptionsBuilder {
+        private String fenodes;
+        private String username;
+        private String password;
+
+        public DorisConnectionOptionsBuilder withFenodes(String fenodes) {
+            this.fenodes = fenodes;
+            return this;
+        }
+
+        public DorisConnectionOptionsBuilder withUsername(String username) {
+            this.username = username;
+            return this;
+        }
+
+        public DorisConnectionOptionsBuilder withPassword(String password) {
+            this.password = password;
+            return this;
+        }
+
+        public DorisConnectionOptions build() {
+            return new DorisConnectionOptions(fenodes, username, password);
+        }
+    }
+
+}
diff --git a/src/main/java/org/apache/doris/flink/cfg/DorisExecutionOptions.java b/src/main/java/org/apache/doris/flink/cfg/DorisExecutionOptions.java
new file mode 100644
index 0000000..ee8b09e
--- /dev/null
+++ b/src/main/java/org/apache/doris/flink/cfg/DorisExecutionOptions.java
@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.doris.flink.cfg;
+
+
+import java.io.Serializable;
+
+/**
+ * JDBC sink batch options.
+ */
+public class DorisExecutionOptions  implements Serializable {
+    private static final long serialVersionUID = 1L;
+
+    private final Integer batchSize;
+    private final Integer maxRetries;
+
+    public DorisExecutionOptions(Integer batchSize, Integer maxRetries) {
+        this.batchSize = batchSize;
+        this.maxRetries = maxRetries;
+    }
+
+    public Integer getBatchSize() {
+        return batchSize;
+    }
+
+    public Integer getMaxRetries() {
+        return maxRetries;
+    }
+
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    /**
+     * Builder of {@link DorisExecutionOptions}.
+     */
+    public static class Builder {
+        private Integer batchSize;
+        private Integer maxRetries;
+
+        public Builder setBatchSize(Integer batchSize) {
+            this.batchSize = batchSize;
+            return this;
+        }
+
+        public Builder setMaxRetries(Integer maxRetries) {
+            this.maxRetries = maxRetries;
+            return this;
+        }
+
+        public DorisExecutionOptions build() {
+            return new DorisExecutionOptions(batchSize,maxRetries);
+        }
+    }
+
+
+}
diff --git a/src/main/java/org/apache/doris/flink/cfg/DorisOptions.java b/src/main/java/org/apache/doris/flink/cfg/DorisOptions.java
new file mode 100644
index 0000000..c542d6b
--- /dev/null
+++ b/src/main/java/org/apache/doris/flink/cfg/DorisOptions.java
@@ -0,0 +1,103 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.doris.flink.cfg;
+
+import org.apache.doris.flink.util.IOUtils;
+
+import java.util.Properties;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * Options for the Doris connector.
+ */
+public class DorisOptions extends DorisConnectionOptions{
+
+    private static final long serialVersionUID = 1L;
+
+    private String tableIdentifier;
+
+
+    public DorisOptions(String fenodes, String username, String password, String tableIdentifier) {
+        super(fenodes, username, password);
+        this.tableIdentifier = tableIdentifier;
+    }
+
+    public String getTableIdentifier() {
+        return tableIdentifier;
+    }
+
+    public String save() throws IllegalArgumentException {
+        Properties copy = new Properties();
+        return IOUtils.propsToString(copy);
+    }
+
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    /**
+     * Builder of {@link DorisOptions}.
+     */
+    public static class Builder {
+        private String fenodes;
+        private String username;
+        private String password;
+        private String tableIdentifier;
+
+        /**
+         * required, tableIdentifier
+         */
+        public Builder setTableIdentifier(String tableIdentifier) {
+            this.tableIdentifier = tableIdentifier;
+            return this;
+        }
+
+        /**
+         * optional, user name.
+         */
+        public Builder setUsername(String username) {
+            this.username = username;
+            return this;
+        }
+
+        /**
+         * optional, password.
+         */
+        public Builder setPassword(String password) {
+            this.password = password;
+            return this;
+        }
+
+        /**
+         * required, JDBC DB url.
+         */
+        public Builder setFenodes(String fenodes) {
+            this.fenodes = fenodes;
+            return this;
+        }
+
+
+        public DorisOptions build() {
+            checkNotNull(fenodes, "No fenodes supplied.");
+            checkNotNull(tableIdentifier, "No tableIdentifier supplied.");
+            return new DorisOptions(fenodes, username, password, tableIdentifier);
+        }
+    }
+
+
+}
diff --git a/src/main/java/org/apache/doris/flink/cfg/DorisReadOptions.java b/src/main/java/org/apache/doris/flink/cfg/DorisReadOptions.java
new file mode 100644
index 0000000..833ecf5
--- /dev/null
+++ b/src/main/java/org/apache/doris/flink/cfg/DorisReadOptions.java
@@ -0,0 +1,185 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.doris.flink.cfg;
+
+
+import java.io.Serializable;
+
+/**
+ * Doris read Options
+ */
+public class DorisReadOptions  implements Serializable {
+
+    private static final long serialVersionUID = 1L;
+
+    private String readFields;
+    private String filterQuery;
+    private Integer requestTabletSize;
+    private Integer requestConnectTimeoutMs;
+    private Integer requestReadTimeoutMs;
+    private Integer requestQueryTimeoutS;
+    private Integer requestRetries;
+    private Integer requestBatchSize;
+    private Long execMemLimit;
+    private Integer  deserializeQueueSize;
+    private Boolean deserializeArrowAsync;
+
+    public DorisReadOptions(String readFields, String filterQuery, Integer requestTabletSize, Integer requestConnectTimeoutMs, Integer requestReadTimeoutMs,
+                            Integer requestQueryTimeoutS, Integer requestRetries, Integer requestBatchSize, Long execMemLimit,
+                            Integer deserializeQueueSize, Boolean deserializeArrowAsync) {
+        this.readFields = readFields;
+        this.filterQuery = filterQuery;
+        this.requestTabletSize = requestTabletSize;
+        this.requestConnectTimeoutMs = requestConnectTimeoutMs;
+        this.requestReadTimeoutMs = requestReadTimeoutMs;
+        this.requestQueryTimeoutS = requestQueryTimeoutS;
+        this.requestRetries = requestRetries;
+        this.requestBatchSize = requestBatchSize;
+        this.execMemLimit = execMemLimit;
+        this.deserializeQueueSize = deserializeQueueSize;
+        this.deserializeArrowAsync = deserializeArrowAsync;
+    }
+
+    public String getReadFields() {
+        return readFields;
+    }
+
+    public String getFilterQuery() {
+        return filterQuery;
+    }
+
+    public Integer getRequestTabletSize() {
+        return requestTabletSize;
+    }
+
+    public Integer getRequestConnectTimeoutMs() {
+        return requestConnectTimeoutMs;
+    }
+
+    public Integer getRequestReadTimeoutMs() {
+        return requestReadTimeoutMs;
+    }
+
+    public Integer getRequestRetries() {
+        return requestRetries;
+    }
+
+    public Integer getRequestBatchSize() {
+        return requestBatchSize;
+    }
+
+    public Integer getRequestQueryTimeoutS() {
+        return requestQueryTimeoutS;
+    }
+
+    public Long getExecMemLimit() {
+        return execMemLimit;
+    }
+
+    public Integer getDeserializeQueueSize() {
+        return deserializeQueueSize;
+    }
+
+    public Boolean getDeserializeArrowAsync() {
+        return deserializeArrowAsync;
+    }
+
+
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    /**
+     * Builder of {@link DorisReadOptions}.
+     */
+    public static class Builder {
+
+        private String readFields;
+        private String filterQuery;
+        private Integer requestTabletSize;
+        private Integer requestConnectTimeoutMs;
+        private Integer requestReadTimeoutMs;
+        private Integer requestQueryTimeoutS;
+        private Integer requestRetries;
+        private Integer requestBatchSize;
+        private Long execMemLimit;
+        private Integer  deserializeQueueSize;
+        private Boolean deserializeArrowAsync;
+
+
+        public Builder setReadFields(String readFields) {
+            this.readFields = readFields;
+            return this;
+        }
+
+        public Builder setFilterQuery(String filterQuery) {
+            this.filterQuery = filterQuery;
+            return this;
+        }
+
+        public Builder setRequestTabletSize(Integer requestTabletSize) {
+            this.requestTabletSize = requestTabletSize;
+            return this;
+        }
+
+        public Builder setRequestConnectTimeoutMs(Integer requestConnectTimeoutMs) {
+            this.requestConnectTimeoutMs = requestConnectTimeoutMs;
+            return this;
+        }
+
+        public Builder setRequestReadTimeoutMs(Integer requestReadTimeoutMs) {
+            this.requestReadTimeoutMs = requestReadTimeoutMs;
+            return this;
+        }
+
+        public Builder setRequestQueryTimeoutS(Integer requesQueryTimeoutS) {
+            this.requestQueryTimeoutS = requesQueryTimeoutS;
+            return this;
+        }
+
+        public Builder setRequestRetries(Integer requestRetries) {
+            this.requestRetries = requestRetries;
+            return this;
+        }
+
+        public Builder setRequestBatchSize(Integer requestBatchSize) {
+            this.requestBatchSize = requestBatchSize;
+            return this;
+        }
+
+        public Builder setExecMemLimit(Long execMemLimit) {
+            this.execMemLimit = execMemLimit;
+            return this;
+        }
+
+        public Builder setDeserializeQueueSize(Integer deserializeQueueSize) {
+            this.deserializeQueueSize = deserializeQueueSize;
+            return this;
+        }
+
+        public Builder setDeserializeArrowAsync(Boolean deserializeArrowAsync) {
+            this.deserializeArrowAsync = deserializeArrowAsync;
+            return this;
+        }
+
+        public DorisReadOptions build() {
+            return new DorisReadOptions(readFields,filterQuery,requestTabletSize,requestConnectTimeoutMs,requestReadTimeoutMs,requestQueryTimeoutS,requestRetries,requestBatchSize,execMemLimit,deserializeQueueSize,deserializeArrowAsync);
+        }
+    }
+
+
+}
diff --git a/src/main/java/org/apache/doris/flink/cfg/DorisStreamOptions.java b/src/main/java/org/apache/doris/flink/cfg/DorisStreamOptions.java
new file mode 100644
index 0000000..016b29a
--- /dev/null
+++ b/src/main/java/org/apache/doris/flink/cfg/DorisStreamOptions.java
@@ -0,0 +1,74 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.doris.flink.cfg;
+
+
+import java.io.Serializable;
+import java.util.Properties;
+
+
+/**
+ * Options for the Doris stream connector.
+ */
+public class DorisStreamOptions implements Serializable {
+    private static final long serialVersionUID = 1L;
+
+    private Properties prop;
+    private DorisOptions options;
+    private DorisReadOptions readOptions;
+
+    public DorisStreamOptions(Properties prop) {
+         this.prop = prop;
+         init();
+    }
+
+    /**
+     * convert DorisStreamOptions to DorisOptions and DorisReadOptions
+     */
+    private void init() {
+        DorisOptions.Builder optionsBuilder = DorisOptions.builder()
+                .setFenodes(prop.getProperty(ConfigurationOptions.DORIS_FENODES))
+                .setUsername(prop.getProperty(ConfigurationOptions.DORIS_USER))
+                .setPassword(prop.getProperty(ConfigurationOptions.DORIS_PASSWORD))
+                .setTableIdentifier(prop.getProperty(ConfigurationOptions.TABLE_IDENTIFIER));
+
+        DorisReadOptions.Builder readOptionsBuilder = DorisReadOptions.builder()
+                .setDeserializeArrowAsync(Boolean.valueOf(prop.getProperty(ConfigurationOptions.DORIS_DESERIALIZE_ARROW_ASYNC,ConfigurationOptions.DORIS_DESERIALIZE_ARROW_ASYNC_DEFAULT.toString())))
+                .setDeserializeQueueSize(Integer.valueOf(prop.getProperty(ConfigurationOptions.DORIS_DESERIALIZE_QUEUE_SIZE,ConfigurationOptions.DORIS_DESERIALIZE_QUEUE_SIZE_DEFAULT.toString())))
+                .setExecMemLimit(Long.valueOf(prop.getProperty(ConfigurationOptions.DORIS_EXEC_MEM_LIMIT,ConfigurationOptions.DORIS_EXEC_MEM_LIMIT_DEFAULT.toString())))
+                .setFilterQuery(prop.getProperty(ConfigurationOptions.DORIS_FILTER_QUERY))
+                .setReadFields(prop.getProperty(ConfigurationOptions.DORIS_READ_FIELD))
+                .setRequestQueryTimeoutS(Integer.valueOf(prop.getProperty(ConfigurationOptions.DORIS_REQUEST_QUERY_TIMEOUT_S,ConfigurationOptions.DORIS_REQUEST_QUERY_TIMEOUT_S_DEFAULT.toString())))
+                .setRequestBatchSize(Integer.valueOf(prop.getProperty(ConfigurationOptions.DORIS_BATCH_SIZE,ConfigurationOptions.DORIS_BATCH_SIZE_DEFAULT.toString())))
+                .setRequestConnectTimeoutMs(Integer.valueOf(prop.getProperty(ConfigurationOptions.DORIS_REQUEST_CONNECT_TIMEOUT_MS,ConfigurationOptions.DORIS_REQUEST_CONNECT_TIMEOUT_MS_DEFAULT.toString())))
+                .setRequestReadTimeoutMs(Integer.valueOf(prop.getProperty(ConfigurationOptions.DORIS_REQUEST_READ_TIMEOUT_MS,ConfigurationOptions.DORIS_REQUEST_READ_TIMEOUT_MS_DEFAULT.toString())))
+                .setRequestRetries(Integer.valueOf(prop.getProperty(ConfigurationOptions.DORIS_REQUEST_RETRIES,ConfigurationOptions.DORIS_REQUEST_RETRIES_DEFAULT.toString())))
+                .setRequestTabletSize(Integer.valueOf(prop.getProperty(ConfigurationOptions.DORIS_TABLET_SIZE,ConfigurationOptions.DORIS_TABLET_SIZE_DEFAULT.toString())));
+
+        this.options = optionsBuilder.build();
+        this.readOptions = readOptionsBuilder.build();
+
+    }
+
+    public DorisOptions getOptions() {
+        return options;
+    }
+
+    public DorisReadOptions getReadOptions() {
+        return readOptions;
+    }
+}
diff --git a/src/main/java/org/apache/doris/flink/datastream/DorisSourceFunction.java b/src/main/java/org/apache/doris/flink/datastream/DorisSourceFunction.java
new file mode 100644
index 0000000..5c580db
--- /dev/null
+++ b/src/main/java/org/apache/doris/flink/datastream/DorisSourceFunction.java
@@ -0,0 +1,82 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.doris.flink.datastream;
+
+import org.apache.doris.flink.cfg.DorisOptions;
+import org.apache.doris.flink.cfg.DorisReadOptions;
+import org.apache.doris.flink.cfg.DorisStreamOptions;
+import org.apache.doris.flink.deserialization.DorisDeserializationSchema;
+import org.apache.doris.flink.rest.PartitionDefinition;
+import org.apache.doris.flink.rest.RestService;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.functions.source.RichSourceFunction;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+
+
+/**
+ * DorisSource
+ **/
+
+public class DorisSourceFunction<T> extends RichSourceFunction<T> implements ResultTypeQueryable<T> {
+
+    private  static final Logger logger = LoggerFactory.getLogger(DorisSourceFunction.class);
+
+    private DorisDeserializationSchema deserializer;
+    private DorisOptions options;
+    private DorisReadOptions readOptions;
+    private List<PartitionDefinition>  dorisPartitions;
+    private ScalaValueReader scalaValueReader;
+
+    public DorisSourceFunction(DorisStreamOptions streamOptions, DorisDeserializationSchema deserializer)  {
+        this.deserializer = deserializer;
+        this.options = streamOptions.getOptions();
+        this.readOptions = streamOptions.getReadOptions();
+    }
+
+    @Override
+    public void open(Configuration parameters) throws Exception {
+        super.open(parameters);
+        this.dorisPartitions =  RestService.findPartitions(options,readOptions,logger);
+    }
+
+    @Override
+    public void run(SourceContext sourceContext) throws Exception{
+        for(PartitionDefinition partitions : dorisPartitions){
+            scalaValueReader = new ScalaValueReader(partitions, options,readOptions);
+            while (scalaValueReader.hasNext()){
+                Object next = scalaValueReader.next();
+                sourceContext.collect(next);
+            }
+        }
+    }
+
+    @Override
+    public void cancel() {
+        scalaValueReader.close();
+    }
+
+
+    @Override
+    public TypeInformation<T> getProducedType() {
+        return this.deserializer.getProducedType();
+    }
+}
diff --git a/src/main/java/org/apache/doris/flink/deserialization/DorisDeserializationSchema.java b/src/main/java/org/apache/doris/flink/deserialization/DorisDeserializationSchema.java
new file mode 100644
index 0000000..ba0921b
--- /dev/null
+++ b/src/main/java/org/apache/doris/flink/deserialization/DorisDeserializationSchema.java
@@ -0,0 +1,25 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.doris.flink.deserialization;
+
+import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
+
+import java.io.Serializable;
+
+
+public interface DorisDeserializationSchema<T> extends Serializable, ResultTypeQueryable<T> {
+}
diff --git a/src/main/java/org/apache/doris/flink/deserialization/SimpleListDeserializationSchema.java b/src/main/java/org/apache/doris/flink/deserialization/SimpleListDeserializationSchema.java
new file mode 100644
index 0000000..6fd68ec
--- /dev/null
+++ b/src/main/java/org/apache/doris/flink/deserialization/SimpleListDeserializationSchema.java
@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.doris.flink.deserialization;
+
+
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import java.util.List;
+
+
+public class SimpleListDeserializationSchema implements DorisDeserializationSchema{
+
+    @Override
+    public TypeInformation getProducedType() {
+        return TypeInformation.of(List.class);
+    }
+}
diff --git a/src/main/java/org/apache/doris/flink/exception/ConnectedFailedException.java b/src/main/java/org/apache/doris/flink/exception/ConnectedFailedException.java
new file mode 100644
index 0000000..e25d1a5
--- /dev/null
+++ b/src/main/java/org/apache/doris/flink/exception/ConnectedFailedException.java
@@ -0,0 +1,28 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.exception;
+
+public class ConnectedFailedException extends DorisException {
+    public ConnectedFailedException(String server, Throwable cause) {
+        super("Connect to " + server + "failed.", cause);
+    }
+
+    public ConnectedFailedException(String server, int statusCode, Throwable cause) {
+        super("Connect to " + server + "failed, status code is " + statusCode + ".", cause);
+    }
+}
diff --git a/src/main/java/org/apache/doris/flink/exception/DorisException.java b/src/main/java/org/apache/doris/flink/exception/DorisException.java
new file mode 100644
index 0000000..26e11e5
--- /dev/null
+++ b/src/main/java/org/apache/doris/flink/exception/DorisException.java
@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.exception;
+
+public class DorisException extends Exception {
+    public DorisException() {
+        super();
+    }
+    public DorisException(String message) {
+        super(message);
+    }
+    public DorisException(String message, Throwable cause) {
+        super(message, cause);
+    }
+    public DorisException(Throwable cause) {
+        super(cause);
+    }
+    protected DorisException(String message, Throwable cause,
+                        boolean enableSuppression,
+                        boolean writableStackTrace) {
+        super(message, cause, enableSuppression, writableStackTrace);
+    }
+}
diff --git a/src/main/java/org/apache/doris/flink/exception/DorisInternalException.java b/src/main/java/org/apache/doris/flink/exception/DorisInternalException.java
new file mode 100644
index 0000000..eadd860
--- /dev/null
+++ b/src/main/java/org/apache/doris/flink/exception/DorisInternalException.java
@@ -0,0 +1,29 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.exception;
+
+import org.apache.doris.thrift.TStatusCode;
+
+import java.util.List;
+
+public class DorisInternalException extends DorisException {
+    public DorisInternalException(String server, TStatusCode statusCode, List<String> errorMsgs) {
+        super("Doris server " + server + " internal failed, status code [" + statusCode + "] error message is " + errorMsgs);
+    }
+
+}
diff --git a/src/main/java/org/apache/doris/flink/exception/IllegalArgumentException.java b/src/main/java/org/apache/doris/flink/exception/IllegalArgumentException.java
new file mode 100644
index 0000000..4c0ae09
--- /dev/null
+++ b/src/main/java/org/apache/doris/flink/exception/IllegalArgumentException.java
@@ -0,0 +1,28 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.exception;
+
+public class IllegalArgumentException extends DorisException {
+    public IllegalArgumentException(String msg, Throwable cause) {
+        super(msg, cause);
+    }
+
+    public IllegalArgumentException(String arg, String value) {
+        super("argument '" + arg + "' is illegal, value is '" + value + "'.");
+    }
+}
diff --git a/src/main/java/org/apache/doris/flink/exception/ShouldNeverHappenException.java b/src/main/java/org/apache/doris/flink/exception/ShouldNeverHappenException.java
new file mode 100644
index 0000000..307c398
--- /dev/null
+++ b/src/main/java/org/apache/doris/flink/exception/ShouldNeverHappenException.java
@@ -0,0 +1,20 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.exception;
+
+public class ShouldNeverHappenException extends DorisException { }
diff --git a/src/main/java/org/apache/doris/flink/exception/StreamLoadException.java b/src/main/java/org/apache/doris/flink/exception/StreamLoadException.java
new file mode 100644
index 0000000..6d7a192
--- /dev/null
+++ b/src/main/java/org/apache/doris/flink/exception/StreamLoadException.java
@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.exception;
+
+public class StreamLoadException extends Exception {
+    public StreamLoadException() {
+        super();
+    }
+    public StreamLoadException(String message) {
+        super(message);
+    }
+    public StreamLoadException(String message, Throwable cause) {
+        super(message, cause);
+    }
+    public StreamLoadException(Throwable cause) {
+        super(cause);
+    }
+    protected StreamLoadException(String message, Throwable cause,
+                                  boolean enableSuppression,
+                                  boolean writableStackTrace) {
+        super(message, cause, enableSuppression, writableStackTrace);
+    }
+}
diff --git a/src/main/java/org/apache/doris/flink/rest/PartitionDefinition.java b/src/main/java/org/apache/doris/flink/rest/PartitionDefinition.java
new file mode 100644
index 0000000..19edd21
--- /dev/null
+++ b/src/main/java/org/apache/doris/flink/rest/PartitionDefinition.java
@@ -0,0 +1,150 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.rest;
+
+import org.apache.doris.flink.cfg.DorisOptions;
+
+import java.io.Serializable;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Objects;
+import java.util.Set;
+
+/**
+ * Doris  partition info.
+ */
+public class PartitionDefinition implements Serializable, Comparable<PartitionDefinition> {
+    private final String database;
+    private final String table;
+
+    private final String beAddress;
+    private final Set<Long> tabletIds;
+    private final String queryPlan;
+    private final String serializedSettings;
+
+    public PartitionDefinition(String database, String table,
+                               DorisOptions settings, String beAddress, Set<Long> tabletIds, String queryPlan)
+            throws IllegalArgumentException {
+        if (settings != null) {
+            this.serializedSettings = settings.save();
+        } else {
+            this.serializedSettings = null;
+        }
+        this.database = database;
+        this.table = table;
+        this.beAddress = beAddress;
+        this.tabletIds = tabletIds;
+        this.queryPlan = queryPlan;
+    }
+
+    public String getBeAddress() {
+        return beAddress;
+    }
+
+    public Set<Long> getTabletIds() {
+        return tabletIds;
+    }
+
+    public String getDatabase() {
+        return database;
+    }
+
+    public String getTable() {
+        return table;
+    }
+
+    public String getQueryPlan() {
+        return queryPlan;
+    }
+
+
+    @Override
+    public int compareTo(PartitionDefinition o) {
+        int cmp = database.compareTo(o.database);
+        if (cmp != 0) {
+            return cmp;
+        }
+        cmp = table.compareTo(o.table);
+        if (cmp != 0) {
+            return cmp;
+        }
+        cmp = beAddress.compareTo(o.beAddress);
+        if (cmp != 0) {
+            return cmp;
+        }
+        cmp = queryPlan.compareTo(o.queryPlan);
+        if (cmp != 0) {
+            return cmp;
+        }
+
+        cmp = tabletIds.size() - o.tabletIds.size();
+        if (cmp != 0) {
+            return cmp;
+        }
+
+        Set<Long> similar = new HashSet<>(tabletIds);
+        Set<Long> diffSelf = new HashSet<>(tabletIds);
+        Set<Long> diffOther = new HashSet<>(o.tabletIds);
+        similar.retainAll(o.tabletIds);
+        diffSelf.removeAll(similar);
+        diffOther.removeAll(similar);
+        if  (diffSelf.size() == 0) {
+            return 0;
+        }
+        long diff = Collections.min(diffSelf) - Collections.min(diffOther);
+        return diff < 0 ? -1 : 1;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        PartitionDefinition that = (PartitionDefinition) o;
+        return Objects.equals(database, that.database) &&
+                Objects.equals(table, that.table) &&
+                Objects.equals(beAddress, that.beAddress) &&
+                Objects.equals(tabletIds, that.tabletIds) &&
+                Objects.equals(queryPlan, that.queryPlan) &&
+                Objects.equals(serializedSettings, that.serializedSettings);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = database.hashCode();
+        result = 31 * result + table.hashCode();
+        result = 31 * result + beAddress.hashCode();
+        result = 31 * result + queryPlan.hashCode();
+        result = 31 * result + tabletIds.hashCode();
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        return "PartitionDefinition{" +
+                ", database='" + database + '\'' +
+                ", table='" + table + '\'' +
+                ", beAddress='" + beAddress + '\'' +
+                ", tabletIds=" + tabletIds +
+                ", queryPlan='" + queryPlan + '\'' +
+                '}';
+    }
+}
diff --git a/src/main/java/org/apache/doris/flink/rest/RestService.java b/src/main/java/org/apache/doris/flink/rest/RestService.java
new file mode 100644
index 0000000..469f1aa
--- /dev/null
+++ b/src/main/java/org/apache/doris/flink/rest/RestService.java
@@ -0,0 +1,573 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.rest;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.commons.io.IOUtils;
+import org.apache.doris.flink.cfg.DorisOptions;
+import org.apache.doris.flink.cfg.DorisReadOptions;
+import org.apache.doris.flink.exception.IllegalArgumentException;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.doris.flink.cfg.ConfigurationOptions;
+import org.apache.doris.flink.exception.ConnectedFailedException;
+import org.apache.doris.flink.exception.DorisException;
+import org.apache.doris.flink.exception.ShouldNeverHappenException;
+import org.apache.doris.flink.rest.models.Backend;
+import org.apache.doris.flink.rest.models.BackendRow;
+import org.apache.doris.flink.rest.models.QueryPlan;
+import org.apache.doris.flink.rest.models.Schema;
+import org.apache.doris.flink.rest.models.Tablet;
+import org.apache.flink.shaded.guava18.com.google.common.annotations.VisibleForTesting;
+import org.apache.http.HttpStatus;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpRequestBase;
+import org.apache.http.entity.StringEntity;
+
+
+import org.slf4j.Logger;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.PrintWriter;
+import java.io.Serializable;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+
+
+import static org.apache.doris.flink.cfg.ConfigurationOptions.DORIS_TABLET_SIZE;
+import static org.apache.doris.flink.cfg.ConfigurationOptions.DORIS_TABLET_SIZE_DEFAULT;
+import static org.apache.doris.flink.cfg.ConfigurationOptions.DORIS_TABLET_SIZE_MIN;
+import static org.apache.doris.flink.util.ErrorMessages.CONNECT_FAILED_MESSAGE;
+import static org.apache.doris.flink.util.ErrorMessages.ILLEGAL_ARGUMENT_MESSAGE;
+import static org.apache.doris.flink.util.ErrorMessages.SHOULD_NOT_HAPPEN_MESSAGE;
+
+
+/**
+ * Service for communicate with Doris FE.
+ */
+public class RestService implements Serializable {
+    public final static int REST_RESPONSE_STATUS_OK = 200;
+    public final static int REST_RESPONSE_CODE_OK = 0;
+    private final static String REST_RESPONSE_BE_ROWS_KEY = "rows";
+    private static final String API_PREFIX = "/api";
+    private static final String SCHEMA = "_schema";
+    private static final String QUERY_PLAN = "_query_plan";
+    private static final String BACKENDS = "/rest/v1/system?path=//backends";
+    private static final String FE_LOGIN = "/rest/v1/login";
+
+    /**
+     * send request to Doris FE and get response json string.
+     * @param options configuration of request
+     * @param request {@link HttpRequestBase} real request
+     * @param logger {@link Logger}
+     * @return Doris FE response in json string
+     * @throws ConnectedFailedException throw when cannot connect to Doris FE
+     */
+    private static String send(DorisOptions options,DorisReadOptions readOptions, HttpRequestBase request, Logger logger) throws
+            ConnectedFailedException {
+        int connectTimeout = readOptions.getRequestConnectTimeoutMs() == null ? ConfigurationOptions.DORIS_REQUEST_CONNECT_TIMEOUT_MS_DEFAULT : readOptions.getRequestConnectTimeoutMs();
+        int socketTimeout = readOptions.getRequestReadTimeoutMs() == null ? ConfigurationOptions.DORIS_REQUEST_READ_TIMEOUT_MS_DEFAULT : readOptions.getRequestReadTimeoutMs();
+        int retries = readOptions.getRequestRetries() == null ? ConfigurationOptions.DORIS_REQUEST_RETRIES_DEFAULT : readOptions.getRequestRetries();
+        logger.trace("connect timeout set to '{}'. socket timeout set to '{}'. retries set to '{}'.",
+                connectTimeout, socketTimeout, retries);
+
+        RequestConfig requestConfig = RequestConfig.custom()
+                .setConnectTimeout(connectTimeout)
+                .setSocketTimeout(socketTimeout)
+                .build();
+
+        request.setConfig(requestConfig);
+
+
+        logger.info("Send request to Doris FE '{}' with user '{}'.", request.getURI(), options.getUsername());
+
+        IOException ex = null;
+        int statusCode = -1;
+
+        for (int attempt = 0; attempt < retries; attempt++) {
+            logger.debug("Attempt {} to request {}.", attempt, request.getURI());
+            try {
+                String response;
+                if(request instanceof HttpGet){
+                    response = getConnectionGet(request.getURI().toString(), options.getUsername(), options.getPassword(),logger);
+                }else{
+                    response = getConnection(request,  options.getUsername(), options.getPassword(),logger);
+                }
+                logger.trace("Success get response from Doris FE: {}, response is: {}.",
+                        request.getURI(), response);
+                //Handle the problem of inconsistent data format returned by http v1 and v2
+                ObjectMapper mapper = new ObjectMapper();
+                Map map = mapper.readValue(response, Map.class);
+                if(map.containsKey("code") && map.containsKey("msg")) {
+                    Object data = map.get("data");
+                    return mapper.writeValueAsString(data);
+                } else {
+                    return response;
+                }
+            } catch (IOException e) {
+                ex = e;
+                logger.warn(CONNECT_FAILED_MESSAGE, request.getURI(), e);
+            }
+        }
+
+        logger.error(CONNECT_FAILED_MESSAGE, request.getURI(), ex);
+        throw new ConnectedFailedException(request.getURI().toString(), statusCode, ex);
+    }
+
+    private static String getConnection(HttpRequestBase request,String user, String passwd,Logger logger) throws IOException {
+        URL url = new URL(request.getURI().toString());
+        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+        conn.setInstanceFollowRedirects(false);
+        conn.setRequestMethod("POST");
+        String authEncoding = Base64.getEncoder().encodeToString(String.format("%s:%s", user, passwd).getBytes(StandardCharsets.UTF_8));
+        conn.setRequestProperty("Authorization", "Basic " + authEncoding);
+
+        InputStream content = ((HttpPost)request).getEntity().getContent();
+        String res = IOUtils.toString(content);
+        conn.setDoOutput(true);
+        conn.setDoInput(true);
+        PrintWriter out = new PrintWriter(conn.getOutputStream());
+        // send request params
+        out.print(res);
+        // flush
+        out.flush();
+        // read response
+        return parseResponse(conn,logger);
+    }
+
+    private static String getConnectionGet(String request,String user, String passwd,Logger logger) throws IOException {
+        URL realUrl = new URL(request);
+        // open connection
+        HttpURLConnection connection = (HttpURLConnection)realUrl.openConnection();
+        String authEncoding = Base64.getEncoder().encodeToString(String.format("%s:%s", user, passwd).getBytes(StandardCharsets.UTF_8));
+        connection.setRequestProperty("Authorization", "Basic " + authEncoding);
+
+        connection.connect();
+        return parseResponse(connection,logger);
+    }
+
+    private static String parseResponse(HttpURLConnection connection,Logger logger) throws IOException {
+        if (connection.getResponseCode() != HttpStatus.SC_OK) {
+            logger.warn("Failed to get response from Doris  {}, http code is {}",
+                    connection.getURL(), connection.getResponseCode());
+            throw new IOException("Failed to get response from Doris");
+        }
+        String result = "";
+        BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream(), "utf-8"));
+        String line;
+        while ((line = in.readLine()) != null) {
+            result += line;
+        }
+        if (in != null) {
+            in.close();
+        }
+        return result;
+    }
+
+    /**
+     * parse table identifier to array.
+     * @param tableIdentifier table identifier string
+     * @param logger {@link Logger}
+     * @return first element is db name, second element is table name
+     * @throws IllegalArgumentException table identifier is illegal
+     */
+    @VisibleForTesting
+    static String[] parseIdentifier(String tableIdentifier, Logger logger) throws IllegalArgumentException {
+        logger.trace("Parse identifier '{}'.", tableIdentifier);
+        if (StringUtils.isEmpty(tableIdentifier)) {
+            logger.error(ILLEGAL_ARGUMENT_MESSAGE, "table.identifier", tableIdentifier);
+            throw new IllegalArgumentException("table.identifier", tableIdentifier);
+        }
+        String[] identifier = tableIdentifier.split("\\.");
+        if (identifier.length != 2) {
+            logger.error(ILLEGAL_ARGUMENT_MESSAGE, "table.identifier", tableIdentifier);
+            throw new IllegalArgumentException("table.identifier", tableIdentifier);
+        }
+        return identifier;
+    }
+
+    /**
+     * choice a Doris FE node to request.
+     * @param feNodes Doris FE node list, separate be comma
+     * @param logger slf4j logger
+     * @return the chosen one Doris FE node
+     * @throws IllegalArgumentException fe nodes is illegal
+     */
+    @VisibleForTesting
+    static String randomEndpoint(String feNodes, Logger logger) throws IllegalArgumentException {
+        logger.trace("Parse fenodes '{}'.", feNodes);
+        if (StringUtils.isEmpty(feNodes)) {
+            logger.error(ILLEGAL_ARGUMENT_MESSAGE, "fenodes", feNodes);
+            throw new IllegalArgumentException("fenodes", feNodes);
+        }
+        List<String> nodes = Arrays.asList(feNodes.split(","));
+        Collections.shuffle(nodes);
+        return nodes.get(0).trim();
+    }
+
+    /**
+     * choice a Doris BE node to request.
+     * @param options configuration of request
+     * @param logger slf4j logger
+     * @return the chosen one Doris BE node
+     * @throws IllegalArgumentException BE nodes is illegal
+     */
+    @VisibleForTesting
+    public static String randomBackend(DorisOptions options,DorisReadOptions readOptions ,Logger logger) throws DorisException, IOException {
+        List<BackendRow> backends = getBackends(options,readOptions, logger);
+        logger.trace("Parse beNodes '{}'.", backends);
+        if (backends == null || backends.isEmpty()) {
+            logger.error(ILLEGAL_ARGUMENT_MESSAGE, "beNodes", backends);
+            throw new IllegalArgumentException("beNodes", String.valueOf(backends));
+        }
+        Collections.shuffle(backends);
+        BackendRow backend = backends.get(0);
+        return backend.getIP() + ":" + backend.getHttpPort();
+    }
+
+    /**
+     * get  Doris BE nodes to request.
+     * @param options configuration of request
+     * @param logger slf4j logger
+     * @return the chosen one Doris BE node
+     * @throws IllegalArgumentException BE nodes is illegal
+     */
+    @VisibleForTesting
+    static List<BackendRow> getBackends(DorisOptions options,DorisReadOptions readOptions, Logger logger) throws DorisException, IOException {
+        String feNodes = options.getFenodes();
+        String feNode = randomEndpoint(feNodes, logger);
+        String beUrl = "http://" + feNode + BACKENDS;
+        HttpGet httpGet = new HttpGet(beUrl);
+        String response = send(options, readOptions,httpGet, logger);
+        logger.info("Backend Info:{}",response);
+        List<BackendRow> backends = parseBackend(response, logger);
+        return backends;
+    }
+
+    static List<BackendRow> parseBackend(String response, Logger logger) throws DorisException, IOException {
+        ObjectMapper mapper = new ObjectMapper();
+        Backend backend;
+        try {
+            backend = mapper.readValue(response, Backend.class);
+        } catch (JsonParseException e) {
+            String errMsg = "Doris BE's response is not a json. res: " + response;
+            logger.error(errMsg, e);
+            throw new DorisException(errMsg, e);
+        } catch (JsonMappingException e) {
+            String errMsg = "Doris BE's response cannot map to schema. res: " + response;
+            logger.error(errMsg, e);
+            throw new DorisException(errMsg, e);
+        } catch (IOException e) {
+            String errMsg = "Parse Doris BE's response to json failed. res: " + response;
+            logger.error(errMsg, e);
+            throw new DorisException(errMsg, e);
+        }
+
+        if (backend == null) {
+            logger.error(SHOULD_NOT_HAPPEN_MESSAGE);
+            throw new ShouldNeverHappenException();
+        }
+        List<BackendRow> backendRows = backend.getRows().stream().filter(v -> v.getAlive()).collect(Collectors.toList());
+        logger.debug("Parsing schema result is '{}'.", backendRows);
+        return backendRows;
+    }
+
+    /**
+     * get a valid URI to connect Doris FE.
+     * @param options configuration of request
+     * @param logger {@link Logger}
+     * @return uri string
+     * @throws IllegalArgumentException throw when configuration is illegal
+     */
+    @VisibleForTesting
+    static String getUriStr(DorisOptions options, Logger logger) throws IllegalArgumentException {
+        String[] identifier = parseIdentifier(options.getTableIdentifier(), logger);
+        return "http://" +
+                randomEndpoint(options.getFenodes(), logger) + API_PREFIX +
+                "/" + identifier[0] +
+                "/" + identifier[1] +
+                "/";
+    }
+
+    /**
+     * discover Doris table schema from Doris FE.
+     * @param options configuration of request
+     * @param logger slf4j logger
+     * @return Doris table schema
+     * @throws DorisException throw when discover failed
+     */
+    public static Schema getSchema(DorisOptions options,DorisReadOptions readOptions, Logger logger)
+            throws DorisException {
+        logger.trace("Finding schema.");
+        HttpGet httpGet = new HttpGet(getUriStr(options, logger) + SCHEMA);
+        String response = send(options, readOptions,httpGet, logger);
+        logger.debug("Find schema response is '{}'.", response);
+        return parseSchema(response, logger);
+    }
+
+    /**
+     * translate Doris FE response to inner {@link Schema} struct.
+     * @param response Doris FE response
+     * @param logger {@link Logger}
+     * @return inner {@link Schema} struct
+     * @throws DorisException throw when translate failed
+     */
+    @VisibleForTesting
+    public static Schema parseSchema(String response, Logger logger) throws DorisException {
+        logger.trace("Parse response '{}' to schema.", response);
+        ObjectMapper mapper = new ObjectMapper();
+        Schema schema;
+        try {
+            schema = mapper.readValue(response, Schema.class);
+        } catch (JsonParseException e) {
+            String errMsg = "Doris FE's response is not a json. res: " + response;
+            logger.error(errMsg, e);
+            throw new DorisException(errMsg, e);
+        } catch (JsonMappingException e) {
+            String errMsg = "Doris FE's response cannot map to schema. res: " + response;
+            logger.error(errMsg, e);
+            throw new DorisException(errMsg, e);
+        } catch (IOException e) {
+            String errMsg = "Parse Doris FE's response to json failed. res: " + response;
+            logger.error(errMsg, e);
+            throw new DorisException(errMsg, e);
+        }
+
+        if (schema == null) {
+            logger.error(SHOULD_NOT_HAPPEN_MESSAGE);
+            throw new ShouldNeverHappenException();
+        }
+
+        if (schema.getStatus() != REST_RESPONSE_STATUS_OK) {
+            String errMsg = "Doris FE's response is not OK, status is " + schema.getStatus();
+            logger.error(errMsg);
+            throw new DorisException(errMsg);
+        }
+        logger.debug("Parsing schema result is '{}'.", schema);
+        return schema;
+    }
+
+    /**
+     * find Doris RDD partitions from Doris FE.
+     * @param options configuration of request
+     * @param logger {@link Logger}
+     * @return an list of Doris RDD partitions
+     * @throws DorisException throw when find partition failed
+     */
+    public static List<PartitionDefinition> findPartitions(DorisOptions options, DorisReadOptions readOptions, Logger logger) throws DorisException {
+        String[] tableIdentifiers = parseIdentifier(options.getTableIdentifier(), logger);
+        String readFields = StringUtils.isBlank(readOptions.getReadFields()) ? "*" :readOptions.getReadFields();
+        String sql = "select " + readFields +
+                " from `" + tableIdentifiers[0] + "`.`" + tableIdentifiers[1] + "`";
+        if (!StringUtils.isEmpty(readOptions.getFilterQuery())) {
+            sql += " where " + readOptions.getFilterQuery();
+        }
+        logger.debug("Query SQL Sending to Doris FE is: '{}'.", sql);
+
+        HttpPost httpPost = new HttpPost(getUriStr(options, logger) + QUERY_PLAN);
+        String entity = "{\"sql\": \""+ sql +"\"}";
+        logger.debug("Post body Sending to Doris FE is: '{}'.", entity);
+        StringEntity stringEntity = new StringEntity(entity, StandardCharsets.UTF_8);
+        stringEntity.setContentEncoding("UTF-8");
+        stringEntity.setContentType("application/json");
+        httpPost.setEntity(stringEntity);
+
+        String resStr = send(options, readOptions,httpPost, logger);
+        logger.debug("Find partition response is '{}'.", resStr);
+        QueryPlan queryPlan = getQueryPlan(resStr, logger);
+        Map<String, List<Long>> be2Tablets = selectBeForTablet(queryPlan, logger);
+        return tabletsMapToPartition(
+                options,
+                readOptions,
+                be2Tablets,
+                queryPlan.getOpaqued_query_plan(),
+                tableIdentifiers[0],
+                tableIdentifiers[1],
+                logger);
+    }
+
+    /**
+     * translate Doris FE response string to inner {@link QueryPlan} struct.
+     * @param response Doris FE response string
+     * @param logger {@link Logger}
+     * @return inner {@link QueryPlan} struct
+     * @throws DorisException throw when translate failed.
+     */
+    @VisibleForTesting
+    static QueryPlan getQueryPlan(String response, Logger logger) throws DorisException {
+        ObjectMapper mapper = new ObjectMapper();
+        QueryPlan queryPlan;
+        try {
+            queryPlan = mapper.readValue(response, QueryPlan.class);
+        } catch (JsonParseException e) {
+            String errMsg = "Doris FE's response is not a json. res: " + response;
+            logger.error(errMsg, e);
+            throw new DorisException(errMsg, e);
+        } catch (JsonMappingException e) {
+            String errMsg = "Doris FE's response cannot map to schema. res: " + response;
+            logger.error(errMsg, e);
+            throw new DorisException(errMsg, e);
+        } catch (IOException e) {
+            String errMsg = "Parse Doris FE's response to json failed. res: " + response;
+            logger.error(errMsg, e);
+            throw new DorisException(errMsg, e);
+        }
+
+        if (queryPlan == null) {
+            logger.error(SHOULD_NOT_HAPPEN_MESSAGE);
+            throw new ShouldNeverHappenException();
+        }
+
+        if (queryPlan.getStatus() != REST_RESPONSE_STATUS_OK) {
+            String errMsg = "Doris FE's response is not OK, status is " + queryPlan.getStatus();
+            logger.error(errMsg);
+            throw new DorisException(errMsg);
+        }
+        logger.debug("Parsing partition result is '{}'.", queryPlan);
+        return queryPlan;
+    }
+
+    /**
+     * select which Doris BE to get tablet data.
+     * @param queryPlan {@link QueryPlan} translated from Doris FE response
+     * @param logger {@link Logger}
+     * @return BE to tablets {@link Map}
+     * @throws DorisException throw when select failed.
+     */
+    @VisibleForTesting
+    static  Map<String, List<Long>> selectBeForTablet(QueryPlan queryPlan, Logger logger) throws DorisException {
+        Map<String, List<Long>> be2Tablets = new HashMap<>();
+        for (Map.Entry<String, Tablet> part : queryPlan.getPartitions().entrySet()) {
+            logger.debug("Parse tablet info: '{}'.", part);
+            long tabletId;
+            try {
+                tabletId = Long.parseLong(part.getKey());
+            } catch (NumberFormatException e) {
+                String errMsg = "Parse tablet id '" + part.getKey() + "' to long failed.";
+                logger.error(errMsg, e);
+                throw new DorisException(errMsg, e);
+            }
+            String target = null;
+            int tabletCount = Integer.MAX_VALUE;
+            for (String candidate : part.getValue().getRoutings()) {
+                logger.trace("Evaluate Doris BE '{}' to tablet '{}'.", candidate, tabletId);
+                if (!be2Tablets.containsKey(candidate)) {
+                    logger.debug("Choice a new Doris BE '{}' for tablet '{}'.", candidate, tabletId);
+                    List<Long> tablets = new ArrayList<>();
+                    be2Tablets.put(candidate, tablets);
+                    target = candidate;
+                    break;
+                } else {
+                    if (be2Tablets.get(candidate).size() < tabletCount) {
+                        target = candidate;
+                        tabletCount = be2Tablets.get(candidate).size();
+                        logger.debug("Current candidate Doris BE to tablet '{}' is '{}' with tablet count {}.",
+                                tabletId, target, tabletCount);
+                    }
+                }
+            }
+            if (target == null) {
+                String errMsg = "Cannot choice Doris BE for tablet " + tabletId;
+                logger.error(errMsg);
+                throw new DorisException(errMsg);
+            }
+
+            logger.debug("Choice Doris BE '{}' for tablet '{}'.", target, tabletId);
+            be2Tablets.get(target).add(tabletId);
+        }
+        return be2Tablets;
+    }
+
+    /**
+     * tablet count limit for one Doris RDD partition
+     * @param readOptions configuration of request
+     * @param logger {@link Logger}
+     * @return tablet count limit
+     */
+    @VisibleForTesting
+    static int tabletCountLimitForOnePartition(DorisReadOptions readOptions, Logger logger) {
+        int tabletsSize = DORIS_TABLET_SIZE_DEFAULT;
+        if (readOptions.getRequestTabletSize() != null) {
+            tabletsSize = readOptions.getRequestTabletSize();
+        }
+        if (tabletsSize < DORIS_TABLET_SIZE_MIN) {
+            logger.warn("{} is less than {}, set to default value {}.",
+                    DORIS_TABLET_SIZE, DORIS_TABLET_SIZE_MIN, DORIS_TABLET_SIZE_MIN);
+            tabletsSize = DORIS_TABLET_SIZE_MIN;
+        }
+        logger.debug("Tablet size is set to {}.", tabletsSize);
+        return tabletsSize;
+    }
+
+    /**
+     * translate BE tablets map to Doris RDD partition.
+     * @param options configuration of request
+     * @param be2Tablets BE to tablets {@link Map}
+     * @param opaquedQueryPlan Doris BE execute plan getting from Doris FE
+     * @param database database name of Doris table
+     * @param table table name of Doris table
+     * @param logger {@link Logger}
+     * @return Doris RDD partition {@link List}
+     * @throws IllegalArgumentException throw when translate failed
+     */
+    @VisibleForTesting
+    static List<PartitionDefinition> tabletsMapToPartition(DorisOptions options,DorisReadOptions readOptions, Map<String, List<Long>> be2Tablets,
+            String opaquedQueryPlan, String database, String table, Logger logger)
+            throws IllegalArgumentException {
+        int tabletsSize = tabletCountLimitForOnePartition(readOptions, logger);
+        List<PartitionDefinition> partitions = new ArrayList<>();
+        for (Map.Entry<String, List<Long>> beInfo : be2Tablets.entrySet()) {
+            logger.debug("Generate partition with beInfo: '{}'.", beInfo);
+            HashSet<Long> tabletSet = new HashSet<>(beInfo.getValue());
+            beInfo.getValue().clear();
+            beInfo.getValue().addAll(tabletSet);
+            int first = 0;
+            while (first < beInfo.getValue().size()) {
+                Set<Long> partitionTablets = new HashSet<>(beInfo.getValue().subList(
+                        first, Math.min(beInfo.getValue().size(), first + tabletsSize)));
+                first = first + tabletsSize;
+                PartitionDefinition partitionDefinition =
+                        new PartitionDefinition(database, table, options,
+                                beInfo.getKey(), partitionTablets, opaquedQueryPlan);
+                logger.debug("Generate one PartitionDefinition '{}'.", partitionDefinition);
+                partitions.add(partitionDefinition);
+            }
+        }
+        return partitions;
+    }
+
+
+}
diff --git a/src/main/java/org/apache/doris/flink/rest/SchemaUtils.java b/src/main/java/org/apache/doris/flink/rest/SchemaUtils.java
new file mode 100644
index 0000000..46c54c0
--- /dev/null
+++ b/src/main/java/org/apache/doris/flink/rest/SchemaUtils.java
@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.doris.flink.rest;
+
+import org.apache.doris.flink.rest.models.Field;
+import org.apache.doris.flink.rest.models.Schema;
+import org.apache.doris.thrift.TScanColumnDesc;
+
+import java.util.List;
+
+
+public class SchemaUtils {
+
+    /**
+     * convert Doris return schema to inner schema struct.
+     * @param tscanColumnDescs Doris BE return schema
+     * @return inner schema struct
+     */
+    public static Schema convertToSchema(List<TScanColumnDesc> tscanColumnDescs ){
+        Schema schema = new Schema(tscanColumnDescs.size());
+        tscanColumnDescs.stream().forEach(desc -> schema.put(new Field(desc.getName(), desc.getType().name(), "", 0, 0)));
+        return schema;
+    }
+}
diff --git a/src/main/java/org/apache/doris/flink/rest/models/Backend.java b/src/main/java/org/apache/doris/flink/rest/models/Backend.java
new file mode 100644
index 0000000..d74e46f
--- /dev/null
+++ b/src/main/java/org/apache/doris/flink/rest/models/Backend.java
@@ -0,0 +1,41 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.rest.models;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.List;
+
+/**
+ * Be response model
+ **/
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class Backend {
+
+    @JsonProperty(value = "rows")
+    private List<BackendRow> rows;
+
+    public List<BackendRow> getRows() {
+        return rows;
+    }
+
+    public void setRows(List<BackendRow> rows) {
+        this.rows = rows;
+    }
+}
diff --git a/src/main/java/org/apache/doris/flink/rest/models/BackendRow.java b/src/main/java/org/apache/doris/flink/rest/models/BackendRow.java
new file mode 100644
index 0000000..5b7df99
--- /dev/null
+++ b/src/main/java/org/apache/doris/flink/rest/models/BackendRow.java
@@ -0,0 +1,67 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.rest.models;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class BackendRow {
+
+    @JsonProperty(value = "HttpPort")
+    private String HttpPort;
+
+    @JsonProperty(value = "IP")
+    private String IP;
+
+    @JsonProperty(value = "Alive")
+    private Boolean Alive;
+
+    public String getHttpPort() {
+        return HttpPort;
+    }
+
+    public void setHttpPort(String httpPort) {
+        HttpPort = httpPort;
+    }
+
+    public String getIP() {
+        return IP;
+    }
+
+    public void setIP(String IP) {
+        this.IP = IP;
+    }
+
+    public Boolean getAlive() {
+        return Alive;
+    }
+
+    public void setAlive(Boolean alive) {
+        Alive = alive;
+    }
+
+    @Override
+    public String toString() {
+        return "BackendRow{" +
+                "HttpPort='" + HttpPort + '\'' +
+                ", IP='" + IP + '\'' +
+                ", Alive=" + Alive +
+                '}';
+    }
+}
diff --git a/src/main/java/org/apache/doris/flink/rest/models/Field.java b/src/main/java/org/apache/doris/flink/rest/models/Field.java
new file mode 100644
index 0000000..4ac66be
--- /dev/null
+++ b/src/main/java/org/apache/doris/flink/rest/models/Field.java
@@ -0,0 +1,110 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.rest.models;
+
+import java.util.Objects;
+
+public class Field {
+    private String name;
+    private String type;
+    private String comment;
+    private int precision;
+    private int scale;
+
+    public Field() { }
+
+    public Field(String name, String type, String comment, int precision, int scale) {
+        this.name = name;
+        this.type = type;
+        this.comment = comment;
+        this.precision = precision;
+        this.scale = scale;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    public String getType() {
+        return type;
+    }
+
+    public void setType(String type) {
+        this.type = type;
+    }
+
+    public String getComment() {
+        return comment;
+    }
+
+    public void setComment(String comment) {
+        this.comment = comment;
+    }
+
+    public int getPrecision() {
+        return precision;
+    }
+
+    public void setPrecision(int precision) {
+        this.precision = precision;
+    }
+
+    public int getScale() {
+        return scale;
+    }
+
+    public void setScale(int scale) {
+        this.scale = scale;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        Field field = (Field) o;
+        return precision == field.precision &&
+                scale == field.scale &&
+                Objects.equals(name, field.name) &&
+                Objects.equals(type, field.type) &&
+                Objects.equals(comment, field.comment);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(name, type, comment, precision, scale);
+    }
+
+    @Override
+    public String toString() {
+        return "Field{" +
+                "name='" + name + '\'' +
+                ", type='" + type + '\'' +
+                ", comment='" + comment + '\'' +
+                ", precision=" + precision +
+                ", scale=" + scale +
+                '}';
+    }
+}
diff --git a/src/main/java/org/apache/doris/flink/rest/models/QueryPlan.java b/src/main/java/org/apache/doris/flink/rest/models/QueryPlan.java
new file mode 100644
index 0000000..e65175c
--- /dev/null
+++ b/src/main/java/org/apache/doris/flink/rest/models/QueryPlan.java
@@ -0,0 +1,70 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.rest.models;
+
+import java.util.Map;
+import java.util.Objects;
+
+public class QueryPlan {
+    private int status;
+    private String opaqued_query_plan;
+    private Map<String, Tablet> partitions;
+
+    public int getStatus() {
+        return status;
+    }
+
+    public void setStatus(int status) {
+        this.status = status;
+    }
+
+    public String getOpaqued_query_plan() {
+        return opaqued_query_plan;
+    }
+
+    public void setOpaqued_query_plan(String opaqued_query_plan) {
+        this.opaqued_query_plan = opaqued_query_plan;
+    }
+
+    public Map<String, Tablet> getPartitions() {
+        return partitions;
+    }
+
+    public void setPartitions(Map<String, Tablet> partitions) {
+        this.partitions = partitions;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        QueryPlan queryPlan = (QueryPlan) o;
+        return status == queryPlan.status &&
+                Objects.equals(opaqued_query_plan, queryPlan.opaqued_query_plan) &&
+                Objects.equals(partitions, queryPlan.partitions);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(status, opaqued_query_plan, partitions);
+    }
+}
diff --git a/src/main/java/org/apache/doris/flink/rest/models/RespContent.java b/src/main/java/org/apache/doris/flink/rest/models/RespContent.java
new file mode 100644
index 0000000..b86b3dd
--- /dev/null
+++ b/src/main/java/org/apache/doris/flink/rest/models/RespContent.java
@@ -0,0 +1,96 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.doris.flink.rest.models;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class RespContent {
+
+    @JsonProperty(value = "TxnId")
+    private int TxnId;
+
+    @JsonProperty(value = "Label")
+    private String Label;
+
+    @JsonProperty(value = "Status")
+    private String Status;
+
+    @JsonProperty(value = "ExistingJobStatus")
+    private String ExistingJobStatus;
+
+    @JsonProperty(value = "Message")
+    private String Message;
+
+    @JsonProperty(value = "NumberTotalRows")
+    private long NumberTotalRows;
+
+    @JsonProperty(value = "NumberLoadedRows")
+    private long NumberLoadedRows;
+
+    @JsonProperty(value = "NumberFilteredRows")
+    private int NumberFilteredRows;
+
+    @JsonProperty(value = "NumberUnselectedRows")
+    private int NumberUnselectedRows;
+
+    @JsonProperty(value = "LoadBytes")
+    private long LoadBytes;
+
+    @JsonProperty(value = "LoadTimeMs")
+    private int LoadTimeMs;
+
+    @JsonProperty(value = "BeginTxnTimeMs")
+    private int BeginTxnTimeMs;
+
+    @JsonProperty(value = "StreamLoadPutTimeMs")
+    private int StreamLoadPutTimeMs;
+
+    @JsonProperty(value = "ReadDataTimeMs")
+    private int ReadDataTimeMs;
+
+    @JsonProperty(value = "WriteDataTimeMs")
+    private int WriteDataTimeMs;
+
+    @JsonProperty(value = "CommitAndPublishTimeMs")
+    private int CommitAndPublishTimeMs;
+
+    @JsonProperty(value = "ErrorURL")
+    private String ErrorURL;
+
+    public String getStatus() {
+        return Status;
+    }
+
+    public String getMessage() {
+        return Message;
+    }
+
+    @Override
+    public String toString() {
+        ObjectMapper mapper = new ObjectMapper();
+        try {
+            return mapper.writeValueAsString(this);
+        } catch (JsonProcessingException e) {
+            return "";
+        }
+
+    }
+}
diff --git a/src/main/java/org/apache/doris/flink/rest/models/Schema.java b/src/main/java/org/apache/doris/flink/rest/models/Schema.java
new file mode 100644
index 0000000..314aa65
--- /dev/null
+++ b/src/main/java/org/apache/doris/flink/rest/models/Schema.java
@@ -0,0 +1,96 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.rest.models;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+public class Schema {
+    private int status = 0;
+    private List<Field> properties;
+
+    public Schema() {
+        properties = new ArrayList<>();
+    }
+
+    public Schema(int fieldCount) {
+        properties = new ArrayList<>(fieldCount);
+    }
+
+    public int getStatus() {
+        return status;
+    }
+
+    public void setStatus(int status) {
+        this.status = status;
+    }
+
+    public List<Field> getProperties() {
+        return properties;
+    }
+
+    public void setProperties(List<Field> properties) {
+        this.properties = properties;
+    }
+
+    public void put(String name, String type, String comment, int scale, int precision) {
+        properties.add(new Field(name, type, comment, scale, precision));
+    }
+
+    public void put(Field f) {
+        properties.add(f);
+    }
+
+    public Field get(int index) {
+        if (index >= properties.size()) {
+            throw new IndexOutOfBoundsException("Index: " + index + ", Fields size:" + properties.size());
+        }
+        return properties.get(index);
+    }
+
+    public int size() {
+        return properties.size();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        Schema schema = (Schema) o;
+        return status == schema.status &&
+                Objects.equals(properties, schema.properties);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(status, properties);
+    }
+
+    @Override
+    public String toString() {
+        return "Schema{" +
+                "status=" + status +
+                ", properties=" + properties +
+                '}';
+    }
+}
diff --git a/src/main/java/org/apache/doris/flink/rest/models/Tablet.java b/src/main/java/org/apache/doris/flink/rest/models/Tablet.java
new file mode 100644
index 0000000..70b0f13
--- /dev/null
+++ b/src/main/java/org/apache/doris/flink/rest/models/Tablet.java
@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.rest.models;
+
+import java.util.List;
+import java.util.Objects;
+
+public class Tablet {
+    private List<String> routings;
+    private int version;
+    private long versionHash;
+    private long schemaHash;
+
+    public List<String> getRoutings() {
+        return routings;
+    }
+
+    public void setRoutings(List<String> routings) {
+        this.routings = routings;
+    }
+
+    public int getVersion() {
+        return version;
+    }
+
+    public void setVersion(int version) {
+        this.version = version;
+    }
+
+    public long getVersionHash() {
+        return versionHash;
+    }
+
+    public void setVersionHash(long versionHash) {
+        this.versionHash = versionHash;
+    }
+
+    public long getSchemaHash() {
+        return schemaHash;
+    }
+
+    public void setSchemaHash(long schemaHash) {
+        this.schemaHash = schemaHash;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        Tablet tablet = (Tablet) o;
+        return version == tablet.version &&
+                versionHash == tablet.versionHash &&
+                schemaHash == tablet.schemaHash &&
+                Objects.equals(routings, tablet.routings);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(routings, version, versionHash, schemaHash);
+    }
+}
diff --git a/src/main/java/org/apache/doris/flink/serialization/Routing.java b/src/main/java/org/apache/doris/flink/serialization/Routing.java
new file mode 100644
index 0000000..25fdbe5
--- /dev/null
+++ b/src/main/java/org/apache/doris/flink/serialization/Routing.java
@@ -0,0 +1,70 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.serialization;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.doris.flink.exception.IllegalArgumentException;
+
+import static org.apache.doris.flink.util.ErrorMessages.PARSE_NUMBER_FAILED_MESSAGE;
+
+/**
+ * present an Doris BE address.
+ */
+public class Routing {
+    private static Logger logger = LoggerFactory.getLogger(Routing.class);
+
+    private String host;
+    private int port;
+
+    public Routing(String routing) throws IllegalArgumentException {
+        parseRouting(routing);
+    }
+
+    private void parseRouting(String routing) throws IllegalArgumentException {
+        logger.debug("Parse Doris BE address: '{}'.", routing);
+        String[] hostPort = routing.split(":");
+        if (hostPort.length != 2) {
+            logger.error("Format of Doris BE address '{}' is illegal.", routing);
+            throw new IllegalArgumentException("Doris BE", routing);
+        }
+        this.host = hostPort[0];
+        try {
+            this.port = Integer.parseInt(hostPort[1]);
+        } catch (NumberFormatException e) {
+            logger.error(PARSE_NUMBER_FAILED_MESSAGE, "Doris BE's port", hostPort[1]);
+            throw new IllegalArgumentException("Doris BE", routing);
+        }
+    }
+
+    public String getHost() {
+        return host;
+    }
+
+    public int getPort() {
+        return port;
+    }
+
+    @Override
+    public String toString() {
+        return "Doris BE{" +
+                "host='" + host + '\'' +
+                ", port=" + port +
+                '}';
+    }
+}
diff --git a/src/main/java/org/apache/doris/flink/serialization/RowBatch.java b/src/main/java/org/apache/doris/flink/serialization/RowBatch.java
new file mode 100644
index 0000000..8d81a37
--- /dev/null
+++ b/src/main/java/org/apache/doris/flink/serialization/RowBatch.java
@@ -0,0 +1,308 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.serialization;
+
+import org.apache.arrow.memory.RootAllocator;
+
+import org.apache.arrow.vector.BigIntVector;
+import org.apache.arrow.vector.BitVector;
+import org.apache.arrow.vector.DecimalVector;
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.Float4Vector;
+import org.apache.arrow.vector.Float8Vector;
+import org.apache.arrow.vector.IntVector;
+import org.apache.arrow.vector.SmallIntVector;
+import org.apache.arrow.vector.TinyIntVector;
+import org.apache.arrow.vector.VarBinaryVector;
+import org.apache.arrow.vector.VarCharVector;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.ipc.ArrowStreamReader;
+import org.apache.arrow.vector.types.Types;
+import org.apache.doris.flink.exception.DorisException;
+import org.apache.doris.flink.rest.models.Schema;
+import org.apache.doris.thrift.TScanBatchResult;
+
+import org.apache.flink.table.data.DecimalData;
+import org.apache.flink.table.data.StringData;
+import org.apache.flink.util.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+/**
+ * row batch data container.
+ */
+public class RowBatch {
+    private static Logger logger = LoggerFactory.getLogger(RowBatch.class);
+
+    public static class Row {
+        private List<Object> cols;
+
+        Row(int colCount) {
+            this.cols = new ArrayList<>(colCount);
+        }
+
+        public  List<Object> getCols() {
+            return cols;
+        }
+
+        public void put(Object o) {
+            cols.add(o);
+        }
+    }
+
+    // offset for iterate the rowBatch
+    private int offsetInRowBatch = 0;
+    private int rowCountInOneBatch = 0;
+    private int readRowCount = 0;
+    private List<Row> rowBatch = new ArrayList<>();
+    private final ArrowStreamReader arrowStreamReader;
+    private VectorSchemaRoot root;
+    private List<FieldVector> fieldVectors;
+    private RootAllocator rootAllocator;
+    private final Schema schema;
+
+    public List<Row> getRowBatch() {
+        return rowBatch;
+    }
+
+    public RowBatch(TScanBatchResult nextResult, Schema schema){
+        this.schema = schema;
+        this.rootAllocator = new RootAllocator(Integer.MAX_VALUE);
+        this.arrowStreamReader = new ArrowStreamReader(
+                new ByteArrayInputStream(nextResult.getRows()),
+                rootAllocator
+                );
+        this.offsetInRowBatch = 0;
+    }
+
+    public RowBatch readArrow() throws DorisException {
+        try {
+            this.root = arrowStreamReader.getVectorSchemaRoot();
+            while (arrowStreamReader.loadNextBatch()) {
+                fieldVectors = root.getFieldVectors();
+                if (fieldVectors.size() != schema.size()) {
+                    logger.error("Schema size '{}' is not equal to arrow field size '{}'.",
+                            fieldVectors.size(), schema.size());
+                    throw new DorisException("Load Doris data failed, schema size of fetch data is wrong.");
+                }
+                if (fieldVectors.size() == 0 || root.getRowCount() == 0) {
+                    logger.debug("One batch in arrow has no data.");
+                    continue;
+                }
+                rowCountInOneBatch = root.getRowCount();
+                // init the rowBatch
+                for (int i = 0; i < rowCountInOneBatch; ++i) {
+                    rowBatch.add(new Row(fieldVectors.size()));
+                }
+                convertArrowToRowBatch();
+                readRowCount += root.getRowCount();
+            }
+            return this;
+        } catch (Exception e) {
+            logger.error("Read Doris Data failed because: ", e);
+            throw new DorisException(e.getMessage());
+        } finally {
+            close();
+        }
+    }
+
+    public boolean hasNext() {
+        if (offsetInRowBatch < readRowCount) {
+            return true;
+        }
+        return false;
+    }
+
+    private void addValueToRow(int rowIndex, Object obj) {
+        if (rowIndex > rowCountInOneBatch) {
+            String errMsg = "Get row offset: " + rowIndex + " larger than row size: " +
+                    rowCountInOneBatch;
+            logger.error(errMsg);
+            throw new NoSuchElementException(errMsg);
+        }
+        rowBatch.get(readRowCount + rowIndex).put(obj);
+    }
+
+    public void convertArrowToRowBatch() throws DorisException {
+        try {
+            for (int col = 0; col < fieldVectors.size(); col++) {
+                FieldVector curFieldVector = fieldVectors.get(col);
+                Types.MinorType mt = curFieldVector.getMinorType();
+
+                final String currentType = schema.get(col).getType();
+                switch (currentType) {
+                    case "NULL_TYPE":
+                        for (int rowIndex = 0; rowIndex < rowCountInOneBatch; rowIndex++) {
+                            addValueToRow(rowIndex, null);
+                        }
+                        break;
+                    case "BOOLEAN":
+                        Preconditions.checkArgument(mt.equals(Types.MinorType.BIT),
+                                typeMismatchMessage(currentType, mt));
+                        BitVector bitVector = (BitVector) curFieldVector;
+                        for (int rowIndex = 0; rowIndex < rowCountInOneBatch; rowIndex++) {
+                            Object fieldValue = bitVector.isNull(rowIndex) ? null : bitVector.get(rowIndex) != 0;
+                            addValueToRow(rowIndex, fieldValue);
+                        }
+                        break;
+                    case "TINYINT":
+                        Preconditions.checkArgument(mt.equals(Types.MinorType.TINYINT),
+                                typeMismatchMessage(currentType, mt));
+                        TinyIntVector tinyIntVector = (TinyIntVector) curFieldVector;
+                        for (int rowIndex = 0; rowIndex < rowCountInOneBatch; rowIndex++) {
+                            Object fieldValue = tinyIntVector.isNull(rowIndex) ? null : tinyIntVector.get(rowIndex);
+                            addValueToRow(rowIndex, fieldValue);
+                        }
+                        break;
+                    case "SMALLINT":
+                        Preconditions.checkArgument(mt.equals(Types.MinorType.SMALLINT),
+                                typeMismatchMessage(currentType, mt));
+                        SmallIntVector smallIntVector = (SmallIntVector) curFieldVector;
+                        for (int rowIndex = 0; rowIndex < rowCountInOneBatch; rowIndex++) {
+                            Object fieldValue = smallIntVector.isNull(rowIndex) ? null : smallIntVector.get(rowIndex);
+                            addValueToRow(rowIndex, fieldValue);
+                        }
+                        break;
+                    case "INT":
+                        Preconditions.checkArgument(mt.equals(Types.MinorType.INT),
+                                typeMismatchMessage(currentType, mt));
+                        IntVector intVector = (IntVector) curFieldVector;
+                        for (int rowIndex = 0; rowIndex < rowCountInOneBatch; rowIndex++) {
+                            Object fieldValue = intVector.isNull(rowIndex) ? null : intVector.get(rowIndex);
+                            addValueToRow(rowIndex, fieldValue);
+                        }
+                        break;
+                    case "BIGINT":
+
+                        Preconditions.checkArgument(mt.equals(Types.MinorType.BIGINT),
+                                typeMismatchMessage(currentType, mt));
+                        BigIntVector bigIntVector = (BigIntVector) curFieldVector;
+                        for (int rowIndex = 0; rowIndex < rowCountInOneBatch; rowIndex++) {
+                            Object fieldValue = bigIntVector.isNull(rowIndex) ? null : bigIntVector.get(rowIndex);
+                            addValueToRow(rowIndex, fieldValue);
+                        }
+                        break;
+                    case "FLOAT":
+                        Preconditions.checkArgument(mt.equals(Types.MinorType.FLOAT4),
+                                typeMismatchMessage(currentType, mt));
+                        Float4Vector float4Vector = (Float4Vector) curFieldVector;
+                        for (int rowIndex = 0; rowIndex < rowCountInOneBatch; rowIndex++) {
+                            Object fieldValue = float4Vector.isNull(rowIndex) ? null : float4Vector.get(rowIndex);
+                            addValueToRow(rowIndex, fieldValue);
+                        }
+                        break;
+                    case "TIME":
+                    case "DOUBLE":
+                        Preconditions.checkArgument(mt.equals(Types.MinorType.FLOAT8),
+                                typeMismatchMessage(currentType, mt));
+                        Float8Vector float8Vector = (Float8Vector) curFieldVector;
+                        for (int rowIndex = 0; rowIndex < rowCountInOneBatch; rowIndex++) {
+                            Object fieldValue = float8Vector.isNull(rowIndex) ? null : float8Vector.get(rowIndex);
+                            addValueToRow(rowIndex, fieldValue);
+                        }
+                        break;
+                    case "BINARY":
+                        Preconditions.checkArgument(mt.equals(Types.MinorType.VARBINARY),
+                                typeMismatchMessage(currentType, mt));
+                        VarBinaryVector varBinaryVector = (VarBinaryVector) curFieldVector;
+                        for (int rowIndex = 0; rowIndex < rowCountInOneBatch; rowIndex++) {
+                            Object fieldValue = varBinaryVector.isNull(rowIndex) ? null : varBinaryVector.get(rowIndex);
+                            addValueToRow(rowIndex, fieldValue);
+                        }
+                        break;
+                    case "DECIMAL":
+                    case "DECIMALV2":
+                        Preconditions.checkArgument(mt.equals(Types.MinorType.DECIMAL),
+                                typeMismatchMessage(currentType, mt));
+                        DecimalVector decimalVector = (DecimalVector) curFieldVector;
+                        for (int rowIndex = 0; rowIndex < rowCountInOneBatch; rowIndex++) {
+                            if (decimalVector.isNull(rowIndex)) {
+                                addValueToRow(rowIndex, null);
+                                continue;
+                            }
+                            BigDecimal value = decimalVector.getObject(rowIndex).stripTrailingZeros();
+                            addValueToRow(rowIndex, DecimalData.fromBigDecimal(value,value.precision(),value.scale()));
+                        }
+                        break;
+                    case "DATE":
+                    case "LARGEINT":
+                    case "DATETIME":
+                    case "CHAR":
+                    case "VARCHAR":
+                        Preconditions.checkArgument(mt.equals(Types.MinorType.VARCHAR),
+                                typeMismatchMessage(currentType, mt));
+                        VarCharVector varCharVector = (VarCharVector) curFieldVector;
+                        for (int rowIndex = 0; rowIndex < rowCountInOneBatch; rowIndex++) {
+                            if (varCharVector.isNull(rowIndex)) {
+                                addValueToRow(rowIndex, null);
+                                continue;
+                            }
+                            String value = new String(varCharVector.get(rowIndex));
+                            addValueToRow(rowIndex, StringData.fromString(value));
+                        }
+                        break;
+                    default:
+                        String errMsg = "Unsupported type " + schema.get(col).getType();
+                        logger.error(errMsg);
+                        throw new DorisException(errMsg);
+                }
+            }
+        } catch (Exception e) {
+            close();
+            throw e;
+        }
+    }
+
+    public List<Object> next() throws DorisException {
+        if (!hasNext()) {
+            String errMsg = "Get row offset:" + offsetInRowBatch + " larger than row size: " + readRowCount;
+            logger.error(errMsg);
+            throw new NoSuchElementException(errMsg);
+        }
+        return rowBatch.get(offsetInRowBatch++).getCols();
+    }
+
+    private String typeMismatchMessage(final String sparkType, final Types.MinorType arrowType) {
+        final String messageTemplate = "FLINK type is %1$s, but arrow type is %2$s.";
+        return String.format(messageTemplate, sparkType, arrowType.name());
+    }
+
+    public int getReadRowCount() {
+        return readRowCount;
+    }
+
+    public void close() {
+        try {
+            if (arrowStreamReader != null) {
+                arrowStreamReader.close();
+            }
+            if (rootAllocator != null) {
+                rootAllocator.close();
+            }
+        } catch (IOException ioe) {
+            // do nothing
+        }
+    }
+}
diff --git a/src/main/java/org/apache/doris/flink/table/DorisDynamicOutputFormat.java b/src/main/java/org/apache/doris/flink/table/DorisDynamicOutputFormat.java
new file mode 100644
index 0000000..baa7b07
--- /dev/null
+++ b/src/main/java/org/apache/doris/flink/table/DorisDynamicOutputFormat.java
@@ -0,0 +1,200 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.doris.flink.table;
+
+import org.apache.doris.flink.cfg.DorisExecutionOptions;
+import org.apache.doris.flink.cfg.DorisOptions;
+import org.apache.doris.flink.cfg.DorisReadOptions;
+import org.apache.doris.flink.exception.DorisException;
+import org.apache.doris.flink.exception.StreamLoadException;
+import org.apache.doris.flink.rest.RestService;
+import org.apache.flink.api.common.io.RichOutputFormat;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.StringJoiner;
+
+
+/**
+ * DorisDynamicOutputFormat
+ **/
+public class DorisDynamicOutputFormat extends RichOutputFormat<RowData>  {
+
+    private static final Logger LOG = LoggerFactory.getLogger(DorisDynamicOutputFormat.class);
+
+    private  DorisOptions options ;
+    private  DorisReadOptions readOptions;
+    private  DorisExecutionOptions executionOptions;
+    private DorisStreamLoad dorisStreamLoad;
+    private final String fieldDelimiter = "\t";
+    private final String lineDelimiter = "\n";
+    private final List<String> batch = new ArrayList<>();
+    private transient volatile boolean closed = false;
+
+    public DorisDynamicOutputFormat(DorisOptions option,DorisReadOptions readOptions,DorisExecutionOptions executionOptions) {
+        this.options = option;
+        this.readOptions = readOptions;
+        this.executionOptions = executionOptions;
+    }
+
+    @Override
+    public void configure(Configuration configuration) {
+
+    }
+
+    @Override
+    public void open(int taskNumber, int numTasks) throws IOException {
+        dorisStreamLoad = new DorisStreamLoad(
+                getBackend(),
+                options.getTableIdentifier().split("\\.")[0],
+                options.getTableIdentifier().split("\\.")[1],
+                options.getUsername(),
+                options.getPassword());
+        LOG.info("Steamload BE:{}",dorisStreamLoad.getLoadUrlStr());
+    }
+
+    @Override
+    public  void writeRecord(RowData row) throws IOException {
+        addBatch(row);
+        if (executionOptions.getBatchSize() > 0 && batch.size() >= executionOptions.getBatchSize()) {
+            flush();
+        }
+    }
+
+    private void addBatch(RowData row) {
+        StringJoiner value = new StringJoiner(this.fieldDelimiter);
+        GenericRowData rowData = (GenericRowData) row;
+        for(int i = 0; i < row.getArity(); ++i) {
+            value.add(rowData.getField(i).toString());
+        }
+        batch.add(value.toString());
+    }
+
+    @Override
+    public  void close() throws IOException {
+        if (!closed) {
+            closed = true;
+            if (batch.size() > 0) {
+                try {
+                    flush();
+                } catch (Exception e) {
+                    LOG.warn("Writing records to doris failed.", e);
+                    throw new RuntimeException("Writing records to doris failed.", e);
+                }
+            }
+        }
+    }
+
+
+    public  void flush() throws IOException {
+        for (int i = 0; i <= executionOptions.getMaxRetries(); i++) {
+            try {
+                dorisStreamLoad.load(String.join(lineDelimiter,batch));
+                batch.clear();
+                break;
+            } catch (StreamLoadException e) {
+                LOG.error("doris sink error, retry times = {}", i, e);
+                if (i >= executionOptions.getMaxRetries()) {
+                    throw new IOException(e);
+                }
+                try {
+                    dorisStreamLoad.setHostPort(getBackend());
+                    LOG.warn("streamload error,switch be: {}",dorisStreamLoad.getLoadUrlStr(), e);
+                    Thread.sleep(1000 * i);
+                } catch (InterruptedException ex) {
+                    Thread.currentThread().interrupt();
+                    throw new IOException("unable to flush; interrupted while doing another attempt", e);
+                }
+            }
+        }
+    }
+
+    private String getBackend() throws IOException{
+        try {
+            //get be url from fe
+           return  RestService.randomBackend(options,readOptions, LOG);
+        } catch (IOException | DorisException e) {
+            LOG.error("get backends info fail");
+            throw new IOException(e);
+        }
+    }
+
+
+    /**
+     * A builder used to set parameters to the output format's configuration in a fluent way.
+     *
+     * @return builder
+     */
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    /**
+     * Builder for {@link DorisDynamicOutputFormat}.
+     */
+    public static class Builder {
+        private DorisOptions.Builder optionsBuilder;
+        private  DorisReadOptions readOptions;
+        private  DorisExecutionOptions executionOptions;
+
+        public Builder() {
+            this.optionsBuilder = DorisOptions.builder();
+        }
+
+        public Builder setFenodes(String fenodes) {
+            this.optionsBuilder.setFenodes(fenodes);
+            return this;
+        }
+
+        public Builder setUsername(String username) {
+            this.optionsBuilder.setUsername(username);
+            return this;
+        }
+
+        public Builder setPassword(String password) {
+            this.optionsBuilder.setPassword(password);
+            return this;
+        }
+
+        public Builder setTableIdentifier(String tableIdentifier) {
+            this.optionsBuilder.setTableIdentifier(tableIdentifier);
+            return this;
+        }
+
+        public Builder setReadOptions(DorisReadOptions readOptions) {
+            this.readOptions = readOptions;
+            return this;
+        }
+
+        public Builder setExecutionOptions(DorisExecutionOptions executionOptions) {
+            this.executionOptions = executionOptions;
+            return this;
+        }
+
+        public DorisDynamicOutputFormat build() {
+            return new DorisDynamicOutputFormat(
+                    optionsBuilder.build(),readOptions,executionOptions
+            );
+        }
+    }
+}
diff --git a/src/main/java/org/apache/doris/flink/table/DorisDynamicTableFactory.java b/src/main/java/org/apache/doris/flink/table/DorisDynamicTableFactory.java
new file mode 100644
index 0000000..aecda37
--- /dev/null
+++ b/src/main/java/org/apache/doris/flink/table/DorisDynamicTableFactory.java
@@ -0,0 +1,250 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.table;
+
+import org.apache.doris.flink.cfg.DorisExecutionOptions;
+import org.apache.doris.flink.cfg.DorisOptions;
+import org.apache.doris.flink.cfg.DorisReadOptions;
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.ConfigOptions;
+import org.apache.flink.configuration.ReadableConfig;
+import org.apache.flink.table.api.TableSchema;
+import org.apache.flink.table.connector.sink.DynamicTableSink;
+import org.apache.flink.table.connector.source.DynamicTableSource;
+import org.apache.flink.table.factories.DynamicTableSinkFactory;
+import org.apache.flink.table.factories.DynamicTableSourceFactory;
+import org.apache.flink.table.factories.FactoryUtil;
+import org.apache.flink.table.types.DataType;
+import org.apache.flink.table.utils.TableSchemaUtils;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.apache.doris.flink.cfg.ConfigurationOptions.DORIS_BATCH_SIZE_DEFAULT;
+import static org.apache.doris.flink.cfg.ConfigurationOptions.DORIS_DESERIALIZE_ARROW_ASYNC_DEFAULT;
+import static org.apache.doris.flink.cfg.ConfigurationOptions.DORIS_DESERIALIZE_QUEUE_SIZE_DEFAULT;
+import static org.apache.doris.flink.cfg.ConfigurationOptions.DORIS_EXEC_MEM_LIMIT_DEFAULT;
+import static org.apache.doris.flink.cfg.ConfigurationOptions.DORIS_REQUEST_CONNECT_TIMEOUT_MS_DEFAULT;
+import static org.apache.doris.flink.cfg.ConfigurationOptions.DORIS_REQUEST_QUERY_TIMEOUT_S_DEFAULT;
+import static org.apache.doris.flink.cfg.ConfigurationOptions.DORIS_REQUEST_READ_TIMEOUT_MS_DEFAULT;
+import static org.apache.doris.flink.cfg.ConfigurationOptions.DORIS_REQUEST_RETRIES_DEFAULT;
+import static org.apache.doris.flink.cfg.ConfigurationOptions.DORIS_TABLET_SIZE_DEFAULT;
+
+/**
+ * The {@link DorisDynamicTableFactory} translates the catalog table to a table source.
+ *
+ * <p>Because the table source requires a decoding format, we are discovering the format using the
+ * provided {@link FactoryUtil} for convenience.
+ */
+public final class DorisDynamicTableFactory implements  DynamicTableSourceFactory , DynamicTableSinkFactory {
+
+	public static final ConfigOption<String> FENODES = ConfigOptions.key("fenodes").stringType().noDefaultValue().withDescription("doris fe http address.");
+	public static final ConfigOption<String> TABLE_IDENTIFIER = ConfigOptions.key("table.identifier").stringType().noDefaultValue().withDescription("the jdbc table name.");
+	public static final ConfigOption<String> USERNAME = ConfigOptions.key("username").stringType().noDefaultValue().withDescription("the jdbc user name.");
+	public static final ConfigOption<String> PASSWORD = ConfigOptions.key("password").stringType().noDefaultValue().withDescription("the jdbc password.");
+
+	// doris options
+	private static final ConfigOption<String> DORIS_READ_FIELD = ConfigOptions
+			.key("doris.read.field")
+			.stringType()
+			.noDefaultValue()
+			.withDescription("List of column names in the Doris table, separated by commas");
+
+	private static final ConfigOption<String> DORIS_FILTER_QUERY = ConfigOptions
+			.key("doris.filter.query")
+			.stringType()
+			.noDefaultValue()
+			.withDescription("Filter expression of the query, which is transparently transmitted to Doris. Doris uses this expression to complete source-side data filtering");
+
+	private static final ConfigOption<Integer> DORIS_TABLET_SIZE = ConfigOptions
+			.key("doris.request.tablet.size")
+			.intType()
+			.defaultValue(DORIS_TABLET_SIZE_DEFAULT)
+			.withDescription("");
+
+	private static final ConfigOption<Integer> DORIS_REQUEST_CONNECT_TIMEOUT_MS = ConfigOptions
+			.key("doris.request.connect.timeout.ms")
+			.intType()
+			.defaultValue(DORIS_REQUEST_CONNECT_TIMEOUT_MS_DEFAULT)
+			.withDescription("");
+
+	private static final ConfigOption<Integer> DORIS_REQUEST_READ_TIMEOUT_MS = ConfigOptions
+			.key("doris.request.read.timeout.ms")
+			.intType()
+			.defaultValue(DORIS_REQUEST_READ_TIMEOUT_MS_DEFAULT)
+			.withDescription("");
+
+	private static final ConfigOption<Integer> DORIS_REQUEST_QUERY_TIMEOUT_S = ConfigOptions
+			.key("doris.request.query.timeout.s")
+			.intType()
+			.defaultValue(DORIS_REQUEST_QUERY_TIMEOUT_S_DEFAULT)
+			.withDescription("");
+
+	private static final ConfigOption<Integer> DORIS_REQUEST_RETRIES = ConfigOptions
+			.key("doris.request.retries")
+			.intType()
+			.defaultValue(DORIS_REQUEST_RETRIES_DEFAULT)
+			.withDescription("");
+
+	private static final ConfigOption<Boolean> DORIS_DESERIALIZE_ARROW_ASYNC = ConfigOptions
+			.key("doris.deserialize.arrow.async")
+			.booleanType()
+			.defaultValue(DORIS_DESERIALIZE_ARROW_ASYNC_DEFAULT)
+			.withDescription("");
+
+	private static final ConfigOption<Integer> DORIS_DESERIALIZE_QUEUE_SIZE = ConfigOptions
+			.key("doris.request.retriesdoris.deserialize.queue.size")
+			.intType()
+			.defaultValue(DORIS_DESERIALIZE_QUEUE_SIZE_DEFAULT)
+			.withDescription("");
+
+
+	private static final ConfigOption<Integer> DORIS_BATCH_SIZE = ConfigOptions
+			.key("doris.batch.size")
+			.intType()
+			.defaultValue(DORIS_BATCH_SIZE_DEFAULT)
+			.withDescription("");
+
+	private static final ConfigOption<Long> DORIS_EXEC_MEM_LIMIT = ConfigOptions
+			.key("doris.exec.mem.limit")
+			.longType()
+			.defaultValue(DORIS_EXEC_MEM_LIMIT_DEFAULT)
+			.withDescription("");
+
+	// flink write config options
+	private static final ConfigOption<Integer> SINK_BUFFER_FLUSH_MAX_ROWS = ConfigOptions
+			.key("sink.batch.size")
+			.intType()
+			.defaultValue(100)
+			.withDescription("the flush max size (includes all append, upsert and delete records), over this number" +
+					" of records, will flush data. The default value is 100.");
+
+	private static final ConfigOption<Integer> SINK_MAX_RETRIES = ConfigOptions
+			.key("sink.max-retries")
+			.intType()
+			.defaultValue(3)
+			.withDescription("the max retry times if writing records to database failed.");
+
+
+	@Override
+	public String factoryIdentifier() {
+		return "doris"; // used for matching to `connector = '...'`
+	}
+
+	@Override
+	public Set<ConfigOption<?>> requiredOptions() {
+		final Set<ConfigOption<?>> options = new HashSet<>();
+		options.add(FENODES);
+		options.add(TABLE_IDENTIFIER);
+		return options;
+	}
+
+	@Override
+	public Set<ConfigOption<?>> optionalOptions() {
+		final Set<ConfigOption<?>> options = new HashSet<>();
+		options.add(FENODES);
+		options.add(TABLE_IDENTIFIER);
+		options.add(USERNAME);
+		options.add(PASSWORD);
+
+		options.add(DORIS_READ_FIELD);
+		options.add(DORIS_FILTER_QUERY);
+		options.add(DORIS_TABLET_SIZE);
+		options.add(DORIS_REQUEST_CONNECT_TIMEOUT_MS);
+		options.add(DORIS_REQUEST_READ_TIMEOUT_MS);
+		options.add(DORIS_REQUEST_QUERY_TIMEOUT_S);
+		options.add(DORIS_REQUEST_RETRIES);
+		options.add(DORIS_DESERIALIZE_ARROW_ASYNC);
+		options.add(DORIS_DESERIALIZE_QUEUE_SIZE);
+		options.add(DORIS_BATCH_SIZE);
+		options.add(DORIS_EXEC_MEM_LIMIT);
+
+		options.add(SINK_BUFFER_FLUSH_MAX_ROWS);
+		options.add(SINK_MAX_RETRIES);
+		return options;
+	}
+
+	@Override
+	public DynamicTableSource createDynamicTableSource(Context context) {
+		// either implement your custom validation logic here ...
+		// or use the provided helper utility
+		final FactoryUtil.TableFactoryHelper helper = FactoryUtil.createTableFactoryHelper(this, context);
+		// validate all options
+		helper.validate();
+		// get the validated options
+		final ReadableConfig options = helper.getOptions();
+		// derive the produced data type (excluding computed columns) from the catalog table
+		final DataType producedDataType = context.getCatalogTable().getSchema().toPhysicalRowDataType();
+		TableSchema physicalSchema = TableSchemaUtils.getPhysicalSchema(context.getCatalogTable().getSchema());
+		// create and return dynamic table source
+		return new DorisDynamicTableSource(
+				getDorisOptions(helper.getOptions()),
+				getDorisReadOptions(helper.getOptions()),
+				physicalSchema);
+	}
+
+	private DorisOptions getDorisOptions(ReadableConfig readableConfig) {
+		final String fenodes = readableConfig.get(FENODES);
+		final DorisOptions.Builder builder = DorisOptions.builder()
+				.setFenodes(fenodes)
+				.setTableIdentifier(readableConfig.get(TABLE_IDENTIFIER));
+
+		readableConfig.getOptional(USERNAME).ifPresent(builder::setUsername);
+		readableConfig.getOptional(PASSWORD).ifPresent(builder::setPassword);
+		return builder.build();
+	}
+
+	private DorisReadOptions getDorisReadOptions(ReadableConfig readableConfig) {
+		final DorisReadOptions.Builder builder = DorisReadOptions.builder();
+		builder.setDeserializeArrowAsync(readableConfig.get(DORIS_DESERIALIZE_ARROW_ASYNC))
+				.setDeserializeQueueSize(readableConfig.get(DORIS_DESERIALIZE_QUEUE_SIZE))
+				.setExecMemLimit(readableConfig.get(DORIS_EXEC_MEM_LIMIT))
+				.setFilterQuery(readableConfig.get(DORIS_FILTER_QUERY))
+				.setReadFields(readableConfig.get(DORIS_READ_FIELD))
+				.setRequestQueryTimeoutS(readableConfig.get(DORIS_REQUEST_QUERY_TIMEOUT_S))
+				.setRequestBatchSize(readableConfig.get(DORIS_BATCH_SIZE))
+				.setRequestConnectTimeoutMs(readableConfig.get(DORIS_REQUEST_CONNECT_TIMEOUT_MS))
+				.setRequestReadTimeoutMs(readableConfig.get(DORIS_REQUEST_READ_TIMEOUT_MS))
+				.setRequestRetries(readableConfig.get(DORIS_REQUEST_RETRIES))
+				.setRequestTabletSize(readableConfig.get(DORIS_TABLET_SIZE));
+		return builder.build();
+	}
+
+	private DorisExecutionOptions getDorisExecutionOptions(ReadableConfig readableConfig) {
+		final DorisExecutionOptions.Builder builder = DorisExecutionOptions.builder();
+		builder.setBatchSize(readableConfig.get(SINK_BUFFER_FLUSH_MAX_ROWS));
+		builder.setMaxRetries(readableConfig.get(SINK_MAX_RETRIES));
+		return builder.build();
+	}
+
+
+	@Override
+	public DynamicTableSink createDynamicTableSink(Context context) {
+		// either implement your custom validation logic here ...
+		// or use the provided helper utility
+		final FactoryUtil.TableFactoryHelper helper = FactoryUtil.createTableFactoryHelper(this, context);
+		// validate all options
+		helper.validate();
+		// create and return dynamic table source
+		return new DorisDynamicTableSink(
+				getDorisOptions(helper.getOptions()),
+				getDorisReadOptions(helper.getOptions()),
+				getDorisExecutionOptions(helper.getOptions())
+		);
+	}
+}
diff --git a/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSink.java b/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSink.java
new file mode 100644
index 0000000..feeab96
--- /dev/null
+++ b/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSink.java
@@ -0,0 +1,73 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.doris.flink.table;
+
+import org.apache.doris.flink.cfg.DorisExecutionOptions;
+import org.apache.doris.flink.cfg.DorisOptions;
+import org.apache.doris.flink.cfg.DorisReadOptions;
+import org.apache.flink.table.connector.ChangelogMode;
+import org.apache.flink.table.connector.sink.DynamicTableSink;
+import org.apache.flink.table.connector.sink.OutputFormatProvider;
+import org.apache.flink.types.RowKind;
+
+/**
+ * DorisDynamicTableSink
+ **/
+public class DorisDynamicTableSink implements DynamicTableSink {
+
+    private final DorisOptions options;
+    private final DorisReadOptions readOptions;
+    private final DorisExecutionOptions executionOptions;
+
+    public DorisDynamicTableSink(DorisOptions options, DorisReadOptions readOptions, DorisExecutionOptions executionOptions) {
+        this.options = options;
+        this.readOptions = readOptions;
+        this.executionOptions = executionOptions;
+    }
+
+    @Override
+    public ChangelogMode getChangelogMode(ChangelogMode changelogMode) {
+        return ChangelogMode.newBuilder()
+                .addContainedKind(RowKind.INSERT)
+                .addContainedKind(RowKind.DELETE)
+                .addContainedKind(RowKind.UPDATE_AFTER)
+                .build();
+    }
+
+    @Override
+    public SinkRuntimeProvider getSinkRuntimeProvider(Context context) {
+        DorisDynamicOutputFormat.Builder builder = DorisDynamicOutputFormat.builder()
+                .setFenodes(options.getFenodes())
+                .setUsername(options.getUsername())
+                .setPassword(options.getPassword())
+                .setTableIdentifier(options.getTableIdentifier())
+                .setReadOptions(readOptions)
+                .setExecutionOptions(executionOptions);
+
+        return OutputFormatProvider.of(builder.build());
+    }
+
+    @Override
+    public DynamicTableSink copy() {
+        return new DorisDynamicTableSink(options,readOptions,executionOptions);
+    }
+
+    @Override
+    public String asSummaryString() {
+        return "Doris Table Sink";
+    }
+}
diff --git a/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSource.java b/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSource.java
new file mode 100644
index 0000000..ab523aa
--- /dev/null
+++ b/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSource.java
@@ -0,0 +1,101 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.table;
+
+import org.apache.doris.flink.cfg.DorisOptions;
+import org.apache.doris.flink.cfg.DorisReadOptions;
+import org.apache.doris.flink.exception.DorisException;
+import org.apache.doris.flink.rest.PartitionDefinition;
+import org.apache.doris.flink.rest.RestService;
+import org.apache.flink.api.common.serialization.DeserializationSchema;
+import org.apache.flink.streaming.api.functions.source.SourceFunction;
+import org.apache.flink.table.api.TableSchema;
+import org.apache.flink.table.connector.ChangelogMode;
+import org.apache.flink.table.connector.source.DynamicTableSource;
+import org.apache.flink.table.connector.source.InputFormatProvider;
+import org.apache.flink.table.connector.source.LookupTableSource;
+import org.apache.flink.table.connector.source.ScanTableSource;
+import org.apache.flink.table.connector.source.abilities.SupportsFilterPushDown;
+import org.apache.flink.table.connector.source.abilities.SupportsProjectionPushDown;
+import org.apache.flink.table.data.RowData;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+
+/**
+ * The {@link DorisDynamicTableSource} is used during planning.
+ *
+ * <p>In our example, we don't implement any of the available ability interfaces such as {@link SupportsFilterPushDown}
+ * or {@link SupportsProjectionPushDown}. Therefore, the main logic can be found in {@link #getScanRuntimeProvider(ScanContext)}
+ * where we instantiate the required {@link SourceFunction} and its {@link DeserializationSchema} for
+ * runtime. Both instances are parameterized to return internal data structures (i.e. {@link RowData}).
+ */
+public final class DorisDynamicTableSource implements ScanTableSource ,LookupTableSource {
+
+	private final DorisOptions options;
+	private final  DorisReadOptions readOptions;
+	private TableSchema physicalSchema;
+	private static final Logger LOG = LoggerFactory.getLogger(DorisRowDataInputFormat.class);
+
+	public DorisDynamicTableSource(DorisOptions options, DorisReadOptions readOptions,TableSchema physicalSchema) {
+		 this.options = options;
+		 this.readOptions = readOptions;
+		 this.physicalSchema = physicalSchema;
+	}
+
+	@Override
+	public ChangelogMode getChangelogMode() {
+		// in our example the format decides about the changelog mode
+		// but it could also be the source itself
+		return ChangelogMode.insertOnly();
+	}
+
+	@Override
+	public ScanRuntimeProvider getScanRuntimeProvider(ScanContext runtimeProviderContext) {
+		List<PartitionDefinition> dorisPartitions ;
+		try {
+			dorisPartitions = RestService.findPartitions(options,readOptions,LOG);
+		} catch (DorisException e) {
+			throw new RuntimeException("can not fetch partitions");
+		}
+		DorisRowDataInputFormat.Builder builder = DorisRowDataInputFormat.builder()
+				.setFenodes(options.getFenodes())
+				.setUsername(options.getUsername())
+				.setPassword(options.getPassword())
+				.setTableIdentifier(options.getTableIdentifier())
+				.setPartitions(dorisPartitions)
+				.setReadOptions(readOptions);
+		return InputFormatProvider.of(builder.build());
+	}
+
+	@Override
+	public LookupRuntimeProvider getLookupRuntimeProvider(LookupContext lookupContext) {
+		return null;
+	}
+
+	@Override
+	public DynamicTableSource copy() {
+		return new DorisDynamicTableSource(options,readOptions,physicalSchema);
+	}
+
+	@Override
+	public String asSummaryString() {
+		return "Doris Table Source";
+	}
+}
diff --git a/src/main/java/org/apache/doris/flink/table/DorisRowDataInputFormat.java b/src/main/java/org/apache/doris/flink/table/DorisRowDataInputFormat.java
new file mode 100644
index 0000000..75e7fc9
--- /dev/null
+++ b/src/main/java/org/apache/doris/flink/table/DorisRowDataInputFormat.java
@@ -0,0 +1,227 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.doris.flink.table;
+
+import org.apache.doris.flink.cfg.DorisOptions;
+import org.apache.doris.flink.cfg.DorisReadOptions;
+import org.apache.doris.flink.datastream.ScalaValueReader;
+import org.apache.doris.flink.rest.PartitionDefinition;
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.io.DefaultInputSplitAssigner;
+import org.apache.flink.api.common.io.InputFormat;
+import org.apache.flink.api.common.io.RichInputFormat;
+import org.apache.flink.api.common.io.statistics.BaseStatistics;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.core.io.InputSplitAssigner;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.sql.PreparedStatement;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * InputFormat for {@link DorisDynamicTableSource}.
+ */
+@Internal
+public class DorisRowDataInputFormat extends RichInputFormat<RowData, DorisTableInputSplit> implements ResultTypeQueryable<RowData> {
+
+	private static final long serialVersionUID = 1L;
+	private static final Logger LOG = LoggerFactory.getLogger(DorisRowDataInputFormat.class);
+
+	private DorisOptions options;
+	private DorisReadOptions readOptions;
+	private List<PartitionDefinition>  dorisPartitions;
+	private TypeInformation<RowData> rowDataTypeInfo;
+
+	private ScalaValueReader scalaValueReader;
+	private transient boolean hasNext;
+
+	public DorisRowDataInputFormat(DorisOptions options,List<PartitionDefinition>  dorisPartitions,DorisReadOptions readOptions) {
+		this.options = options;
+		this.dorisPartitions = dorisPartitions;
+		this.readOptions = readOptions;
+	}
+
+	@Override
+	public void configure(Configuration parameters) {
+		//do nothing here
+	}
+
+	@Override
+	public void openInputFormat() {
+		//called once per inputFormat (on open)
+	}
+
+	@Override
+	public void closeInputFormat() {
+		//called once per inputFormat (on close)
+	}
+
+	/**
+	 * Connects to the source database and executes the query in a <b>parallel
+	 * fashion</b> if
+	 * this {@link InputFormat} is built using a parameterized query (i.e. using
+	 * a {@link PreparedStatement})
+	 * and a proper {@link  }, in a <b>non-parallel
+	 * fashion</b> otherwise.
+	 *
+	 * @param inputSplit which is ignored if this InputFormat is executed as a
+	 *                   non-parallel source,
+	 *                   a "hook" to the query parameters otherwise (using its
+	 *                   <i>splitNumber</i>)
+	 * @throws IOException if there's an error during the execution of the query
+	 */
+	@Override
+	public void open(DorisTableInputSplit inputSplit) throws IOException {
+		scalaValueReader = new ScalaValueReader(inputSplit.partition, options,readOptions);
+		hasNext = scalaValueReader.hasNext();
+	}
+
+	/**
+	 * Closes all resources used.
+	 *
+	 * @throws IOException Indicates that a resource could not be closed.
+	 */
+	@Override
+	public void close() throws IOException {
+
+	}
+
+	@Override
+	public TypeInformation<RowData> getProducedType() {
+		return rowDataTypeInfo;
+	}
+
+	/**
+	 * Checks whether all data has been read.
+	 *
+	 * @return boolean value indication whether all data has been read.
+	 * @throws IOException
+	 */
+	@Override
+	public boolean reachedEnd() throws IOException {
+		return !hasNext;
+	}
+
+	/**
+	 * Stores the next resultSet row in a tuple.
+	 *
+	 * @param reuse row to be reused.
+	 * @return row containing next {@link RowData}
+	 * @throws IOException
+	 */
+	@Override
+	public RowData nextRecord(RowData reuse) throws IOException {
+		if (!hasNext) {
+			return null;
+		}
+		List next = (List)scalaValueReader.next();
+		GenericRowData genericRowData = new GenericRowData(next.size());
+		for(int i =0;i<next.size();i++){
+			genericRowData.setField(i, next.get(i));
+		}
+		//update hasNext after we've read the record
+		hasNext = scalaValueReader.hasNext();
+		return genericRowData;
+	}
+
+	@Override
+	public BaseStatistics getStatistics(BaseStatistics cachedStatistics) throws IOException {
+		return cachedStatistics;
+	}
+
+	@Override
+	public DorisTableInputSplit[] createInputSplits(int minNumSplits) throws IOException {
+		List<DorisTableInputSplit> dorisSplits = new ArrayList<>();
+		int splitNum = 0;
+		for (PartitionDefinition partition : dorisPartitions) {
+			dorisSplits.add(new DorisTableInputSplit(splitNum++,partition));
+		}
+		LOG.info("DorisTableInputSplit Num:{}",dorisSplits.size());
+		return dorisSplits.toArray(new DorisTableInputSplit[0]);
+	}
+
+	@Override
+	public InputSplitAssigner getInputSplitAssigner(DorisTableInputSplit[] inputSplits) {
+		return new DefaultInputSplitAssigner(inputSplits);
+	}
+
+	/**
+	 * A builder used to set parameters to the output format's configuration in a fluent way.
+	 *
+	 * @return builder
+	 */
+	public static Builder builder() {
+		return new Builder();
+	}
+
+	/**
+	 * Builder for {@link DorisRowDataInputFormat}.
+	 */
+	public static class Builder {
+		private DorisOptions.Builder optionsBuilder;
+		private List<PartitionDefinition> partitions;
+		private DorisReadOptions readOptions;
+
+
+		public Builder() {
+			this.optionsBuilder = DorisOptions.builder();
+		}
+
+		public Builder setFenodes(String fenodes) {
+			this.optionsBuilder.setFenodes(fenodes);
+			return this;
+		}
+
+		public Builder setUsername(String username) {
+			this.optionsBuilder.setUsername(username);
+			return this;
+		}
+
+		public Builder setPassword(String password) {
+			this.optionsBuilder.setPassword(password);
+			return this;
+		}
+
+		public Builder setTableIdentifier(String tableIdentifier) {
+			this.optionsBuilder.setTableIdentifier(tableIdentifier);
+			return this;
+		}
+
+		public Builder setPartitions(List<PartitionDefinition> partitions) {
+			this.partitions = partitions;
+			return this;
+		}
+
+		public Builder setReadOptions(DorisReadOptions readOptions) {
+			this.readOptions = readOptions;
+			return this;
+		}
+
+		public DorisRowDataInputFormat build() {
+			return new DorisRowDataInputFormat(
+					optionsBuilder.build(),partitions,readOptions
+				 );
+		}
+	}
+}
diff --git a/src/main/java/org/apache/doris/flink/table/DorisStreamLoad.java b/src/main/java/org/apache/doris/flink/table/DorisStreamLoad.java
new file mode 100644
index 0000000..fa001a6
--- /dev/null
+++ b/src/main/java/org/apache/doris/flink/table/DorisStreamLoad.java
@@ -0,0 +1,179 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.doris.flink.table;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.doris.flink.exception.StreamLoadException;
+import org.apache.doris.flink.rest.models.RespContent;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedOutputStream;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Serializable;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Calendar;
+import java.util.List;
+import java.util.UUID;
+
+/**
+ * DorisStreamLoad
+ **/
+public class DorisStreamLoad implements Serializable{
+
+    private static final Logger LOG = LoggerFactory.getLogger(DorisStreamLoad.class);
+
+    private final static List<String> DORIS_SUCCESS_STATUS = new ArrayList<>(Arrays.asList("Success", "Publish Timeout"));
+    private static String loadUrlPattern = "http://%s/api/%s/%s/_stream_load?";
+    private String user;
+    private String passwd;
+    private String loadUrlStr;
+    private String hostPort;
+    private String db;
+    private String tbl;
+    private String authEncoding;
+
+    public DorisStreamLoad(String hostPort, String db, String tbl, String user, String passwd) {
+        this.hostPort = hostPort;
+        this.db = db;
+        this.tbl = tbl;
+        this.user = user;
+        this.passwd = passwd;
+        this.loadUrlStr = String.format(loadUrlPattern, hostPort, db, tbl);
+        this.authEncoding = Base64.getEncoder().encodeToString(String.format("%s:%s", user, passwd).getBytes(StandardCharsets.UTF_8));
+    }
+
+    public String getLoadUrlStr() {
+        return loadUrlStr;
+    }
+
+    public String getHostPort() {
+        return hostPort;
+    }
+
+    public void setHostPort(String hostPort) {
+        this.hostPort = hostPort;
+        this.loadUrlStr = String.format(loadUrlPattern, hostPort, this.db, this.tbl);
+    }
+
+
+    private HttpURLConnection getConnection(String urlStr, String label) throws IOException {
+        URL url = new URL(urlStr);
+        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+        conn.setInstanceFollowRedirects(false);
+        conn.setRequestMethod("PUT");
+        String authEncoding = Base64.getEncoder().encodeToString(String.format("%s:%s", user, passwd).getBytes(StandardCharsets.UTF_8));
+        conn.setRequestProperty("Authorization", "Basic " + authEncoding);
+        conn.addRequestProperty("Expect", "100-continue");
+        conn.addRequestProperty("Content-Type", "text/plain; charset=UTF-8");
+        conn.addRequestProperty("label", label);
+        conn.setDoOutput(true);
+        conn.setDoInput(true);
+        return conn;
+    }
+
+    public static class LoadResponse {
+        public int status;
+        public String respMsg;
+        public String respContent;
+
+        public LoadResponse(int status, String respMsg, String respContent) {
+            this.status = status;
+            this.respMsg = respMsg;
+            this.respContent = respContent;
+        }
+        @Override
+        public String toString() {
+            StringBuilder sb = new StringBuilder();
+            sb.append("status: ").append(status);
+            sb.append(", resp msg: ").append(respMsg);
+            sb.append(", resp content: ").append(respContent);
+            return sb.toString();
+        }
+    }
+
+    public void load(String value) throws StreamLoadException {
+        LoadResponse loadResponse = loadBatch(value);
+        LOG.info("Streamload Response:{}",loadResponse);
+        if(loadResponse.status != 200){
+            throw new StreamLoadException("stream load error: " + loadResponse.respContent);
+        }else{
+            ObjectMapper obj = new ObjectMapper();
+            try {
+                RespContent respContent = obj.readValue(loadResponse.respContent, RespContent.class);
+                if(!DORIS_SUCCESS_STATUS.contains(respContent.getStatus())){
+                    throw new StreamLoadException("stream load error: " + respContent.getMessage());
+                }
+            } catch (IOException e) {
+                throw new StreamLoadException(e);
+            }
+        }
+    }
+
+    private LoadResponse loadBatch(String value) {
+        Calendar calendar = Calendar.getInstance();
+        String label = String.format("audit_%s%02d%02d_%02d%02d%02d_%s",
+                calendar.get(Calendar.YEAR), calendar.get(Calendar.MONTH) + 1, calendar.get(Calendar.DAY_OF_MONTH),
+                calendar.get(Calendar.HOUR_OF_DAY), calendar.get(Calendar.MINUTE), calendar.get(Calendar.SECOND),
+                UUID.randomUUID().toString().replaceAll("-", ""));
+
+        HttpURLConnection feConn = null;
+        HttpURLConnection beConn = null;
+        try {
+            // build request and send to new be location
+            beConn = getConnection(loadUrlStr, label);
+            // send data to be
+            BufferedOutputStream bos = new BufferedOutputStream(beConn.getOutputStream());
+            bos.write(value.getBytes());
+            bos.close();
+
+            // get respond
+            int status = beConn.getResponseCode();
+            String respMsg = beConn.getResponseMessage();
+            InputStream stream = (InputStream) beConn.getContent();
+            BufferedReader br = new BufferedReader(new InputStreamReader(stream));
+            StringBuilder response = new StringBuilder();
+            String line;
+            while ((line = br.readLine()) != null) {
+                response.append(line);
+            }
+//            log.info("AuditLoader plugin load with label: {}, response code: {}, msg: {}, content: {}",label, status, respMsg, response.toString());
+            return new LoadResponse(status, respMsg, response.toString());
+
+        } catch (Exception e) {
+            e.printStackTrace();
+            String err = "failed to load audit via AuditLoader plugin with label: " + label;
+            LOG.warn(err, e);
+            return new LoadResponse(-1, e.getMessage(), err);
+        } finally {
+            if (feConn != null) {
+                feConn.disconnect();
+            }
+            if (beConn != null) {
+                beConn.disconnect();
+            }
+        }
+    }
+}
diff --git a/src/main/java/org/apache/doris/flink/table/DorisTableInputSplit.java b/src/main/java/org/apache/doris/flink/table/DorisTableInputSplit.java
new file mode 100644
index 0000000..f245dac
--- /dev/null
+++ b/src/main/java/org/apache/doris/flink/table/DorisTableInputSplit.java
@@ -0,0 +1,44 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.doris.flink.table;
+
+import org.apache.doris.flink.rest.PartitionDefinition;
+import org.apache.flink.core.io.InputSplit;
+
+/**
+ * DorisTableInputSplit
+ **/
+public class DorisTableInputSplit implements InputSplit, java.io.Serializable{
+
+    /** The number of the split. */
+    private final int splitNumber;
+
+    protected final PartitionDefinition partition;
+
+    public DorisTableInputSplit(int splitNumber,PartitionDefinition partition) {
+        super();
+        this.splitNumber = splitNumber;
+        this.partition = partition;
+    }
+
+
+    @Override
+    public int getSplitNumber() {
+        return splitNumber;
+    }
+
+}
diff --git a/src/main/java/org/apache/doris/flink/util/ErrorMessages.java b/src/main/java/org/apache/doris/flink/util/ErrorMessages.java
new file mode 100644
index 0000000..2e02daa
--- /dev/null
+++ b/src/main/java/org/apache/doris/flink/util/ErrorMessages.java
@@ -0,0 +1,27 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.util;
+
+public abstract class ErrorMessages {
+    public static final String PARSE_NUMBER_FAILED_MESSAGE = "Parse '{}' to number failed. Original string is '{}'.";
+    public static final String PARSE_BOOL_FAILED_MESSAGE = "Parse '{}' to boolean failed. Original string is '{}'.";
+    public static final String CONNECT_FAILED_MESSAGE = "Connect to doris {} failed.";
+    public static final String ILLEGAL_ARGUMENT_MESSAGE = "argument '{}' is illegal, value is '{}'.";
+    public static final String SHOULD_NOT_HAPPEN_MESSAGE = "Should not come here.";
+    public static final String DORIS_INTERNAL_FAIL_MESSAGE = "Doris server '{}' internal failed, status is '{}', error message is '{}'";
+}
diff --git a/src/main/java/org/apache/doris/flink/util/IOUtils.java b/src/main/java/org/apache/doris/flink/util/IOUtils.java
new file mode 100644
index 0000000..203010c
--- /dev/null
+++ b/src/main/java/org/apache/doris/flink/util/IOUtils.java
@@ -0,0 +1,49 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.util;
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.io.StringWriter;
+import java.util.Properties;
+
+public class IOUtils {
+    public static String propsToString(Properties props) throws IllegalArgumentException {
+        StringWriter sw = new StringWriter();
+        if (props != null) {
+            try {
+                props.store(sw, "");
+            } catch (IOException ex) {
+                throw new IllegalArgumentException("Cannot parse props to String.", ex);
+            }
+        }
+        return sw.toString();
+    }
+
+    public static Properties propsFromString(String source) throws IllegalArgumentException {
+        Properties copy = new Properties();
+        if (source != null) {
+            try {
+                copy.load(new StringReader(source));
+            } catch (IOException ex) {
+                throw new IllegalArgumentException("Cannot parse props from String.", ex);
+            }
+        }
+        return copy;
+    }
+}
diff --git a/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory b/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory
new file mode 100644
index 0000000..e625cc7
--- /dev/null
+++ b/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+org.apache.doris.flink.table.DorisDynamicTableFactory
\ No newline at end of file
diff --git a/src/main/resources/log4j.properties b/src/main/resources/log4j.properties
new file mode 100644
index 0000000..da32ea0
--- /dev/null
+++ b/src/main/resources/log4j.properties
@@ -0,0 +1,23 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+log4j.rootLogger=INFO, console
+
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n
diff --git a/src/main/scala/org/apache/doris/flink/datastream/ScalaValueReader.scala b/src/main/scala/org/apache/doris/flink/datastream/ScalaValueReader.scala
new file mode 100644
index 0000000..bdf9487
--- /dev/null
+++ b/src/main/scala/org/apache/doris/flink/datastream/ScalaValueReader.scala
@@ -0,0 +1,223 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.datastream
+
+import java.util.concurrent._
+import java.util.concurrent.atomic.AtomicBoolean
+
+import org.apache.doris.flink.backend.BackendClient
+import org.apache.doris.flink.cfg.ConfigurationOptions._
+import org.apache.doris.flink.cfg.{DorisOptions, DorisReadOptions}
+import org.apache.doris.flink.exception.ShouldNeverHappenException
+import org.apache.doris.flink.rest.{PartitionDefinition, SchemaUtils}
+import org.apache.doris.flink.rest.models.Schema
+import org.apache.doris.flink.serialization.{Routing, RowBatch}
+import org.apache.doris.flink.util.ErrorMessages
+import org.apache.doris.flink.util.ErrorMessages._
+import org.apache.doris.thrift.{TScanCloseParams, TScanNextBatchParams, TScanOpenParams, TScanOpenResult}
+import org.apache.log4j.Logger
+
+import scala.collection.JavaConversions._
+import scala.util.Try
+import scala.util.control.Breaks
+
+/**
+ * read data from Doris BE to array.
+ * @param partition Doris RDD partition
+ * @param options request configuration
+ */
+class ScalaValueReader(partition: PartitionDefinition, options: DorisOptions, readOptions: DorisReadOptions) {
+  protected val logger = Logger.getLogger(classOf[ScalaValueReader])
+
+  protected val client = new BackendClient(new Routing(partition.getBeAddress), readOptions)
+  protected var offset = 0
+  protected var eos: AtomicBoolean = new AtomicBoolean(false)
+  protected var rowBatch: RowBatch = _
+  // flag indicate if support deserialize Arrow to RowBatch asynchronously
+  protected var deserializeArrowToRowBatchAsync: java.lang.Boolean = Try {
+    if(readOptions.getDeserializeArrowAsync == null ) DORIS_DESERIALIZE_ARROW_ASYNC_DEFAULT else readOptions.getDeserializeArrowAsync
+  } getOrElse {
+    logger.warn(ErrorMessages.PARSE_BOOL_FAILED_MESSAGE, DORIS_DESERIALIZE_ARROW_ASYNC, readOptions.getDeserializeArrowAsync)
+    DORIS_DESERIALIZE_ARROW_ASYNC_DEFAULT
+  }
+
+  protected var rowBatchBlockingQueue: BlockingQueue[RowBatch] = {
+    val blockingQueueSize = Try {
+      if(readOptions.getDeserializeQueueSize == null) DORIS_DESERIALIZE_QUEUE_SIZE_DEFAULT else readOptions.getDeserializeQueueSize
+    } getOrElse {
+      logger.warn(ErrorMessages.PARSE_NUMBER_FAILED_MESSAGE, DORIS_DESERIALIZE_QUEUE_SIZE, readOptions.getDeserializeQueueSize)
+      DORIS_DESERIALIZE_QUEUE_SIZE_DEFAULT
+    }
+
+    var queue: BlockingQueue[RowBatch] = null
+    if (deserializeArrowToRowBatchAsync) {
+      queue = new ArrayBlockingQueue(blockingQueueSize)
+    }
+    queue
+  }
+
+  private val openParams: TScanOpenParams = {
+    val params = new TScanOpenParams
+    params.cluster = DORIS_DEFAULT_CLUSTER
+    params.database = partition.getDatabase
+    params.table = partition.getTable
+
+    params.tablet_ids = partition.getTabletIds.toList
+    params.opaqued_query_plan = partition.getQueryPlan
+
+    // max row number of one read batch
+    val batchSize = Try {
+      if(readOptions.getRequestBatchSize == null)  DORIS_BATCH_SIZE_DEFAULT else readOptions.getRequestBatchSize;
+    } getOrElse {
+        logger.warn(ErrorMessages.PARSE_NUMBER_FAILED_MESSAGE, DORIS_BATCH_SIZE, readOptions.getRequestBatchSize)
+        DORIS_BATCH_SIZE_DEFAULT
+    }
+
+    val queryDorisTimeout = Try {
+      if(readOptions.getRequestQueryTimeoutS == null) DORIS_REQUEST_QUERY_TIMEOUT_S_DEFAULT else readOptions.getRequestQueryTimeoutS
+    } getOrElse {
+      logger.warn(ErrorMessages.PARSE_NUMBER_FAILED_MESSAGE, DORIS_REQUEST_QUERY_TIMEOUT_S, readOptions.getRequestQueryTimeoutS)
+      DORIS_REQUEST_QUERY_TIMEOUT_S_DEFAULT
+    }
+
+    val execMemLimit = Try {
+      if(readOptions.getExecMemLimit == null) DORIS_EXEC_MEM_LIMIT_DEFAULT else readOptions.getExecMemLimit
+    } getOrElse {
+      logger.warn(ErrorMessages.PARSE_NUMBER_FAILED_MESSAGE, DORIS_EXEC_MEM_LIMIT, readOptions.getExecMemLimit)
+      DORIS_EXEC_MEM_LIMIT_DEFAULT
+    }
+
+    params.setBatch_size(batchSize)
+    params.setQuery_timeout(queryDorisTimeout)
+    params.setMem_limit(execMemLimit)
+    params.setUser(options.getUsername)
+    params.setPasswd(options.getPassword)
+
+    logger.debug(s"Open scan params is, " +
+        s"cluster: ${params.getCluster}, " +
+        s"database: ${params.getDatabase}, " +
+        s"table: ${params.getTable}, " +
+        s"tabletId: ${params.getTablet_ids}, " +
+        s"batch size: $batchSize, " +
+        s"query timeout: $queryDorisTimeout, " +
+        s"execution memory limit: $execMemLimit, " +
+        s"user: ${params.getUser}, " +
+        s"query plan: ${params.opaqued_query_plan}")
+
+    params
+  }
+
+  protected val openResult: TScanOpenResult = client.openScanner(openParams)
+  protected val contextId: String = openResult.getContext_id
+  protected val schema: Schema =
+    SchemaUtils.convertToSchema(openResult.getSelected_columns)
+
+  protected val asyncThread: Thread = new Thread {
+    override def run {
+      val nextBatchParams = new TScanNextBatchParams
+      nextBatchParams.setContext_id(contextId)
+      while (!eos.get) {
+        nextBatchParams.setOffset(offset)
+        val nextResult = client.getNext(nextBatchParams)
+        eos.set(nextResult.isEos)
+        if (!eos.get) {
+          val rowBatch = new RowBatch(nextResult, schema).readArrow()
+          offset += rowBatch.getReadRowCount
+          rowBatch.close
+          rowBatchBlockingQueue.put(rowBatch)
+        }
+      }
+    }
+  }
+
+  protected val asyncThreadStarted: Boolean = {
+    var started = false
+    if (deserializeArrowToRowBatchAsync) {
+      asyncThread.start
+      started = true
+    }
+    started
+  }
+
+  logger.debug(s"Open scan result is, contextId: $contextId, schema: $schema.")
+
+  /**
+   * read data and cached in rowBatch.
+   * @return true if hax next value
+   */
+  def hasNext: Boolean = {
+    var hasNext = false
+    if (deserializeArrowToRowBatchAsync && asyncThreadStarted) {
+      // support deserialize Arrow to RowBatch asynchronously
+      if (rowBatch == null || !rowBatch.hasNext) {
+        val loop = new Breaks
+        loop.breakable {
+          while (!eos.get || !rowBatchBlockingQueue.isEmpty) {
+            if (!rowBatchBlockingQueue.isEmpty) {
+              rowBatch = rowBatchBlockingQueue.take
+              hasNext = true
+              loop.break
+            } else {
+              // wait for rowBatch put in queue or eos change
+              Thread.sleep(5)
+            }
+          }
+        }
+      } else {
+        hasNext = true
+      }
+    } else {
+      // Arrow data was acquired synchronously during the iterative process
+      if (!eos.get && (rowBatch == null || !rowBatch.hasNext)) {
+        if (rowBatch != null) {
+          offset += rowBatch.getReadRowCount
+          rowBatch.close
+        }
+        val nextBatchParams = new TScanNextBatchParams
+        nextBatchParams.setContext_id(contextId)
+        nextBatchParams.setOffset(offset)
+        val nextResult = client.getNext(nextBatchParams)
+        eos.set(nextResult.isEos)
+        if (!eos.get) {
+          rowBatch = new RowBatch(nextResult, schema).readArrow()
+        }
+      }
+      hasNext = !eos.get
+    }
+    hasNext
+  }
+
+  /**
+   * get next value.
+   * @return next value
+   */
+  def next: AnyRef = {
+    if (!hasNext) {
+      logger.error(SHOULD_NOT_HAPPEN_MESSAGE)
+      throw new ShouldNeverHappenException
+    }
+    rowBatch.next
+  }
+
+  def close(): Unit = {
+    val closeParams = new TScanCloseParams
+    closeParams.context_id = contextId
+    client.closeScanner(closeParams)
+  }
+
+}
diff --git a/src/main/thrift/doris/DorisExternalService.thrift b/src/main/thrift/doris/DorisExternalService.thrift
new file mode 100644
index 0000000..c169874
--- /dev/null
+++ b/src/main/thrift/doris/DorisExternalService.thrift
@@ -0,0 +1,122 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+namespace java org.apache.doris.thrift
+namespace cpp doris
+
+include "Types.thrift"
+include "Status.thrift"
+
+
+// Parameters to open().
+struct TScanOpenParams {
+
+  1: required string cluster
+
+  2: required string database
+
+  3: required string table
+
+  // tablets to scan
+  4: required list<i64> tablet_ids
+
+  // base64 encoded binary plan fragment
+  5: required string opaqued_query_plan
+
+  // A string specified for the table that is passed to the external data source.
+  // Always set, may be an empty string.
+  6: optional i32 batch_size
+
+  // reserved params for use
+  7: optional map<string,string> properties
+
+  // The query limit, if specified.
+  8: optional i64 limit
+
+  // The authenticated user name. Always set.
+  // maybe usefullless
+  9: optional string user
+
+  10: optional string passwd
+    // max keep alive time min
+  11: optional i16 keep_alive_min
+
+  12: optional i32 query_timeout
+
+  // memory limit for a single query
+  13: optional i64 mem_limit
+}
+
+struct TScanColumnDesc {
+  // The column name
+  1: optional string name
+  // The column type. Always set.
+  2: optional Types.TPrimitiveType type
+}
+
+// Returned by open().
+struct TScanOpenResult {
+  1: required Status.TStatus status
+  // An opaque context_id used in subsequent getNext()/close() calls. Required.
+  2: optional string context_id
+  // selected fields
+  3: optional list<TScanColumnDesc> selected_columns
+
+}
+
+// Parameters to getNext()
+struct TScanNextBatchParams {
+  // The opaque handle returned by the previous open() call. Always set.
+  1: optional string context_id    // doris olap engine context id
+  2: optional i64 offset            // doris should check the offset to prevent duplicate rpc calls
+}
+
+// Returned by getNext().
+struct TScanBatchResult {
+  1: required Status.TStatus status
+
+  // If true, reached the end of the result stream; subsequent calls to
+  // getNext() won’t return any more results. Required.
+  2: optional bool eos
+
+  // A batch of rows of arrow format to return, if any exist. The number of rows in the batch
+  // should be less than or equal to the batch_size specified in TOpenParams.
+  3: optional binary rows
+}
+
+// Parameters to close()
+struct TScanCloseParams {
+  // The opaque handle returned by the previous open() call. Always set.
+  1: optional string context_id
+}
+
+// Returned by close().
+struct TScanCloseResult {
+  1: required Status.TStatus status
+}
+
+// scan service expose ability of scanning data ability to other compute system
+service TDorisExternalService {
+    // doris will build  a scan context for this session, context_id returned if success
+    TScanOpenResult open_scanner(1: TScanOpenParams params);
+
+    // return the batch_size of data
+    TScanBatchResult get_next(1: TScanNextBatchParams params);
+
+    // release the context resource associated with the context_id
+    TScanCloseResult close_scanner(1: TScanCloseParams params);
+}
diff --git a/src/main/thrift/doris/Status.thrift b/src/main/thrift/doris/Status.thrift
new file mode 100644
index 0000000..2966a8a
--- /dev/null
+++ b/src/main/thrift/doris/Status.thrift
@@ -0,0 +1,66 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+namespace cpp doris
+namespace java org.apache.doris.thrift
+
+enum TStatusCode {
+    OK,
+    CANCELLED,
+    ANALYSIS_ERROR,
+    NOT_IMPLEMENTED_ERROR,
+    RUNTIME_ERROR,
+    MEM_LIMIT_EXCEEDED,
+    INTERNAL_ERROR,
+    THRIFT_RPC_ERROR,
+    TIMEOUT,
+    KUDU_NOT_ENABLED, // Deprecated
+    KUDU_NOT_SUPPORTED_ON_OS, // Deprecated
+    MEM_ALLOC_FAILED,
+    BUFFER_ALLOCATION_FAILED,
+    MINIMUM_RESERVATION_UNAVAILABLE,
+    PUBLISH_TIMEOUT,
+    LABEL_ALREADY_EXISTS,
+    ES_INTERNAL_ERROR,
+    ES_INDEX_NOT_FOUND,
+    ES_SHARD_NOT_FOUND,
+    ES_INVALID_CONTEXTID,
+    ES_INVALID_OFFSET,
+    ES_REQUEST_ERROR,
+
+    // end of file
+    END_OF_FILE = 30,
+    NOT_FOUND = 31,
+    CORRUPTION = 32,
+    INVALID_ARGUMENT = 33,
+    IO_ERROR = 34,
+    ALREADY_EXIST = 35,
+    NETWORK_ERROR = 36,
+    ILLEGAL_STATE = 37,
+    NOT_AUTHORIZED = 38,
+    ABORTED = 39,
+    REMOTE_ERROR = 40,
+    SERVICE_UNAVAILABLE = 41,
+    UNINITIALIZED = 42,
+    CONFIGURATION_ERROR = 43,
+    INCOMPLETE = 44
+}
+
+struct TStatus {
+  1: required TStatusCode status_code
+  2: optional list<string> error_msgs
+}
diff --git a/src/main/thrift/doris/Types.thrift b/src/main/thrift/doris/Types.thrift
new file mode 100644
index 0000000..2d902ba
--- /dev/null
+++ b/src/main/thrift/doris/Types.thrift
@@ -0,0 +1,370 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+namespace cpp doris
+namespace java org.apache.doris.thrift
+
+
+typedef i64 TTimestamp
+typedef i32 TPlanNodeId
+typedef i32 TTupleId
+typedef i32 TSlotId
+typedef i64 TTableId
+typedef i64 TTabletId
+typedef i64 TVersion
+typedef i64 TVersionHash
+typedef i32 TSchemaHash
+typedef i32 TPort
+typedef i64 TCount
+typedef i64 TSize
+typedef i32 TClusterId
+typedef i64 TEpoch
+
+// add for real time load, partitionid is not defined previously, define it here
+typedef i64 TTransactionId
+typedef i64 TPartitionId
+
+enum TStorageType {
+    ROW,
+    COLUMN,
+}
+
+enum TStorageMedium {
+    HDD,
+    SSD,
+}
+
+enum TVarType {
+    SESSION,
+    GLOBAL
+}
+
+enum TPrimitiveType {
+  INVALID_TYPE,
+  NULL_TYPE,
+  BOOLEAN,
+  TINYINT,
+  SMALLINT,
+  INT,
+  BIGINT,
+  FLOAT,
+  DOUBLE,
+  DATE,
+  DATETIME,
+  BINARY,
+  DECIMAL,
+  // CHAR(n). Currently only supported in UDAs
+  CHAR,
+  LARGEINT,
+  VARCHAR,
+  HLL,
+  DECIMALV2,
+  TIME
+}
+
+enum TTypeNodeType {
+    SCALAR,
+    ARRAY,
+    MAP,
+    STRUCT
+}
+
+struct TScalarType {
+    1: required TPrimitiveType type
+
+    // Only set if type == CHAR or type == VARCHAR
+    2: optional i32 len
+
+    // Only set for DECIMAL
+    3: optional i32 precision
+    4: optional i32 scale
+}
+
+// Represents a field in a STRUCT type.
+// TODO: Model column stats for struct fields.
+struct TStructField {
+    1: required string name
+    2: optional string comment
+}
+
+struct TTypeNode {
+    1: required TTypeNodeType type
+
+    // only set for scalar types
+    2: optional TScalarType scalar_type
+
+    // only used for structs; has struct_fields.size() corresponding child types
+    3: optional list<TStructField> struct_fields
+}
+
+// A flattened representation of a tree of column types obtained by depth-first
+// traversal. Complex types such as map, array and struct have child types corresponding
+// to the map key/value, array item type, and struct fields, respectively.
+// For scalar types the list contains only a single node.
+// Note: We cannot rename this to TType because it conflicts with Thrift's internal TType
+// and the generated Python thrift files will not work.
+// Note: TTypeDesc in impala is TColumnType, but we already use TColumnType, so we name this
+// to TTypeDesc. In future, we merge these two to one
+struct TTypeDesc {
+    1: list<TTypeNode> types
+}
+
+enum TAggregationType {
+    SUM,
+    MAX,
+    MIN,
+    REPLACE,
+    HLL_UNION,
+    NONE
+}
+
+enum TPushType {
+    LOAD,
+    DELETE,
+    LOAD_DELETE
+}
+
+enum TTaskType {
+    CREATE,
+    DROP,
+    PUSH,
+    CLONE,
+    STORAGE_MEDIUM_MIGRATE,
+    ROLLUP,
+    SCHEMA_CHANGE,
+    CANCEL_DELETE,  // Deprecated
+    MAKE_SNAPSHOT,
+    RELEASE_SNAPSHOT,
+    CHECK_CONSISTENCY,
+    UPLOAD,
+    DOWNLOAD,
+    CLEAR_REMOTE_FILE,
+    MOVE
+    REALTIME_PUSH,
+    PUBLISH_VERSION,
+    CLEAR_ALTER_TASK,
+    CLEAR_TRANSACTION_TASK,
+    RECOVER_TABLET,
+    STREAM_LOAD,
+    UPDATE_TABLET_META_INFO,
+    ALTER_TASK
+}
+
+enum TStmtType {
+  QUERY,
+  DDL,  // Data definition, e.g. CREATE TABLE (includes read-only functions e.g. SHOW)
+  DML,  // Data modification e.g. INSERT
+  EXPLAIN   // EXPLAIN 
+}
+
+// level of verboseness for "explain" output
+// TODO: should this go somewhere else?
+enum TExplainLevel {
+  NORMAL,
+  VERBOSE
+}
+
+struct TColumnType {
+  1: required TPrimitiveType type
+  // Only set if type == CHAR_ARRAY
+  2: optional i32 len
+  3: optional i32 index_len
+  4: optional i32 precision
+  5: optional i32 scale
+}
+
+// A TNetworkAddress is the standard host, port representation of a
+// network address. The hostname field must be resolvable to an IPv4
+// address.
+struct TNetworkAddress {
+  1: required string hostname
+  2: required i32 port
+}
+
+// Wire format for UniqueId
+struct TUniqueId {
+  1: required i64 hi
+  2: required i64 lo
+}
+
+enum QueryState {
+  CREATED,
+  INITIALIZED,
+  COMPILED,
+  RUNNING,
+  FINISHED,
+  EXCEPTION
+}
+
+enum TFunctionType {
+  SCALAR,
+  AGGREGATE,
+}
+
+enum TFunctionBinaryType {
+  // Palo builtin. We can either run this interpreted or via codegen
+  // depending on the query option.
+  BUILTIN,
+
+  // Hive UDFs, loaded from *.jar
+  HIVE,
+
+  // Native-interface, precompiled UDFs loaded from *.so
+  NATIVE,
+
+  // Native-interface, precompiled to IR; loaded from *.ll
+  IR,
+}
+
+// Represents a fully qualified function name.
+struct TFunctionName {
+  // Name of the function's parent database. Not set if in global
+  // namespace (e.g. builtins)
+  1: optional string db_name
+
+  // Name of the function
+  2: required string function_name
+}
+
+struct TScalarFunction {
+    // Symbol for the function
+    1: required string symbol
+    2: optional string prepare_fn_symbol
+    3: optional string close_fn_symbol
+}
+
+struct TAggregateFunction {
+  1: required TTypeDesc intermediate_type
+  2: optional string update_fn_symbol
+  3: optional string init_fn_symbol
+  4: optional string serialize_fn_symbol
+  5: optional string merge_fn_symbol
+  6: optional string finalize_fn_symbol
+  8: optional string get_value_fn_symbol
+  9: optional string remove_fn_symbol
+  10: optional bool is_analytic_only_fn = false
+}
+
+// Represents a function in the Catalog.
+struct TFunction {
+  // Fully qualified function name.
+  1: required TFunctionName name
+
+  // Type of the udf. e.g. hive, native, ir
+  2: required TFunctionBinaryType binary_type
+
+  // The types of the arguments to the function
+  3: required list<TTypeDesc> arg_types
+
+  // Return type for the function.
+  4: required TTypeDesc ret_type
+
+  // If true, this function takes var args.
+  5: required bool has_var_args
+
+  // Optional comment to attach to the function
+  6: optional string comment
+
+  7: optional string signature
+
+  // HDFS path for the function binary. This binary must exist at the time the
+  // function is created.
+  8: optional string hdfs_location
+
+  // One of these should be set.
+  9: optional TScalarFunction scalar_fn
+  10: optional TAggregateFunction aggregate_fn
+
+  11: optional i64 id
+  12: optional string checksum
+}
+
+enum TLoadJobState {
+    PENDING,
+    ETL,
+    LOADING,
+    FINISHED,
+    CANCELLED
+}   
+
+enum TEtlState {
+	RUNNING,
+	FINISHED,
+	CANCELLED,
+    UNKNOWN
+}
+
+enum TTableType {
+    MYSQL_TABLE,
+    OLAP_TABLE,
+    SCHEMA_TABLE,
+    KUDU_TABLE, // Deprecated
+    BROKER_TABLE,
+    ES_TABLE
+}
+
+enum TKeysType {
+    PRIMARY_KEYS,
+    DUP_KEYS,
+    UNIQUE_KEYS,
+    AGG_KEYS
+}
+
+enum TPriority {
+    NORMAL,
+    HIGH
+}
+
+struct TBackend {
+    1: required string host
+    2: required TPort be_port
+    3: required TPort http_port
+}
+
+struct TResourceInfo {
+    1: required string user
+    2: required string group
+}
+
+enum TExportState {
+    RUNNING,
+    FINISHED,
+    CANCELLED,
+    UNKNOWN
+}
+
+enum TFileType {
+    FILE_LOCAL,
+    FILE_BROKER,
+    FILE_STREAM,    // file content is streaming in the buffer
+}
+
+struct TTabletCommitInfo {
+    1: required i64 tabletId
+    2: required i64 backendId
+}
+
+enum TLoadType {
+    MANUL_LOAD,
+    ROUTINE_LOAD,
+    MINI_LOAD
+}
+
+enum TLoadSourceType {
+    RAW,
+    KAFKA,
+}
diff --git a/src/test/java/org/apache/doris/flink/DorisSinkExample.java b/src/test/java/org/apache/doris/flink/DorisSinkExample.java
new file mode 100644
index 0000000..dde97de
--- /dev/null
+++ b/src/test/java/org/apache/doris/flink/DorisSinkExample.java
@@ -0,0 +1,56 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.doris.flink;
+
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.streaming.api.datastream.DataStreamSource;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.flink.table.api.Expressions.$;
+
+public class DorisSinkExample {
+
+    public static void main(String[] args) {
+        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+        env.setParallelism(1);
+        final StreamTableEnvironment tEnv = StreamTableEnvironment.create(env);
+
+        List<Tuple2<String, Integer>> data = new ArrayList<>();
+        data.add(new Tuple2<>("doris",1));
+        DataStreamSource<Tuple2<String, Integer>> source = env.fromCollection(data);
+        tEnv.createTemporaryView("doris_test",source,$("name"),$("age"));
+
+        tEnv.executeSql(
+                "CREATE TABLE doris_test_sink (" +
+                        "name STRING," +
+                        "age INT" +
+                        ") " +
+                        "WITH (\n" +
+                        "  'connector' = 'doris',\n" +
+                        "  'fenodes' = 'FE_IP:8030',\n" +
+                        "  'table.identifier' = 'db.table',\n" +
+                        "  'username' = 'root',\n" +
+                        "  'password' = ''\n" +
+                        ")");
+
+        tEnv.executeSql("INSERT INTO doris_test_sink select name,age from doris_test");
+    }
+}
diff --git a/src/test/java/org/apache/doris/flink/DorisSourceDataStream.java b/src/test/java/org/apache/doris/flink/DorisSourceDataStream.java
new file mode 100644
index 0000000..6815234
--- /dev/null
+++ b/src/test/java/org/apache/doris/flink/DorisSourceDataStream.java
@@ -0,0 +1,46 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink;
+
+import org.apache.doris.flink.cfg.DorisStreamOptions;
+import org.apache.doris.flink.datastream.DorisSourceFunction;
+import org.apache.doris.flink.deserialization.SimpleListDeserializationSchema;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+
+import java.util.Properties;
+
+
+
+public class DorisSourceDataStream {
+
+    public static void main(String[] args) throws Exception {
+        Properties properties = new Properties();
+        properties.put("fenodes","FE_IP:8030");
+        properties.put("username","root");
+        properties.put("password","");
+        properties.put("table.identifier","db.table");
+        properties.put("doris.read.field","id,code,name");
+        properties.put("doris.filter.query","name='doris'");
+        DorisStreamOptions  options = new DorisStreamOptions(properties);
+
+        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+        env.setParallelism(2);
+        env.addSource(new DorisSourceFunction(options,new SimpleListDeserializationSchema())).print();
+        env.execute("Flink doris test");
+    }
+}
diff --git a/src/test/java/org/apache/doris/flink/DorisSourceExample.java b/src/test/java/org/apache/doris/flink/DorisSourceExample.java
new file mode 100644
index 0000000..eb1d819
--- /dev/null
+++ b/src/test/java/org/apache/doris/flink/DorisSourceExample.java
@@ -0,0 +1,64 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.doris.flink;
+
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.table.api.Table;
+import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
+import org.apache.flink.types.Row;
+
+public class DorisSourceExample {
+
+    public static void main(String[] args) throws Exception {
+
+        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+        env.setParallelism(1); // source only supports parallelism of 1
+
+        final StreamTableEnvironment tEnv = StreamTableEnvironment.create(env);
+
+        // register a table in the catalog
+        tEnv.executeSql(
+                "CREATE TABLE doris_source (" +
+                        "bigint_1 BIGINT," +
+                        "char_1 STRING," +
+                        "date_1 STRING," +
+                        "datetime_1 STRING," +
+                        "decimal_1 DECIMAL(5,2)," +
+                        "double_1 DOUBLE," +
+                        "float_1 FLOAT ," +
+                        "int_1 INT ," +
+                        "largeint_1 STRING, " +
+                        "smallint_1 SMALLINT, " +
+                        "tinyint_1 TINYINT, " +
+                        "varchar_1 STRING " +
+                        ") " +
+                        "WITH (\n" +
+                        "  'connector' = 'doris',\n" +
+                        "  'fenodes' = 'FE_IP:8030',\n" +
+                        "  'table.identifier' = 'db.table',\n" +
+                        "  'username' = 'root',\n" +
+                        "  'password' = ''\n" +
+                        ")");
+
+        // define a dynamic aggregating query
+        final Table result = tEnv.sqlQuery("SELECT * from doris_source  ");
+
+        // print the result to the console
+        tEnv.toRetractStream(result, Row.class).print();
+        env.execute();
+    }
+}
diff --git a/src/test/java/org/apache/doris/flink/DorisSourceSinkExample.java b/src/test/java/org/apache/doris/flink/DorisSourceSinkExample.java
new file mode 100644
index 0000000..c4ce1a5
--- /dev/null
+++ b/src/test/java/org/apache/doris/flink/DorisSourceSinkExample.java
@@ -0,0 +1,64 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink;
+
+import org.apache.flink.table.api.EnvironmentSettings;
+import org.apache.flink.table.api.TableEnvironment;
+
+public class DorisSourceSinkExample {
+
+    public static void main(String[] args) {
+        EnvironmentSettings settings = EnvironmentSettings.newInstance()
+                .useBlinkPlanner()
+                .inStreamingMode()
+                .build();
+        TableEnvironment tEnv = TableEnvironment.create(settings);
+        tEnv.executeSql(
+                "CREATE TABLE doris_test (" +
+                        "name STRING," +
+                        "age INT," +
+                        "price DECIMAL(5,2)," +
+                        "sale DOUBLE" +
+                        ") " +
+                        "WITH (\n" +
+                        "  'connector' = 'doris',\n" +
+                        "  'fenodes' = 'FE_IP:8030',\n" +
+                        "  'table.identifier' = 'db.table',\n" +
+                        "  'username' = 'root',\n" +
+                        "  'password' = ''" +
+                        ")");
+        tEnv.executeSql(
+                "CREATE TABLE doris_test_sink (" +
+                        "name STRING," +
+                        "age INT," +
+                        "price DECIMAL(5,2)," +
+                        "sale DOUBLE" +
+                        ") " +
+                        "WITH (\n" +
+                        "  'connector' = 'doris',\n" +
+                        "  'fenodes' = 'FE_IP:8030',\n" +
+                        "  'table.identifier' = 'db.table',\n" +
+                        "  'username' = 'root',\n" +
+                        "  'password' = '',\n" +
+                        "  'sink.batch.size' = '3',\n" +
+                        "  'sink.max-retries' = '2'\n" +
+                        ")");
+
+        tEnv.executeSql("INSERT INTO doris_test_sink select name,age,price,sale from doris_test");
+    }
+}
diff --git a/src/test/java/org/apache/doris/flink/serialization/TestRowBatch.java b/src/test/java/org/apache/doris/flink/serialization/TestRowBatch.java
new file mode 100644
index 0000000..a9d2492
--- /dev/null
+++ b/src/test/java/org/apache/doris/flink/serialization/TestRowBatch.java
@@ -0,0 +1,439 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.serialization;
+
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.BigIntVector;
+import org.apache.arrow.vector.BitVector;
+import org.apache.arrow.vector.DecimalVector;
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.Float4Vector;
+import org.apache.arrow.vector.Float8Vector;
+import org.apache.arrow.vector.IntVector;
+import org.apache.arrow.vector.SmallIntVector;
+import org.apache.arrow.vector.TinyIntVector;
+import org.apache.arrow.vector.VarBinaryVector;
+import org.apache.arrow.vector.VarCharVector;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.dictionary.DictionaryProvider;
+import org.apache.arrow.vector.ipc.ArrowStreamWriter;
+import org.apache.arrow.vector.types.FloatingPointPrecision;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.doris.flink.rest.RestService;
+import org.apache.doris.flink.rest.models.Schema;
+import org.apache.doris.thrift.TScanBatchResult;
+import org.apache.doris.thrift.TStatus;
+import org.apache.doris.thrift.TStatusCode;
+import org.apache.flink.shaded.guava18.com.google.common.collect.ImmutableList;
+import org.apache.flink.shaded.guava18.com.google.common.collect.Lists;
+import org.apache.flink.table.data.DecimalData;
+import org.apache.flink.table.data.StringData;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayOutputStream;
+import java.math.BigDecimal;
+import java.util.Arrays;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+import static org.hamcrest.core.StringStartsWith.startsWith;
+
+public class TestRowBatch {
+    private static Logger logger = LoggerFactory.getLogger(TestRowBatch.class);
+
+    @Rule
+    public ExpectedException thrown = ExpectedException.none();
+
+    @Test
+    public void testRowBatch() throws Exception {
+        // schema
+        ImmutableList.Builder<Field> childrenBuilder = ImmutableList.builder();
+        childrenBuilder.add(new Field("k0", FieldType.nullable(new ArrowType.Bool()), null));
+        childrenBuilder.add(new Field("k1", FieldType.nullable(new ArrowType.Int(8, true)), null));
+        childrenBuilder.add(new Field("k2", FieldType.nullable(new ArrowType.Int(16, true)), null));
+        childrenBuilder.add(new Field("k3", FieldType.nullable(new ArrowType.Int(32, true)), null));
+        childrenBuilder.add(new Field("k4", FieldType.nullable(new ArrowType.Int(64, true)), null));
+        childrenBuilder.add(new Field("k9", FieldType.nullable(new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)), null));
+        childrenBuilder.add(new Field("k8", FieldType.nullable(new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), null));
+        childrenBuilder.add(new Field("k10", FieldType.nullable(new ArrowType.Utf8()), null));
+        childrenBuilder.add(new Field("k11", FieldType.nullable(new ArrowType.Utf8()), null));
+        childrenBuilder.add(new Field("k5", FieldType.nullable(new ArrowType.Decimal(9,2)), null));
+        childrenBuilder.add(new Field("k6", FieldType.nullable(new ArrowType.Utf8()), null));
+
+        VectorSchemaRoot root = VectorSchemaRoot.create(
+                new org.apache.arrow.vector.types.pojo.Schema(childrenBuilder.build(), null),
+                new RootAllocator(Integer.MAX_VALUE));
+        ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+        ArrowStreamWriter arrowStreamWriter = new ArrowStreamWriter(
+                root,
+                new DictionaryProvider.MapDictionaryProvider(),
+                outputStream);
+
+        arrowStreamWriter.start();
+        root.setRowCount(3);
+
+        FieldVector vector = root.getVector("k0");
+        BitVector bitVector = (BitVector)vector;
+        bitVector.setInitialCapacity(3);
+        bitVector.allocateNew(3);
+        bitVector.setSafe(0, 1);
+        bitVector.setSafe(1, 0);
+        bitVector.setSafe(2, 1);
+        vector.setValueCount(3);
+
+        vector = root.getVector("k1");
+        TinyIntVector tinyIntVector = (TinyIntVector)vector;
+        tinyIntVector.setInitialCapacity(3);
+        tinyIntVector.allocateNew(3);
+        tinyIntVector.setSafe(0, 1);
+        tinyIntVector.setSafe(1, 2);
+        tinyIntVector.setSafe(2, 3);
+        vector.setValueCount(3);
+
+        vector = root.getVector("k2");
+        SmallIntVector smallIntVector = (SmallIntVector)vector;
+        smallIntVector.setInitialCapacity(3);
+        smallIntVector.allocateNew(3);
+        smallIntVector.setSafe(0, 1);
+        smallIntVector.setSafe(1, 2);
+        smallIntVector.setSafe(2, 3);
+        vector.setValueCount(3);
+
+        vector = root.getVector("k3");
+        IntVector intVector = (IntVector)vector;
+        intVector.setInitialCapacity(3);
+        intVector.allocateNew(3);
+        intVector.setSafe(0, 1);
+        intVector.setNull(1);
+        intVector.setSafe(2, 3);
+        vector.setValueCount(3);
+
+        vector = root.getVector("k4");
+        BigIntVector bigIntVector = (BigIntVector)vector;
+        bigIntVector.setInitialCapacity(3);
+        bigIntVector.allocateNew(3);
+        bigIntVector.setSafe(0, 1);
+        bigIntVector.setSafe(1, 2);
+        bigIntVector.setSafe(2, 3);
+        vector.setValueCount(3);
+
+        vector = root.getVector("k5");
+        DecimalVector decimalVector = (DecimalVector)vector;
+        decimalVector.setInitialCapacity(3);
+        decimalVector.allocateNew();
+        decimalVector.setIndexDefined(0);
+        decimalVector.setSafe(0, new BigDecimal("12.34"));
+        decimalVector.setIndexDefined(1);
+        decimalVector.setSafe(1, new BigDecimal("88.88"));
+        decimalVector.setIndexDefined(2);
+        decimalVector.setSafe(2, new BigDecimal("10.22"));
+        vector.setValueCount(3);
+
+        vector = root.getVector("k6");
+        VarCharVector charVector = (VarCharVector)vector;
+        charVector.setInitialCapacity(3);
+        charVector.allocateNew();
+        charVector.setIndexDefined(0);
+        charVector.setValueLengthSafe(0, 5);
+        charVector.setSafe(0, "char1".getBytes());
+        charVector.setIndexDefined(1);
+        charVector.setValueLengthSafe(1, 5);
+        charVector.setSafe(1, "char2".getBytes());
+        charVector.setIndexDefined(2);
+        charVector.setValueLengthSafe(2, 5);
+        charVector.setSafe(2, "char3".getBytes());
+        vector.setValueCount(3);
+
+        vector = root.getVector("k8");
+        Float8Vector float8Vector = (Float8Vector)vector;
+        float8Vector.setInitialCapacity(3);
+        float8Vector.allocateNew(3);
+        float8Vector.setSafe(0, 1.1);
+        float8Vector.setSafe(1, 2.2);
+        float8Vector.setSafe(2, 3.3);
+        vector.setValueCount(3);
+
+        vector = root.getVector("k9");
+        Float4Vector float4Vector = (Float4Vector)vector;
+        float4Vector.setInitialCapacity(3);
+        float4Vector.allocateNew(3);
+        float4Vector.setSafe(0, 1.1f);
+        float4Vector.setSafe(1, 2.2f);
+        float4Vector.setSafe(2, 3.3f);
+        vector.setValueCount(3);
+
+        vector = root.getVector("k10");
+        VarCharVector datecharVector = (VarCharVector)vector;
+        datecharVector.setInitialCapacity(3);
+        datecharVector.allocateNew();
+        datecharVector.setIndexDefined(0);
+        datecharVector.setValueLengthSafe(0, 5);
+        datecharVector.setSafe(0, "2008-08-08".getBytes());
+        datecharVector.setIndexDefined(1);
+        datecharVector.setValueLengthSafe(1, 5);
+        datecharVector.setSafe(1, "1900-08-08".getBytes());
+        datecharVector.setIndexDefined(2);
+        datecharVector.setValueLengthSafe(2, 5);
+        datecharVector.setSafe(2, "2100-08-08".getBytes());
+        vector.setValueCount(3);
+
+        vector = root.getVector("k11");
+        VarCharVector timecharVector = (VarCharVector)vector;
+        timecharVector.setInitialCapacity(3);
+        timecharVector.allocateNew();
+        timecharVector.setIndexDefined(0);
+        timecharVector.setValueLengthSafe(0, 5);
+        timecharVector.setSafe(0, "2008-08-08 00:00:00".getBytes());
+        timecharVector.setIndexDefined(1);
+        timecharVector.setValueLengthSafe(1, 5);
+        timecharVector.setSafe(1, "1900-08-08 00:00:00".getBytes());
+        timecharVector.setIndexDefined(2);
+        timecharVector.setValueLengthSafe(2, 5);
+        timecharVector.setSafe(2, "2100-08-08 00:00:00".getBytes());
+        vector.setValueCount(3);
+
+        arrowStreamWriter.writeBatch();
+
+        arrowStreamWriter.end();
+        arrowStreamWriter.close();
+
+        TStatus status = new TStatus();
+        status.setStatus_code(TStatusCode.OK);
+        TScanBatchResult scanBatchResult = new TScanBatchResult();
+        scanBatchResult.setStatus(status);
+        scanBatchResult.setEos(false);
+        scanBatchResult.setRows(outputStream.toByteArray());
+
+        String schemaStr = "{\"properties\":[{\"type\":\"BOOLEAN\",\"name\":\"k0\",\"comment\":\"\"},"
+                + "{\"type\":\"TINYINT\",\"name\":\"k1\",\"comment\":\"\"},{\"type\":\"SMALLINT\",\"name\":\"k2\","
+                + "\"comment\":\"\"},{\"type\":\"INT\",\"name\":\"k3\",\"comment\":\"\"},{\"type\":\"BIGINT\","
+                + "\"name\":\"k4\",\"comment\":\"\"},{\"type\":\"FLOAT\",\"name\":\"k9\",\"comment\":\"\"},"
+                + "{\"type\":\"DOUBLE\",\"name\":\"k8\",\"comment\":\"\"},{\"type\":\"DATE\",\"name\":\"k10\","
+                + "\"comment\":\"\"},{\"type\":\"DATETIME\",\"name\":\"k11\",\"comment\":\"\"},"
+                + "{\"name\":\"k5\",\"scale\":\"9\",\"comment\":\"\","
+                + "\"type\":\"DECIMAL\",\"precision\":\"2\"},{\"type\":\"CHAR\",\"name\":\"k6\",\"comment\":\"\"}],"
+                + "\"status\":200}";
+
+        Schema schema = RestService.parseSchema(schemaStr, logger);
+
+        RowBatch rowBatch = new RowBatch(scanBatchResult, schema).readArrow();
+
+        List<Object> expectedRow1 = Lists.newArrayList(
+                Boolean.TRUE,
+                (byte) 1,
+                (short) 1,
+                1,
+                1L,
+                (float) 1.1,
+                (double) 1.1,
+                StringData.fromString("2008-08-08"),
+                StringData.fromString("2008-08-08 00:00:00"),
+                DecimalData.fromBigDecimal(new BigDecimal(12.34), 4, 2),
+                StringData.fromString("char1")
+        );
+
+        List<Object> expectedRow2 = Arrays.asList(
+                Boolean.FALSE,
+                (byte) 2,
+                (short) 2,
+                null,
+                2L,
+                (float) 2.2,
+                (double) 2.2,
+                StringData.fromString("1900-08-08"),
+                StringData.fromString("1900-08-08 00:00:00"),
+                DecimalData.fromBigDecimal(new BigDecimal(88.88), 4, 2),
+                StringData.fromString("char2")
+        );
+
+        List<Object> expectedRow3 = Arrays.asList(
+                Boolean.TRUE,
+                (byte) 3,
+                (short) 3,
+                3,
+                3L,
+                (float) 3.3,
+                (double) 3.3,
+                StringData.fromString("2100-08-08"),
+                StringData.fromString("2100-08-08 00:00:00"),
+                DecimalData.fromBigDecimal(new BigDecimal(10.22), 4, 2),
+                StringData.fromString("char3")
+        );
+
+        Assert.assertTrue(rowBatch.hasNext());
+        List<Object> actualRow1 = rowBatch.next();
+        Assert.assertEquals(expectedRow1, actualRow1);
+
+        Assert.assertTrue(rowBatch.hasNext());
+        List<Object> actualRow2 = rowBatch.next();
+        Assert.assertEquals(expectedRow2, actualRow2);
+
+        Assert.assertTrue(rowBatch.hasNext());
+        List<Object> actualRow3 = rowBatch.next();
+        Assert.assertEquals(expectedRow3, actualRow3);
+
+        Assert.assertFalse(rowBatch.hasNext());
+        thrown.expect(NoSuchElementException.class);
+        thrown.expectMessage(startsWith("Get row offset:"));
+        rowBatch.next();
+    }
+
+    @Test
+    public void testBinary() throws Exception {
+        byte[] binaryRow0 = {'a', 'b', 'c'};
+        byte[] binaryRow1 = {'d', 'e', 'f'};
+        byte[] binaryRow2 = {'g', 'h', 'i'};
+
+        ImmutableList.Builder<Field> childrenBuilder = ImmutableList.builder();
+        childrenBuilder.add(new Field("k7", FieldType.nullable(new ArrowType.Binary()), null));
+
+        VectorSchemaRoot root = VectorSchemaRoot.create(
+                new org.apache.arrow.vector.types.pojo.Schema(childrenBuilder.build(), null),
+                new RootAllocator(Integer.MAX_VALUE));
+        ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+        ArrowStreamWriter arrowStreamWriter = new ArrowStreamWriter(
+                root,
+                new DictionaryProvider.MapDictionaryProvider(),
+                outputStream);
+
+        arrowStreamWriter.start();
+        root.setRowCount(3);
+
+        FieldVector vector = root.getVector("k7");
+        VarBinaryVector varBinaryVector = (VarBinaryVector) vector;
+        varBinaryVector.setInitialCapacity(3);
+        varBinaryVector.allocateNew();
+        varBinaryVector.setIndexDefined(0);
+        varBinaryVector.setValueLengthSafe(0, 3);
+        varBinaryVector.setSafe(0, binaryRow0);
+        varBinaryVector.setIndexDefined(1);
+        varBinaryVector.setValueLengthSafe(1, 3);
+        varBinaryVector.setSafe(1, binaryRow1);
+        varBinaryVector.setIndexDefined(2);
+        varBinaryVector.setValueLengthSafe(2, 3);
+        varBinaryVector.setSafe(2, binaryRow2);
+        vector.setValueCount(3);
+
+        arrowStreamWriter.writeBatch();
+
+        arrowStreamWriter.end();
+        arrowStreamWriter.close();
+
+        TStatus status = new TStatus();
+        status.setStatus_code(TStatusCode.OK);
+        TScanBatchResult scanBatchResult = new TScanBatchResult();
+        scanBatchResult.setStatus(status);
+        scanBatchResult.setEos(false);
+        scanBatchResult.setRows(outputStream.toByteArray());
+
+        String schemaStr = "{\"properties\":[{\"type\":\"BINARY\",\"name\":\"k7\",\"comment\":\"\"}], \"status\":200}";
+
+        Schema schema = RestService.parseSchema(schemaStr, logger);
+
+        RowBatch rowBatch = new RowBatch(scanBatchResult, schema).readArrow();
+
+        Assert.assertTrue(rowBatch.hasNext());
+        List<Object> actualRow0 = rowBatch.next();
+        Assert.assertArrayEquals(binaryRow0, (byte[])actualRow0.get(0));
+
+        Assert.assertTrue(rowBatch.hasNext());
+        List<Object> actualRow1 = rowBatch.next();
+        Assert.assertArrayEquals(binaryRow1, (byte[])actualRow1.get(0));
+
+        Assert.assertTrue(rowBatch.hasNext());
+        List<Object> actualRow2 = rowBatch.next();
+        Assert.assertArrayEquals(binaryRow2, (byte[])actualRow2.get(0));
+
+        Assert.assertFalse(rowBatch.hasNext());
+        thrown.expect(NoSuchElementException.class);
+        thrown.expectMessage(startsWith("Get row offset:"));
+        rowBatch.next();
+    }
+
+    @Test
+    public void testDecimalV2() throws Exception {
+        ImmutableList.Builder<Field> childrenBuilder = ImmutableList.builder();
+        childrenBuilder.add(new Field("k7", FieldType.nullable(new ArrowType.Decimal(27, 9)), null));
+
+        VectorSchemaRoot root = VectorSchemaRoot.create(
+                new org.apache.arrow.vector.types.pojo.Schema(childrenBuilder.build(), null),
+                new RootAllocator(Integer.MAX_VALUE));
+        ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+        ArrowStreamWriter arrowStreamWriter = new ArrowStreamWriter(
+                root,
+                new DictionaryProvider.MapDictionaryProvider(),
+                outputStream);
+
+        arrowStreamWriter.start();
+        root.setRowCount(3);
+
+        FieldVector vector = root.getVector("k7");
+        DecimalVector decimalVector = (DecimalVector) vector;
+        decimalVector.setInitialCapacity(3);
+        decimalVector.allocateNew(3);
+        decimalVector.setSafe(0, new BigDecimal("12.340000000"));
+        decimalVector.setSafe(1, new BigDecimal("88.880000000"));
+        decimalVector.setSafe(2, new BigDecimal("10.000000000"));
+        vector.setValueCount(3);
+
+        arrowStreamWriter.writeBatch();
+
+        arrowStreamWriter.end();
+        arrowStreamWriter.close();
+
+        TStatus status = new TStatus();
+        status.setStatus_code(TStatusCode.OK);
+        TScanBatchResult scanBatchResult = new TScanBatchResult();
+        scanBatchResult.setStatus(status);
+        scanBatchResult.setEos(false);
+        scanBatchResult.setRows(outputStream.toByteArray());
+
+        String schemaStr = "{\"properties\":[{\"type\":\"DECIMALV2\",\"scale\": 0,"
+                + "\"precision\": 9, \"name\":\"k7\",\"comment\":\"\"}], "
+                + "\"status\":200}";
+
+        Schema schema = RestService.parseSchema(schemaStr, logger);
+
+        RowBatch rowBatch = new RowBatch(scanBatchResult, schema).readArrow();
+
+        Assert.assertTrue(rowBatch.hasNext());
+        List<Object> actualRow0 = rowBatch.next();
+        Assert.assertEquals(DecimalData.fromBigDecimal(new BigDecimal(12.340000000), 11, 9), actualRow0.get(0));
+
+        Assert.assertTrue(rowBatch.hasNext());
+        List<Object> actualRow1 = rowBatch.next();
+
+        Assert.assertEquals(DecimalData.fromBigDecimal(new BigDecimal(88.880000000), 11, 9),  actualRow1.get(0));
+
+        Assert.assertTrue(rowBatch.hasNext());
+        List<Object> actualRow2 = rowBatch.next();
+        Assert.assertEquals(DecimalData.fromBigDecimal(new BigDecimal(10.000000000),11, 9), actualRow2.get(0));
+
+        Assert.assertFalse(rowBatch.hasNext());
+        thrown.expect(NoSuchElementException.class);
+        thrown.expectMessage(startsWith("Get row offset:"));
+        rowBatch.next();
+    }
+}

---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org