You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@rocketmq.apache.org by sh...@apache.org on 2024/12/11 01:58:49 UTC
(rocketmq-connect) branch master updated: [Improve]Improve doris connector (#539)
This is an automated email from the ASF dual-hosted git repository.
shenlin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
The following commit(s) were added to refs/heads/master by this push:
new 85da68c9 [Improve]Improve doris connector (#539)
85da68c9 is described below
commit 85da68c92f599f0a289c0e63de919ca417289f69
Author: wudongliang <46...@users.noreply.github.com>
AuthorDate: Wed Dec 11 09:58:45 2024 +0800
[Improve]Improve doris connector (#539)
* [Improve]Improve doris connector
---
connectors/rocketmq-connect-doris/pom.xml | 68 ++-
.../rocketmq/connect/doris/DorisSinkConnector.java | 66 +++
.../rocketmq/connect/doris/DorisSinkTask.java | 66 +++
.../rocketmq/connect/doris/cfg/DorisOptions.java | 387 ++++++++++++++
.../doris/cfg/DorisSinkConnectorConfig.java | 134 +++++
.../connect/doris/config/AbstractConfig.java | 132 -----
.../doris/connection/ConnectionProvider.java | 35 ++
.../doris/connection/JdbcConnectionProvider.java | 120 +++++
.../connect/doris/connector/DorisSinkConfig.java | 331 ------------
.../doris/connector/DorisSinkConnector.java | 74 ---
.../connect/doris/connector/DorisSinkTask.java | 107 ----
.../connect/doris/converter/ConverterMode.java | 45 ++
.../connect/doris/converter/RecordDescriptor.java | 258 +++++++++
.../connect/doris/converter/RecordService.java | 334 ++++++++++++
.../doris/converter/RecordTypeRegister.java | 122 +++++
.../converter/schema/SchemaChangeManager.java | 189 +++++++
.../converter/schema/SchemaEvolutionMode.java | 44 ++
.../doris/converter/type/AbstractDateType.java | 34 ++
.../doris/converter/type/AbstractGeometryType.java | 30 ++
.../doris/converter/type/AbstractTemporalType.java | 54 ++
.../doris/converter/type/AbstractTimeType.java | 51 ++
.../converter/type/AbstractTimestampType.java | 46 ++
.../connect/doris/converter/type/AbstractType.java | 70 +++
.../connect/doris/converter/type/Type.java | 58 ++
.../type/connect/AbstractConnectMapType.java | 42 ++
.../type/connect/AbstractConnectSchemaType.java | 25 +
.../converter/type/connect/ConnectBooleanType.java | 38 ++
.../converter/type/connect/ConnectBytesType.java | 70 +++
.../converter/type/connect/ConnectDateType.java | 49 ++
.../converter/type/connect/ConnectDecimalType.java | 55 ++
.../converter/type/connect/ConnectFloat32Type.java | 43 ++
.../converter/type/connect/ConnectFloat64Type.java | 43 ++
.../converter/type/connect/ConnectInt16Type.java | 43 ++
.../converter/type/connect/ConnectInt32Type.java | 43 ++
.../converter/type/connect/ConnectInt64Type.java | 43 ++
.../converter/type/connect/ConnectInt8Type.java | 43 ++
.../connect/ConnectMapToConnectStringType.java | 42 ++
.../converter/type/connect/ConnectStringType.java | 49 ++
.../converter/type/connect/ConnectTimeType.java | 57 ++
.../type/connect/ConnectTimestampType.java | 50 ++
.../type/debezium/AbstractDebeziumTimeType.java | 45 ++
.../debezium/AbstractDebeziumTimestampType.java | 43 ++
.../doris/converter/type/debezium/ArrayType.java | 107 ++++
.../doris/converter/type/debezium/DateType.java | 50 ++
.../converter/type/debezium/GeographyType.java | 32 ++
.../converter/type/debezium/GeometryType.java | 47 ++
.../converter/type/debezium/MicroTimeType.java | 39 ++
.../type/debezium/MicroTimestampType.java | 39 ++
.../converter/type/debezium/NanoTimeType.java | 39 ++
.../converter/type/debezium/NanoTimestampType.java | 43 ++
.../doris/converter/type/debezium/PointType.java | 46 ++
.../doris/converter/type/debezium/TimeType.java | 39 ++
.../converter/type/debezium/TimestampType.java | 40 ++
.../type/debezium/VariableScaleDecimalType.java | 75 +++
.../converter/type/debezium/ZonedTimeType.java | 59 ++
.../type/debezium/ZonedTimestampType.java | 53 ++
.../doris/converter/type/doris/DorisType.java | 44 ++
.../converter/type/doris/DorisTypeProperties.java | 32 ++
.../doris/converter/type/util/DateTimeUtils.java | 121 +++++
.../doris/converter/type/util/GeoUtils.java | 68 +++
.../doris/converter/type/util/SchemaUtils.java | 59 ++
.../doris/exception/ArgumentsException.java | 38 ++
.../connect/doris/exception/ConfigException.java | 36 --
.../doris/exception/ConnectedFailedException.java | 35 ++
.../connect/doris/exception/CopyLoadException.java | 38 ++
.../doris/exception/DataConverterException.java | 47 ++
.../doris/exception/DataFormatException.java | 38 ++
.../connect/doris/exception/DorisException.java | 40 +-
.../doris/exception/SchemaChangeException.java | 49 ++
.../doris/exception/StreamLoadException.java | 47 ++
.../exception/TableAlterOrCreateException.java | 34 --
.../connect/doris/exception/UploadException.java | 38 ++
.../connect/doris/metrics/DorisConnectMonitor.java | 178 ++++++
.../connect/doris/metrics/MetricsJmxReporter.java | 105 ++++
.../connect/doris/metrics/MetricsUtil.java | 49 ++
.../rocketmq/connect/doris/model/BackendV2.java | 89 +++
.../rocketmq/connect/doris/model/BaseResponse.java | 42 ++
.../connect/doris/model/ColumnDescriptor.java | 91 ++++
.../rocketmq/connect/doris/model/CopyIntoResp.java | 43 ++
.../connect/doris/model/KafkaRespContent.java | 60 +++
.../connect/doris/model/LoadOperation.java | 32 ++
.../rocketmq/connect/doris/model/RespContent.java | 117 ++++
.../connect/doris/model/TableDescriptor.java | 101 ++++
.../rocketmq/connect/doris/model/doris/Field.java | 149 ++++++
.../rocketmq/connect/doris/model/doris/Schema.java | 110 ++++
.../doris/schema/column/ColumnDefAdjuster.java | 72 ---
.../doris/schema/column/ColumnDefinition.java | 337 ------------
.../connect/doris/schema/column/ColumnId.java | 105 ----
.../schema/column/parser/DateColumnParser.java | 61 ---
.../schema/column/parser/TimeColumnParser.java | 63 ---
.../column/parser/TimestampColumnParser.java | 54 --
.../connect/doris/schema/db/DbStructure.java | 50 --
.../doris/schema/table/TableDefinition.java | 63 ---
.../doris/schema/table/TableDefinitions.java | 79 ---
.../connect/doris/schema/table/TableId.java | 144 -----
.../doris/serialization/ByteArrayDeserializer.java | 25 -
.../doris/serialization/ByteArraySerializer.java | 24 -
.../serialization/ByteBufferDeserializer.java | 30 --
.../doris/serialization/ByteBufferSerializer.java | 41 --
.../connect/doris/serialization/Deserializer.java | 63 ---
.../doris/serialization/DoubleDeserializer.java | 38 --
.../doris/serialization/DoubleSerializer.java | 38 --
.../doris/serialization/FloatDeserializer.java | 38 --
.../doris/serialization/FloatSerializer.java | 34 --
.../doris/serialization/IntegerDeserializer.java | 38 --
.../doris/serialization/IntegerSerializer.java | 33 --
.../doris/serialization/JsonDeserializer.java | 44 --
.../doris/serialization/JsonSerializer.java | 50 --
.../doris/serialization/LongDeserializer.java | 38 --
.../doris/serialization/LongSerializer.java | 37 --
.../connect/doris/serialization/Serializer.java | 61 ---
.../doris/serialization/ShortDeserializer.java | 39 --
.../doris/serialization/ShortSerializer.java | 31 --
.../doris/serialization/StringDeserializer.java | 49 --
.../doris/serialization/StringSerializer.java | 51 --
.../doris/service/DorisDefaultSinkService.java | 125 +++++
.../connect/doris/service/DorisSinkService.java | 43 ++
.../doris/service/DorisSinkServiceFactory.java | 32 ++
.../connect/doris/service/DorisSystemService.java | 116 ++++
.../connect/doris/service/RestService.java | 361 +++++++++++++
.../connect/doris/sink/BufferedRecords.java | 234 --------
.../rocketmq/connect/doris/sink/DorisDialect.java | 125 -----
.../connect/doris/sink/DorisStreamLoader.java | 145 -----
.../connect/doris/sink/RecordValidator.java | 101 ----
.../rocketmq/connect/doris/sink/Updater.java | 74 ---
.../doris/sink/metadata/FieldsMetadata.java | 304 -----------
.../connect/doris/sink/metadata/SchemaPair.java | 64 ---
.../doris/sink/metadata/SinkRecordField.java | 90 ----
.../rocketmq/connect/doris/util/BytesUtil.java | 30 --
.../connect/doris/util/ConnectorGroupUtils.java | 53 --
.../rocketmq/connect/doris/util/DateTimeUtils.java | 71 ---
.../connect/doris/util/ExpressionBuilder.java | 595 ---------------------
.../connect/doris/util/IdentifierRules.java | 186 -------
.../connect/doris/util/JdbcDriverInfo.java | 136 -----
.../connect/doris/util/NumericMapping.java | 54 --
.../rocketmq/connect/doris/util/QuoteMethod.java | 42 --
.../rocketmq/connect/doris/util/TableType.java | 85 ---
.../rocketmq/connect/doris/util/TableUtil.java | 34 --
.../rocketmq/connect/doris/utils/BackendUtils.java | 72 +++
.../connect/doris/utils/BackoffAndRetryUtils.java | 70 +++
.../connect/doris/utils/ConfigCheckUtils.java | 334 ++++++++++++
.../connect/doris/utils/ConnectRecordUtil.java | 36 ++
.../connect/doris/utils/FileNameUtils.java | 119 +++++
.../connect/doris/utils/HttpGetWithEntity.java | 37 ++
.../connect/doris/utils/HttpPostBuilder.java | 71 +++
.../connect/doris/utils/HttpPutBuilder.java | 126 +++++
.../rocketmq/connect/doris/utils/HttpUtils.java | 89 +++
.../connect/doris/writer/CopyIntoWriter.java | 159 ++++++
.../connect/doris/writer/CopySQLBuilder.java | 72 +++
.../connect/doris/writer/DeliveryGuarantee.java | 44 ++
.../rocketmq/connect/doris/writer/DorisWriter.java | 184 +++++++
.../connect/doris/writer/LabelGenerator.java | 54 ++
.../connect/doris/writer/LoadConstants.java | 34 ++
.../rocketmq/connect/doris/writer/LoadStatus.java | 30 ++
.../connect/doris/writer/PartitionBuffer.java | 141 +++++
.../connect/doris/writer/RecordBuffer.java | 57 ++
.../connect/doris/writer/ResponseUtil.java | 37 ++
.../connect/doris/writer/StreamLoadWriter.java | 190 +++++++
.../doris/writer/commit/DorisCommittable.java | 93 ++++
.../doris/writer/commit/DorisCommitter.java | 139 +++++
.../connect/doris/writer/load/CopyLoad.java | 274 ++++++++++
.../connect/doris/writer/load/DataLoad.java | 31 ++
.../connect/doris/writer/load/DorisStreamLoad.java | 149 ++++++
.../connect/doris/writer/load/GroupCommitMode.java | 28 +
.../connect/doris/writer/load/LoadModel.java | 44 ++
.../rocketmq/connect/doris/DorisSinkTaskTest.java | 156 ++++++
166 files changed, 9115 insertions(+), 5001 deletions(-)
diff --git a/connectors/rocketmq-connect-doris/pom.xml b/connectors/rocketmq-connect-doris/pom.xml
index 7ed0e952..660aa460 100644
--- a/connectors/rocketmq-connect-doris/pom.xml
+++ b/connectors/rocketmq-connect-doris/pom.xml
@@ -49,10 +49,13 @@
<!--rocket connect api-->
<openmessaging-connector.version>0.1.4</openmessaging-connector.version>
<openmessaging-api.version>0.3.1-alpha</openmessaging-api.version>
-
- <!--fast json-->
- <fastjson.version>1.2.83</fastjson.version>
-
+ <metrics.version>4.2.25</metrics.version>
+ <jackson.version>2.13.2.1</jackson.version>
+ <rocketmq-connect.version>0.0.1-SNAPSHOT</rocketmq-connect.version>
+ <debezium.version>1.9.8.Final</debezium.version>
+ <geometry.version>2.2.0</geometry.version>
+ <commons-io.version>2.3</commons-io.version>
+ <mysql-connector.version>8.0.33</mysql-connector.version>
</properties>
<dependencies>
@@ -94,27 +97,60 @@
<scope>test</scope>
</dependency>
- <!--fast json version-->
<dependency>
- <groupId>com.alibaba</groupId>
- <artifactId>fastjson</artifactId>
- <version>${fastjson.version}</version>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${jackson.version}</version>
</dependency>
<dependency>
- <groupId>ch.qos.logback</groupId>
- <artifactId>logback-classic</artifactId>
- <version>1.2.0</version>
- </dependency>
- <dependency>
- <groupId>ch.qos.logback</groupId>
- <artifactId>logback-core</artifactId>
- <version>1.2.9</version>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-connect-runtime</artifactId>
+ <version>${rocketmq-connect.version}</version>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.13</version>
</dependency>
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ <version>${metrics.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-jmx</artifactId>
+ <version>${metrics.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>io.debezium</groupId>
+ <artifactId>debezium-core</artifactId>
+ <version>${debezium.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.esri.geometry</groupId>
+ <artifactId>esri-geometry-api</artifactId>
+ <version>${geometry.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>${commons-io.version}</version>
+ </dependency>
+ <!-- https://mvnrepository.com/artifact/mysql/mysql-connector-java -->
+ <dependency>
+ <groupId>mysql</groupId>
+ <artifactId>mysql-connector-java</artifactId>
+ <version>${mysql-connector.version}</version>
+ </dependency>
</dependencies>
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/DorisSinkConnector.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/DorisSinkConnector.java
new file mode 100644
index 00000000..88244efa
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/DorisSinkConnector.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris;
+
+import io.openmessaging.KeyValue;
+import io.openmessaging.connector.api.component.task.Task;
+import io.openmessaging.connector.api.component.task.sink.SinkConnector;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.rocketmq.connect.doris.cfg.DorisSinkConnectorConfig;
+import org.apache.rocketmq.connect.doris.utils.ConfigCheckUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class DorisSinkConnector extends SinkConnector {
+ private static final Logger LOG = LoggerFactory.getLogger(DorisSinkConnector.class);
+ private KeyValue keyValue;
+
+ @Override
+ public void start(KeyValue keyValue) {
+ this.keyValue = DorisSinkConnectorConfig.convertToLowercase(keyValue);
+ DorisSinkConnectorConfig.setDefaultValues(this.keyValue);
+ ConfigCheckUtils.validateConfig(this.keyValue);
+ }
+
+ /**
+ * stop DorisSinkConnector
+ */
+ @Override
+ public void stop() {
+ LOG.info("doris sink connector stop");
+ }
+
+ @Override
+ public Class<? extends Task> taskClass() {
+ return DorisSinkTask.class;
+ }
+
+ @Override
+ public List<KeyValue> taskConfigs(final int maxTasks) {
+ List<KeyValue> taskConfigs = new ArrayList<>(maxTasks);
+ for (int i = 0; i < maxTasks; i++) {
+ keyValue.put("task_id", i + "");
+ taskConfigs.add(this.keyValue);
+ }
+ return taskConfigs;
+ }
+
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/DorisSinkTask.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/DorisSinkTask.java
new file mode 100644
index 00000000..fa14a6a1
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/DorisSinkTask.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris;
+
+import io.openmessaging.KeyValue;
+import io.openmessaging.connector.api.component.task.sink.SinkTask;
+import io.openmessaging.connector.api.data.ConnectRecord;
+import io.openmessaging.connector.api.data.RecordOffset;
+import io.openmessaging.connector.api.data.RecordPartition;
+import io.openmessaging.connector.api.errors.ConnectException;
+import java.util.List;
+import java.util.Map;
+import org.apache.rocketmq.connect.doris.service.DorisSinkService;
+import org.apache.rocketmq.connect.doris.service.DorisSinkServiceFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class DorisSinkTask extends SinkTask {
+ private static final Logger LOG = LoggerFactory.getLogger(DorisSinkTask.class);
+ private DorisSinkService sink;
+
+ @Override
+ public void start(KeyValue keyValue) {
+ LOG.info("rocketmq doris sink task start");
+ this.sink = DorisSinkServiceFactory.getDorisSinkService(keyValue);
+ }
+
+ @Override
+ public void put(List<ConnectRecord> sinkRecords) throws ConnectException {
+ LOG.info("Read {} records from Kafka", sinkRecords.size());
+ sink.insert(sinkRecords);
+ }
+
+ /**
+ * Support doris's two-phase commit
+ */
+ @Override
+ public void flush(Map<RecordPartition, RecordOffset> currentOffsets) throws ConnectException {
+ if (sink == null || sink.getDorisWriterSize() == 0) {
+ return;
+ }
+ sink.commit(currentOffsets);
+ }
+
+ @Override
+ public void stop() {
+ LOG.info("rocketmq doris sink task stopped");
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/cfg/DorisOptions.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/cfg/DorisOptions.java
new file mode 100644
index 00000000..a1164d7c
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/cfg/DorisOptions.java
@@ -0,0 +1,387 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.cfg;
+
+import io.openmessaging.KeyValue;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Properties;
+import java.util.stream.Collectors;
+import org.apache.rocketmq.connect.doris.converter.ConverterMode;
+import org.apache.rocketmq.connect.doris.converter.schema.SchemaEvolutionMode;
+import org.apache.rocketmq.connect.doris.exception.DorisException;
+import org.apache.rocketmq.connect.doris.utils.ConfigCheckUtils;
+import org.apache.rocketmq.connect.doris.writer.DeliveryGuarantee;
+import org.apache.rocketmq.connect.doris.writer.load.LoadModel;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class DorisOptions {
+ private static final Logger LOG = LoggerFactory.getLogger(DorisOptions.class);
+ private final String name;
+ private final String urls;
+ private final int queryPort;
+ private final int httpPort;
+ private final String user;
+ private final String password;
+ private final String database;
+ private final Map<String, String> topicMap;
+ private final int fileSize;
+ private final int recordNum;
+ private final long flushTime;
+ private final boolean enableCustomJMX;
+ private final int taskId;
+ private final boolean enableDelete;
+ private boolean enable2PC = true;
+ private boolean force2PC;
+ private boolean autoRedirect = true;
+ private int requestReadTimeoutMs;
+ private int requestConnectTimeoutMs;
+ private final boolean enableGroupCommit;
+ private boolean customCluster;
+ private ProxyConfig proxyConfig;
+ /**
+ * Properties for the StreamLoad.
+ */
+ private final Properties streamLoadProp;
+ private final String databaseTimeZone;
+ private final LoadModel loadModel;
+ private final DeliveryGuarantee deliveryGuarantee;
+ private final ConverterMode converterMode;
+ private final SchemaEvolutionMode schemaEvolutionMode;
+
+ public DorisOptions(KeyValue config) {
+ this.name = config.getString(DorisSinkConnectorConfig.NAME);
+ this.urls = config.getString(DorisSinkConnectorConfig.DORIS_URLS);
+ this.queryPort = Integer.parseInt(config.getString(DorisSinkConnectorConfig.DORIS_QUERY_PORT));
+ this.httpPort = Integer.parseInt(config.getString(DorisSinkConnectorConfig.DORIS_HTTP_PORT));
+ this.user = config.getString(DorisSinkConnectorConfig.DORIS_USER);
+ this.password = config.getString(DorisSinkConnectorConfig.DORIS_PASSWORD);
+ this.database = config.getString(DorisSinkConnectorConfig.DORIS_DATABASE);
+ this.taskId = Integer.parseInt(config.getString(ConfigCheckUtils.TASK_ID));
+ this.databaseTimeZone = config.getString(DorisSinkConnectorConfig.DATABASE_TIME_ZONE);
+ this.loadModel = LoadModel.of(config.getString(DorisSinkConnectorConfig.LOAD_MODEL));
+ this.deliveryGuarantee =
+ DeliveryGuarantee.of(config.getString(DorisSinkConnectorConfig.DELIVERY_GUARANTEE));
+ this.converterMode = ConverterMode.of(config.getString(DorisSinkConnectorConfig.CONVERTER_MODE));
+ this.schemaEvolutionMode =
+ SchemaEvolutionMode.of(
+ config.getString(DorisSinkConnectorConfig.DEBEZIUM_SCHEMA_EVOLUTION));
+ this.fileSize = Integer.parseInt(config.getString(DorisSinkConnectorConfig.BUFFER_SIZE_BYTES));
+ this.recordNum =
+ Integer.parseInt(config.getString(DorisSinkConnectorConfig.BUFFER_COUNT_RECORDS));
+
+ this.flushTime = Long.parseLong(config.getString(DorisSinkConnectorConfig.BUFFER_FLUSH_TIME_SEC));
+ this.topicMap = getTopicToTableMap(config);
+
+ if (config.containsKey(DorisSinkConnectorConfig.ENABLE_2PC)) {
+ if (Boolean.parseBoolean(config.getString(DorisSinkConnectorConfig.ENABLE_2PC))) {
+ this.enable2PC = true;
+ this.force2PC = true;
+ } else {
+ this.enable2PC = false;
+ }
+ }
+ this.enableCustomJMX = Boolean.parseBoolean(config.getString(DorisSinkConnectorConfig.JMX_OPT));
+ this.enableDelete =
+ Boolean.parseBoolean(config.getString(DorisSinkConnectorConfig.ENABLE_DELETE));
+ this.requestConnectTimeoutMs =
+ DorisSinkConnectorConfig.DORIS_REQUEST_CONNECT_TIMEOUT_MS_DEFAULT;
+ this.requestReadTimeoutMs = DorisSinkConnectorConfig.DORIS_REQUEST_READ_TIMEOUT_MS_DEFAULT;
+ if (config.containsKey(DorisSinkConnectorConfig.AUTO_REDIRECT)) {
+ this.autoRedirect =
+ Boolean.parseBoolean(config.getString(DorisSinkConnectorConfig.AUTO_REDIRECT));
+ }
+ if (config.containsKey(DorisSinkConnectorConfig.REQUEST_CONNECT_TIMEOUT_MS)) {
+ this.requestConnectTimeoutMs =
+ Integer.parseInt(
+ config.getString(DorisSinkConnectorConfig.REQUEST_CONNECT_TIMEOUT_MS));
+ }
+ if (config.containsKey(DorisSinkConnectorConfig.REQUEST_READ_TIMEOUT_MS)) {
+ this.requestReadTimeoutMs =
+ Integer.parseInt(config.getString(DorisSinkConnectorConfig.REQUEST_READ_TIMEOUT_MS));
+ }
+ if (config.containsKey(DorisSinkConnectorConfig.DORIS_CUSTOM_CLUSTER)) {
+ this.customCluster = Boolean.parseBoolean(config.getString(DorisSinkConnectorConfig.DORIS_CUSTOM_CLUSTER));
+ parseClusterProxyConfig(config);
+ }
+ this.streamLoadProp = getStreamLoadPropFromConfig(config);
+ this.enableGroupCommit = ConfigCheckUtils.validateGroupCommitMode(this);
+ }
+
+ private void parseClusterProxyConfig(KeyValue config) {
+ if (customCluster) {
+ String socks5Endpoint = config.getString(DorisSinkConnectorConfig.SOCKS5_ENDPOINT);
+ String socks5UserName = config.getString(DorisSinkConnectorConfig.SOCKS5_USERNAME);
+ String socks5Password = config.getString(DorisSinkConnectorConfig.SOCKET5_PASSWORD);
+ if (socks5Endpoint == null || socks5UserName == null || socks5Password == null) {
+ throw new DorisException(
+ "Currently it is doris custom cluster mode, and socks5Endpoint, socks5UserName, socks5Password need to be provided.");
+ }
+ this.proxyConfig = new ProxyConfig(socks5Endpoint, socks5UserName, socks5Password);
+ }
+ }
+
+ private Properties getStreamLoadPropFromConfig(KeyValue config) {
+ Properties properties = new Properties();
+ properties.putAll(getStreamLoadDefaultValues());
+ for (String key : config.keySet()) {
+ if (key.startsWith(DorisSinkConnectorConfig.STREAM_LOAD_PROP_PREFIX)) {
+ String subKey = key.substring(DorisSinkConnectorConfig.STREAM_LOAD_PROP_PREFIX.length());
+ properties.put(subKey, config.getString(key));
+ }
+ }
+ return properties;
+ }
+
+ private Properties getStreamLoadDefaultValues() {
+ Properties properties = new Properties();
+ properties.setProperty("format", "json");
+ properties.setProperty("read_json_by_line", "true");
+ return properties;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public String getUrls() {
+ return urls;
+ }
+
+ public int getQueryPort() {
+ return queryPort;
+ }
+
+ public int getHttpPort() {
+ return httpPort;
+ }
+
+ public String getUser() {
+ return user;
+ }
+
+ public String getPassword() {
+ return password;
+ }
+
+ public String getDatabase() {
+ return database;
+ }
+
+ public int getFileSize() {
+ return fileSize;
+ }
+
+ public int getRecordNum() {
+ return recordNum;
+ }
+
+ public long getFlushTime() {
+ return flushTime;
+ }
+
+ public String getTopicMapTable(String topic) {
+ return topicMap.get(topic);
+ }
+
+ public boolean force2PC() {
+ return force2PC;
+ }
+
+ public void setEnable2PC(boolean enable2PC) {
+ this.enable2PC = enable2PC;
+ }
+
+ public boolean enableGroupCommit() {
+ return enableGroupCommit;
+ }
+
+ public boolean enable2PC() {
+ return enable2PC;
+ }
+
+ public Map<String, String> getTopicMap() {
+ return topicMap;
+ }
+
+ public String getQueryUrl() {
+ List<String> queryUrls = getQueryUrls();
+ return queryUrls.get(0);
+ }
+
+ public String getHttpUrl() {
+ List<String> httpUrls = getHttpUrls();
+ return httpUrls.get(0);
+ }
+
+ public List<String> getQueryUrls() {
+ List<String> queryUrls = new ArrayList<>();
+ if (urls.contains(",")) {
+ queryUrls =
+ Arrays.stream(urls.split(","))
+ .map(
+ url -> {
+ return url.trim() + ":" + queryPort;
+ })
+ .collect(Collectors.toList());
+ Collections.shuffle(queryUrls);
+ return queryUrls;
+ }
+ queryUrls.add(urls + ":" + queryPort);
+ return queryUrls;
+ }
+
+ public List<String> getHttpUrls() {
+ List<String> httpUrls = new ArrayList<>();
+ if (urls.contains(",")) {
+ httpUrls =
+ Arrays.stream(urls.split(","))
+ .map(
+ url -> {
+ return url.trim() + ":" + httpPort;
+ })
+ .collect(Collectors.toList());
+ Collections.shuffle(httpUrls);
+ return httpUrls;
+ }
+ httpUrls.add(urls + ":" + httpPort);
+ return httpUrls;
+ }
+
+ public Integer getRequestReadTimeoutMs() {
+ return this.requestReadTimeoutMs;
+ }
+
+ public Integer getRequestConnectTimeoutMs() {
+ return this.requestConnectTimeoutMs;
+ }
+
+ public Properties getStreamLoadProp() {
+ return streamLoadProp;
+ }
+
+ public boolean isEnableCustomJMX() {
+ return enableCustomJMX;
+ }
+
+ public int getTaskId() {
+ return taskId;
+ }
+
+ public LoadModel getLoadModel() {
+ return loadModel;
+ }
+
+ public DeliveryGuarantee getDeliveryGuarantee() {
+ return this.deliveryGuarantee;
+ }
+
+ public ConverterMode getConverterMode() {
+ return this.converterMode;
+ }
+
+ public SchemaEvolutionMode getSchemaEvolutionMode() {
+ return this.schemaEvolutionMode;
+ }
+
+ public boolean isAutoRedirect() {
+ return autoRedirect;
+ }
+
+ public String getDatabaseTimeZone() {
+ return databaseTimeZone;
+ }
+
+ public boolean isEnableDelete() {
+ return enableDelete;
+ }
+
+ public boolean customCluster() {
+ return customCluster;
+ }
+
+ public Optional<ProxyConfig> getProxyConfig() {
+ return Optional.ofNullable(proxyConfig);
+ }
+
+ /**
+ * parse topic to table map
+ *
+ * @param config connector config file
+ * @return result map
+ */
+ private Map<String, String> getTopicToTableMap(KeyValue config) {
+ if (config.containsKey(DorisSinkConnectorConfig.TOPICS_TABLES_MAP)) {
+ Map<String, String> result =
+ ConfigCheckUtils.parseTopicToTableMap(
+ config.getString(DorisSinkConnectorConfig.TOPICS_TABLES_MAP));
+ if (result != null) {
+ return result;
+ }
+ LOG.error("Invalid Input, Topic2Table Map disabled");
+ }
+ return new HashMap<>();
+ }
+
+ public class ProxyConfig {
+ private final String socks5Endpoint;
+ private final String socks5UserName;
+ private final String socks5Password;
+ private final String socks5Host;
+ private final Integer socks5Port;
+
+ public ProxyConfig(String socks5Endpoint, String socks5UserName, String socks5Password) {
+ this.socks5Endpoint = socks5Endpoint;
+ this.socks5UserName = socks5UserName;
+ this.socks5Password = socks5Password;
+ String[] splitEndpoint = socks5Endpoint.split(":");
+ socks5Host = splitEndpoint[0];
+ socks5Port = Integer.parseInt(splitEndpoint[1]);
+ assert socks5Host != null;
+ }
+
+ public String getSocks5Endpoint() {
+ return socks5Endpoint;
+ }
+
+ public String getSocks5Host() {
+ return socks5Host;
+ }
+
+ public String getSocks5Password() {
+ return socks5Password;
+ }
+
+ public Integer getSocks5Port() {
+ return socks5Port;
+ }
+
+ public String getSocks5UserName() {
+ return socks5UserName;
+ }
+ }
+
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/cfg/DorisSinkConnectorConfig.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/cfg/DorisSinkConnectorConfig.java
new file mode 100644
index 00000000..f5232535
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/cfg/DorisSinkConnectorConfig.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.cfg;
+
+import io.openmessaging.KeyValue;
+import io.openmessaging.internal.DefaultKeyValue;
+import java.util.Locale;
+import org.apache.rocketmq.connect.doris.DorisSinkConnector;
+import org.apache.rocketmq.connect.doris.converter.ConverterMode;
+import org.apache.rocketmq.connect.doris.converter.schema.SchemaEvolutionMode;
+import org.apache.rocketmq.connect.doris.writer.DeliveryGuarantee;
+import org.apache.rocketmq.connect.doris.writer.load.LoadModel;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Doris SinkConnectorConfig
+ */
+public class DorisSinkConnectorConfig {
+ private static final Logger LOG = LoggerFactory.getLogger(DorisSinkConnector.class);
+
+ public static final String NAME = "name";
+ public static final String TOPICS = "topics";
+ public static final String TOPICS_REGEX = "topics.regex";
+
+ // Connector config
+ public static final String BUFFER_COUNT_RECORDS = "buffer.count.records";
+ public static final long BUFFER_COUNT_RECORDS_DEFAULT = 10000;
+ public static final String BUFFER_SIZE_BYTES = "buffer.size.bytes";
+ public static final long BUFFER_SIZE_BYTES_DEFAULT = 5000000;
+ public static final long BUFFER_SIZE_BYTES_MIN = 1;
+ public static final String TOPICS_TABLES_MAP = "doris.topic2table.map";
+ public static final String LABEL_PREFIX = "label.prefix";
+
+ // Time in seconds
+ public static final long BUFFER_FLUSH_TIME_SEC_MIN = 10;
+ public static final long BUFFER_FLUSH_TIME_SEC_DEFAULT = 120;
+ public static final String BUFFER_FLUSH_TIME_SEC = "buffer.flush.time";
+
+ private static final String DORIS_INFO = "Doris Info";
+
+ // doris config
+ public static final String DORIS_URLS = "doris.urls";
+ public static final String DORIS_QUERY_PORT = "doris.query.port";
+ public static final String DORIS_HTTP_PORT = "doris.http.port";
+ public static final String DORIS_USER = "doris.user";
+ public static final String DORIS_PASSWORD = "doris.password";
+ public static final String DORIS_DATABASE = "doris.database";
+ public static final String REQUEST_READ_TIMEOUT_MS = "request.read.timeout.ms";
+ public static final String REQUEST_CONNECT_TIMEOUT_MS = "request.connect.timeout.ms";
+ public static final Integer DORIS_REQUEST_READ_TIMEOUT_MS_DEFAULT = 30 * 1000;
+ public static final Integer DORIS_REQUEST_CONNECT_TIMEOUT_MS_DEFAULT = 30 * 1000;
+ public static final String DATABASE_TIME_ZONE = "database.time_zone";
+ public static final String DATABASE_TIME_ZONE_DEFAULT = "UTC";
+ public static final String LOAD_MODEL = "load.model";
+ public static final String LOAD_MODEL_DEFAULT = LoadModel.STREAM_LOAD.name();
+ public static final String AUTO_REDIRECT = "auto.redirect";
+ public static final String DELIVERY_GUARANTEE = "delivery.guarantee";
+ public static final String DELIVERY_GUARANTEE_DEFAULT = DeliveryGuarantee.AT_LEAST_ONCE.name();
+ public static final String CONVERTER_MODE = "converter.mode";
+ public static final String CONVERT_MODE_DEFAULT = ConverterMode.NORMAL.getName();
+
+ // Prefix for Doris StreamLoad specific properties.
+ public static final String STREAM_LOAD_PROP_PREFIX = "sink.properties.";
+ public static final String DEBEZIUM_SCHEMA_EVOLUTION = "debezium.schema.evolution";
+ public static final String DEBEZIUM_SCHEMA_EVOLUTION_DEFAULT =
+ SchemaEvolutionMode.NONE.getName();
+
+ // custom cluster config
+ public static final String DORIS_CUSTOM_CLUSTER = "doris.custom.cluster";
+ public static final String DORIS_CUSTOM_CLUSTER_DEFAULT = "false";
+ public static final String SOCKS5_ENDPOINT = "socks5Endpoint";
+ public static final String SOCKS5_USERNAME = "socks5UserName";
+ public static final String SOCKET5_PASSWORD = "socks5Password";
+
+ // metrics
+ public static final String JMX_OPT = "jmx";
+ public static final boolean JMX_OPT_DEFAULT = true;
+
+ public static final String ENABLE_DELETE = "enable.delete";
+ public static final boolean ENABLE_DELETE_DEFAULT = false;
+ public static final String ENABLE_2PC = "enable.2pc";
+ public static final boolean ENABLE_2PC_DEFAULT = true;
+
+ public static void setDefaultValues(KeyValue config) {
+ setFieldToDefaultValues(
+ config, BUFFER_COUNT_RECORDS, String.valueOf(BUFFER_COUNT_RECORDS_DEFAULT));
+ setFieldToDefaultValues(
+ config, BUFFER_SIZE_BYTES, String.valueOf(BUFFER_SIZE_BYTES_DEFAULT));
+ setFieldToDefaultValues(
+ config, BUFFER_FLUSH_TIME_SEC, String.valueOf(BUFFER_FLUSH_TIME_SEC_DEFAULT));
+ setFieldToDefaultValues(config, DATABASE_TIME_ZONE, DATABASE_TIME_ZONE_DEFAULT);
+ setFieldToDefaultValues(config, LOAD_MODEL, LOAD_MODEL_DEFAULT);
+ setFieldToDefaultValues(config, DELIVERY_GUARANTEE, DELIVERY_GUARANTEE_DEFAULT);
+ setFieldToDefaultValues(config, CONVERTER_MODE, CONVERT_MODE_DEFAULT);
+ setFieldToDefaultValues(
+ config, DEBEZIUM_SCHEMA_EVOLUTION, DEBEZIUM_SCHEMA_EVOLUTION_DEFAULT);
+ setFieldToDefaultValues(config, JMX_OPT, String.valueOf(JMX_OPT_DEFAULT));
+ setFieldToDefaultValues(config, DORIS_CUSTOM_CLUSTER, DORIS_CUSTOM_CLUSTER_DEFAULT);
+ }
+
+ private static void setFieldToDefaultValues(KeyValue config, String field, String value) {
+ if (!config.containsKey(field)) {
+ config.put(field, value);
+ LOG.info("Set the default value of {} to {}", field, value);
+ }
+ }
+
+ public static KeyValue convertToLowercase(KeyValue config) {
+ KeyValue newConfig = new DefaultKeyValue();
+ for (String key : config.keySet()) {
+ String value = config.getString(key);
+ newConfig.put(key.toLowerCase(Locale.ROOT), value);
+ }
+ return newConfig;
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/config/AbstractConfig.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/config/AbstractConfig.java
deleted file mode 100644
index 5ff2b2ae..00000000
--- a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/config/AbstractConfig.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.connect.doris.config;
-import io.openmessaging.KeyValue;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Objects;
-import java.util.regex.Pattern;
-
-/**
- * abstract config
- */
-public abstract class AbstractConfig {
-
- private static final Pattern COMMA_WITH_WHITESPACE = Pattern.compile("\\s*,\\s*");
-
- // connection url
- public static final String CONNECTION_PREFIX = "connection.";
- public static final String CONNECTION_URL_CONFIG = CONNECTION_PREFIX + "url";
- // connection user
- public static final String CONNECTION_USER_CONFIG = CONNECTION_PREFIX + "user";
- private static final String CONNECTION_USER_DOC = "JDBC connection user.";
- // connection password
- public static final String CONNECTION_PASSWORD_CONFIG = CONNECTION_PREFIX + "password";
- private static final String CONNECTION_PASSWORD_DOC = "JDBC connection password.";
- // connection attempts
- public static final String CONNECTION_ATTEMPTS_CONFIG = CONNECTION_PREFIX + "attempts";
- public static final String CONNECTION_ATTEMPTS_DOC = "Maximum number of attempts to retrieve a valid JDBC connection.Must be a positive integer.";
- public static final int CONNECTION_ATTEMPTS_DEFAULT = 3;
- // backoff ms
- public static final String CONNECTION_BACKOFF_CONFIG = CONNECTION_PREFIX + "backoff.ms";
- public static final String CONNECTION_BACKOFF_DOC = "Backoff time in milliseconds between connection attempts.";
- public static final long CONNECTION_BACKOFF_DEFAULT = 10000L;
- /**
- * quote.sql.identifiers
- */
- public static final String QUOTE_SQL_IDENTIFIERS_CONFIG = "quote.sql.identifiers";
- public static final String QUOTE_SQL_IDENTIFIERS_DOC =
- "When to quote table names, column names, and other identifiers in SQL statements. "
- + "For backward compatibility, the default is ``always``.";
-
-
- private String connectionDbUrl;
- private String connectionDbUser;
- private String connectionDbPassword;
- private Integer attempts;
- private Long backoffMs;
- private String quoteSqlIdentifiers;
-
- public AbstractConfig(KeyValue config) {
- connectionDbUrl = config.getString(CONNECTION_URL_CONFIG);
- connectionDbUser = config.getString(CONNECTION_USER_CONFIG);
- connectionDbPassword = config.getString(CONNECTION_PASSWORD_CONFIG);
- attempts = config.getInt(CONNECTION_ATTEMPTS_CONFIG, CONNECTION_ATTEMPTS_DEFAULT);
- backoffMs = config.getLong(CONNECTION_BACKOFF_CONFIG, CONNECTION_BACKOFF_DEFAULT);
- }
-
-
- public String getConnectionDbUrl() {
- return connectionDbUrl;
- }
-
- public String getConnectionDbUser() {
- return connectionDbUser;
- }
-
- public String getConnectionDbPassword() {
- return connectionDbPassword;
- }
-
- public Integer getAttempts() {
- return attempts;
- }
-
- public Long getBackoffMs() {
- return backoffMs;
- }
-
- public String getQuoteSqlIdentifiers() {
- return quoteSqlIdentifiers;
- }
-
- /**
- * get list
- *
- * @param config
- * @param key
- * @return
- */
- protected List<String> getList(KeyValue config, String key) {
- if (!config.containsKey(key) || Objects.isNull(config.getString(key))) {
- return new ArrayList<>();
- }
- return Arrays.asList(COMMA_WITH_WHITESPACE.split(config.getString(key), -1));
- }
-
- /**
- * get list
- *
- * @param config
- * @param key
- * @return
- */
- protected List<String> getList(KeyValue config, String key, String defaultValue) {
- if (config.containsKey(key) || Objects.isNull(config.getString(key))) {
- return Collections.singletonList(defaultValue);
- }
- return Arrays.asList(COMMA_WITH_WHITESPACE.split(config.getString(key), -1));
- }
-
- protected Boolean getBoolean(KeyValue config, String key, Boolean defaultValue) {
- return config.containsKey(key) ? Boolean.parseBoolean(config.getString(key)) : defaultValue;
- }
-
-}
-
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/connection/ConnectionProvider.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/connection/ConnectionProvider.java
new file mode 100644
index 00000000..a77d3c5f
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/connection/ConnectionProvider.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.connection;
+
+import java.sql.Connection;
+
+public interface ConnectionProvider {
+
+ /**
+ * Get existing connection or establish an new one if there is none.
+ */
+ Connection getOrEstablishConnection() throws Exception;
+
+ /**
+ * Close possible existing connection.
+ */
+ void closeConnection();
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/connection/JdbcConnectionProvider.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/connection/JdbcConnectionProvider.java
new file mode 100644
index 00000000..4207b7f0
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/connection/JdbcConnectionProvider.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.connection;
+
+import java.io.Serializable;
+import java.net.Authenticator;
+import java.net.PasswordAuthentication;
+import java.net.Proxy;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.NoSuchElementException;
+import java.util.Objects;
+import org.apache.rocketmq.connect.doris.cfg.DorisOptions;
+import org.apache.rocketmq.connect.doris.cfg.DorisOptions.ProxyConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class JdbcConnectionProvider implements ConnectionProvider, Serializable {
+
+ private static final Logger LOG = LoggerFactory.getLogger(JdbcConnectionProvider.class);
+ protected final String driverName = "com.mysql.jdbc.Driver";
+ protected final String cjDriverName = "com.mysql.cj.jdbc.Driver";
+ private static final String JDBC_URL_TEMPLATE = "jdbc:mysql://%s";
+ private static final String JDBC_URL_PROXY_TEMPLATE = "jdbc:mysql://%s?useSSL=false&proxyHost=%s&proxyPort=%d";
+
+ private static final long serialVersionUID = 1L;
+
+ private final DorisOptions options;
+
+ private transient Connection connection;
+
+ public JdbcConnectionProvider(DorisOptions options) {
+ this.options = options;
+ }
+
+ @Override
+ public Connection getOrEstablishConnection() throws ClassNotFoundException, SQLException {
+ if (connection != null && !connection.isClosed() && connection.isValid(10)) {
+ return connection;
+ }
+ try {
+ Class.forName(cjDriverName);
+ } catch (ClassNotFoundException ex) {
+ LOG.warn(
+ "can not found class com.mysql.cj.jdbc.Driver, use class com.mysql.jdbc.Driver");
+ Class.forName(driverName);
+ }
+ String jdbcUrl;
+ if (options.customCluster()) {
+ ProxyConfig proxyConfig = options.getProxyConfig()
+ .orElseThrow(() -> new NoSuchElementException("Failed to get ProxyConfig."));
+ String proxyHost = proxyConfig.getSocks5Host();
+ int proxyPort = proxyConfig.getSocks5Port();
+ String queryUrl = options.getQueryUrl();
+ jdbcUrl = String.format(JDBC_URL_PROXY_TEMPLATE, queryUrl, proxyHost, proxyPort);
+ configureSocks5ProxyAuthentication();
+ } else {
+ jdbcUrl = String.format(JDBC_URL_TEMPLATE, options.getQueryUrl());
+ }
+
+ if (!Objects.isNull(options.getUser())) {
+ connection =
+ DriverManager.getConnection(jdbcUrl, options.getUser(), options.getPassword());
+ } else {
+ connection = DriverManager.getConnection(jdbcUrl);
+ }
+ return connection;
+ }
+
+ private void configureSocks5ProxyAuthentication() {
+ if (options.customCluster()) {
+ ProxyConfig proxyConfig = options.getProxyConfig()
+ .orElseThrow(() -> new NoSuchElementException("Failed to get ProxyConfig."));
+ String socks5UserName = proxyConfig.getSocks5UserName();
+ String socks5Password = proxyConfig.getSocks5Password();
+ Authenticator.setDefault(new Authenticator() {
+ @Override
+ protected PasswordAuthentication getPasswordAuthentication() {
+ if (Proxy.Type.SOCKS.name().equalsIgnoreCase(getRequestingProtocol())) {
+ return new PasswordAuthentication(socks5UserName, socks5Password.toCharArray());
+ }
+ return null;
+ }
+ });
+ System.setProperty("java.net.socks.username", socks5UserName);
+ System.setProperty("java.net.socks.password", socks5Password);
+ }
+ }
+
+ @Override
+ public void closeConnection() {
+ if (connection != null) {
+ try {
+ connection.close();
+ } catch (SQLException e) {
+ LOG.warn("JDBC connection close failed.", e);
+ } finally {
+ connection = null;
+ }
+ }
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/connector/DorisSinkConfig.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/connector/DorisSinkConfig.java
deleted file mode 100644
index 5cbe294d..00000000
--- a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/connector/DorisSinkConfig.java
+++ /dev/null
@@ -1,331 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.connect.doris.connector;
-
-import io.openmessaging.KeyValue;
-import org.apache.rocketmq.connect.doris.config.AbstractConfig;
-import org.apache.rocketmq.connect.doris.exception.ConfigException;
-import org.apache.rocketmq.connect.doris.schema.table.TableId;
-import org.apache.rocketmq.connect.doris.util.TableType;
-import org.apache.rocketmq.connect.doris.util.TableUtil;
-
-import java.time.ZoneId;
-import java.util.EnumSet;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Objects;
-import java.util.Set;
-import java.util.TimeZone;
-import java.util.stream.Collectors;
-
-/**
- * jdbc sink config
- */
-public class DorisSinkConfig extends AbstractConfig {
-
- public enum InsertMode {
- INSERT,
- UPSERT,
- UPDATE;
-
- }
-
- public enum PrimaryKeyMode {
- NONE,
- RECORD_KEY,
- RECORD_VALUE;
- }
- public static final String HOST = "host";
- public static final String PORT = "port";
- public static final String DATABASE = "database";
- public static final String USER = "user";
- public static final String PASSWD = "passwd";
- public static final String TABLE_NAME_FORMAT = "table.name.format";
- public static final String TABLE_NAME_FORMAT_DEFAULT = "${topic}";
- private static final String TABLE_NAME_FORMAT_DISPLAY = "Table Name Format";
-
- /**
- * table name from header
- */
- public static final String TABLE_NAME_FROM_HEADER = "table.name.from.header";
- private static final boolean TABLE_NAME_FROM_HEADER_DEFAULT = Boolean.FALSE;
- private static final String TABLE_NAME_FROM_HEADER_DISPLAY = "Table from header";
-
- /**
- * max retries
- */
- public static final String MAX_RETRIES = "max.retries";
- private static final int MAX_RETRIES_DEFAULT = 10;
- private static final String MAX_RETRIES_DOC =
- "The maximum number of times to retry on errors before failing the task.";
-
- public static final String RETRY_BACKOFF_MS = "retry.backoff.ms";
- private static final int RETRY_BACKOFF_MS_DEFAULT = 3000;
-
- public static final String BATCH_SIZE = "batch.size";
- private static final int BATCH_SIZE_DEFAULT = 100;
-
-
- public static final String DELETE_ENABLED = "delete.enabled";
- private static final boolean DELETE_ENABLED_DEFAULT = false;
-
-
- public static final String AUTO_CREATE = "auto.create";
- private static final boolean AUTO_CREATE_DEFAULT = false;
-
- public static final String AUTO_EVOLVE = "auto.evolve";
- private static final boolean AUTO_EVOLVE_DEFAULT = false;
- private static final String AUTO_EVOLVE_DOC =
- "Whether to automatically add columns in the table schema when found to be missing relative "
- + "to the record schema by issuing ``ALTER``.";
-
- public static final String INSERT_MODE = "insert.mode";
- private static final String INSERT_MODE_DEFAULT = "insert";
- private static final String INSERT_MODE_DOC =
- "The insertion mode to use. Supported modes are:\n"
- + "``insert``\n"
- + " Use standard SQL ``INSERT`` statements.\n"
- + "``upsert``\n"
- + " Use the appropriate upsert semantics for the target database if it is supported by "
- + "the connector, e.g. ``INSERT OR IGNORE``.\n"
- + "``update``\n"
- + " Use the appropriate update semantics for the target database if it is supported by "
- + "the connector, e.g. ``UPDATE``.";
- private static final String INSERT_MODE_DISPLAY = "Insert Mode";
-
-
- public static final String PK_FIELDS = "pk.fields";
- private static final String PK_FIELDS_DEFAULT = "";
- private static final String PK_FIELDS_DOC =
- "List of comma-separated primary key field names. The runtime interpretation of this config"
- + " depends on the ``pk.mode``:\n"
- + "``none``\n"
- + " Ignored as no fields are used as primary key in this mode.\n"
- + "``record_key``\n"
- + " If empty, all fields from the key struct will be used, otherwise used to extract the"
- + " desired fields - for primitive key only a single field name must be configured.\n"
- + "``record_value``\n"
- + " If empty, all fields from the value struct will be used, otherwise used to extract "
- + "the desired fields.";
- private static final String PK_FIELDS_DISPLAY = "Primary Key Fields";
-
- public static final String PK_MODE = "pk.mode";
- private static final String PK_MODE_DEFAULT = "none";
- private static final String PK_MODE_DOC =
- "The primary key mode, also refer to ``" + PK_FIELDS + "`` documentation for interplay. "
- + "Supported modes are:\n"
- + "``none``\n"
- + " No keys utilized.\n"
- + "``record_value``\n"
- + " Field(s) from the record value are used, which must be a struct.";
- private static final String PK_MODE_DISPLAY = "Primary Key Mode";
-
- public static final String FIELDS_WHITELIST = "fields.whitelist";
- private static final String FIELDS_WHITELIST_DEFAULT = "";
- private static final String FIELDS_WHITELIST_DOC =
- "List of comma-separated record value field names. If empty, all fields from the record "
- + "value are utilized, otherwise used to filter to the desired fields.\n"
- + "Note that ``" + PK_FIELDS + "`` is applied independently in the context of which field"
- + "(s) form the primary key columns in the destination database,"
- + " while this configuration is applicable for the other columns.";
- private static final String FIELDS_WHITELIST_DISPLAY = "Fields Whitelist";
-
-
- public static final String DIALECT_NAME_CONFIG = "dialect.name";
- public static final String DIALECT_NAME_DEFAULT = "";
-
-
- public static final String DB_TIMEZONE_CONFIG = "db.timezone";
- public static final String DB_TIMEZONE_DEFAULT = "UTC";
-
- // table types
- public static final String TABLE_TYPES_CONFIG = "table.types";
- public static final String TABLE_TYPES_DEFAULT = TableType.TABLE.toString();
- private static final String TABLE_TYPES_DOC =
- "The comma-separated types of database tables to which the sink connector can write. "
- + "By default this is ``" + TableType.TABLE + "``, but any combination of ``"
- + TableType.TABLE + "`` and ``" + TableType.VIEW + "`` is allowed. Not all databases "
- + "support writing to views, and when they do the the sink connector will fail if the "
- + "view definition does not match the records' schemas (regardless of ``"
- + AUTO_EVOLVE + "``).";
-
- // white list tables
- public static final String TABLE_WHITE_LIST_CONFIG = "tables.whitelist";
- public static final String TABLE_NAME = "tableName";
- public static final String TABLE_WHITE_LIST_DEFAULT = "";
- private static final String TABLE_WHITE_LIST_DOC =
- "Table white list.<br>db1.table01,db1.table02</br>";
-
-
- private String tableNameFormat;
- private boolean tableFromHeader;
- private int maxRetries;
- private int retryBackoffMs;
- private int batchSize;
- private boolean deleteEnabled;
- private boolean autoCreate;
- private boolean autoEvolve;
- private InsertMode insertMode;
- public final PrimaryKeyMode pkMode;
- private List<String> pkFields;
- private Set<String> fieldsWhitelist;
- private Set<String> tableWhitelist;
- private TimeZone timeZone;
- private EnumSet<TableType> tableTypes;
- public String getHost() {
- return host;
- }
-
- public int getPort() {
- return port;
- }
-
- public String getDatabase() {
- return database;
- }
-
- public String getUser() {
- return user;
- }
-
- public String getPasswd() {
- return passwd;
- }
-
- private String host;
- private int port;
- private String database;
- private String user;
- private String passwd;
-
- public DorisSinkConfig(KeyValue config) {
- super(config);
- host = config.getString(HOST).trim();
- port = config.getInt(PORT);
- database = config.getString(DATABASE).trim();
- user = config.getString(USER).trim();
- passwd = config.getString(PASSWD).trim();
-
- tableNameFormat = config.getString(TABLE_NAME_FORMAT, TABLE_NAME_FORMAT_DEFAULT).trim();
- tableFromHeader = getBoolean(config, TABLE_NAME_FROM_HEADER, false);
- batchSize = config.getInt(BATCH_SIZE, BATCH_SIZE_DEFAULT);
-
- maxRetries = config.getInt(MAX_RETRIES, MAX_RETRIES_DEFAULT);
- retryBackoffMs = config.getInt(RETRY_BACKOFF_MS, RETRY_BACKOFF_MS_DEFAULT);
- autoCreate = getBoolean(config, AUTO_CREATE, AUTO_CREATE_DEFAULT);
- autoEvolve = getBoolean(config, AUTO_EVOLVE, AUTO_EVOLVE_DEFAULT);
- if (Objects.nonNull(config.getString(INSERT_MODE))) {
- insertMode = InsertMode.valueOf(config.getString(INSERT_MODE, INSERT_MODE_DEFAULT).toUpperCase());
- }
- deleteEnabled = getBoolean(config, DELETE_ENABLED, DELETE_ENABLED_DEFAULT);
- pkMode = PrimaryKeyMode.valueOf(config.getString(PK_MODE, PK_MODE_DEFAULT).toUpperCase());
- pkFields = getList(config, PK_FIELDS);
- if (deleteEnabled && pkMode != PrimaryKeyMode.RECORD_KEY) {
- throw new ConfigException(
- "Primary key mode must be 'record_key' when delete support is enabled");
- }
- fieldsWhitelist = new HashSet<>(getList(config, FIELDS_WHITELIST));
- // table white list
- tableWhitelist = new HashSet<>(getList(config, TABLE_WHITE_LIST_CONFIG));
- String dbTimeZone = config.getString(DB_TIMEZONE_CONFIG, DB_TIMEZONE_DEFAULT);
- timeZone = TimeZone.getTimeZone(ZoneId.of(dbTimeZone));
- tableTypes = TableType.parse(getList(config, TABLE_TYPES_CONFIG, TABLE_TYPES_DEFAULT));
- }
-
- public String getTableNameFormat() {
- return tableNameFormat;
- }
-
- public boolean isTableFromHeader() {
- return tableFromHeader;
- }
-
- public int getMaxRetries() {
- return maxRetries;
- }
-
- public int getRetryBackoffMs() {
- return retryBackoffMs;
- }
-
- public int getBatchSize() {
- return batchSize;
- }
-
- public boolean isDeleteEnabled() {
- return deleteEnabled;
- }
-
- public boolean isAutoCreate() {
- return autoCreate;
- }
-
- public boolean isAutoEvolve() {
- return autoEvolve;
- }
-
- public InsertMode getInsertMode() {
- return insertMode;
- }
-
- public PrimaryKeyMode getPkMode() {
- return pkMode;
- }
-
- public List<String> getPkFields() {
- return pkFields;
- }
-
- public Set<String> getFieldsWhitelist() {
- return fieldsWhitelist;
- }
-
- public Set<String> getTableWhitelist() {
- return tableWhitelist;
- }
-
- public TimeZone getTimeZone() {
- return timeZone;
- }
-
- public EnumSet<TableType> getTableTypes() {
- return tableTypes;
- }
-
- public boolean filterWhiteTable(TableId tableId) {
- // not filter table
- if (tableWhitelist.isEmpty()) {
- return true;
- }
- for (String tableName : tableWhitelist) {
- TableId table = TableUtil.parseToTableId(tableName);
- if (table.catalogName() != null && table.catalogName().equals(tableId.catalogName())) {
- return true;
- }
- if (table.tableName().equals(tableId.tableName())) {
- return true;
- }
- }
- return false;
- }
-
- public Set<String> tableTypeNames() {
- return tableTypes.stream().map(TableType::toString).collect(Collectors.toSet());
- }
-
-}
-
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/connector/DorisSinkConnector.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/connector/DorisSinkConnector.java
deleted file mode 100644
index a2e3119f..00000000
--- a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/connector/DorisSinkConnector.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.connect.doris.connector;
-
-import io.openmessaging.KeyValue;
-import io.openmessaging.connector.api.component.task.Task;
-import io.openmessaging.connector.api.component.task.sink.SinkConnector;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class DorisSinkConnector extends SinkConnector {
-
- private static final Logger log = LoggerFactory.getLogger(DorisSinkConnector.class);
- private KeyValue connectConfig;
-
- public void start(KeyValue config) {
- this.connectConfig = config;
- }
-
- /**
- * Should invoke before start the connector.
- *
- * @param config
- * @return error message
- */
- @Override
- public void validate(KeyValue config) {
- // do validate config
- }
-
- @Override
- public void stop() {
- this.connectConfig = null;
- }
-
- /**
- * Returns a set of configurations for Tasks based on the current configuration,
- * producing at most count configurations.
- *
- * @param maxTasks maximum number of configurations to generate
- * @return configurations for Tasks
- */
- @Override
- public List<KeyValue> taskConfigs(int maxTasks) {
- log.info("Starting task config !!! ");
- List<KeyValue> configs = new ArrayList<>();
- for (int i = 0; i < maxTasks; i++) {
- configs.add(this.connectConfig);
- }
- return configs;
- }
-
- @Override
- public Class<? extends Task> taskClass() {
- return DorisSinkTask.class;
- }
-}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/connector/DorisSinkTask.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/connector/DorisSinkTask.java
deleted file mode 100644
index f775b092..00000000
--- a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/connector/DorisSinkTask.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.rocketmq.connect.doris.connector;
-
-import io.openmessaging.KeyValue;
-import io.openmessaging.connector.api.component.task.sink.ErrorRecordReporter;
-import io.openmessaging.connector.api.component.task.sink.SinkTask;
-import io.openmessaging.connector.api.component.task.sink.SinkTaskContext;
-import io.openmessaging.connector.api.data.ConnectRecord;
-import io.openmessaging.connector.api.errors.ConnectException;
-import io.openmessaging.connector.api.errors.RetriableException;
-import org.apache.rocketmq.connect.doris.exception.TableAlterOrCreateException;
-import org.apache.rocketmq.connect.doris.sink.Updater;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.sql.SQLException;
-import java.util.List;
-
-/**
- * jdbc sink task
- */
-public class DorisSinkTask extends SinkTask {
-
- private static final Logger log = LoggerFactory.getLogger(DorisSinkTask.class);
- private SinkTaskContext context;
- private ErrorRecordReporter errorRecordReporter;
- private KeyValue originalConfig;
- private DorisSinkConfig config;
- int remainingRetries;
- private Updater updater;
-
-
- /**
- * Put the records to the sink
- *
- * @param records
- */
- @Override
- public void put(List<ConnectRecord> records) throws ConnectException {
- if (records.isEmpty()) {
- return;
- }
- final int recordsCount = records.size();
- log.debug("Received {} records.", recordsCount);
- try {
- updater.write(records);
- } catch (TableAlterOrCreateException tace) {
- throw tace;
- } catch (SQLException sqle) {
- SQLException sqlAllMessagesException = getAllMessagesException(sqle);
- if (remainingRetries > 0) {
-// updater.closeQuietly();
- start(originalConfig);
- remainingRetries--;
- throw new RetriableException(sqlAllMessagesException);
- }
-
- }
- remainingRetries = config.getMaxRetries();
- }
-
- private SQLException getAllMessagesException(SQLException sqle) {
- String sqleAllMessages = "Exception chain:" + System.lineSeparator();
- for (Throwable e : sqle) {
- sqleAllMessages += e + System.lineSeparator();
- }
- SQLException sqlAllMessagesException = new SQLException(sqleAllMessages);
- sqlAllMessagesException.setNextException(sqle);
- return sqlAllMessagesException;
- }
-
- /**
- * Start the component
- *
- * @param keyValue
- */
- @Override
- public void start(KeyValue keyValue) {
- originalConfig = keyValue;
- config = new DorisSinkConfig(keyValue);
- remainingRetries = config.getMaxRetries();
- log.info("Initializing doris writer");
- this.updater = new Updater(config);
- }
-
- @Override
- public void stop() {
- log.info("Stopping task");
- }
-}
-
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/ConverterMode.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/ConverterMode.java
new file mode 100644
index 00000000..4755166d
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/ConverterMode.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter;
+
+public enum ConverterMode {
+ NORMAL("normal"),
+
+ // kafka upstream data comes from debezium
+ DEBEZIUM_INGESTION("debezium_ingestion");
+
+ private final String name;
+
+ ConverterMode(String name) {
+ this.name = name;
+ }
+
+ public static ConverterMode of(String name) {
+ return ConverterMode.valueOf(name.toUpperCase());
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public static String[] instances() {
+ return new String[] {NORMAL.name, DEBEZIUM_INGESTION.name};
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/RecordDescriptor.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/RecordDescriptor.java
new file mode 100644
index 00000000..a7d43f04
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/RecordDescriptor.java
@@ -0,0 +1,258 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter;
+
+import io.openmessaging.connector.api.data.ConnectRecord;
+import io.openmessaging.connector.api.data.Field;
+import io.openmessaging.connector.api.data.Schema;
+import io.openmessaging.connector.api.data.Struct;
+import io.openmessaging.connector.api.errors.ConnectException;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import org.apache.rocketmq.connect.doris.converter.type.Type;
+import org.apache.rocketmq.connect.doris.utils.ConnectRecordUtil;
+
+public class RecordDescriptor {
+ private final ConnectRecord record;
+ private final String topicName;
+ private final List<String> keyFieldNames;
+ private final List<String> nonKeyFieldNames;
+ private final Map<String, FieldDescriptor> fields;
+ private final boolean flattened;
+
+ private RecordDescriptor(
+ ConnectRecord record,
+ String topicName,
+ List<String> keyFieldNames,
+ List<String> nonKeyFieldNames,
+ Map<String, FieldDescriptor> fields,
+ boolean flattened) {
+ this.record = record;
+ this.topicName = topicName;
+ this.keyFieldNames = keyFieldNames;
+ this.nonKeyFieldNames = nonKeyFieldNames;
+ this.fields = fields;
+ this.flattened = flattened;
+ }
+
+ public String getTopicName() {
+ return topicName;
+ }
+
+ public long getQueueOffset() {
+ return ConnectRecordUtil.getQueueOffset(record.getPosition().getOffset());
+ }
+
+ public List<String> getKeyFieldNames() {
+ return keyFieldNames;
+ }
+
+ public List<String> getNonKeyFieldNames() {
+ return nonKeyFieldNames;
+ }
+
+ public Map<String, FieldDescriptor> getFields() {
+ return fields;
+ }
+
+ public boolean isDebeziumSinkRecord() {
+ return !flattened;
+ }
+
+ public boolean isTombstone() {
+ // Debezium TOMBSTONE has both value and valueSchema to null.
+ return record.getData() == null && record.getSchema().getValueSchema() == null;
+ }
+
+ public boolean isDelete() {
+ if (!isDebeziumSinkRecord()) {
+ return record.getData() == null;
+ } else if (record.getData() != null) {
+ final Struct value = (Struct) record.getData();
+ return "d".equals(value.getString("op"));
+ }
+ return false;
+ }
+
+ public Struct getAfterStruct() {
+ if (isDebeziumSinkRecord()) {
+ return ((Struct) record.getData()).getStruct("after");
+ } else {
+ return ((Struct) record.getData());
+ }
+ }
+
+ public Struct getBeforeStruct() {
+ if (isDebeziumSinkRecord()) {
+ return ((Struct) record.getData()).getStruct("before");
+ } else {
+ return ((Struct) record.getData());
+ }
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public static class FieldDescriptor {
+ private final Schema schema;
+ private final String name;
+ private final Map<String, Type> typeRegistry;
+ private final Type type;
+ private final String typeName;
+ private final String schemaName;
+ private String comment;
+ private String defaultValue;
+
+ public FieldDescriptor(Schema schema, String name, Map<String, Type> typeRegistry) {
+ this.schema = schema;
+ this.name = name;
+ this.typeRegistry = typeRegistry;
+ this.schemaName = schema.getName();
+ this.type =
+ typeRegistry.getOrDefault(
+ schema.getName(), typeRegistry.get(schema.getFieldType().name()));
+ if (this.type == null) {
+ throw new IllegalArgumentException(
+ "Type not found in registry for schema: " + schema);
+ }
+ this.typeName = type.getTypeName(schema);
+ }
+
+ public FieldDescriptor(
+ Schema schema,
+ String name,
+ Map<String, Type> typeRegistry,
+ String comment,
+ String defaultValue) {
+ this(schema, name, typeRegistry);
+ this.comment = comment;
+ this.defaultValue = defaultValue;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public Type getType() {
+ return type;
+ }
+
+ public String getTypeName() {
+ return typeName;
+ }
+
+ public String getSchemaName() {
+ return schemaName;
+ }
+
+ public Schema getSchema() {
+ return schema;
+ }
+
+ public String getComment() {
+ return comment;
+ }
+
+ public String getDefaultValue() {
+ return defaultValue;
+ }
+ }
+
+ public static class Builder {
+
+ private ConnectRecord sinkRecord;
+ private Map<String, Type> typeRegistry;
+
+ // Internal build state
+ private final List<String> keyFieldNames = new ArrayList<>();
+ private final List<String> nonKeyFieldNames = new ArrayList<>();
+ private final Map<String, FieldDescriptor> allFields = new LinkedHashMap<>();
+
+ public Builder withSinkRecord(ConnectRecord record) {
+ this.sinkRecord = record;
+ return this;
+ }
+
+ public Builder withTypeRegistry(Map<String, Type> typeRegistry) {
+ this.typeRegistry = typeRegistry;
+ return this;
+ }
+
+ public RecordDescriptor build() {
+ Objects.requireNonNull(sinkRecord, "The sink record must be provided.");
+
+ final boolean flattened = !isTombstone(sinkRecord) && isFlattened(sinkRecord);
+ readSinkRecordNonKeyData(sinkRecord, flattened);
+
+ return new RecordDescriptor(
+ sinkRecord,
+ ConnectRecordUtil.getTopicName(sinkRecord.getPosition().getPartition()),
+ keyFieldNames,
+ nonKeyFieldNames,
+ allFields,
+ flattened);
+ }
+
+ private boolean isFlattened(ConnectRecord record) {
+ return record.getSchema().getValueSchema().getName() == null || !record.getSchema().getValueSchema()
+ .getName().contains("Envelope");
+ }
+
+ private boolean isTombstone(ConnectRecord record) {
+ return record.getData() == null && record.getSchema() == null;
+ }
+
+ private void readSinkRecordNonKeyData(ConnectRecord record, boolean flattened) {
+ final Schema valueSchema = record.getSchema().getValueSchema();
+ if (valueSchema != null) {
+ if (flattened) {
+ // In a flattened event type, it's safe to read the field names directly
+ // from the schema as this isn't a complex Debezium message type.
+ applyNonKeyFields(valueSchema);
+ } else {
+ Field after = valueSchema.getField("after");
+ if (after == null) {
+ throw new ConnectException(
+ "Received an unexpected message type that does not have an 'after' Debezium block");
+ }
+ applyNonKeyFields(after.getSchema());
+ }
+ }
+ }
+
+ private void applyNonKeyFields(Schema schema) {
+ for (Field field : schema.getFields()) {
+ if (!keyFieldNames.contains(field.getName())) {
+ applyNonKeyField(field.getName(), field.getSchema());
+ }
+ }
+ }
+
+ private void applyNonKeyField(String name, Schema schema) {
+ FieldDescriptor fieldDescriptor = new FieldDescriptor(schema, name, typeRegistry);
+ nonKeyFieldNames.add(fieldDescriptor.getName());
+ allFields.put(fieldDescriptor.getName(), fieldDescriptor);
+ }
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/RecordService.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/RecordService.java
new file mode 100644
index 00000000..c08ff1d9
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/RecordService.java
@@ -0,0 +1,334 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.openmessaging.connector.api.data.ConnectRecord;
+import io.openmessaging.connector.api.data.Struct;
+import io.openmessaging.connector.api.errors.ConnectException;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.StringJoiner;
+import java.util.stream.Collectors;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.connect.doris.cfg.DorisOptions;
+import org.apache.rocketmq.connect.doris.converter.schema.SchemaChangeManager;
+import org.apache.rocketmq.connect.doris.converter.schema.SchemaEvolutionMode;
+import org.apache.rocketmq.connect.doris.converter.type.Type;
+import org.apache.rocketmq.connect.doris.exception.DataFormatException;
+import org.apache.rocketmq.connect.doris.exception.DorisException;
+import org.apache.rocketmq.connect.doris.exception.SchemaChangeException;
+import org.apache.rocketmq.connect.doris.model.ColumnDescriptor;
+import org.apache.rocketmq.connect.doris.model.TableDescriptor;
+import org.apache.rocketmq.connect.doris.model.doris.Schema;
+import org.apache.rocketmq.connect.doris.service.DorisSystemService;
+import org.apache.rocketmq.connect.doris.service.RestService;
+import org.apache.rocketmq.connect.doris.utils.ConnectRecordUtil;
+import org.apache.rocketmq.connect.doris.writer.LoadConstants;
+import org.apache.rocketmq.connect.doris.writer.RecordBuffer;
+import org.apache.rocketmq.connect.runtime.converter.record.json.JsonConverter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class RecordService {
+ private static final Logger LOG = LoggerFactory.getLogger(RecordService.class);
+ public static final String SCHEMA_CHANGE_VALUE = "SchemaChangeValue";
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+ private final JsonConverter converter;
+ private DorisSystemService dorisSystemService;
+ private SchemaChangeManager schemaChangeManager;
+ private DorisOptions dorisOptions;
+ private RecordTypeRegister recordTypeRegister;
+ private Map<String, TableDescriptor> dorisTableDescriptorCache;
+
+ public RecordService() {
+ this.converter = new JsonConverter();
+ Map<String, Object> config = new HashMap<>();
+ config.put("converterConfig", "false");
+ this.converter.configure(config);
+ }
+
+ public RecordService(DorisOptions dorisOptions) {
+ this();
+ this.dorisOptions = dorisOptions;
+ this.recordTypeRegister = new RecordTypeRegister(dorisOptions);
+ this.dorisSystemService = new DorisSystemService(dorisOptions);
+ this.schemaChangeManager = new SchemaChangeManager(dorisOptions);
+ this.dorisTableDescriptorCache = new HashMap<>();
+ }
+
+ /**
+ * process struct record from debezium: { "schema": { "type": "struct", "fields": [ ...... ],
+ * "optional": false, "name": "" }, "payload": { "name": "doris", "__deleted": "true" } }
+ */
+ public String processStructRecord(ConnectRecord record) {
+ String processedRecord;
+ String topicName = ConnectRecordUtil.getTopicName(record.getPosition().getPartition());
+ if (ConverterMode.DEBEZIUM_INGESTION == dorisOptions.getConverterMode()) {
+ validate(record);
+ RecordDescriptor recordDescriptor = buildRecordDescriptor(record);
+ if (recordDescriptor.isTombstone()) {
+ return null;
+ }
+ String tableName = dorisOptions.getTopicMapTable(recordDescriptor.getTopicName());
+ checkAndApplyTableChangesIfNeeded(tableName, recordDescriptor);
+
+ List<String> nonKeyFieldNames = recordDescriptor.getNonKeyFieldNames();
+ if (recordDescriptor.isDelete()) {
+ processedRecord =
+ parseFieldValues(
+ recordDescriptor,
+ recordDescriptor.getBeforeStruct(),
+ nonKeyFieldNames,
+ true);
+ } else {
+ processedRecord =
+ parseFieldValues(
+ recordDescriptor,
+ recordDescriptor.getAfterStruct(),
+ nonKeyFieldNames,
+ false);
+ }
+ } else {
+ byte[] bytes =
+ converter.fromConnectData(topicName, record.getSchema().getValueSchema(), record.getData());
+ processedRecord = new String(bytes, StandardCharsets.UTF_8);
+ }
+ return processedRecord;
+ }
+
+ private void validate(ConnectRecord record) {
+ if (isSchemaChange(record)) {
+ LOG.warn(
+ "Schema change records are not supported by doris-kafka-connector. Adjust `topics` or `topics.regex` to exclude schema change topic.");
+ throw new DorisException(
+ "Schema change records are not supported by doris-kafka-connector. Adjust `topics` or `topics.regex` to exclude schema change topic.");
+ }
+ }
+
+ private static boolean isSchemaChange(final ConnectRecord record) {
+ return record.getSchema().getValueSchema() != null
+ && StringUtils.isNotEmpty(record.getSchema().getValueSchema().getName())
+ && record.getSchema().getValueSchema().getName().contains(SCHEMA_CHANGE_VALUE);
+ }
+
+ private void checkAndApplyTableChangesIfNeeded(
+ String tableName, RecordDescriptor recordDescriptor) {
+ if (!hasTable(tableName)) {
+ // TODO Table does not exist, lets attempt to create it.
+ LOG.warn("The {} table does not exist, please create it manually.", tableName);
+ throw new DorisException(
+ "The " + tableName + " table does not exist, please create it manually.");
+ } else {
+ // Table exists, lets attempt to alter it if necessary.
+ alterTableIfNeeded(tableName, recordDescriptor);
+ }
+ }
+
+ private boolean hasTable(String tableName) {
+ if (!dorisTableDescriptorCache.containsKey(tableName)) {
+ boolean exist = dorisSystemService.tableExists(dorisOptions.getDatabase(), tableName);
+ if (exist) {
+ dorisTableDescriptorCache.put(tableName, null);
+ }
+ return exist;
+ }
+ return true;
+ }
+
+ private void alterTableIfNeeded(String tableName, RecordDescriptor record) {
+ // Resolve table metadata from the database
+ final TableDescriptor table = fetchDorisTableDescriptor(tableName);
+
+ Set<RecordDescriptor.FieldDescriptor> missingFields = resolveMissingFields(record, table);
+ if (missingFields.isEmpty()) {
+ // There are no missing fields, simply return
+ // TODO should we check column type changes or default value changes?
+ return;
+ }
+
+ LOG.info(
+ "Find some miss columns in {} table, try to alter add this columns={}.",
+ tableName,
+ missingFields.stream()
+ .map(RecordDescriptor.FieldDescriptor::getName)
+ .collect(Collectors.toList()));
+ if (SchemaEvolutionMode.NONE.equals(dorisOptions.getSchemaEvolutionMode())) {
+ LOG.warn(
+ "Table '{}' cannot be altered because schema evolution is disabled.",
+ tableName);
+ throw new SchemaChangeException(
+ "Cannot alter table " + table + " because schema evolution is disabled");
+ }
+ for (RecordDescriptor.FieldDescriptor missingField : missingFields) {
+ schemaChangeManager.addColumnDDL(tableName, missingField);
+ }
+ TableDescriptor newTableDescriptor = obtainTableSchema(tableName);
+ dorisTableDescriptorCache.put(tableName, newTableDescriptor);
+ }
+
+ private Set<RecordDescriptor.FieldDescriptor> resolveMissingFields(
+ RecordDescriptor record, TableDescriptor table) {
+ Set<RecordDescriptor.FieldDescriptor> missingFields = new HashSet<>();
+ for (Map.Entry<String, RecordDescriptor.FieldDescriptor> entry :
+ record.getFields().entrySet()) {
+ String filedName = entry.getKey();
+ if (!table.hasColumn(filedName)) {
+ missingFields.add(entry.getValue());
+ }
+ }
+ return missingFields;
+ }
+
+ private TableDescriptor fetchDorisTableDescriptor(String tableName) {
+ if (!dorisTableDescriptorCache.containsKey(tableName)
+ || Objects.isNull(dorisTableDescriptorCache.get(tableName))) {
+ TableDescriptor tableDescriptor = obtainTableSchema(tableName);
+ dorisTableDescriptorCache.put(tableName, tableDescriptor);
+ }
+ return dorisTableDescriptorCache.get(tableName);
+ }
+
+ private TableDescriptor obtainTableSchema(String tableName) {
+ Schema schema =
+ RestService.getSchema(dorisOptions, dorisOptions.getDatabase(), tableName, LOG);
+ List<ColumnDescriptor> columnDescriptors = new ArrayList<>();
+ schema.getProperties()
+ .forEach(
+ column -> {
+ ColumnDescriptor columnDescriptor =
+ ColumnDescriptor.builder()
+ .columnName(column.getName())
+ .typeName(column.getType())
+ .comment(column.getComment())
+ .build();
+ columnDescriptors.add(columnDescriptor);
+ });
+ return TableDescriptor.builder()
+ .tableName(tableName)
+ .type(schema.getKeysType())
+ .columns(columnDescriptors)
+ .build();
+ }
+
+ /**
+ * process list record from kafka [{"name":"doris1"},{"name":"doris2"}]
+ */
+ public String processListRecord(ConnectRecord record) {
+ try {
+ StringJoiner sj = new StringJoiner(RecordBuffer.LINE_SEPARATOR);
+ List recordList = (List) record.getData();
+ for (Object item : recordList) {
+ sj.add(MAPPER.writeValueAsString(item));
+ }
+ return sj.toString();
+ } catch (IOException e) {
+ LOG.error("process list record failed: {}", record.getData());
+ throw new DataFormatException("process list record failed");
+ }
+ }
+
+ /**
+ * process map record from kafka {"name":"doris"}
+ */
+ public String processMapRecord(ConnectRecord record) {
+ try {
+ return MAPPER.writeValueAsString(record.getData());
+ } catch (IOException e) {
+ LOG.error("process map record failed: {}", record.getData());
+ throw new DataFormatException("process map record failed");
+ }
+ }
+
+ private String parseFieldValues(
+ RecordDescriptor record, Struct source, List<String> fields, boolean isDelete) {
+ Map<String, Object> filedMapping = new LinkedHashMap<>();
+ String filedResult = null;
+ for (String fieldName : fields) {
+ final RecordDescriptor.FieldDescriptor field = record.getFields().get(fieldName);
+ Type type = field.getType();
+ Object value =
+ field.getSchema().isOptional()
+ ? source.getWithoutDefault(fieldName)
+ : source.get(fieldName);
+ Object convertValue = type.getValue(value, field.getSchema());
+ if (Objects.nonNull(convertValue) && !type.isNumber()) {
+ filedMapping.put(fieldName, convertValue.toString());
+ } else {
+ filedMapping.put(fieldName, convertValue);
+ }
+ }
+ try {
+ if (isDelete) {
+ filedMapping.put(LoadConstants.DORIS_DELETE_SIGN, LoadConstants.DORIS_DEL_TRUE);
+ } else {
+ filedMapping.put(LoadConstants.DORIS_DELETE_SIGN, LoadConstants.DORIS_DEL_FALSE);
+ }
+ filedResult = MAPPER.writeValueAsString(filedMapping);
+ } catch (JsonProcessingException e) {
+ LOG.error("parse record failed, cause by parse json error: {}", filedMapping);
+ }
+ return filedResult;
+ }
+
+ /**
+ * Given a single Record from put API, process it and convert it into a Json String.
+ *
+ * @param record record from Kafka
+ * @return Json String
+ */
+ public String getProcessedRecord(ConnectRecord record) {
+ String processedRecord;
+ if (record.getData() instanceof Struct) {
+ processedRecord = processStructRecord(record);
+ } else if (record.getData() instanceof List) {
+ processedRecord = processListRecord(record);
+ } else if (record.getData() instanceof Map) {
+ processedRecord = processMapRecord(record);
+ } else {
+ processedRecord = record.getData().toString();
+ }
+ return processedRecord;
+ }
+
+ private RecordDescriptor buildRecordDescriptor(ConnectRecord record) {
+ RecordDescriptor recordDescriptor;
+ try {
+ recordDescriptor =
+ RecordDescriptor.builder()
+ .withSinkRecord(record)
+ .withTypeRegistry(recordTypeRegister.getTypeRegistry())
+ .build();
+ } catch (Exception e) {
+ throw new ConnectException("Failed to process a sink record", e);
+ }
+ return recordDescriptor;
+ }
+
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/RecordTypeRegister.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/RecordTypeRegister.java
new file mode 100644
index 00000000..ced15e62
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/RecordTypeRegister.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.rocketmq.connect.doris.cfg.DorisOptions;
+import org.apache.rocketmq.connect.doris.converter.type.Type;
+import org.apache.rocketmq.connect.doris.converter.type.connect.ConnectBooleanType;
+import org.apache.rocketmq.connect.doris.converter.type.connect.ConnectBytesType;
+import org.apache.rocketmq.connect.doris.converter.type.connect.ConnectDateType;
+import org.apache.rocketmq.connect.doris.converter.type.connect.ConnectDecimalType;
+import org.apache.rocketmq.connect.doris.converter.type.connect.ConnectFloat32Type;
+import org.apache.rocketmq.connect.doris.converter.type.connect.ConnectFloat64Type;
+import org.apache.rocketmq.connect.doris.converter.type.connect.ConnectInt16Type;
+import org.apache.rocketmq.connect.doris.converter.type.connect.ConnectInt32Type;
+import org.apache.rocketmq.connect.doris.converter.type.connect.ConnectInt64Type;
+import org.apache.rocketmq.connect.doris.converter.type.connect.ConnectInt8Type;
+import org.apache.rocketmq.connect.doris.converter.type.connect.ConnectMapToConnectStringType;
+import org.apache.rocketmq.connect.doris.converter.type.connect.ConnectStringType;
+import org.apache.rocketmq.connect.doris.converter.type.connect.ConnectTimeType;
+import org.apache.rocketmq.connect.doris.converter.type.connect.ConnectTimestampType;
+import org.apache.rocketmq.connect.doris.converter.type.debezium.ArrayType;
+import org.apache.rocketmq.connect.doris.converter.type.debezium.DateType;
+import org.apache.rocketmq.connect.doris.converter.type.debezium.GeographyType;
+import org.apache.rocketmq.connect.doris.converter.type.debezium.GeometryType;
+import org.apache.rocketmq.connect.doris.converter.type.debezium.MicroTimeType;
+import org.apache.rocketmq.connect.doris.converter.type.debezium.MicroTimestampType;
+import org.apache.rocketmq.connect.doris.converter.type.debezium.NanoTimeType;
+import org.apache.rocketmq.connect.doris.converter.type.debezium.NanoTimestampType;
+import org.apache.rocketmq.connect.doris.converter.type.debezium.PointType;
+import org.apache.rocketmq.connect.doris.converter.type.debezium.TimeType;
+import org.apache.rocketmq.connect.doris.converter.type.debezium.TimestampType;
+import org.apache.rocketmq.connect.doris.converter.type.debezium.VariableScaleDecimalType;
+import org.apache.rocketmq.connect.doris.converter.type.debezium.ZonedTimeType;
+import org.apache.rocketmq.connect.doris.converter.type.debezium.ZonedTimestampType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class RecordTypeRegister {
+
+ private static final Logger LOG = LoggerFactory.getLogger(RecordTypeRegister.class);
+
+ private final Map<String, Type> typeRegistry = new HashMap<>();
+ private final DorisOptions dorisOptions;
+
+ public RecordTypeRegister(DorisOptions dorisOptions) {
+ this.dorisOptions = dorisOptions;
+ registerTypes();
+ }
+
+ protected void registerTypes() {
+ // Supported common Debezium data types
+ registerType(DateType.INSTANCE);
+ registerType(TimeType.INSTANCE);
+ registerType(MicroTimeType.INSTANCE);
+ registerType(TimestampType.INSTANCE);
+ registerType(MicroTimestampType.INSTANCE);
+ registerType(NanoTimeType.INSTANCE);
+ registerType(NanoTimestampType.INSTANCE);
+ registerType(ZonedTimeType.INSTANCE);
+ registerType(ZonedTimestampType.INSTANCE);
+ registerType(VariableScaleDecimalType.INSTANCE);
+ registerType(PointType.INSTANCE);
+ registerType(GeographyType.INSTANCE);
+ registerType(GeometryType.INSTANCE);
+ registerType(ArrayType.INSTANCE);
+
+ // Supported connect data types
+ registerType(ConnectBooleanType.INSTANCE);
+ registerType(ConnectBytesType.INSTANCE);
+ registerType(ConnectDateType.INSTANCE);
+ registerType(ConnectDecimalType.INSTANCE);
+ registerType(ConnectFloat32Type.INSTANCE);
+ registerType(ConnectFloat64Type.INSTANCE);
+ registerType(ConnectInt8Type.INSTANCE);
+ registerType(ConnectInt16Type.INSTANCE);
+ registerType(ConnectInt32Type.INSTANCE);
+ registerType(ConnectInt64Type.INSTANCE);
+ registerType(ConnectStringType.INSTANCE);
+ registerType(ConnectTimestampType.INSTANCE);
+ registerType(ConnectTimeType.INSTANCE);
+ registerType(ConnectMapToConnectStringType.INSTANCE);
+ }
+
+ protected void registerType(Type type) {
+ type.configure(dorisOptions);
+ for (String key : type.getRegistrationKeys()) {
+ final Type existing = typeRegistry.put(key, type);
+ if (existing != null) {
+ LOG.debug(
+ "Type replaced [{}]: {} -> {}",
+ key,
+ existing.getClass().getName(),
+ type.getClass().getName());
+ } else {
+ LOG.debug("Type registered [{}]: {}", key, type.getClass().getName());
+ }
+ }
+ }
+
+ public Map<String, Type> getTypeRegistry() {
+ return typeRegistry;
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/schema/SchemaChangeManager.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/schema/SchemaChangeManager.java
new file mode 100644
index 00000000..52765d47
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/schema/SchemaChangeManager.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.schema;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.io.IOException;
+import java.io.Serializable;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.http.HttpHeaders;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.util.EntityUtils;
+import org.apache.rocketmq.connect.doris.cfg.DorisOptions;
+import org.apache.rocketmq.connect.doris.converter.RecordDescriptor;
+import org.apache.rocketmq.connect.doris.exception.SchemaChangeException;
+import org.apache.rocketmq.connect.doris.service.DorisSystemService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static java.net.HttpURLConnection.HTTP_OK;
+
+public class SchemaChangeManager implements Serializable {
+ private static final long serialVersionUID = 1L;
+ private static final Logger LOG = LoggerFactory.getLogger(SchemaChangeManager.class);
+ private static final String ADD_DDL = "ALTER TABLE %s ADD COLUMN %s %s";
+ private static final String SCHEMA_CHANGE_API = "http://%s/api/query/default_cluster/%s";
+ private final ObjectMapper objectMapper = new ObjectMapper();
+ private final DorisOptions dorisOptions;
+ private final DorisSystemService dorisSystemService;
+
+ public SchemaChangeManager(DorisOptions dorisOptions) {
+ this.dorisOptions = dorisOptions;
+ this.dorisSystemService = new DorisSystemService(dorisOptions);
+ }
+
+ private boolean handleSchemaChange(Map<String, Object> responseMap, String responseEntity) {
+ String code = responseMap.getOrDefault("code", "-1").toString();
+ if (code.equals("0")) {
+ return true;
+ } else {
+ throw new SchemaChangeException("Failed to schemaChange, response: " + responseEntity);
+ }
+ }
+
+ public void addColumnDDL(String tableName, RecordDescriptor.FieldDescriptor field) {
+ try {
+ // check the add column whether exist in table.
+ if (dorisSystemService.isColumnExist(
+ dorisOptions.getDatabase(), tableName, field.getName())) {
+ LOG.warn(
+ "The column {} already exists in table {}, no need to add it again",
+ field.getName(),
+ tableName);
+ return;
+ }
+
+ String addColumnDDL = buildAddColumnDDL(dorisOptions.getDatabase(), tableName, field);
+ boolean status = execute(addColumnDDL, dorisOptions.getDatabase());
+ LOG.info(
+ "Add missing column for {} table, ddl={}, status={}",
+ tableName,
+ addColumnDDL,
+ status);
+ } catch (Exception e) {
+ LOG.warn("Failed to add column for {}, cause by: ", tableName, e);
+ throw new SchemaChangeException(
+ "Failed to add column for " + tableName + ", cause by:", e);
+ }
+ }
+
+ public static String buildAddColumnDDL(
+ String database, String tableName, RecordDescriptor.FieldDescriptor field) {
+ String name = field.getName();
+ String typeName = field.getTypeName();
+ String comment = field.getComment();
+ String defaultValue = field.getDefaultValue();
+
+ String addDDL =
+ String.format(
+ ADD_DDL,
+ identifier(database) + "." + identifier(tableName),
+ identifier(name),
+ typeName);
+ if (defaultValue != null) {
+ addDDL = addDDL + " DEFAULT " + quoteDefaultValue(defaultValue);
+ }
+ if (StringUtils.isNotEmpty(comment)) {
+ addDDL = addDDL + " COMMENT '" + quoteComment(comment) + "'";
+ }
+ return addDDL;
+ }
+
+ private static String quoteComment(String comment) {
+ return comment.replaceAll("'", "\\\\'");
+ }
+
+ private static String identifier(String name) {
+ return "`" + name + "`";
+ }
+
+ private static String quoteDefaultValue(String defaultValue) {
+ // DEFAULT current_timestamp not need quote
+ if (defaultValue.equalsIgnoreCase("current_timestamp")) {
+ return defaultValue;
+ }
+ return "'" + defaultValue + "'";
+ }
+
+ /**
+ * execute sql in doris.
+ */
+ public boolean execute(String ddl, String database)
+ throws IOException, IllegalArgumentException {
+ if (StringUtils.isEmpty(ddl)) {
+ return false;
+ }
+ LOG.info("Execute SQL: {}", ddl);
+ HttpPost httpPost = buildHttpPost(ddl, database);
+ String responseEntity = "";
+ Map<String, Object> responseMap = handleResponse(httpPost, responseEntity);
+ return handleSchemaChange(responseMap, responseEntity);
+ }
+
+ public HttpPost buildHttpPost(String ddl, String database)
+ throws IllegalArgumentException, IOException {
+ Map<String, String> param = new HashMap<>();
+ param.put("stmt", ddl);
+ String requestUrl = String.format(SCHEMA_CHANGE_API, dorisOptions.getHttpUrl(), database);
+ HttpPost httpPost = new HttpPost(requestUrl);
+ httpPost.setHeader(HttpHeaders.AUTHORIZATION, authHeader());
+ httpPost.setHeader(HttpHeaders.CONTENT_TYPE, "application/json");
+ httpPost.setEntity(new StringEntity(objectMapper.writeValueAsString(param)));
+ return httpPost;
+ }
+
+ private Map<String, Object> handleResponse(HttpUriRequest request, String responseEntity) {
+ try (CloseableHttpClient httpclient = HttpClients.createDefault()) {
+ CloseableHttpResponse response = httpclient.execute(request);
+ final int statusCode = response.getStatusLine().getStatusCode();
+ final String reasonPhrase = response.getStatusLine().getReasonPhrase();
+ if (statusCode == HTTP_OK && response.getEntity() != null) {
+ responseEntity = EntityUtils.toString(response.getEntity());
+ return objectMapper.readValue(responseEntity, Map.class);
+ } else {
+ throw new SchemaChangeException(
+ "Failed to schemaChange, status: "
+ + statusCode
+ + ", reason: "
+ + reasonPhrase);
+ }
+ } catch (Exception e) {
+ LOG.error("SchemaChange request error,", e);
+ throw new SchemaChangeException("SchemaChange request error with " + e.getMessage());
+ }
+ }
+
+ private String authHeader() {
+ return "Basic "
+ + new String(
+ Base64.encodeBase64(
+ (dorisOptions.getUser() + ":" + dorisOptions.getPassword())
+ .getBytes(StandardCharsets.UTF_8)));
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/schema/SchemaEvolutionMode.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/schema/SchemaEvolutionMode.java
new file mode 100644
index 00000000..fc98b7c9
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/schema/SchemaEvolutionMode.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.schema;
+
+public enum SchemaEvolutionMode {
+ NONE("none"),
+
+ BASIC("basic");
+
+ private final String name;
+
+ SchemaEvolutionMode(String name) {
+ this.name = name;
+ }
+
+ public static SchemaEvolutionMode of(String name) {
+ return SchemaEvolutionMode.valueOf(name.toUpperCase());
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public static String[] instances() {
+ return new String[] {NONE.name, BASIC.name};
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/AbstractDateType.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/AbstractDateType.java
new file mode 100644
index 00000000..ef6eea01
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/AbstractDateType.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.type;
+
+import io.openmessaging.connector.api.data.Schema;
+import org.apache.rocketmq.connect.doris.converter.type.doris.DorisType;
+
+/**
+ * An abstract base class for all temporal date implementations of {@link Type}.
+ */
+public abstract class AbstractDateType extends AbstractTemporalType {
+
+ @Override
+ public String getTypeName(Schema schema) {
+ return DorisType.DATE;
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/AbstractGeometryType.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/AbstractGeometryType.java
new file mode 100644
index 00000000..ac688754
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/AbstractGeometryType.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.type;
+
+import io.openmessaging.connector.api.data.Schema;
+import org.apache.rocketmq.connect.doris.converter.type.doris.DorisType;
+
+public abstract class AbstractGeometryType extends AbstractType {
+ @Override
+ public String getTypeName(Schema schema) {
+ return DorisType.STRING;
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/AbstractTemporalType.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/AbstractTemporalType.java
new file mode 100644
index 00000000..67afcb29
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/AbstractTemporalType.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.type;
+
+import java.time.ZoneId;
+import java.util.TimeZone;
+import org.apache.rocketmq.connect.doris.cfg.DorisOptions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * An abstract base class for all temporal implementations of {@link Type}.
+ */
+public abstract class AbstractTemporalType extends AbstractType {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(AbstractTemporalType.class);
+
+ private TimeZone databaseTimeZone;
+
+ @Override
+ public void configure(DorisOptions dorisOptions) {
+ final String databaseTimeZone = dorisOptions.getDatabaseTimeZone();
+ try {
+ this.databaseTimeZone = TimeZone.getTimeZone(ZoneId.of(databaseTimeZone));
+ } catch (Exception e) {
+ LOGGER.error(
+ "Failed to resolve time zone '{}', please specify a correct time zone value",
+ databaseTimeZone,
+ e);
+ throw e;
+ }
+ }
+
+ protected TimeZone getDatabaseTimeZone() {
+ return databaseTimeZone;
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/AbstractTimeType.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/AbstractTimeType.java
new file mode 100644
index 00000000..1100ee06
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/AbstractTimeType.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.type;
+
+import io.openmessaging.connector.api.data.Schema;
+import java.util.Optional;
+import org.apache.rocketmq.connect.doris.converter.type.doris.DorisType;
+import org.apache.rocketmq.connect.doris.converter.type.doris.DorisTypeProperties;
+
+/**
+ * An abstract temporal implementation of {@link Type} for {@code TIME} based columns.
+ */
+public abstract class AbstractTimeType extends AbstractTemporalType {
+
+ @Override
+ public String getTypeName(Schema schema) {
+ // NOTE:
+ // The MySQL connector does not use the __debezium.source.column.scale parameter to pass
+ // the time column's precision but instead uses the __debezium.source.column.length key
+ // which differs from all other connector implementations.
+ //
+ final int precision = getTimePrecision(schema);
+ return String.format(
+ "%s(%s)",
+ DorisType.DATETIME,
+ Math.min(precision, DorisTypeProperties.MAX_SUPPORTED_DATE_TIME_PRECISION));
+ }
+
+ protected int getTimePrecision(Schema schema) {
+ final String length = getSourceColumnLength(schema).orElse("0");
+ final Optional<String> scale = getSourceColumnPrecision(schema);
+ return scale.map(Integer::parseInt).orElseGet(() -> Integer.parseInt(length));
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/AbstractTimestampType.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/AbstractTimestampType.java
new file mode 100644
index 00000000..9f13d7cb
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/AbstractTimestampType.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.type;
+
+import io.openmessaging.connector.api.data.Schema;
+import java.util.Optional;
+import org.apache.rocketmq.connect.doris.converter.type.doris.DorisType;
+import org.apache.rocketmq.connect.doris.converter.type.doris.DorisTypeProperties;
+
+/**
+ * An abstract temporal implementation of {@link Type} for {@code TIMESTAMP} based columns.
+ */
+public abstract class AbstractTimestampType extends AbstractTemporalType {
+
+ @Override
+ public String getTypeName(Schema schema) {
+ final int precision = getTimePrecision(schema);
+ return String.format(
+ "%s(%s)",
+ DorisType.DATETIME,
+ Math.min(precision, DorisTypeProperties.MAX_SUPPORTED_DATE_TIME_PRECISION));
+ }
+
+ protected int getTimePrecision(Schema schema) {
+ final String length = getSourceColumnLength(schema).orElse("0");
+ final Optional<String> scale = getSourceColumnPrecision(schema);
+ return scale.map(Integer::parseInt).orElseGet(() -> Integer.parseInt(length));
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/AbstractType.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/AbstractType.java
new file mode 100644
index 00000000..4dd9e71a
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/AbstractType.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.type;
+
+import io.openmessaging.connector.api.data.Schema;
+import java.util.Objects;
+import java.util.Optional;
+import org.apache.rocketmq.connect.doris.cfg.DorisOptions;
+import org.apache.rocketmq.connect.doris.converter.type.util.SchemaUtils;
+
+/**
+ * An abstract implementation of {@link Type}, which all types should extend.
+ */
+public abstract class AbstractType implements Type {
+
+ @Override
+ public void configure(DorisOptions dorisOptions) {
+ }
+
+ @Override
+ public Object getValue(Object sourceValue) {
+ return sourceValue;
+ }
+
+ @Override
+ public boolean isNumber() {
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName();
+ }
+
+ protected Optional<String> getSchemaParameter(Schema schema, String parameterName) {
+ if (!Objects.isNull(schema.getParameters())) {
+ return Optional.ofNullable(schema.getParameters().get(parameterName));
+ }
+ return Optional.empty();
+ }
+
+ protected Optional<String> getSourceColumnType(Schema schema) {
+ return SchemaUtils.getSourceColumnType(schema);
+ }
+
+ protected Optional<String> getSourceColumnLength(Schema schema) {
+ return SchemaUtils.getSourceColumnLength(schema);
+ }
+
+ protected Optional<String> getSourceColumnPrecision(Schema schema) {
+ return SchemaUtils.getSourceColumnPrecision(schema);
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/Type.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/Type.java
new file mode 100644
index 00000000..2ada4bca
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/Type.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.type;
+
+import io.openmessaging.connector.api.data.Schema;
+import org.apache.rocketmq.connect.doris.cfg.DorisOptions;
+
+/**
+ * A type indicates the type of each column of kafka record, including various column types of
+ * debezium and connect.
+ */
+public interface Type {
+
+ /**
+ * Allows a type to perform initialization/configuration tasks based on user configs.
+ */
+ void configure(DorisOptions dorisOptions);
+
+ /**
+ * Returns the names that this type will be mapped as.
+ *
+ * <p>For example, when creating a custom mapping for {@code io.debezium.data.Bits}, a type
+ * could be registered using the {@code LOGICAL_NAME} of the schema if the type is to be used
+ * when a schema name is identified; otherwise it could be registered as the raw column type
+ * when column type propagation is enabled.
+ */
+ String[] getRegistrationKeys();
+
+ /**
+ * Get the actual converted value based on the column type.
+ */
+ Object getValue(Object sourceValue);
+
+ default Object getValue(Object sourceValue, Schema schema) {
+ return getValue(sourceValue);
+ }
+
+ String getTypeName(Schema schema);
+
+ boolean isNumber();
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/AbstractConnectMapType.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/AbstractConnectMapType.java
new file mode 100644
index 00000000..9de7b72a
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/AbstractConnectMapType.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.type.connect;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.openmessaging.connector.api.errors.ConnectException;
+
+public abstract class AbstractConnectMapType extends AbstractConnectSchemaType {
+
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+
+ @Override
+ public String[] getRegistrationKeys() {
+ return new String[] {"MAP"};
+ }
+
+ protected String mapToJsonString(Object value) {
+ try {
+ return MAPPER.writeValueAsString(value);
+ } catch (JsonProcessingException e) {
+ throw new ConnectException("Failed to deserialize MAP data to JSON", e);
+ }
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/AbstractConnectSchemaType.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/AbstractConnectSchemaType.java
new file mode 100644
index 00000000..f6560d68
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/AbstractConnectSchemaType.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.type.connect;
+
+import org.apache.rocketmq.connect.doris.converter.type.AbstractType;
+
+public abstract class AbstractConnectSchemaType extends AbstractType {
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/ConnectBooleanType.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/ConnectBooleanType.java
new file mode 100644
index 00000000..5fc351db
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/ConnectBooleanType.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.type.connect;
+
+import io.openmessaging.connector.api.data.Schema;
+import org.apache.rocketmq.connect.doris.converter.type.doris.DorisType;
+
+public class ConnectBooleanType extends AbstractConnectSchemaType {
+
+ public static final ConnectBooleanType INSTANCE = new ConnectBooleanType();
+
+ @Override
+ public String[] getRegistrationKeys() {
+ return new String[] {"BOOLEAN"};
+ }
+
+ @Override
+ public String getTypeName(Schema schema) {
+ return DorisType.BOOLEAN;
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/ConnectBytesType.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/ConnectBytesType.java
new file mode 100644
index 00000000..ab18ea41
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/ConnectBytesType.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.type.connect;
+
+import io.openmessaging.connector.api.data.Schema;
+import java.nio.ByteBuffer;
+import org.apache.rocketmq.connect.doris.converter.type.doris.DorisType;
+
+public class ConnectBytesType extends AbstractConnectSchemaType {
+
+ public static final ConnectBytesType INSTANCE = new ConnectBytesType();
+
+ @Override
+ public String[] getRegistrationKeys() {
+ return new String[] {"BYTES"};
+ }
+
+ @Override
+ public Object getValue(Object sourceValue) {
+ if (sourceValue == null) {
+ return null;
+ }
+ return bytesToHexString(getByteArrayFromValue(sourceValue));
+ }
+
+ @Override
+ public String getTypeName(Schema schema) {
+ return DorisType.STRING;
+ }
+
+ private byte[] getByteArrayFromValue(Object value) {
+ byte[] byteArray = null;
+ if (value instanceof ByteBuffer) {
+ final ByteBuffer buffer = ((ByteBuffer) value).slice();
+ byteArray = new byte[buffer.remaining()];
+ buffer.get(byteArray);
+ } else if (value instanceof byte[]) {
+ byteArray = (byte[]) value;
+ }
+ return byteArray;
+ }
+
+ /**
+ * Convert hexadecimal byte array to string
+ */
+ private String bytesToHexString(byte[] bytes) {
+ StringBuilder sb = new StringBuilder();
+ for (byte b : bytes) {
+ sb.append(String.format("%02X", b));
+ }
+ return sb.toString();
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/ConnectDateType.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/ConnectDateType.java
new file mode 100644
index 00000000..26f97d6a
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/ConnectDateType.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.type.connect;
+
+import io.openmessaging.connector.api.data.logical.Date;
+import io.openmessaging.connector.api.errors.ConnectException;
+import org.apache.rocketmq.connect.doris.converter.type.AbstractDateType;
+import org.apache.rocketmq.connect.doris.converter.type.util.DateTimeUtils;
+
+public class ConnectDateType extends AbstractDateType {
+
+ public static final ConnectDateType INSTANCE = new ConnectDateType();
+
+ @Override
+ public String[] getRegistrationKeys() {
+ return new String[] {Date.LOGICAL_NAME};
+ }
+
+ @Override
+ public Object getValue(Object sourceValue) {
+ if (sourceValue == null) {
+ return null;
+ }
+ if (sourceValue instanceof java.util.Date) {
+ return DateTimeUtils.toLocalDateFromDate((java.util.Date) sourceValue);
+ }
+ throw new ConnectException(
+ String.format(
+ "Unexpected %s value '%s' with type '%s'",
+ getClass().getSimpleName(), sourceValue, sourceValue.getClass().getName()));
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/ConnectDecimalType.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/ConnectDecimalType.java
new file mode 100644
index 00000000..c33a87dd
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/ConnectDecimalType.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.type.connect;
+
+import io.openmessaging.connector.api.data.Schema;
+import io.openmessaging.connector.api.data.logical.Decimal;
+import org.apache.rocketmq.connect.doris.converter.type.AbstractType;
+import org.apache.rocketmq.connect.doris.converter.type.doris.DorisType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ConnectDecimalType extends AbstractType {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(ConnectDecimalType.class);
+
+ public static final ConnectDecimalType INSTANCE = new ConnectDecimalType();
+
+ @Override
+ public String[] getRegistrationKeys() {
+ return new String[] {Decimal.LOGICAL_NAME};
+ }
+
+ @Override
+ public String getTypeName(Schema schema) {
+ int scale = Integer.parseInt(getSchemaParameter(schema, "scale").orElse("0"));
+ int precision =
+ Integer.parseInt(
+ getSchemaParameter(schema, "connect.decimal.precision").orElse("0"));
+ return precision <= 38
+ ? String.format("%s(%s,%s)", DorisType.DECIMAL, precision, Math.max(scale, 0))
+ : DorisType.STRING;
+ }
+
+ @Override
+ public boolean isNumber() {
+ return true;
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/ConnectFloat32Type.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/ConnectFloat32Type.java
new file mode 100644
index 00000000..c6222339
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/ConnectFloat32Type.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.type.connect;
+
+import io.openmessaging.connector.api.data.Schema;
+import org.apache.rocketmq.connect.doris.converter.type.doris.DorisType;
+
+public class ConnectFloat32Type extends AbstractConnectSchemaType {
+
+ public static final ConnectFloat32Type INSTANCE = new ConnectFloat32Type();
+
+ @Override
+ public String[] getRegistrationKeys() {
+ return new String[] {"FLOAT32"};
+ }
+
+ @Override
+ public String getTypeName(Schema schema) {
+ return DorisType.FLOAT;
+ }
+
+ @Override
+ public boolean isNumber() {
+ return true;
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/ConnectFloat64Type.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/ConnectFloat64Type.java
new file mode 100644
index 00000000..a9e01769
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/ConnectFloat64Type.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.type.connect;
+
+import io.openmessaging.connector.api.data.Schema;
+import org.apache.rocketmq.connect.doris.converter.type.doris.DorisType;
+
+public class ConnectFloat64Type extends AbstractConnectSchemaType {
+
+ public static final ConnectFloat64Type INSTANCE = new ConnectFloat64Type();
+
+ @Override
+ public String[] getRegistrationKeys() {
+ return new String[] {"FLOAT64"};
+ }
+
+ @Override
+ public String getTypeName(Schema schema) {
+ return DorisType.DOUBLE;
+ }
+
+ @Override
+ public boolean isNumber() {
+ return true;
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/ConnectInt16Type.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/ConnectInt16Type.java
new file mode 100644
index 00000000..57919fd5
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/ConnectInt16Type.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.type.connect;
+
+import io.openmessaging.connector.api.data.Schema;
+import org.apache.rocketmq.connect.doris.converter.type.doris.DorisType;
+
+public class ConnectInt16Type extends AbstractConnectSchemaType {
+
+ public static final ConnectInt16Type INSTANCE = new ConnectInt16Type();
+
+ @Override
+ public String[] getRegistrationKeys() {
+ return new String[] {"INT16"};
+ }
+
+ @Override
+ public String getTypeName(Schema schema) {
+ return DorisType.SMALLINT;
+ }
+
+ @Override
+ public boolean isNumber() {
+ return true;
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/ConnectInt32Type.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/ConnectInt32Type.java
new file mode 100644
index 00000000..70c2c2c8
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/ConnectInt32Type.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.type.connect;
+
+import io.openmessaging.connector.api.data.Schema;
+import org.apache.rocketmq.connect.doris.converter.type.doris.DorisType;
+
+public class ConnectInt32Type extends AbstractConnectSchemaType {
+
+ public static final ConnectInt32Type INSTANCE = new ConnectInt32Type();
+
+ @Override
+ public String[] getRegistrationKeys() {
+ return new String[] {"INT32"};
+ }
+
+ @Override
+ public String getTypeName(Schema schema) {
+ return DorisType.INT;
+ }
+
+ @Override
+ public boolean isNumber() {
+ return true;
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/ConnectInt64Type.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/ConnectInt64Type.java
new file mode 100644
index 00000000..96c220ff
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/ConnectInt64Type.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.type.connect;
+
+import io.openmessaging.connector.api.data.Schema;
+import org.apache.rocketmq.connect.doris.converter.type.doris.DorisType;
+
+public class ConnectInt64Type extends AbstractConnectSchemaType {
+
+ public static final ConnectInt64Type INSTANCE = new ConnectInt64Type();
+
+ @Override
+ public String[] getRegistrationKeys() {
+ return new String[] {"INT64"};
+ }
+
+ @Override
+ public String getTypeName(Schema schema) {
+ return DorisType.BIGINT;
+ }
+
+ @Override
+ public boolean isNumber() {
+ return true;
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/ConnectInt8Type.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/ConnectInt8Type.java
new file mode 100644
index 00000000..1839ed36
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/ConnectInt8Type.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.type.connect;
+
+import io.openmessaging.connector.api.data.Schema;
+import org.apache.rocketmq.connect.doris.converter.type.doris.DorisType;
+
+public class ConnectInt8Type extends AbstractConnectSchemaType {
+
+ public static final ConnectInt8Type INSTANCE = new ConnectInt8Type();
+
+ @Override
+ public String[] getRegistrationKeys() {
+ return new String[] {"INT8"};
+ }
+
+ @Override
+ public String getTypeName(Schema schema) {
+ return DorisType.TINYINT;
+ }
+
+ @Override
+ public boolean isNumber() {
+ return true;
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/ConnectMapToConnectStringType.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/ConnectMapToConnectStringType.java
new file mode 100644
index 00000000..e2f32d35
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/ConnectMapToConnectStringType.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.type.connect;
+
+import io.openmessaging.connector.api.data.Schema;
+import java.util.Map;
+
+public class ConnectMapToConnectStringType extends AbstractConnectMapType {
+
+ public static final ConnectMapToConnectStringType INSTANCE =
+ new ConnectMapToConnectStringType();
+
+ @Override
+ public String getTypeName(Schema schema) {
+ return ConnectStringType.INSTANCE.getTypeName(schema);
+ }
+
+ @Override
+ public Object getValue(Object sourceValue) {
+ if (sourceValue instanceof Map) {
+ sourceValue = mapToJsonString(sourceValue);
+ }
+ return ConnectStringType.INSTANCE.getValue(sourceValue);
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/ConnectStringType.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/ConnectStringType.java
new file mode 100644
index 00000000..9809601b
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/ConnectStringType.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.type.connect;
+
+import io.openmessaging.connector.api.data.Schema;
+import org.apache.rocketmq.connect.doris.converter.type.doris.DorisType;
+import org.apache.rocketmq.connect.doris.converter.type.doris.DorisTypeProperties;
+
+public class ConnectStringType extends AbstractConnectSchemaType {
+
+ public static final ConnectStringType INSTANCE = new ConnectStringType();
+
+ @Override
+ public String getTypeName(Schema schema) {
+ int columnLength = getColumnLength(schema);
+ if (columnLength > 0) {
+ return columnLength * 3 > DorisTypeProperties.MAX_VARCHAR_SIZE
+ ? DorisType.STRING
+ : String.format("%s(%s)", DorisType.VARCHAR, columnLength * 3);
+ }
+ return DorisType.STRING;
+ }
+
+ @Override
+ public String[] getRegistrationKeys() {
+ return new String[] {"STRING"};
+ }
+
+ private int getColumnLength(Schema schema) {
+ return Integer.parseInt(getSourceColumnLength(schema).orElse("0"));
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/ConnectTimeType.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/ConnectTimeType.java
new file mode 100644
index 00000000..6ccc9448
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/ConnectTimeType.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.type.connect;
+
+import io.openmessaging.connector.api.data.logical.Time;
+import io.openmessaging.connector.api.errors.ConnectException;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.util.Date;
+import org.apache.rocketmq.connect.doris.converter.type.AbstractTimeType;
+import org.apache.rocketmq.connect.doris.converter.type.util.DateTimeUtils;
+
+public class ConnectTimeType extends AbstractTimeType {
+
+ public static final ConnectTimeType INSTANCE = new ConnectTimeType();
+
+ @Override
+ public String[] getRegistrationKeys() {
+ return new String[] {Time.LOGICAL_NAME};
+ }
+
+ @Override
+ public Object getValue(Object sourceValue) {
+ if (sourceValue == null) {
+ return null;
+ }
+ if (sourceValue instanceof Date) {
+
+ final LocalTime localTime = DateTimeUtils.toLocalTimeFromUtcDate((Date) sourceValue);
+ final LocalDateTime localDateTime = localTime.atDate(LocalDate.now());
+ return localDateTime.toLocalTime();
+ }
+
+ throw new ConnectException(
+ String.format(
+ "Unexpected %s value '%s' with type '%s'",
+ getClass().getSimpleName(), sourceValue, sourceValue.getClass().getName()));
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/ConnectTimestampType.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/ConnectTimestampType.java
new file mode 100644
index 00000000..748ba442
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/ConnectTimestampType.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.type.connect;
+
+import io.openmessaging.connector.api.data.logical.Timestamp;
+import io.openmessaging.connector.api.errors.ConnectException;
+import org.apache.rocketmq.connect.doris.converter.type.AbstractTimestampType;
+import org.apache.rocketmq.connect.doris.converter.type.util.DateTimeUtils;
+
+public class ConnectTimestampType extends AbstractTimestampType {
+
+ public static final ConnectTimestampType INSTANCE = new ConnectTimestampType();
+
+ @Override
+ public String[] getRegistrationKeys() {
+ return new String[] {Timestamp.LOGICAL_NAME};
+ }
+
+ @Override
+ public Object getValue(Object sourceValue) {
+ if (sourceValue == null) {
+ return null;
+ }
+ if (sourceValue instanceof java.util.Date) {
+ return DateTimeUtils.toLocalDateTimeFromDate((java.util.Date) sourceValue);
+ }
+
+ throw new ConnectException(
+ String.format(
+ "Unexpected %s value '%s' with type '%s'",
+ getClass().getSimpleName(), sourceValue, sourceValue.getClass().getName()));
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/AbstractDebeziumTimeType.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/AbstractDebeziumTimeType.java
new file mode 100644
index 00000000..fa28a710
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/AbstractDebeziumTimeType.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.type.debezium;
+
+import io.openmessaging.connector.api.errors.ConnectException;
+import java.time.LocalTime;
+import java.time.format.DateTimeFormatter;
+import org.apache.rocketmq.connect.doris.converter.type.AbstractTimeType;
+
+public abstract class AbstractDebeziumTimeType extends AbstractTimeType {
+
+ @Override
+ public Object getValue(Object sourceValue) {
+ if (sourceValue == null) {
+ return null;
+ }
+ if (sourceValue instanceof Number) {
+ final LocalTime localTime = getLocalTime((Number) sourceValue);
+ return String.format("%s", DateTimeFormatter.ISO_TIME.format(localTime));
+ }
+ throw new ConnectException(
+ String.format(
+ "Unexpected %s value '%s' with type '%s'",
+ getClass().getSimpleName(), sourceValue, sourceValue.getClass().getName()));
+ }
+
+ protected abstract LocalTime getLocalTime(Number value);
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/AbstractDebeziumTimestampType.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/AbstractDebeziumTimestampType.java
new file mode 100644
index 00000000..8c844ede
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/AbstractDebeziumTimestampType.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.type.debezium;
+
+import io.openmessaging.connector.api.errors.ConnectException;
+import java.time.LocalDateTime;
+import org.apache.rocketmq.connect.doris.converter.type.AbstractTimestampType;
+
+public abstract class AbstractDebeziumTimestampType extends AbstractTimestampType {
+
+ @Override
+ public Object getValue(Object sourceValue) {
+ if (sourceValue == null) {
+ return null;
+ }
+ if (sourceValue instanceof Number) {
+ return getLocalDateTime(((Number) sourceValue).longValue());
+ }
+ throw new ConnectException(
+ String.format(
+ "Unexpected %s value '%s' with type '%s'",
+ getClass().getSimpleName(), sourceValue, sourceValue.getClass().getName()));
+ }
+
+ protected abstract LocalDateTime getLocalDateTime(long value);
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/ArrayType.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/ArrayType.java
new file mode 100644
index 00000000..8fd97de8
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/ArrayType.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.type.debezium;
+
+import io.openmessaging.connector.api.data.Schema;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import org.apache.rocketmq.connect.doris.cfg.DorisOptions;
+import org.apache.rocketmq.connect.doris.converter.RecordTypeRegister;
+import org.apache.rocketmq.connect.doris.converter.type.AbstractType;
+import org.apache.rocketmq.connect.doris.converter.type.Type;
+import org.apache.rocketmq.connect.doris.converter.type.doris.DorisType;
+
+public class ArrayType extends AbstractType {
+ private static final String ARRAY_TYPE_TEMPLATE = "%s<%s>";
+ public static final ArrayType INSTANCE = new ArrayType();
+ private DorisOptions dorisOptions;
+ private RecordTypeRegister recordTypeRegister;
+
+ @Override
+ public void configure(DorisOptions dorisOptions) {
+ if (this.dorisOptions == null && this.recordTypeRegister == null) {
+ this.dorisOptions = dorisOptions;
+ registerNestedArrayType();
+ }
+ }
+
+ @Override
+ public String[] getRegistrationKeys() {
+ return new String[] {"ARRAY"};
+ }
+
+ @Override
+ public String getTypeName(Schema schema) {
+ if (schema.getValueSchema().isOptional()) {
+ Schema valueSchema = schema.getValueSchema();
+ String type =
+ Objects.nonNull(valueSchema.getName())
+ ? valueSchema.getName()
+ : valueSchema.getFieldType().name();
+ if (recordTypeRegister == null) {
+ registerNestedArrayType();
+ }
+ Type valueType = recordTypeRegister.getTypeRegistry().get(type);
+ if (valueType == null) {
+ return DorisType.STRING;
+ }
+ String typeName = valueType.getTypeName(schema);
+ return String.format(ARRAY_TYPE_TEMPLATE, DorisType.ARRAY, typeName);
+ }
+ return DorisType.STRING;
+ }
+
+ @Override
+ public Object getValue(Object sourceValue, Schema schema) {
+
+ if (sourceValue == null) {
+ return null;
+ }
+ Schema valueSchema = schema.getValueSchema();
+ String type =
+ Objects.nonNull(valueSchema.getName())
+ ? valueSchema.getName()
+ : valueSchema.getFieldType().name();
+
+ if (sourceValue instanceof List) {
+ List<Object> resultList = new ArrayList<>();
+ ArrayList<?> convertedValue = (ArrayList<?>) sourceValue;
+ if (recordTypeRegister == null) {
+ registerNestedArrayType();
+ }
+ Type valueType = recordTypeRegister.getTypeRegistry().get(type);
+ if (valueType == null) {
+ return sourceValue;
+ }
+
+ for (Object value : convertedValue) {
+ resultList.add(valueType.getValue(value, valueSchema));
+ }
+ return resultList;
+ }
+
+ return sourceValue;
+ }
+
+ private void registerNestedArrayType() {
+ this.recordTypeRegister = new RecordTypeRegister(dorisOptions);
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/DateType.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/DateType.java
new file mode 100644
index 00000000..f3c66287
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/DateType.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.type.debezium;
+
+import io.debezium.time.Date;
+import io.openmessaging.connector.api.errors.ConnectException;
+import org.apache.rocketmq.connect.doris.converter.type.AbstractDateType;
+import org.apache.rocketmq.connect.doris.converter.type.util.DateTimeUtils;
+
+public class DateType extends AbstractDateType {
+
+ public static final DateType INSTANCE = new DateType();
+
+ @Override
+ public String[] getRegistrationKeys() {
+ return new String[] {Date.SCHEMA_NAME};
+ }
+
+ @Override
+ public Object getValue(Object sourceValue) {
+ if (sourceValue == null) {
+ return null;
+ }
+ if (sourceValue instanceof Number) {
+ return DateTimeUtils.toLocalDateOfEpochDays(((Number) sourceValue).longValue());
+ }
+
+ throw new ConnectException(
+ String.format(
+ "Unexpected %s value '%s' with type '%s'",
+ getClass().getSimpleName(), sourceValue, sourceValue.getClass().getName()));
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/GeographyType.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/GeographyType.java
new file mode 100644
index 00000000..4af22e70
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/GeographyType.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.type.debezium;
+
+import io.debezium.data.geometry.Geography;
+import org.apache.rocketmq.connect.doris.converter.type.AbstractGeometryType;
+
+public class GeographyType extends AbstractGeometryType {
+ public static final GeographyType INSTANCE = new GeographyType();
+
+ @Override
+ public String[] getRegistrationKeys() {
+ return new String[] {Geography.LOGICAL_NAME};
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/GeometryType.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/GeometryType.java
new file mode 100644
index 00000000..30f6f578
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/GeometryType.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.type.debezium;
+
+import io.debezium.data.geometry.Geometry;
+import io.openmessaging.connector.api.data.Struct;
+import org.apache.rocketmq.connect.doris.converter.type.AbstractGeometryType;
+import org.apache.rocketmq.connect.doris.converter.type.util.GeoUtils;
+
+public class GeometryType extends AbstractGeometryType {
+ public static final GeometryType INSTANCE = new GeometryType();
+
+ @Override
+ public String[] getRegistrationKeys() {
+ return new String[] {Geometry.LOGICAL_NAME};
+ }
+
+ @Override
+ public Object getValue(Object sourceValue) {
+ if (sourceValue == null) {
+ return null;
+ }
+
+ if (sourceValue instanceof Struct) {
+ return GeoUtils.handleGeoStructData(sourceValue);
+ }
+
+ return sourceValue;
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/MicroTimeType.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/MicroTimeType.java
new file mode 100644
index 00000000..28a709eb
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/MicroTimeType.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.type.debezium;
+
+import io.debezium.time.MicroTime;
+import java.time.LocalTime;
+import org.apache.rocketmq.connect.doris.converter.type.util.DateTimeUtils;
+
+public class MicroTimeType extends AbstractDebeziumTimeType {
+
+ public static final MicroTimeType INSTANCE = new MicroTimeType();
+
+ @Override
+ public String[] getRegistrationKeys() {
+ return new String[] {MicroTime.SCHEMA_NAME};
+ }
+
+ @Override
+ protected LocalTime getLocalTime(Number value) {
+ return DateTimeUtils.toLocalTimeFromDurationMicroseconds(value.longValue());
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/MicroTimestampType.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/MicroTimestampType.java
new file mode 100644
index 00000000..a76d276d
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/MicroTimestampType.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.type.debezium;
+
+import io.debezium.time.MicroTimestamp;
+import java.time.LocalDateTime;
+import org.apache.rocketmq.connect.doris.converter.type.util.DateTimeUtils;
+
+public class MicroTimestampType extends AbstractDebeziumTimestampType {
+
+ public static final MicroTimestampType INSTANCE = new MicroTimestampType();
+
+ @Override
+ public String[] getRegistrationKeys() {
+ return new String[] {MicroTimestamp.SCHEMA_NAME};
+ }
+
+ @Override
+ protected LocalDateTime getLocalDateTime(long value) {
+ return DateTimeUtils.toLocalDateTimeFromInstantEpochMicros(value);
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/NanoTimeType.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/NanoTimeType.java
new file mode 100644
index 00000000..2a356e21
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/NanoTimeType.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.type.debezium;
+
+import io.debezium.time.NanoTime;
+import java.time.LocalTime;
+import org.apache.rocketmq.connect.doris.converter.type.util.DateTimeUtils;
+
+public class NanoTimeType extends AbstractDebeziumTimeType {
+
+ public static final NanoTimeType INSTANCE = new NanoTimeType();
+
+ @Override
+ public String[] getRegistrationKeys() {
+ return new String[] {NanoTime.SCHEMA_NAME};
+ }
+
+ @Override
+ protected LocalTime getLocalTime(Number value) {
+ return DateTimeUtils.toLocalTimeFromDurationNanoseconds(value.longValue());
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/NanoTimestampType.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/NanoTimestampType.java
new file mode 100644
index 00000000..3f7e88a9
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/NanoTimestampType.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.type.debezium;
+
+import io.debezium.time.NanoTimestamp;
+import java.time.LocalDateTime;
+import org.apache.rocketmq.connect.doris.converter.type.util.DateTimeUtils;
+
+/**
+ * An implementation of {@link org.apache.doris.kafka.connector.converter.type.Type} for {@link
+ * MicroTimestamp} values.
+ */
+public class NanoTimestampType extends AbstractDebeziumTimestampType {
+
+ public static final NanoTimestampType INSTANCE = new NanoTimestampType();
+
+ @Override
+ public String[] getRegistrationKeys() {
+ return new String[] {NanoTimestamp.SCHEMA_NAME};
+ }
+
+ @Override
+ protected LocalDateTime getLocalDateTime(long value) {
+ return DateTimeUtils.toLocalDateTimeFromInstantEpochNanos(value);
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/PointType.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/PointType.java
new file mode 100644
index 00000000..9268acf8
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/PointType.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.type.debezium;
+
+import io.debezium.data.geometry.Point;
+import io.openmessaging.connector.api.data.Struct;
+import org.apache.rocketmq.connect.doris.converter.type.AbstractGeometryType;
+import org.apache.rocketmq.connect.doris.converter.type.util.GeoUtils;
+
+public class PointType extends AbstractGeometryType {
+ public static final PointType INSTANCE = new PointType();
+
+ @Override
+ public String[] getRegistrationKeys() {
+ return new String[] {Point.LOGICAL_NAME};
+ }
+
+ @Override
+ public Object getValue(Object sourceValue) {
+ if (sourceValue == null) {
+ return null;
+ }
+ if (sourceValue instanceof Struct) {
+ return GeoUtils.handleGeoStructData(sourceValue);
+ }
+
+ return sourceValue;
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/TimeType.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/TimeType.java
new file mode 100644
index 00000000..df099b62
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/TimeType.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.type.debezium;
+
+import io.debezium.time.Time;
+import java.time.LocalTime;
+import org.apache.rocketmq.connect.doris.converter.type.util.DateTimeUtils;
+
+public class TimeType extends AbstractDebeziumTimeType {
+
+ public static final TimeType INSTANCE = new TimeType();
+
+ @Override
+ public String[] getRegistrationKeys() {
+ return new String[] {Time.SCHEMA_NAME};
+ }
+
+ @Override
+ protected LocalTime getLocalTime(Number value) {
+ return DateTimeUtils.toLocalTimeFromDurationMilliseconds(value.longValue());
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/TimestampType.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/TimestampType.java
new file mode 100644
index 00000000..456cb7b0
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/TimestampType.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.type.debezium;
+
+import io.debezium.time.Timestamp;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+
+public class TimestampType extends AbstractDebeziumTimestampType {
+
+ public static final TimestampType INSTANCE = new TimestampType();
+
+ @Override
+ public String[] getRegistrationKeys() {
+ return new String[] {Timestamp.SCHEMA_NAME};
+ }
+
+ @Override
+ protected LocalDateTime getLocalDateTime(long value) {
+ return LocalDateTime.ofInstant(Instant.ofEpochMilli(value), ZoneOffset.UTC);
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/VariableScaleDecimalType.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/VariableScaleDecimalType.java
new file mode 100644
index 00000000..062aa84a
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/VariableScaleDecimalType.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.type.debezium;
+
+import io.debezium.data.SpecialValueDecimal;
+import io.debezium.data.VariableScaleDecimal;
+import io.openmessaging.connector.api.data.Schema;
+import io.openmessaging.connector.api.data.Struct;
+import io.openmessaging.connector.api.errors.ConnectException;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.Optional;
+import org.apache.rocketmq.connect.doris.converter.type.AbstractType;
+import org.apache.rocketmq.connect.doris.converter.type.doris.DorisType;
+
+public class VariableScaleDecimalType extends AbstractType {
+
+ public static final VariableScaleDecimalType INSTANCE = new VariableScaleDecimalType();
+
+ @Override
+ public String[] getRegistrationKeys() {
+ return new String[] {VariableScaleDecimal.LOGICAL_NAME};
+ }
+
+ @Override
+ public Object getValue(Object sourceValue) {
+ if (sourceValue == null) {
+ return null;
+ }
+ if (sourceValue instanceof Struct) {
+ Optional<BigDecimal> bigDecimalValue = toLogical((Struct) sourceValue).getDecimalValue();
+ return bigDecimalValue.get();
+ }
+
+ throw new ConnectException(
+ String.format(
+ "Unexpected %s value '%s' with type '%s'",
+ getClass().getSimpleName(), sourceValue, sourceValue.getClass().getName()));
+ }
+
+ @Override
+ public String getTypeName(Schema schema) {
+ // The data passed by VariableScaleDecimal data types does not provide adequate information to
+ // resolve the precision and scale for the data type, so instead we're going to default to the
+ // maximum double-based data types for the dialect, using DOUBLE.
+ return DorisType.DOUBLE;
+ }
+
+ @Override
+ public boolean isNumber() {
+ return true;
+ }
+
+ private static SpecialValueDecimal toLogical(final Struct value) {
+ return new SpecialValueDecimal(
+ new BigDecimal(new BigInteger(value.getBytes("value")), value.getInt32("scale")));
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/ZonedTimeType.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/ZonedTimeType.java
new file mode 100644
index 00000000..fcd48f69
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/ZonedTimeType.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.type.debezium;
+
+import io.debezium.time.ZonedTime;
+import io.openmessaging.connector.api.errors.ConnectException;
+import java.time.OffsetTime;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import org.apache.rocketmq.connect.doris.converter.type.AbstractTimeType;
+
+public class ZonedTimeType extends AbstractTimeType {
+
+ public static final ZonedTimeType INSTANCE = new ZonedTimeType();
+ // The ZonedTime of debezium type only contains three types of hours, minutes and seconds
+ private final DateTimeFormatter TIME_FORMATTER = DateTimeFormatter.ofPattern("HH:mm:ss");
+
+ @Override
+ public String[] getRegistrationKeys() {
+ return new String[] {ZonedTime.SCHEMA_NAME};
+ }
+
+ @Override
+ public Object getValue(Object sourceValue) {
+ if (sourceValue == null) {
+ return null;
+ }
+ if (sourceValue instanceof String) {
+ OffsetTime offsetTime =
+ OffsetTime.parse((String) sourceValue, ZonedTime.FORMATTER)
+ .withOffsetSameInstant(
+ ZonedDateTime.now(getDatabaseTimeZone().toZoneId())
+ .getOffset());
+ return offsetTime.format(TIME_FORMATTER);
+ }
+
+ throw new ConnectException(
+ String.format(
+ "Unexpected %s value '%s' with type '%s'",
+ getClass().getSimpleName(), sourceValue, sourceValue.getClass().getName()));
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/ZonedTimestampType.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/ZonedTimestampType.java
new file mode 100644
index 00000000..89141459
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/ZonedTimestampType.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.type.debezium;
+
+import io.debezium.time.ZonedTimestamp;
+import io.openmessaging.connector.api.errors.ConnectException;
+import java.time.ZonedDateTime;
+import org.apache.rocketmq.connect.doris.converter.type.AbstractTimestampType;
+
+public class ZonedTimestampType extends AbstractTimestampType {
+
+ public static final ZonedTimestampType INSTANCE = new ZonedTimestampType();
+
+ @Override
+ public String[] getRegistrationKeys() {
+ return new String[] {ZonedTimestamp.SCHEMA_NAME};
+ }
+
+ @Override
+ public Object getValue(Object sourceValue) {
+ if (sourceValue == null) {
+ return null;
+ }
+ if (sourceValue instanceof String) {
+ final ZonedDateTime zdt =
+ ZonedDateTime.parse((String) sourceValue, ZonedTimestamp.FORMATTER)
+ .withZoneSameInstant(getDatabaseTimeZone().toZoneId());
+ return zdt.toOffsetDateTime();
+ }
+
+ throw new ConnectException(
+ String.format(
+ "Unexpected %s value '%s' with type '%s'",
+ getClass().getSimpleName(), sourceValue, sourceValue.getClass().getName()));
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/doris/DorisType.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/doris/DorisType.java
new file mode 100644
index 00000000..49609195
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/doris/DorisType.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.type.doris;
+
+public class DorisType {
+ public static final String BOOLEAN = "BOOLEAN";
+ public static final String TINYINT = "TINYINT";
+ public static final String SMALLINT = "SMALLINT";
+ public static final String INT = "INT";
+ public static final String BIGINT = "BIGINT";
+ public static final String LARGEINT = "LARGEINT";
+ public static final String FLOAT = "FLOAT";
+ public static final String DOUBLE = "DOUBLE";
+ public static final String DECIMAL = "DECIMAL";
+ public static final String DATE = "DATE";
+ public static final String DATETIME = "DATETIME";
+ public static final String CHAR = "CHAR";
+ public static final String VARCHAR = "VARCHAR";
+ public static final String STRING = "STRING";
+ public static final String HLL = "HLL";
+ public static final String BITMAP = "BITMAP";
+ public static final String ARRAY = "ARRAY";
+ public static final String JSONB = "JSONB";
+ public static final String JSON = "JSON";
+ public static final String MAP = "MAP";
+ public static final String STRUCT = "STRUCT";
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/doris/DorisTypeProperties.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/doris/DorisTypeProperties.java
new file mode 100644
index 00000000..b19fa221
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/doris/DorisTypeProperties.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.type.doris;
+
+public class DorisTypeProperties {
+
+ /* Max precision of datetime type of Doris. */
+ public static final int MAX_SUPPORTED_DATE_TIME_PRECISION = 6;
+
+ public static final int TIMESTAMP_TYPE_MAX_PRECISION = 9;
+
+ public static final int MAX_VARCHAR_SIZE = 65533;
+
+ public static final int MAX_CHAR_SIZE = 255;
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/util/DateTimeUtils.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/util/DateTimeUtils.java
new file mode 100644
index 00000000..363841c7
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/util/DateTimeUtils.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ * Copied from
+ * https://github.com/debezium/debezium-connector-jdbc/blob/main/src/main/java/io/debezium/connector/jdbc/util/DateTimeUtils.java
+ * modified by doris.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.type.util;
+
+import io.debezium.time.Conversions;
+import java.sql.Timestamp;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.time.temporal.ChronoUnit;
+import java.util.Date;
+import java.util.TimeZone;
+import java.util.concurrent.TimeUnit;
+
+public class DateTimeUtils {
+
+ private DateTimeUtils() {
+ }
+
+ public static Instant toInstantFromNanos(long epochNanos) {
+ final long epochSeconds = TimeUnit.NANOSECONDS.toSeconds(epochNanos);
+ final long adjustment =
+ TimeUnit.NANOSECONDS.toNanos(epochNanos % TimeUnit.SECONDS.toNanos(1));
+ return Instant.ofEpochSecond(epochSeconds, adjustment);
+ }
+
+ public static ZonedDateTime toZonedDateTimeFromDate(Date date, TimeZone timeZone) {
+ return toZonedDateTimeFromDate(date, timeZone.toZoneId());
+ }
+
+ public static ZonedDateTime toZonedDateTimeFromDate(Date date, ZoneId zoneId) {
+ return date.toInstant().atZone(zoneId);
+ }
+
+ public static ZonedDateTime toZonedDateTimeFromInstantEpochMicros(long epochMicros) {
+ return Conversions.toInstantFromMicros(epochMicros).atZone(ZoneOffset.UTC);
+ }
+
+ public static ZonedDateTime toZonedDateTimeFromInstantEpochNanos(long epochNanos) {
+ return ZonedDateTime.ofInstant(toInstantFromNanos(epochNanos), ZoneOffset.UTC);
+ }
+
+ public static LocalDate toLocalDateOfEpochDays(long epochDays) {
+ return LocalDate.ofEpochDay(epochDays);
+ }
+
+ public static LocalDate toLocalDateFromDate(Date date) {
+ return toLocalDateFromInstantEpochMillis(date.getTime());
+ }
+
+ public static LocalDate toLocalDateFromInstantEpochMillis(long epochMillis) {
+ return LocalDate.ofEpochDay(Duration.ofMillis(epochMillis).toDays());
+ }
+
+ public static LocalTime toLocalTimeFromDurationMilliseconds(long durationMillis) {
+ return LocalTime.ofNanoOfDay(Duration.of(durationMillis, ChronoUnit.MILLIS).toNanos());
+ }
+
+ public static LocalTime toLocalTimeFromDurationMicroseconds(long durationMicros) {
+ return LocalTime.ofNanoOfDay(Duration.of(durationMicros, ChronoUnit.MICROS).toNanos());
+ }
+
+ public static LocalTime toLocalTimeFromDurationNanoseconds(long durationNanos) {
+ return LocalTime.ofNanoOfDay(Duration.of(durationNanos, ChronoUnit.NANOS).toNanos());
+ }
+
+ public static LocalTime toLocalTimeFromUtcDate(Date date) {
+ return date.toInstant().atOffset(ZoneOffset.UTC).toLocalTime();
+ }
+
+ public static LocalDateTime toLocalDateTimeFromDate(Date date) {
+ return toLocalDateTimeFromInstantEpochMillis(date.getTime());
+ }
+
+ public static LocalDateTime toLocalDateTimeFromInstantEpochMillis(long epochMillis) {
+ return LocalDateTime.ofInstant(
+ Conversions.toInstantFromMillis(epochMillis), ZoneOffset.UTC);
+ }
+
+ public static LocalDateTime toLocalDateTimeFromInstantEpochMicros(long epochMicros) {
+ return LocalDateTime.ofInstant(
+ Conversions.toInstantFromMicros(epochMicros), ZoneOffset.UTC);
+ }
+
+ public static LocalDateTime toLocalDateTimeFromInstantEpochNanos(long epochNanos) {
+ return LocalDateTime.ofInstant(toInstantFromNanos(epochNanos), ZoneOffset.UTC);
+ }
+
+ public static Timestamp toTimestampFromMillis(long epochMilliseconds) {
+ final Instant instant = Conversions.toInstantFromMillis(epochMilliseconds);
+ final Timestamp ts = new Timestamp(instant.toEpochMilli());
+ ts.setNanos(instant.getNano());
+ return ts;
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/util/GeoUtils.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/util/GeoUtils.java
new file mode 100644
index 00000000..a7e83164
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/util/GeoUtils.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.type.util;
+
+import com.esri.core.geometry.ogc.OGCGeometry;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.openmessaging.connector.api.data.Struct;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class GeoUtils {
+ private static final Logger LOGGER = LoggerFactory.getLogger(GeoUtils.class);
+
+ private GeoUtils() {
+ }
+
+ public static Object handleGeoStructData(Object sourceValue) {
+ // the Geometry datatype in MySQL will be converted to
+ // a String with Json format
+ final ObjectMapper objectMapper = new ObjectMapper();
+ Struct geometryStruct = (Struct) sourceValue;
+
+ try {
+ byte[] wkb = geometryStruct.getBytes("wkb");
+ String geoJson = OGCGeometry.fromBinary(ByteBuffer.wrap(wkb)).asGeoJson();
+ JsonNode originGeoNode = objectMapper.readTree(geoJson);
+
+ Optional<Integer> srid = Optional.ofNullable(geometryStruct.getInt32("srid"));
+ Map<String, Object> geometryInfo = new HashMap<>();
+ String geometryType = originGeoNode.get("type").asText();
+
+ geometryInfo.put("type", geometryType);
+ if ("GeometryCollection".equals(geometryType)) {
+ geometryInfo.put("geometries", originGeoNode.get("geometries"));
+ } else {
+ geometryInfo.put("coordinates", originGeoNode.get("coordinates"));
+ }
+
+ geometryInfo.put("srid", srid.orElse(0));
+ return geometryInfo;
+ } catch (Exception e) {
+ LOGGER.warn("Failed to parse Geometry datatype, converting the value to null", e);
+ return null;
+ }
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/util/SchemaUtils.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/util/SchemaUtils.java
new file mode 100644
index 00000000..095f90ba
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/util/SchemaUtils.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ * Copied from
+ * https://github.com/debezium/debezium-connector-jdbc/blob/main/src/main/java/io/debezium/connector/jdbc/util/SchemaUtils.java
+ * modified by doris.
+ */
+
+package org.apache.rocketmq.connect.doris.converter.type.util;
+
+import io.openmessaging.connector.api.data.Schema;
+import java.util.Objects;
+import java.util.Optional;
+
+public class SchemaUtils {
+ private static final String SCHEMA_PARAMETER_COLUMN_TYPE = "__debezium.source.column.type";
+ private static final String SCHEMA_PARAMETER_COLUMN_LENGTH = "__debezium.source.column.length";
+ private static final String SCHEMA_PARAMETER_COLUMN_PRECISION =
+ "__debezium.source.column.scale";
+ private static final String SCHEMA_PARAMETER_COLUMN_NAME = "__debezium.source.column.name";
+
+ public static Optional<String> getSourceColumnType(Schema schema) {
+ return getSchemaParameter(schema, SCHEMA_PARAMETER_COLUMN_TYPE);
+ }
+
+ public static Optional<String> getSourceColumnLength(Schema schema) {
+ return getSchemaParameter(schema, SCHEMA_PARAMETER_COLUMN_LENGTH);
+ }
+
+ public static Optional<String> getSourceColumnPrecision(Schema schema) {
+ return getSchemaParameter(schema, SCHEMA_PARAMETER_COLUMN_PRECISION);
+ }
+
+ public static Optional<String> getSourceColumnName(Schema schema) {
+ return getSchemaParameter(schema, SCHEMA_PARAMETER_COLUMN_NAME);
+ }
+
+ public static Optional<String> getSchemaParameter(Schema schema, String parameterName) {
+ if (!Objects.isNull(schema.getParameters())) {
+ return Optional.ofNullable(schema.getParameters().get(parameterName));
+ }
+ return Optional.empty();
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/exception/ArgumentsException.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/exception/ArgumentsException.java
new file mode 100644
index 00000000..96dea18a
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/exception/ArgumentsException.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.exception;
+
+public class ArgumentsException extends DorisException {
+
+ public ArgumentsException() {
+ }
+
+ public ArgumentsException(String message) {
+ super(message);
+ }
+
+ public ArgumentsException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public ArgumentsException(Throwable cause) {
+ super(cause);
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/exception/ConfigException.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/exception/ConfigException.java
deleted file mode 100644
index f74873f5..00000000
--- a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/exception/ConfigException.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.connect.doris.exception;
-
-/**
- * config exception
- */
-public class ConfigException extends RuntimeException {
- private static final long serialVersionUID = 1L;
-
- public ConfigException(String message) {
- super(message);
- }
-
- public ConfigException(String name, Object value) {
- this(name, value, null);
- }
-
- public ConfigException(String name, Object value, String message) {
- super("Invalid value " + value + " for configuration " + name + (message == null ? "" : ": " + message));
- }
-}
\ No newline at end of file
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/exception/ConnectedFailedException.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/exception/ConnectedFailedException.java
new file mode 100644
index 00000000..260b2c6e
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/exception/ConnectedFailedException.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.exception;
+
+public class ConnectedFailedException extends DorisException {
+
+ public ConnectedFailedException(String message) {
+ super(message);
+ }
+
+ public ConnectedFailedException(String server, Throwable cause) {
+ super("Connect to " + server + " failed.", cause);
+ }
+
+ public ConnectedFailedException(String server, int statusCode, Throwable cause) {
+ super("Connect to " + server + " failed, status code is " + statusCode + ".", cause);
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/exception/CopyLoadException.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/exception/CopyLoadException.java
new file mode 100644
index 00000000..958a7d35
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/exception/CopyLoadException.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.exception;
+
+public class CopyLoadException extends DorisException {
+
+ public CopyLoadException() {
+ }
+
+ public CopyLoadException(String message) {
+ super(message);
+ }
+
+ public CopyLoadException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public CopyLoadException(Throwable cause) {
+ super(cause);
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/exception/DataConverterException.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/exception/DataConverterException.java
new file mode 100644
index 00000000..593aa956
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/exception/DataConverterException.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.exception;
+
+public class DataConverterException extends RuntimeException {
+
+ public DataConverterException() {
+ super();
+ }
+
+ public DataConverterException(String message) {
+ super(message);
+ }
+
+ public DataConverterException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public DataConverterException(Throwable cause) {
+ super(cause);
+ }
+
+ protected DataConverterException(
+ String message,
+ Throwable cause,
+ boolean enableSuppression,
+ boolean writableStackTrace) {
+ super(message, cause, enableSuppression, writableStackTrace);
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/exception/DataFormatException.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/exception/DataFormatException.java
new file mode 100644
index 00000000..904d78f4
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/exception/DataFormatException.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.exception;
+
+public class DataFormatException extends DorisException {
+
+ public DataFormatException() {
+ }
+
+ public DataFormatException(String message) {
+ super(message);
+ }
+
+ public DataFormatException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public DataFormatException(Throwable cause) {
+ super(cause);
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/exception/DorisException.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/exception/DorisException.java
index 0c991a91..09f82e71 100644
--- a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/exception/DorisException.java
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/exception/DorisException.java
@@ -1,34 +1,38 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
*/
package org.apache.rocketmq.connect.doris.exception;
public class DorisException extends RuntimeException {
- private static final long serialVersionUID = 2L;
+
+ public DorisException() {
+ }
public DorisException(String message) {
super(message);
}
- public DorisException(String name, Object value) {
- this(name, value, null);
+ public DorisException(String message, Throwable cause) {
+ super(message, cause);
}
- public DorisException(String name, Object value, String message) {
- super("Invalid value " + value + " for configuration " + name + (message == null ? "" : ": " + message));
+ public DorisException(Throwable cause) {
+ super(cause);
}
-}
\ No newline at end of file
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/exception/SchemaChangeException.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/exception/SchemaChangeException.java
new file mode 100644
index 00000000..d7d75658
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/exception/SchemaChangeException.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.exception;
+
+/**
+ * Doris Schema Change run exception.
+ */
+public class SchemaChangeException extends RuntimeException {
+ public SchemaChangeException() {
+ super();
+ }
+
+ public SchemaChangeException(String message) {
+ super(message);
+ }
+
+ public SchemaChangeException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public SchemaChangeException(Throwable cause) {
+ super(cause);
+ }
+
+ protected SchemaChangeException(
+ String message,
+ Throwable cause,
+ boolean enableSuppression,
+ boolean writableStackTrace) {
+ super(message, cause, enableSuppression, writableStackTrace);
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/exception/StreamLoadException.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/exception/StreamLoadException.java
new file mode 100644
index 00000000..d2aaf282
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/exception/StreamLoadException.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.exception;
+
+public class StreamLoadException extends RuntimeException {
+
+ public StreamLoadException() {
+ super();
+ }
+
+ public StreamLoadException(String message) {
+ super(message);
+ }
+
+ public StreamLoadException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public StreamLoadException(Throwable cause) {
+ super(cause);
+ }
+
+ protected StreamLoadException(
+ String message,
+ Throwable cause,
+ boolean enableSuppression,
+ boolean writableStackTrace) {
+ super(message, cause, enableSuppression, writableStackTrace);
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/exception/TableAlterOrCreateException.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/exception/TableAlterOrCreateException.java
deleted file mode 100644
index 288873d2..00000000
--- a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/exception/TableAlterOrCreateException.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.connect.doris.exception;
-
-import io.openmessaging.connector.api.errors.ConnectException;
-
-/**
- * table alter or create exception
- */
-public class TableAlterOrCreateException extends ConnectException {
-
- public TableAlterOrCreateException(String reason) {
- super(reason);
- }
-
- public TableAlterOrCreateException(String reason, Throwable throwable) {
- super(reason, throwable);
- }
-}
-
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/exception/UploadException.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/exception/UploadException.java
new file mode 100644
index 00000000..be599466
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/exception/UploadException.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.exception;
+
+public class UploadException extends DorisException {
+
+ public UploadException() {
+ }
+
+ public UploadException(String message) {
+ super(message);
+ }
+
+ public UploadException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public UploadException(Throwable cause) {
+ super(cause);
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/metrics/DorisConnectMonitor.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/metrics/DorisConnectMonitor.java
new file mode 100644
index 00000000..743a8a54
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/metrics/DorisConnectMonitor.java
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.metrics;
+
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.Histogram;
+import com.codahale.metrics.MetricRegistry;
+import java.util.concurrent.atomic.AtomicLong;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class DorisConnectMonitor {
+
+ private static final Logger LOG = LoggerFactory.getLogger(DorisConnectMonitor.class);
+
+ // committed offset in doris
+ private final AtomicLong committedOffset;
+
+ // total record flushed in doris
+ private final AtomicLong totalNumberOfRecord;
+ private final AtomicLong totalSizeOfData;
+ // total number of data successfully imported to doris through stream-load (or the total number
+ // of data files uploaded through copy-into).
+ private final AtomicLong totalLoadCount;
+
+ // buffer metrics, updated everytime when a buffer is flushed
+ private Histogram partitionBufferSizeBytesHistogram; // in Bytes
+ private Histogram partitionBufferCountHistogram;
+ private final AtomicLong buffMemoryUsage;
+ private final int taskId;
+ private final boolean enableCustomJMX;
+
+ public DorisConnectMonitor(
+ final boolean enableCustomJMXConfig,
+ final Integer taskId,
+ final MetricsJmxReporter metricsJmxReporter) {
+ this.committedOffset = new AtomicLong(-1);
+
+ this.totalLoadCount = new AtomicLong(0);
+ this.totalNumberOfRecord = new AtomicLong(0);
+ this.totalSizeOfData = new AtomicLong(0);
+
+ this.buffMemoryUsage = new AtomicLong(0);
+ this.taskId = taskId;
+ this.enableCustomJMX = enableCustomJMXConfig;
+ if (this.enableCustomJMX) {
+ registerJMXMetrics(metricsJmxReporter);
+ LOG.info("init DorisConnectMonitor, taskId={}", taskId);
+ }
+ }
+
+ /**
+ * Registers all the Metrics inside the metricRegistry.
+ *
+ * @param metricsJmxReporter wrapper class for registering all metrics related to above
+ * connector
+ */
+ private void registerJMXMetrics(MetricsJmxReporter metricsJmxReporter) {
+ MetricRegistry currentMetricRegistry = metricsJmxReporter.getMetricRegistry();
+
+ // Lazily remove all registered metrics from the registry since this can be invoked during
+ // partition reassignment
+ LOG.debug(
+ "Registering metrics existing:{}",
+ metricsJmxReporter.getMetricRegistry().getMetrics().keySet().toString());
+ metricsJmxReporter.removeMetricsFromRegistry(String.valueOf(taskId));
+
+ try {
+ // Offset JMX
+ currentMetricRegistry.register(
+ MetricsUtil.constructMetricName(
+ taskId, MetricsUtil.OFFSET_DOMAIN, MetricsUtil.COMMITTED_OFFSET),
+ (Gauge<Long>) committedOffset::get);
+
+ // Total Processed JMX
+ currentMetricRegistry.register(
+ MetricsUtil.constructMetricName(
+ taskId,
+ MetricsUtil.TOTAL_PROCESSED_DOMAIN,
+ MetricsUtil.TOTAL_LOAD_COUNT),
+ (Gauge<Long>) totalLoadCount::get);
+
+ currentMetricRegistry.register(
+ MetricsUtil.constructMetricName(
+ taskId,
+ MetricsUtil.TOTAL_PROCESSED_DOMAIN,
+ MetricsUtil.TOTAL_RECORD_COUNT),
+ (Gauge<Long>) totalNumberOfRecord::get);
+
+ currentMetricRegistry.register(
+ MetricsUtil.constructMetricName(
+ taskId,
+ MetricsUtil.TOTAL_PROCESSED_DOMAIN,
+ MetricsUtil.TOTAL_DATA_SIZE),
+ (Gauge<Long>) totalSizeOfData::get);
+
+ // Buffer histogram JMX
+ partitionBufferCountHistogram =
+ currentMetricRegistry.histogram(
+ MetricsUtil.constructMetricName(
+ taskId,
+ MetricsUtil.BUFFER_DOMAIN,
+ MetricsUtil.BUFFER_RECORD_COUNT));
+ partitionBufferSizeBytesHistogram =
+ currentMetricRegistry.histogram(
+ MetricsUtil.constructMetricName(
+ taskId,
+ MetricsUtil.BUFFER_DOMAIN,
+ MetricsUtil.BUFFER_SIZE_BYTES));
+ currentMetricRegistry.register(
+ MetricsUtil.constructMetricName(
+ taskId, MetricsUtil.BUFFER_DOMAIN, MetricsUtil.BUFFER_MEMORY_USAGE),
+ (Gauge<Long>) buffMemoryUsage::get);
+ } catch (IllegalArgumentException ex) {
+ LOG.warn("Metrics already present:{}", ex.getMessage());
+ }
+ }
+
+ public void setCommittedOffset(long committedOffset) {
+ if (enableCustomJMX) {
+ this.committedOffset.set(committedOffset);
+ }
+ }
+
+ public void addAndGetLoadCount() {
+ if (enableCustomJMX) {
+ this.totalLoadCount.getAndIncrement();
+ }
+ }
+
+ public void addAndGetTotalNumberOfRecord(long totalNumberOfRecord) {
+ if (enableCustomJMX) {
+ this.totalNumberOfRecord.addAndGet(totalNumberOfRecord);
+ }
+ }
+
+ public void addAndGetTotalSizeOfData(long totalSizeOfData) {
+ if (enableCustomJMX) {
+ this.totalSizeOfData.addAndGet(totalSizeOfData);
+ }
+ }
+
+ public void addAndGetBuffMemoryUsage(long memoryUsage) {
+ if (enableCustomJMX) {
+ this.buffMemoryUsage.addAndGet(memoryUsage);
+ }
+ }
+
+ public void resetMemoryUsage() {
+ if (enableCustomJMX) {
+ this.buffMemoryUsage.set(0L);
+ }
+ }
+
+ public void updateBufferMetrics(long bufferSizeBytes, int numOfRecords) {
+ if (enableCustomJMX) {
+ partitionBufferSizeBytesHistogram.update(bufferSizeBytes);
+ partitionBufferCountHistogram.update(numOfRecords);
+ }
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/metrics/MetricsJmxReporter.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/metrics/MetricsJmxReporter.java
new file mode 100644
index 00000000..a4f38444
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/metrics/MetricsJmxReporter.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.metrics;
+
+import com.codahale.metrics.MetricFilter;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.jmx.JmxReporter;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.concurrent.TimeUnit;
+import javax.management.MalformedObjectNameException;
+import javax.management.ObjectName;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.connect.doris.exception.DorisException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class MetricsJmxReporter {
+ static final Logger LOG = LoggerFactory.getLogger(MetricsJmxReporter.class);
+
+ private final MetricRegistry metricRegistry;
+
+ /**
+ * Wrapper on top of listeners and metricRegistry for codehale. This will be useful to start the
+ * jmx metrics when time is appropriate. (Check {@link MetricsJmxReporter#start()}
+ */
+ private final JmxReporter jmxReporter;
+
+ public MetricsJmxReporter(MetricRegistry metricRegistry, final String connectorName) {
+ this.metricRegistry = metricRegistry;
+ this.jmxReporter = createJMXReporter(connectorName);
+ }
+
+ public MetricRegistry getMetricRegistry() {
+ return metricRegistry;
+ }
+
+ /**
+ * This function will internally register all metrics present inside metric registry and will
+ * register mbeans to the mbeanserver
+ */
+ public void start() {
+ jmxReporter.start();
+ }
+
+ private static ObjectName getObjectName(
+ String connectorName, String jmxDomain, String metricName) {
+ try {
+ StringBuilder sb =
+ new StringBuilder(jmxDomain)
+ .append(":connector=")
+ .append(connectorName)
+ .append(',');
+
+ Iterator<String> tokens = Arrays.stream(StringUtils.split(metricName, "/")).iterator();
+ sb.append("task=").append(tokens.next());
+ sb.append(",category=").append(tokens.next());
+ sb.append(",name=").append(tokens.next());
+
+ return new ObjectName(sb.toString());
+ } catch (MalformedObjectNameException e) {
+ LOG.warn("Could not create Object name for MetricName:{}", metricName);
+ throw new DorisException("Object Name is invalid");
+ }
+ }
+
+ public void removeMetricsFromRegistry(final String prefixFilter) {
+ if (!metricRegistry.getMetrics().isEmpty()) {
+ LOG.debug("Unregistering all metrics:{}", prefixFilter);
+ metricRegistry.removeMatching(MetricFilter.startsWith(prefixFilter));
+ LOG.debug(
+ "Metric registry:{}, size is:{}, names:{}",
+ prefixFilter,
+ metricRegistry.getMetrics().size(),
+ metricRegistry.getMetrics().keySet().toString());
+ }
+ }
+
+ private JmxReporter createJMXReporter(final String connectorName) {
+ return JmxReporter.forRegistry(this.metricRegistry)
+ .inDomain(MetricsUtil.JMX_METRIC_PREFIX)
+ .convertDurationsTo(TimeUnit.SECONDS)
+ .createsObjectNamesWith(
+ (ignoreMeterType, jmxDomain, metricName) ->
+ getObjectName(connectorName, jmxDomain, metricName))
+ .build();
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/metrics/MetricsUtil.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/metrics/MetricsUtil.java
new file mode 100644
index 00000000..34330b83
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/metrics/MetricsUtil.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.metrics;
+
+public class MetricsUtil {
+ public static final String JMX_METRIC_PREFIX = "kafka.connector.doris";
+
+ // Offset related constants
+ public static final String TOTAL_PROCESSED_DOMAIN = "total-processed";
+
+ // total number of data successfully imported to doris through stream-load (or the total number
+ // of data files uploaded through copy-into).
+ public static final String TOTAL_LOAD_COUNT = "total-load-count";
+ public static final String TOTAL_RECORD_COUNT = "total-record-count";
+ public static final String TOTAL_DATA_SIZE = "total-data-size";
+
+ // file count related constants
+ public static final String OFFSET_DOMAIN = "offsets";
+ // Successfully submitted data to doris' offset
+ public static final String COMMITTED_OFFSET = "committed-offset";
+
+ // Buffer related constants
+ public static final String BUFFER_DOMAIN = "buffer";
+ public static final String BUFFER_MEMORY_USAGE = "buffer-memory-usage";
+ public static final String BUFFER_SIZE_BYTES = "buffer-size-bytes";
+ public static final String BUFFER_RECORD_COUNT = "buffer-record-count";
+
+ public static String constructMetricName(
+ final Integer taskId, final String domain, final String metricName) {
+ return String.format("%s/%s/%s", taskId, domain, metricName);
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/model/BackendV2.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/model/BackendV2.java
new file mode 100644
index 00000000..64883080
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/model/BackendV2.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.List;
+
+/**
+ * Be response model.
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class BackendV2 {
+
+ @JsonProperty(value = "backends")
+ private List<BackendRowV2> backends;
+
+ public List<BackendRowV2> getBackends() {
+ return backends;
+ }
+
+ public void setBackends(List<BackendRowV2> backends) {
+ this.backends = backends;
+ }
+
+ public static class BackendRowV2 {
+ @JsonProperty("ip")
+ public String ip;
+
+ @JsonProperty("http_port")
+ public int httpPort;
+
+ @JsonProperty("is_alive")
+ public boolean isAlive;
+
+ public String getIp() {
+ return ip;
+ }
+
+ public void setIp(String ip) {
+ this.ip = ip;
+ }
+
+ public int getHttpPort() {
+ return httpPort;
+ }
+
+ public void setHttpPort(int httpPort) {
+ this.httpPort = httpPort;
+ }
+
+ public boolean isAlive() {
+ return isAlive;
+ }
+
+ public void setAlive(boolean alive) {
+ isAlive = alive;
+ }
+
+ public String toBackendString() {
+ return ip + ":" + httpPort;
+ }
+
+ public static BackendRowV2 of(String ip, int httpPort, boolean alive) {
+ BackendRowV2 rowV2 = new BackendRowV2();
+ rowV2.setIp(ip);
+ rowV2.setHttpPort(httpPort);
+ rowV2.setAlive(alive);
+ return rowV2;
+ }
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/model/BaseResponse.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/model/BaseResponse.java
new file mode 100644
index 00000000..8d29b3d9
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/model/BaseResponse.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class BaseResponse<T> {
+ private int code;
+ private String msg;
+ private T data;
+ private int count;
+
+ public int getCode() {
+ return code;
+ }
+
+ public String getMsg() {
+ return msg;
+ }
+
+ public T getData() {
+ return data;
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/model/ColumnDescriptor.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/model/ColumnDescriptor.java
new file mode 100644
index 00000000..326cf090
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/model/ColumnDescriptor.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.model;
+
+import java.util.Objects;
+
+public class ColumnDescriptor {
+ private final String columnName;
+ private final String typeName;
+ private final String comment;
+ private final String defaultValue;
+
+ private ColumnDescriptor(
+ String columnName, String typeName, String comment, String defaultValue) {
+ this.columnName = columnName;
+ this.typeName = typeName;
+ this.comment = comment;
+ this.defaultValue = defaultValue;
+ }
+
+ public String getColumnName() {
+ return columnName;
+ }
+
+ public String getTypeName() {
+ return typeName;
+ }
+
+ public String getDefaultValue() {
+ return defaultValue;
+ }
+
+ public String getComment() {
+ return comment;
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public static class Builder {
+ private String columnName;
+ private String typeName;
+ private String comment;
+ private String defaultValue;
+
+ public Builder columnName(String columnName) {
+ this.columnName = columnName;
+ return this;
+ }
+
+ public Builder typeName(String typeName) {
+ this.typeName = typeName;
+ return this;
+ }
+
+ public Builder comment(String comment) {
+ this.comment = comment;
+ return this;
+ }
+
+ public Builder defaultValue(String defaultValue) {
+ this.defaultValue = defaultValue;
+ return this;
+ }
+
+ public ColumnDescriptor build() {
+ Objects.requireNonNull(columnName, "A column name is required");
+ Objects.requireNonNull(typeName, "A type name is required");
+
+ return new ColumnDescriptor(columnName, typeName, comment, defaultValue);
+ }
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/model/CopyIntoResp.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/model/CopyIntoResp.java
new file mode 100644
index 00000000..4b270d4d
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/model/CopyIntoResp.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import java.util.Map;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class CopyIntoResp extends BaseResponse {
+ private String code;
+ private String exception;
+
+ private Map<String, String> result;
+
+ public String getDataCode() {
+ return code;
+ }
+
+ public String getException() {
+ return exception;
+ }
+
+ public Map<String, String> getResult() {
+ return result;
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/model/KafkaRespContent.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/model/KafkaRespContent.java
new file mode 100644
index 00000000..b44c78fb
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/model/KafkaRespContent.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.model;
+
+public class KafkaRespContent extends RespContent {
+
+ private String topic;
+ private String database;
+ private String table;
+ private long lastOffset;
+
+ public String getTopic() {
+ return topic;
+ }
+
+ public void setTopic(String topic) {
+ this.topic = topic;
+ }
+
+ public String getDatabase() {
+ return database;
+ }
+
+ public void setDatabase(String database) {
+ this.database = database;
+ }
+
+ public String getTable() {
+ return table;
+ }
+
+ public void setTable(String table) {
+ this.table = table;
+ }
+
+ public long getLastOffset() {
+ return lastOffset;
+ }
+
+ public void setLastOffset(long lastOffset) {
+ this.lastOffset = lastOffset;
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/model/LoadOperation.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/model/LoadOperation.java
new file mode 100644
index 00000000..33f8c2a6
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/model/LoadOperation.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.model;
+
+public enum LoadOperation {
+ SEND_REQUEST_TO_DORIS,
+
+ COMMIT_TRANSACTION,
+
+ GET_UPLOAD_ADDRESS,
+
+ UPLOAD_FILE,
+
+ EXECUTE_COPY;
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/model/RespContent.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/model/RespContent.java
new file mode 100644
index 00000000..6e5caee3
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/model/RespContent.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class RespContent {
+
+ @JsonProperty(value = "TxnId")
+ private long txnId;
+
+ @JsonProperty(value = "Label")
+ private String label;
+
+ @JsonProperty(value = "Status")
+ private String status;
+
+ @JsonProperty(value = "TwoPhaseCommit")
+ private String twoPhaseCommit;
+
+ @JsonProperty(value = "ExistingJobStatus")
+ private String existingJobStatus;
+
+ @JsonProperty(value = "Message")
+ private String message;
+
+ @JsonProperty(value = "NumberTotalRows")
+ private long numberTotalRows;
+
+ @JsonProperty(value = "NumberLoadedRows")
+ private long numberLoadedRows;
+
+ @JsonProperty(value = "NumberFilteredRows")
+ private int numberFilteredRows;
+
+ @JsonProperty(value = "NumberUnselectedRows")
+ private int numberUnselectedRows;
+
+ @JsonProperty(value = "LoadBytes")
+ private long loadBytes;
+
+ @JsonProperty(value = "LoadTimeMs")
+ private int loadTimeMs;
+
+ @JsonProperty(value = "BeginTxnTimeMs")
+ private int beginTxnTimeMs;
+
+ @JsonProperty(value = "StreamLoadPutTimeMs")
+ private int streamLoadPutTimeMs;
+
+ @JsonProperty(value = "ReadDataTimeMs")
+ private int readDataTimeMs;
+
+ @JsonProperty(value = "WriteDataTimeMs")
+ private int writeDataTimeMs;
+
+ @JsonProperty(value = "CommitAndPublishTimeMs")
+ private int commitAndPublishTimeMs;
+
+ @JsonProperty(value = "ErrorURL")
+ private String errorURL;
+
+ public long getTxnId() {
+ return txnId;
+ }
+
+ public String getStatus() {
+ return status;
+ }
+
+ public String getTwoPhaseCommit() {
+ return twoPhaseCommit;
+ }
+
+ public String getMessage() {
+ return message;
+ }
+
+ public String getExistingJobStatus() {
+ return existingJobStatus;
+ }
+
+ @Override
+ public String toString() {
+ ObjectMapper mapper = new ObjectMapper();
+ try {
+ return mapper.writeValueAsString(this);
+ } catch (JsonProcessingException e) {
+ return "";
+ }
+ }
+
+ public String getErrorURL() {
+ return errorURL;
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/model/TableDescriptor.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/model/TableDescriptor.java
new file mode 100644
index 00000000..5f92738c
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/model/TableDescriptor.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.model;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+public class TableDescriptor {
+ private final String tableName;
+ private final String tableType;
+ private final Map<String, ColumnDescriptor> columns = new LinkedHashMap<>();
+
+ private TableDescriptor(String tableName, String tableType, List<ColumnDescriptor> columns) {
+ this.tableName = tableName;
+ this.tableType = tableType;
+ columns.forEach(c -> this.columns.put(c.getColumnName(), c));
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ public String getTableType() {
+ return tableType;
+ }
+
+ public Collection<ColumnDescriptor> getColumns() {
+ return columns.values();
+ }
+
+ public ColumnDescriptor getColumnByName(String columnName) {
+ return columns.get(columnName);
+ }
+
+ public boolean hasColumn(String columnName) {
+ return columns.containsKey(columnName);
+ }
+
+ public static class Builder {
+ private String schemaName;
+ private String tableName;
+ private String tableType;
+ private final List<ColumnDescriptor> columns = new ArrayList<>();
+
+ private Builder() {
+ }
+
+ public Builder schemaName(String schemaName) {
+ this.schemaName = schemaName;
+ return this;
+ }
+
+ public Builder tableName(String tableName) {
+ this.tableName = tableName;
+ return this;
+ }
+
+ public Builder type(String tableType) {
+ this.tableType = tableType;
+ return this;
+ }
+
+ public Builder column(ColumnDescriptor column) {
+ this.columns.add(column);
+ return this;
+ }
+
+ public Builder columns(List<ColumnDescriptor> columns) {
+ this.columns.addAll(columns);
+ return this;
+ }
+
+ public TableDescriptor build() {
+ return new TableDescriptor(tableName, tableType, columns);
+ }
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/model/doris/Field.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/model/doris/Field.java
new file mode 100644
index 00000000..edce95ad
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/model/doris/Field.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.model.doris;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.Objects;
+
+public class Field {
+ @JsonProperty(value = "name")
+ private String name;
+
+ @JsonProperty(value = "type")
+ private String type;
+
+ @JsonProperty(value = "comment")
+ private String comment;
+
+ @JsonProperty(value = "precision")
+ private int precision;
+
+ @JsonProperty(value = "scale")
+ private int scale;
+
+ @JsonProperty(value = "aggregation_type")
+ private String aggregationType;
+
+ public Field() {
+ }
+
+ public Field(
+ String name,
+ String type,
+ String comment,
+ int precision,
+ int scale,
+ String aggregationType) {
+ this.name = name;
+ this.type = type;
+ this.comment = comment;
+ this.precision = precision;
+ this.scale = scale;
+ this.aggregationType = aggregationType;
+ }
+
+ public String getAggregationType() {
+ return aggregationType;
+ }
+
+ public void setAggregationType(String aggregationType) {
+ this.aggregationType = aggregationType;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ public String getComment() {
+ return comment;
+ }
+
+ public void setComment(String comment) {
+ this.comment = comment;
+ }
+
+ public int getPrecision() {
+ return precision;
+ }
+
+ public void setPrecision(int precision) {
+ this.precision = precision;
+ }
+
+ public int getScale() {
+ return scale;
+ }
+
+ public void setScale(int scale) {
+ this.scale = scale;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ Field field = (Field) o;
+ return precision == field.precision
+ && scale == field.scale
+ && Objects.equals(name, field.name)
+ && Objects.equals(type, field.type)
+ && Objects.equals(comment, field.comment);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(name, type, comment, precision, scale);
+ }
+
+ @Override
+ public String toString() {
+ return "Field{"
+ + "name='"
+ + name
+ + '\''
+ + ", type='"
+ + type
+ + '\''
+ + ", comment='"
+ + comment
+ + '\''
+ + ", precision="
+ + precision
+ + ", scale="
+ + scale
+ + '}';
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/model/doris/Schema.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/model/doris/Schema.java
new file mode 100644
index 00000000..60ffb9d3
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/model/doris/Schema.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.model.doris;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+public class Schema {
+ private int status = 0;
+ private String keysType;
+ private List<Field> properties;
+
+ public Schema() {
+ properties = new ArrayList<>();
+ }
+
+ public Schema(int fieldCount) {
+ properties = new ArrayList<>(fieldCount);
+ }
+
+ public int getStatus() {
+ return status;
+ }
+
+ public void setStatus(int status) {
+ this.status = status;
+ }
+
+ public String getKeysType() {
+ return keysType;
+ }
+
+ public void setKeysType(String keysType) {
+ this.keysType = keysType;
+ }
+
+ public List<Field> getProperties() {
+ return properties;
+ }
+
+ public void setProperties(List<Field> properties) {
+ this.properties = properties;
+ }
+
+ public void put(
+ String name,
+ String type,
+ String comment,
+ int scale,
+ int precision,
+ String aggregationType) {
+ properties.add(new Field(name, type, comment, scale, precision, aggregationType));
+ }
+
+ public void put(Field f) {
+ properties.add(f);
+ }
+
+ public Field get(int index) {
+ if (index >= properties.size()) {
+ throw new IndexOutOfBoundsException(
+ "Index: " + index + ", Fields size：" + properties.size());
+ }
+ return properties.get(index);
+ }
+
+ public int size() {
+ return properties.size();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ Schema schema = (Schema) o;
+ return status == schema.status && Objects.equals(properties, schema.properties);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(status, properties);
+ }
+
+ @Override
+ public String toString() {
+ return "Schema{" + "status=" + status + ", properties=" + properties + '}';
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/schema/column/ColumnDefAdjuster.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/schema/column/ColumnDefAdjuster.java
deleted file mode 100644
index 12b01b94..00000000
--- a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/schema/column/ColumnDefAdjuster.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.connect.doris.schema.column;
-
-import java.sql.Connection;
-import java.sql.DatabaseMetaData;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.HashMap;
-import java.util.Map;
-
-public class ColumnDefAdjuster {
- Map<String, ColumnDefinition.Nullability> nullable = new HashMap<>();
-
- public ColumnDefAdjuster() {
- }
-
- public static ColumnDefAdjuster create(Connection conn,
- String catalogPattern,
- String schemaPattern,
- String tablePattern,
- String columnPattern) {
- ColumnDefAdjuster adjuster = new ColumnDefAdjuster();
- try (ResultSet rs = conn.getMetaData().getColumns(
- catalogPattern, schemaPattern, tablePattern, columnPattern)) {
- final int rsColumnCount = rs.getMetaData().getColumnCount();
- while (rs.next()) {
- final String columnName = rs.getString(4);
- ColumnDefinition.Nullability nullability;
- final int nullableValue = rs.getInt(11);
- switch (nullableValue) {
- case DatabaseMetaData.columnNoNulls:
- nullability = ColumnDefinition.Nullability.NOT_NULL;
- break;
- case DatabaseMetaData.columnNullable:
- nullability = ColumnDefinition.Nullability.NULL;
- break;
- case DatabaseMetaData.columnNullableUnknown:
- default:
- nullability = ColumnDefinition.Nullability.UNKNOWN;
- break;
- }
- adjuster.nullable.put(columnName, nullability);
- }
- } catch (SQLException e) {
- //pass
- }
-
- return adjuster;
- }
-
- public ColumnDefinition.Nullability nullable(String columnName) {
- if (nullable == null || !nullable.containsKey(columnName)) {
- return null;
- }
- return nullable.get(columnName);
- }
-}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/schema/column/ColumnDefinition.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/schema/column/ColumnDefinition.java
deleted file mode 100644
index 3bcd3b8c..00000000
--- a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/schema/column/ColumnDefinition.java
+++ /dev/null
@@ -1,337 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.connect.doris.schema.column;
-
-import java.sql.Types;
-import java.util.Objects;
-
-/**
- * column definition
- */
-public class ColumnDefinition {
-
- /**
- * The nullability of a column.
- */
- public enum Nullability {
- NULL, NOT_NULL, UNKNOWN
- }
-
- /**
- * The mutability of a column.
- */
- public enum Mutability {
- READ_ONLY, MAYBE_WRITABLE, WRITABLE, UNKNOWN
- }
-
- private final ColumnId id;
- private final String typeName;
- private final int jdbcType;
- private final int displaySize;
- private final int precision;
- private final int scale;
- private final boolean autoIncremented;
- private final boolean caseSensitive;
- private final boolean searchable;
- private final boolean currency;
- private final boolean signedNumbers;
- private final boolean isPrimaryKey;
- private final Nullability nullability;
- private final Mutability mutability;
- private final String classNameForType;
-
- public ColumnDefinition(
- ColumnId id,
- int jdbcType,
- String typeName,
- String classNameForType,
- Nullability nullability,
- Mutability mutability,
- int precision,
- int scale,
- boolean signedNumbers,
- int displaySize,
- boolean autoIncremented,
- boolean caseSensitive,
- boolean searchable,
- boolean currency,
- boolean isPrimaryKey
- ) {
- this.id = id;
- this.typeName = typeName;
- this.jdbcType = jdbcType;
- this.displaySize = displaySize;
- this.precision = precision;
- this.scale = scale;
- this.autoIncremented = autoIncremented;
- this.caseSensitive = caseSensitive;
- this.searchable = searchable;
- this.currency = currency;
- this.signedNumbers = signedNumbers;
- this.nullability = nullability != null ? nullability : Nullability.UNKNOWN;
- this.mutability = mutability != null ? mutability : Mutability.MAYBE_WRITABLE;
- this.classNameForType = classNameForType;
- this.isPrimaryKey = isPrimaryKey;
- }
-
-
- /**
- * Indicates whether the column is automatically numbered.
- *
- * @return <code>true</code> if so; <code>false</code> otherwise
- */
- public boolean isAutoIncrement() {
- return autoIncremented;
- }
-
- /**
- * Indicates whether the column's case matters.
- *
- * @return <code>true</code> if so; <code>false</code> otherwise
- */
- public boolean isCaseSensitive() {
- return caseSensitive;
- }
-
- /**
- * Indicates whether the column can be used in a where clause.
- *
- * @return <code>true</code> if so; <code>false</code> otherwise
- */
- public boolean isSearchable() {
- return searchable;
- }
-
- /**
- * Indicates whether the column is a cash value.
- *
- * @return <code>true</code> if so; <code>false</code> otherwise
- */
- public boolean isCurrency() {
- return currency;
- }
-
- /**
- * Indicates whether the column is part of the table's primary key.
- *
- * @return <code>true</code> if so; <code>false</code> otherwise
- */
- public boolean isPrimaryKey() {
- return isPrimaryKey;
- }
-
- /**
- * Indicates the nullability of values in the column.
- *
- * @return the nullability status of the given column; never null
- */
- public Nullability nullability() {
- return nullability;
- }
-
- /**
- * Indicates whether values in the column are optional. This is equivalent to calling:
- * <pre>
- * nullability() == Nullability.NULL || nullability() == Nullability.UNKNOWN
- * </pre>
- *
- * @return <code>true</code> if so; <code>false</code> otherwise
- */
- public boolean isOptional() {
- return nullability == Nullability.NULL || nullability == Nullability.UNKNOWN;
- }
-
- /**
- * Indicates whether values in the column are signed numbers.
- *
- * @return <code>true</code> if so; <code>false</code> otherwise
- */
- public boolean isSignedNumber() {
- return signedNumbers;
- }
-
- /**
- * Indicates the column's normal maximum width in characters.
- *
- * @return the normal maximum number of characters allowed as the width of the designated column
- */
- public int displaySize() {
- return displaySize;
- }
-
- /**
- * Get the column's identifier.
- *
- * @return column identifier; never null
- */
- public ColumnId id() {
- return id;
- }
-
- /**
- * Get the column's table identifier.
- *
- * @return the table identifier; never null
- */
-// public TableId tableId() {
-// return id.tableId();
-// }
-
- /**
- * Get the column's specified column size. For numeric data, this is the maximum precision. For
- * character data, this is the length in characters. For datetime datatypes, this is the length in
- * characters of the String representation (assuming the maximum allowed precision of the
- * fractional seconds component). For binary data, this is the length in bytes. For the ROWID
- * datatype, this is the length in bytes. 0 is returned for data types where the column size is
- * not applicable.
- *
- * @return precision
- */
- public int precision() {
- return precision;
- }
-
- /**
- * Gets the column's number of digits to right of the decimal point. 0 is returned for data types
- * where the scale is not applicable.
- *
- * @return scale
- */
- public int scale() {
- return scale;
- }
-
- /**
- * Retrieves the column's JDBC type.
- *
- * @return SQL type from java.sql.Types
- * @see Types
- */
- public int type() {
- return jdbcType;
- }
-
- /**
- * Retrieves the designated column's database-specific type name.
- *
- * @return type name used by the database. If the column type is a user-defined type, then a
- * fully-qualified type name is returned.
- */
- public String typeName() {
- return typeName;
- }
-
- /**
- * Indicates whether the designated column is mutable.
- *
- * @return the mutability; never null
- */
- public Mutability mutability() {
- return mutability;
- }
-
- /**
- * Returns the fully-qualified name of the Java class whose instances are manufactured if the
- * method {@link java.sql.ResultSet#getObject(int)} is called to retrieve a value from the column.
- * {@link java.sql.ResultSet#getObject(int)} may return a subclass of the class returned by this
- * method.
- *
- * @return the fully-qualified name of the class in the Java programming language that would be
- * used by the method <code>ResultSet.getObject</code> to retrieve the value in the specified
- * column. This is the class name used for custom mapping.
- */
- public String classNameForType() {
- return classNameForType;
- }
-
-
- @Override
- public int hashCode() {
- return id.hashCode();
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj == this) {
- return true;
- }
- if (obj instanceof ColumnDefinition) {
- ColumnDefinition that = (ColumnDefinition) obj;
- return Objects.equals(this.id, that.id)
- && this.jdbcType == that.jdbcType
- && this.displaySize == that.displaySize
- && this.precision == that.precision
- && this.scale == that.scale
- && this.autoIncremented == that.autoIncremented
- && this.caseSensitive == that.caseSensitive
- && this.searchable == that.searchable
- && this.currency == that.currency
- && this.signedNumbers == that.signedNumbers
- && this.nullability == that.nullability
- && Objects.equals(this.typeName, that.typeName)
- && Objects.equals(this.classNameForType, that.classNameForType)
- && Objects.equals(this.isPrimaryKey, that.isPrimaryKey);
- }
- return false;
- }
-
- @Override
- public String toString() {
- return "Column{'" + id.name() + '\'' + ", isPrimaryKey=" + isPrimaryKey + ", allowsNull="
- + isOptional() + ", sqlType=" + typeName + '}';
- }
-
- /**
- * Obtain a {@link ColumnDefinition} that has all the same characteristics as this column except
- * that belongs to the specified table.
- *
- * @param tableId the new table ID; may not be null
-// * @return this definition if the specified table ID matches this definition's {@link #tableId()},
- * or a new definition that is a copy of this definition except with an {@link #id() ID} that
- * uses the specified table; never null
- */
-// public ColumnDefinition forTable(TableId tableId) {
-// if (tableId().equals(tableId)) {
-// return this;
-// }
-// ColumnId newId = new ColumnId(tableId, this.id().name());
-// return new ColumnDefinition(newId, jdbcType, typeName, classNameForType, nullability,
-// mutability, precision, scale, signedNumbers, displaySize,
-// autoIncremented, caseSensitive, searchable, currency, isPrimaryKey
-// );
-// }
-
- /**
- * Obtain a {@link ColumnDefinition} that has all the same characteristics as this column except
- * that it does or does not belong to the table's primary key
- *
- * @param isPrimaryKey true if the resulting column definition should be part of the table's
- * primary key, or false otherwise
- * @return a definition that is the same as this definition except it is or is not part of the
- * tables primary key, or may be this object if {@link #isPrimaryKey()} already matches the
- * supplied parameter; never null
- */
- public ColumnDefinition asPartOfPrimaryKey(boolean isPrimaryKey) {
- if (isPrimaryKey == isPrimaryKey()) {
- return this;
- }
- return new ColumnDefinition(id, jdbcType, typeName, classNameForType, nullability, mutability,
- precision, scale, signedNumbers, displaySize, autoIncremented,
- caseSensitive, searchable, currency, isPrimaryKey
- );
- }
-}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/schema/column/ColumnId.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/schema/column/ColumnId.java
deleted file mode 100644
index 39310c98..00000000
--- a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/schema/column/ColumnId.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.connect.doris.schema.column;
-
-import org.apache.rocketmq.connect.doris.schema.table.TableId;
-import org.apache.rocketmq.connect.doris.util.ExpressionBuilder;
-import org.apache.rocketmq.connect.doris.util.QuoteMethod;
-
-import java.util.Objects;
-
-/**
- * column id
- */
-public class ColumnId implements ExpressionBuilder.Expressable {
-
- private final TableId tableId;
- private final String name;
- private final String alias;
- private final int hash;
-
- public ColumnId(
- TableId tableId,
- String columnName
- ) {
- this(tableId, columnName, null);
- }
-
- public ColumnId(
- TableId tableId,
- String columnName,
- String alias
- ) {
- assert columnName != null;
- this.tableId = tableId;
- this.name = columnName;
- this.alias = alias != null && !alias.trim().isEmpty() ? alias : name;
- this.hash = Objects.hash(this.tableId, this.name);
- }
-
- public TableId tableId() {
- return tableId;
- }
-
- public String name() {
- return name;
- }
-
- public String aliasOrName() {
- return alias;
- }
-
- @Override
- public void appendTo(ExpressionBuilder builder, boolean useQuotes) {
- appendTo(builder, useQuotes ? QuoteMethod.ALWAYS : QuoteMethod.NEVER);
- }
-
- @Override
- public void appendTo(
- ExpressionBuilder builder,
- QuoteMethod useQuotes
- ) {
- if (tableId != null) {
- builder.append(tableId);
- builder.appendIdentifierDelimiter();
- }
- builder.appendColumnName(this.name, useQuotes);
- }
-
- @Override
- public int hashCode() {
- return hash;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj == this) {
- return true;
- }
- if (obj instanceof ColumnId) {
- ColumnId that = (ColumnId) obj;
- return Objects.equals(this.name, that.name) && Objects.equals(this.alias, that.alias)
- && Objects.equals(this.tableId, that.tableId);
- }
- return false;
- }
-
- @Override
- public String toString() {
- return ExpressionBuilder.create().append(this).toString();
- }
-}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/schema/column/parser/DateColumnParser.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/schema/column/parser/DateColumnParser.java
deleted file mode 100644
index a089f621..00000000
--- a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/schema/column/parser/DateColumnParser.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.connect.doris.schema.column.parser;
-
-import io.openmessaging.connector.api.data.Schema;
-import io.openmessaging.connector.api.data.SchemaBuilder;
-import io.openmessaging.connector.api.errors.ConnectException;
-
-import java.util.Calendar;
-import java.util.TimeZone;
-
-public class DateColumnParser {
- public static final String LOGICAL_NAME = "org.apache.rocketmq.connect.doris.schema.column.parser.DateColumnParser";
- private static final long MILLIS_PER_DAY = 86400000L;
- private static final TimeZone UTC = TimeZone.getTimeZone("UTC");
- public static final Schema SCHEMA = builder().build();
-
- public DateColumnParser() {
- }
-
- public static SchemaBuilder builder() {
- return SchemaBuilder.int32().name("org.apache.kafka.connect.data.Date");
- }
-
- public static int fromLogical(Schema schema, java.util.Date value) {
- if (!LOGICAL_NAME.equals(schema.getName())) {
- throw new ConnectException("Requested conversion of Date object but the schema does not match.");
- } else {
- Calendar calendar = Calendar.getInstance(UTC);
- calendar.setTime(value);
- if (calendar.get(11) == 0 && calendar.get(12) == 0 && calendar.get(13) == 0 && calendar.get(14) == 0) {
- long unixMillis = calendar.getTimeInMillis();
- return (int) (unixMillis / 86400000L);
- } else {
- throw new ConnectException("Kafka Connect Date type should not have any time fields set to non-zero values.");
- }
- }
- }
-
- public static java.util.Date toLogical(Schema schema, int value) {
- if (!LOGICAL_NAME.equals(schema.getName())) {
- throw new ConnectException("Requested conversion of Date object but the schema does not match.");
- } else {
- return new java.util.Date((long) value * 86400000L);
- }
- }
-}
\ No newline at end of file
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/schema/column/parser/TimeColumnParser.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/schema/column/parser/TimeColumnParser.java
deleted file mode 100644
index d1606739..00000000
--- a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/schema/column/parser/TimeColumnParser.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.rocketmq.connect.doris.schema.column.parser;
-
-import io.openmessaging.connector.api.data.Schema;
-import io.openmessaging.connector.api.data.SchemaBuilder;
-import io.openmessaging.connector.api.errors.ConnectException;
-
-import java.util.Calendar;
-import java.util.Date;
-import java.util.TimeZone;
-
-public class TimeColumnParser {
- public static final String LOGICAL_NAME = "org.apache.rocketmq.connect.doris.schema.column.parser.TimeColumnParser";
- private static final long MILLIS_PER_DAY = 86400000L;
- private static final TimeZone UTC = TimeZone.getTimeZone("UTC");
- public static final Schema SCHEMA = builder().build();
-
- public static SchemaBuilder builder() {
- return SchemaBuilder.int32().name("org.apache.rocketmq.connect.doris.schema.column.parser.TimeColumnParser");
- }
-
- public static int fromLogical(Schema schema, Date value) {
- if (!LOGICAL_NAME.equals(schema.getName())) {
- throw new ConnectException("Requested conversion of Time object but the schema does not match.");
- } else {
- Calendar calendar = Calendar.getInstance(UTC);
- calendar.setTime(value);
- long unixMillis = calendar.getTimeInMillis();
- if (unixMillis >= 0L && unixMillis <= 86400000L) {
- return (int) unixMillis;
- } else {
- throw new ConnectException("Kafka Connect Time type should not have any date fields set to non-zero values.");
- }
- }
- }
-
- public static Date toLogical(Schema schema, int value) {
- if (!LOGICAL_NAME.equals(schema.getName())) {
- throw new ConnectException("Requested conversion of Date object but the schema does not match.");
- } else if (value >= 0 && (long) value <= 86400000L) {
- return new Date((long) value);
- } else {
- throw new ConnectException("Time values must use number of milliseconds greater than 0 and less than 86400000");
- }
- }
-
-}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/schema/column/parser/TimestampColumnParser.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/schema/column/parser/TimestampColumnParser.java
deleted file mode 100644
index a095bded..00000000
--- a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/schema/column/parser/TimestampColumnParser.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.connect.doris.schema.column.parser;
-
-import io.openmessaging.connector.api.data.Schema;
-import io.openmessaging.connector.api.data.SchemaBuilder;
-import io.openmessaging.connector.api.errors.ConnectException;
-
-import java.util.Date;
-
-/**
- * time stamp column parser
- */
-public class TimestampColumnParser {
- public static final String LOGICAL_NAME = "org.apache.rocketmq.connect.doris.schema.column.parser.TimestampColumnParser";
- public static final Schema SCHEMA = builder().build();
-
- public TimestampColumnParser() {
- }
-
- public static SchemaBuilder builder() {
- return SchemaBuilder.int64().name("org.apache.kafka.connect.data.Timestamp");
- }
-
- public static long fromLogical(Schema schema, Date value) {
- if (!LOGICAL_NAME.equals(schema.getName())) {
- throw new ConnectException("Requested conversion of Timestamp object but the schema does not match.");
- } else {
- return value.getTime();
- }
- }
-
- public static Date toLogical(Schema schema, long value) {
- if (!LOGICAL_NAME.equals(schema.getName())) {
- throw new ConnectException("Requested conversion of Timestamp object but the schema does not match.");
- } else {
- return new Date(value);
- }
- }
-}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/schema/db/DbStructure.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/schema/db/DbStructure.java
deleted file mode 100644
index bcc14879..00000000
--- a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/schema/db/DbStructure.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.connect.doris.schema.db;
-
-import org.apache.rocketmq.connect.doris.connector.DorisSinkConfig;
-import org.apache.rocketmq.connect.doris.schema.table.TableId;
-import org.apache.rocketmq.connect.doris.sink.metadata.FieldsMetadata;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import java.sql.SQLException;
-
-/**
- *
- */
-public class DbStructure {
- private static final Logger log = LoggerFactory.getLogger(DbStructure.class);
-
- /**
- * Create or amend table.
- *
- * @param config the connector configuration
-// * @param connection the database connection handle
- * @param tableId the table ID
- * @param fieldsMetadata the fields metadata
- * @return whether a DDL operation was performed
- * @throws SQLException if a DDL operation was deemed necessary but failed
- */
- public boolean createOrAmendIfNecessary(
- final DorisSinkConfig config,
- final TableId tableId,
- final FieldsMetadata fieldsMetadata
- ) throws SQLException {
- // It seems that doris don't support create or amend table via stream load, so do nothing
- return false;
- }
-}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/schema/table/TableDefinition.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/schema/table/TableDefinition.java
deleted file mode 100644
index 68ca5aaf..00000000
--- a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/schema/table/TableDefinition.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.connect.doris.schema.table;
-
-import org.apache.rocketmq.connect.doris.schema.column.ColumnDefinition;
-import org.apache.rocketmq.connect.doris.util.TableType;
-
-import java.util.LinkedHashMap;
-import java.util.Map;
-import java.util.Objects;
-
-
-/**
- * A description of a table.
- */
-public class TableDefinition {
- private final TableId id;
- private final Map<String, String> pkColumnNames = new LinkedHashMap<>();
-
- public TableDefinition(TableId id, Iterable<ColumnDefinition> columns, TableType type) {
- this.id = id;
- }
-
- public TableId id() {
- return id;
- }
-
- @Override
- public int hashCode() {
- return id.hashCode();
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj == null) {
- return true;
- }
- if (obj instanceof TableDefinition) {
- TableDefinition that = (TableDefinition) obj;
- return Objects.equals(this.id(), that.id());
- }
- return false;
- }
-
- @Override
- public String toString() {
- return String.format("Table{name='%s', type=%s columns=%s}", id);
- }
-}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/schema/table/TableDefinitions.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/schema/table/TableDefinitions.java
deleted file mode 100644
index 51a0d153..00000000
--- a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/schema/table/TableDefinitions.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.connect.doris.schema.table;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.sql.SQLException;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * A simple cache of {@link TableDefinition} keyed.
- */
-public class TableDefinitions {
-
- private static final Logger log = LoggerFactory.getLogger(TableDefinitions.class);
-
- private final Map<TableId, TableDefinition> cache = new HashMap<>();
-// private final DatabaseDialect dialect;
-
- /**
- * Create an instance that uses the specified database dialect.
- *
- * @param dialect the database dialect; may not be null
- */
-// public TableDefinitions(DatabaseDialect dialect) {
-// this.dialect = dialect;
-// }
-
- /**
- * Get the {@link TableDefinition} for the given table.
- *
- * @param tableId the table identifier; may not be null
- * @return the cached {@link TableDefinition}, or null if there is no such table
- * @throws SQLException if there is any problem using the connection
- */
- public TableDefinition get(
- final TableId tableId
- ) {
- return cache.get(tableId);
- }
-
- public boolean contains(
- final TableId tableId
- ) {
- return cache.containsKey(tableId);
- }
-
- /**
- * Refresh the cached {@link TableDefinition} for the given table.
- *
- * @param tableId the table identifier; may not be null
- * @return the refreshed {@link TableDefinition}, or null if there is no such table
- * @throws SQLException if there is any problem using the connection
- */
- public TableDefinition refresh(
- TableId tableId
- ) throws SQLException {
- TableDefinition dbTable = null;
- log.info("Refreshing metadata for table {} to {}", tableId, dbTable);
- cache.put(dbTable.id(), dbTable);
- return dbTable;
- }
-}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/schema/table/TableId.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/schema/table/TableId.java
deleted file mode 100644
index 2b0e821f..00000000
--- a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/schema/table/TableId.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.connect.doris.schema.table;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.rocketmq.connect.doris.util.ExpressionBuilder;
-import org.apache.rocketmq.connect.doris.util.QuoteMethod;
-
-import java.util.Objects;
-
-public class TableId implements Comparable<TableId>, ExpressionBuilder.Expressable {
-
- private final String catalogName;
- private final String schemaName;
- private final String tableName;
- private final int hash;
-
- public TableId(
- String catalogName,
- String schemaName,
- String tableName
- ) {
- this.catalogName = catalogName == null || catalogName.isEmpty() ? null : catalogName;
- this.schemaName = schemaName == null || schemaName.isEmpty() ? null : schemaName;
- this.tableName = tableName;
- this.hash = Objects.hash(catalogName, schemaName, tableName);
- }
-
- public String catalogName() {
- return catalogName;
- }
-
- public String schemaName() {
- return schemaName;
- }
-
- public String dbName() {
- if (StringUtils.isNotBlank(catalogName)) {
- return catalogName;
- }
- return schemaName;
- }
-
- public String tableName() {
- return tableName;
- }
-
- @Override
- public void appendTo(ExpressionBuilder builder, boolean useQuotes) {
- appendTo(builder, useQuotes ? QuoteMethod.ALWAYS : QuoteMethod.NEVER);
- }
-
- @Override
- public void appendTo(
- ExpressionBuilder builder,
- QuoteMethod useQuotes
- ) {
- if (catalogName != null) {
- builder.appendIdentifier(catalogName, useQuotes);
- builder.appendIdentifierDelimiter();
- }
- if (schemaName != null) {
- builder.appendIdentifier(schemaName, useQuotes);
- builder.appendIdentifierDelimiter();
- }
- builder.appendTableName(tableName, useQuotes);
- }
-
- @Override
- public int hashCode() {
- return hash;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj == this) {
- return true;
- }
- if (obj instanceof TableId) {
- TableId that = (TableId) obj;
- return Objects.equals(this.catalogName, that.catalogName)
- && Objects.equals(this.schemaName, that.schemaName)
- && Objects.equals(this.tableName, that.tableName);
- }
- return false;
- }
-
- @Override
- public int compareTo(TableId that) {
- if (that == this) {
- return 0;
- }
- int diff = this.tableName.compareTo(that.tableName);
- if (diff != 0) {
- return diff;
- }
- if (this.schemaName == null) {
- if (that.schemaName != null) {
- return -1;
- }
- } else {
- if (that.schemaName == null) {
- return 1;
- }
- diff = this.schemaName.compareTo(that.schemaName);
- if (diff != 0) {
- return diff;
- }
- }
- if (this.catalogName == null) {
- if (that.catalogName != null) {
- return -1;
- }
- } else {
- if (that.catalogName == null) {
- return 1;
- }
- diff = this.catalogName.compareTo(that.catalogName);
- if (diff != 0) {
- return diff;
- }
- }
- return 0;
- }
-
- @Override
- public String toString() {
- return ExpressionBuilder.create().append(this).toString();
- }
-}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/ByteArrayDeserializer.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/ByteArrayDeserializer.java
deleted file mode 100644
index f0f79214..00000000
--- a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/ByteArrayDeserializer.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.connect.doris.serialization;
-
-public class ByteArrayDeserializer implements Deserializer<byte[]> {
-
- @Override
- public byte[] deserialize(String topic, byte[] data) {
- return data;
- }
-}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/ByteArraySerializer.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/ByteArraySerializer.java
deleted file mode 100644
index e7863742..00000000
--- a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/ByteArraySerializer.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.connect.doris.serialization;
-
-public class ByteArraySerializer implements Serializer<byte[]> {
- @Override
- public byte[] serialize(String topic, byte[] data) {
- return data;
- }
-}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/ByteBufferDeserializer.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/ByteBufferDeserializer.java
deleted file mode 100644
index 3aa5996f..00000000
--- a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/ByteBufferDeserializer.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.connect.doris.serialization;
-
-import java.nio.ByteBuffer;
-
-public class ByteBufferDeserializer implements Deserializer<ByteBuffer> {
- @Override
- public ByteBuffer deserialize(String topic, byte[] data) {
- if (data == null) {
- return null;
- }
-
- return ByteBuffer.wrap(data);
- }
-}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/ByteBufferSerializer.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/ByteBufferSerializer.java
deleted file mode 100644
index 807eed3a..00000000
--- a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/ByteBufferSerializer.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.connect.doris.serialization;
-
-import java.nio.ByteBuffer;
-
-public class ByteBufferSerializer implements Serializer<ByteBuffer> {
- @Override
- public byte[] serialize(String topic, ByteBuffer data) {
- if (data == null) {
- return null;
- }
-
- data.rewind();
- if (data.hasArray()) {
- byte[] arr = data.array();
- if (data.arrayOffset() == 0 && arr.length == data.remaining()) {
- return arr;
- }
- }
-
- byte[] ret = new byte[data.remaining()];
- data.get(ret, 0, ret.length);
- data.rewind();
- return ret;
- }
-}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/Deserializer.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/Deserializer.java
deleted file mode 100644
index 69e91891..00000000
--- a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/Deserializer.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.connect.doris.serialization;
-
-import io.openmessaging.KeyValue;
-
-import java.io.Closeable;
-import java.util.Map;
-
-/**
- * An interface for converting bytes to objects.
- *
- * @param <T> Type to be deserialized into.
- */
-public interface Deserializer<T> extends Closeable {
-
- /**
- * Configure this class.
- * @param configs configs in key/value pairs
- */
- default void configure(Map<String, ?> configs) {
- // intentionally left blank
- }
-
- /**
- * Deserialize a record value from a byte array into a value or object.
- * @param topic topic associated with the data
- * @param data serialized bytes; may be null; implementations are recommended to handle null by returning a value or null rather than throwing an exception.
- * @return deserialized typed data; may be null
- */
- T deserialize(String topic, byte[] data);
-
- /**
- * Deserialize a record value from a byte array into a value or object.
- */
- default T deserialize(String topic, KeyValue extensions, byte[] data) {
- return deserialize(topic, data);
- }
-
- /**
- * Close this deserializer.
- * <p>
- * This method must be idempotent as it may be called multiple times.
- */
- @Override
- default void close() {
- // intentionally left blank
- }
-}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/DoubleDeserializer.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/DoubleDeserializer.java
deleted file mode 100644
index 23a5447a..00000000
--- a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/DoubleDeserializer.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.connect.doris.serialization;
-
-import org.apache.commons.lang3.SerializationException;
-
-public class DoubleDeserializer implements Deserializer<Double> {
-
- @Override
- public Double deserialize(String topic, byte[] data) {
- if (data == null)
- return null;
- if (data.length != 8) {
- throw new SerializationException("Size of data received by Deserializer is not 8");
- }
-
- long value = 0;
- for (byte b : data) {
- value <<= 8;
- value |= b & 0xFF;
- }
- return Double.longBitsToDouble(value);
- }
-}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/DoubleSerializer.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/DoubleSerializer.java
deleted file mode 100644
index 86ca81f0..00000000
--- a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/DoubleSerializer.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.connect.doris.serialization;
-
-public class DoubleSerializer implements Serializer<Double> {
- @Override
- public byte[] serialize(String topic, Double data) {
- if (data == null) {
- return null;
- }
-
- long bits = Double.doubleToLongBits(data);
- return new byte[] {
- (byte) (bits >>> 56),
- (byte) (bits >>> 48),
- (byte) (bits >>> 40),
- (byte) (bits >>> 32),
- (byte) (bits >>> 24),
- (byte) (bits >>> 16),
- (byte) (bits >>> 8),
- (byte) bits
- };
- }
-}
\ No newline at end of file
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/FloatDeserializer.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/FloatDeserializer.java
deleted file mode 100644
index b2da4faf..00000000
--- a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/FloatDeserializer.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.connect.doris.serialization;
-
-import org.apache.commons.lang3.SerializationException;
-
-public class FloatDeserializer implements Deserializer<Float> {
- @Override
- public Float deserialize(final String topic, final byte[] data) {
- if (data == null) {
- return null;
- }
- if (data.length != 4) {
- throw new SerializationException("Size of data received by Deserializer is not 4");
- }
-
- int value = 0;
- for (byte b : data) {
- value <<= 8;
- value |= b & 0xFF;
- }
- return Float.intBitsToFloat(value);
- }
-}
\ No newline at end of file
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/FloatSerializer.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/FloatSerializer.java
deleted file mode 100644
index fc06f2be..00000000
--- a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/FloatSerializer.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.connect.doris.serialization;
-
-public class FloatSerializer implements Serializer<Float> {
- @Override
- public byte[] serialize(final String topic, final Float data) {
- if (data == null) {
- return null;
- }
-
- long bits = Float.floatToRawIntBits(data);
- return new byte[] {
- (byte) (bits >>> 24),
- (byte) (bits >>> 16),
- (byte) (bits >>> 8),
- (byte) bits
- };
- }
-}
\ No newline at end of file
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/IntegerDeserializer.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/IntegerDeserializer.java
deleted file mode 100644
index b9ab9eea..00000000
--- a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/IntegerDeserializer.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.connect.doris.serialization;
-
-import org.apache.commons.lang3.SerializationException;
-
-public class IntegerDeserializer implements Deserializer<Integer> {
- @Override
- public Integer deserialize(String topic, byte[] data) {
- if (data == null) {
- return null;
- }
- if (data.length != 4) {
- throw new SerializationException("Size of data received by IntegerDeserializer is not 4");
- }
-
- int value = 0;
- for (byte b : data) {
- value <<= 8;
- value |= b & 0xFF;
- }
- return value;
- }
-}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/IntegerSerializer.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/IntegerSerializer.java
deleted file mode 100644
index 6203ddac..00000000
--- a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/IntegerSerializer.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.connect.doris.serialization;
-
-public class IntegerSerializer implements Serializer<Integer> {
- @Override
- public byte[] serialize(String topic, Integer data) {
- if (data == null) {
- return null;
- }
-
- return new byte[] {
- (byte) (data >>> 24),
- (byte) (data >>> 16),
- (byte) (data >>> 8),
- data.byteValue()
- };
- }
-}
\ No newline at end of file
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/JsonDeserializer.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/JsonDeserializer.java
deleted file mode 100644
index dd1930c7..00000000
--- a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/JsonDeserializer.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.connect.doris.serialization;
-
-import com.alibaba.fastjson.JSON;
-import io.openmessaging.connector.api.errors.ConnectException;
-
-import java.nio.charset.StandardCharsets;
-import java.util.Objects;
-
-/**
- * json deserializer
- */
-public class JsonDeserializer implements Deserializer<Object> {
-
- @Override
- public Object deserialize(String topic, byte[] bytes) {
- if (Objects.isNull(bytes)) {
- return null;
- }
- Object data;
- try {
- String json = new String(bytes, StandardCharsets.UTF_8);
- data = JSON.parse(json);
- } catch (Exception e) {
- throw new ConnectException(e);
- }
- return data;
- }
-}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/JsonSerializer.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/JsonSerializer.java
deleted file mode 100644
index ca794a8e..00000000
--- a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/JsonSerializer.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.connect.doris.serialization;
-
-import com.alibaba.fastjson.JSON;
-import com.alibaba.fastjson.serializer.SerializerFeature;
-import io.openmessaging.connector.api.errors.ConnectException;
-
-import java.nio.charset.StandardCharsets;
-import java.util.Objects;
-
-/**
- * json serializer
- */
-public class JsonSerializer implements Serializer<Object> {
-
-
- /**
- * Convert {@code data} into a byte array.
- *
- * @param topic topic associated with data
- * @param data typed data
- * @return serialized bytes
- */
- @Override
- public byte[] serialize(String topic, Object data) {
- if (Objects.isNull(data)) {
- return null;
- }
- try {
- return JSON.toJSONString(data, SerializerFeature.DisableCircularReferenceDetect, SerializerFeature.WriteMapNullValue).getBytes(StandardCharsets.UTF_8);
- } catch (Exception e) {
- throw new ConnectException("Error serializing JSON message", e);
- }
- }
-}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/LongDeserializer.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/LongDeserializer.java
deleted file mode 100644
index 5022b454..00000000
--- a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/LongDeserializer.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.connect.doris.serialization;
-
-import org.apache.commons.lang3.SerializationException;
-
-public class LongDeserializer implements Deserializer<Long> {
- @Override
- public Long deserialize(String topic, byte[] data) {
- if (data == null) {
- return null;
- }
- if (data.length != 8) {
- throw new SerializationException("Size of data received by LongDeserializer is not 8");
- }
-
- long value = 0;
- for (byte b : data) {
- value <<= 8;
- value |= b & 0xFF;
- }
- return value;
- }
-}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/LongSerializer.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/LongSerializer.java
deleted file mode 100644
index 150e5bdc..00000000
--- a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/LongSerializer.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.connect.doris.serialization;
-
-public class LongSerializer implements Serializer<Long> {
- @Override
- public byte[] serialize(String topic, Long data) {
- if (data == null) {
- return null;
- }
-
- return new byte[] {
- (byte) (data >>> 56),
- (byte) (data >>> 48),
- (byte) (data >>> 40),
- (byte) (data >>> 32),
- (byte) (data >>> 24),
- (byte) (data >>> 16),
- (byte) (data >>> 8),
- data.byteValue()
- };
- }
-}
\ No newline at end of file
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/Serializer.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/Serializer.java
deleted file mode 100644
index c96145e4..00000000
--- a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/Serializer.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.connect.doris.serialization;
-
-import io.openmessaging.KeyValue;
-
-import java.io.Closeable;
-import java.util.Map;
-
-/**
- * An interface for converting objects to bytes.
- */
-public interface Serializer<T> extends Closeable {
-
- /**
- * Configure this class.
- * @param configs configs in key/value pairs
- */
- default void configure(Map<String, ?> configs) {
- // intentionally left blank
- }
-
- /**
- * Convert {@code data} into a byte array.
- *
- * @param topic topic associated with data
- * @param data typed data
- * @return serialized bytes
- */
- byte[] serialize(String topic, T data);
-
- /**
- * Convert data into a byte array.
- * @return serialized bytes
- */
- default byte[] serialize(String topic, KeyValue extensions, T data) {
- return serialize(topic, data);
- }
-
- /**
- * Close this serializer.
- */
- @Override
- default void close() {
- // intentionally left blank
- }
-}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/ShortDeserializer.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/ShortDeserializer.java
deleted file mode 100644
index f5d4fd75..00000000
--- a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/ShortDeserializer.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.connect.doris.serialization;
-
-import org.apache.commons.lang3.SerializationException;
-
-public class ShortDeserializer implements Deserializer<Short> {
-
- @Override
- public Short deserialize(String topic, byte[] data) {
- if (data == null) {
- return null;
- }
- if (data.length != 2) {
- throw new SerializationException("Size of data received by ShortDeserializer is not 2");
- }
-
- short value = 0;
- for (byte b : data) {
- value <<= 8;
- value |= b & 0xFF;
- }
- return value;
- }
-}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/ShortSerializer.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/ShortSerializer.java
deleted file mode 100644
index 1e40b38f..00000000
--- a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/ShortSerializer.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.connect.doris.serialization;
-
-public class ShortSerializer implements Serializer<Short> {
- @Override
- public byte[] serialize(String topic, Short data) {
- if (data == null) {
- return null;
- }
-
- return new byte[] {
- (byte) (data >>> 8),
- data.byteValue()
- };
- }
-}
\ No newline at end of file
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/StringDeserializer.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/StringDeserializer.java
deleted file mode 100644
index d7a42e6b..00000000
--- a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/StringDeserializer.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.connect.doris.serialization;
-
-import java.io.UnsupportedEncodingException;
-import java.util.Map;
-
-/**
- * String encoding defaults to UTF8 and can be customized by setting the property key.deserializer.encoding,
- * value.deserializer.encoding or deserializer.encoding. The first two take precedence over the last.
- */
-public class StringDeserializer implements Deserializer<String> {
- private String encoding = "UTF8";
-
- @Override
- public void configure(Map<String, ?> configs) {
- Object encodingValue = configs.get("deserializer.encoding");
- if (encodingValue instanceof String) {
- encoding = (String) encodingValue;
- }
- }
-
- @Override
- public String deserialize(String topic, byte[] data) {
- try {
- if (data == null) {
- return null;
- } else {
- return new String(data, encoding);
- }
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException("Error when deserializing byte[] to string due to unsupported encoding " + encoding);
- }
- }
-}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/StringSerializer.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/StringSerializer.java
deleted file mode 100644
index f9b569a6..00000000
--- a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/serialization/StringSerializer.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.connect.doris.serialization;
-
-import java.io.UnsupportedEncodingException;
-import java.util.Map;
-
-/**
- * string serializer
- */
-public class StringSerializer implements Serializer<String> {
- private String encoding = "UTF8";
-
- @Override
- public void configure(Map<String, ?> configs) {
- String propertyName = "serializer.encoding";
- if (configs.containsKey(propertyName)) {
- Object encodingValue = configs.get(propertyName);
- if (encodingValue instanceof String) {
- encoding = (String) encodingValue;
- }
- }
- }
-
- @Override
- public byte[] serialize(String topic, String data) {
- try {
- if (data == null) {
- return null;
- } else {
- return data.getBytes(encoding);
- }
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException("Error when serializing string to byte[] due to unsupported encoding " + encoding);
- }
- }
-}
\ No newline at end of file
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/service/DorisDefaultSinkService.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/service/DorisDefaultSinkService.java
new file mode 100644
index 00000000..34d3466e
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/service/DorisDefaultSinkService.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.service;
+
+import com.codahale.metrics.MetricRegistry;
+import io.openmessaging.KeyValue;
+import io.openmessaging.connector.api.data.ConnectRecord;
+import io.openmessaging.connector.api.data.RecordOffset;
+import io.openmessaging.connector.api.data.RecordPartition;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import org.apache.rocketmq.connect.doris.cfg.DorisOptions;
+import org.apache.rocketmq.connect.doris.connection.ConnectionProvider;
+import org.apache.rocketmq.connect.doris.connection.JdbcConnectionProvider;
+import org.apache.rocketmq.connect.doris.metrics.DorisConnectMonitor;
+import org.apache.rocketmq.connect.doris.metrics.MetricsJmxReporter;
+import org.apache.rocketmq.connect.doris.utils.ConnectRecordUtil;
+import org.apache.rocketmq.connect.doris.writer.CopyIntoWriter;
+import org.apache.rocketmq.connect.doris.writer.DorisWriter;
+import org.apache.rocketmq.connect.doris.writer.StreamLoadWriter;
+import org.apache.rocketmq.connect.doris.writer.load.LoadModel;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class DorisDefaultSinkService implements DorisSinkService {
+ private static final Logger LOG = LoggerFactory.getLogger(DorisDefaultSinkService.class);
+ private final ConnectionProvider conn;
+ private final Map<String, DorisWriter> writer;
+ private final DorisOptions dorisOptions;
+ private final MetricsJmxReporter metricsJmxReporter;
+ private final DorisConnectMonitor connectMonitor;
+
+ DorisDefaultSinkService(KeyValue config) {
+ this.dorisOptions = new DorisOptions(config);
+ this.writer = new HashMap<>();
+ this.conn = new JdbcConnectionProvider(dorisOptions);
+ MetricRegistry metricRegistry = new MetricRegistry();
+ this.metricsJmxReporter = new MetricsJmxReporter(metricRegistry, dorisOptions.getName());
+ this.connectMonitor =
+ new DorisConnectMonitor(
+ dorisOptions.isEnableCustomJMX(),
+ dorisOptions.getTaskId(),
+ this.metricsJmxReporter);
+ }
+
+ public void startService(String topic) {
+ LoadModel loadModel = dorisOptions.getLoadModel();
+ DorisWriter dorisWriter =
+ LoadModel.COPY_INTO.equals(loadModel)
+ ? new CopyIntoWriter(
+ topic, dorisOptions, conn, connectMonitor)
+ : new StreamLoadWriter(
+ topic, dorisOptions, conn, connectMonitor);
+ writer.put(topic, dorisWriter);
+ metricsJmxReporter.start();
+ }
+
+ @Override
+ public void insert(final List<ConnectRecord> records) {
+ for (ConnectRecord record : records) {
+ if (Objects.isNull(record.getData())) {
+ RecordPartition partition = record.getPosition().getPartition();
+ LOG.debug(
+ "Null valued record from topic={} brokerName={} queueId={} and offset={} was skipped",
+ ConnectRecordUtil.getTopicName(partition),
+ ConnectRecordUtil.getBrokerName(partition),
+ ConnectRecordUtil.getQueueId(partition),
+ ConnectRecordUtil.getQueueOffset(record.getPosition().getOffset()));
+ continue;
+ }
+ insert(record);
+ }
+ // check all sink writer to see if they need to be flushed
+ for (DorisWriter writer : writer.values()) {
+ // Time based flushing
+ if (writer.shouldFlush()) {
+ writer.flushBuffer();
+ }
+ }
+ }
+
+ @Override
+ public void insert(final ConnectRecord record) {
+ String topicName = ConnectRecordUtil.getTopicName(record.getPosition().getPartition());
+ if (!writer.containsKey(topicName)) {
+ // todo startTask can be initialized in DorisSinkTask, and SinkTask needs to support the initialization method for each RecordPartition.
+ startService(topicName);
+ }
+ writer.get(topicName).insert(record);
+ }
+
+ @Override
+ public void commit(Map<RecordPartition, RecordOffset> currentOffsets) {
+ currentOffsets.keySet()
+ .forEach(
+ rp -> {
+ String topicName = ConnectRecordUtil.getTopicName(rp);
+ writer.get(topicName).commit();
+ });
+ }
+
+ @Override
+ public int getDorisWriterSize() {
+ return writer.size();
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/service/DorisSinkService.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/service/DorisSinkService.java
new file mode 100644
index 00000000..8b73c302
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/service/DorisSinkService.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.service;
+
+import io.openmessaging.connector.api.data.ConnectRecord;
+import io.openmessaging.connector.api.data.RecordOffset;
+import io.openmessaging.connector.api.data.RecordPartition;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Background service of data sink, responsible to create/drop table and insert/delete files
+ */
+public interface DorisSinkService {
+
+ void startService(String topic);
+
+ void insert(final List<ConnectRecord> records);
+
+ void insert(final ConnectRecord record);
+
+ void commit(Map<RecordPartition, RecordOffset> currentOffsets);
+
+ int getDorisWriterSize();
+
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/service/DorisSinkServiceFactory.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/service/DorisSinkServiceFactory.java
new file mode 100644
index 00000000..bb204efa
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/service/DorisSinkServiceFactory.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.service;
+
+import io.openmessaging.KeyValue;
+
+/**
+ * A factory to create {@link DorisSinkService}
+ */
+public class DorisSinkServiceFactory {
+
+ public static DorisSinkService getDorisSinkService(KeyValue connectorConfig) {
+ return new DorisDefaultSinkService(connectorConfig);
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/service/DorisSystemService.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/service/DorisSystemService.java
new file mode 100644
index 00000000..45f98b25
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/service/DorisSystemService.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.service;
+
+import com.google.common.collect.Lists;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+import java.util.function.Predicate;
+import org.apache.rocketmq.connect.doris.cfg.DorisOptions;
+import org.apache.rocketmq.connect.doris.connection.JdbcConnectionProvider;
+import org.apache.rocketmq.connect.doris.exception.DorisException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class DorisSystemService {
+
+ private static final Logger LOG = LoggerFactory.getLogger(DorisSystemService.class);
+ private static final String GET_COLUMN_EXISTS_TEMPLATE =
+ "SELECT COLUMN_NAME FROM information_schema.`COLUMNS` WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? AND COLUMN_NAME = ?";
+ private final JdbcConnectionProvider jdbcConnectionProvider;
+
+ public DorisSystemService(DorisOptions dorisOptions) {
+ this.jdbcConnectionProvider = new JdbcConnectionProvider(dorisOptions);
+ }
+
+ private static final List<String> builtinDatabases =
+ Collections.singletonList("information_schema");
+
+ public boolean tableExists(String database, String table) {
+ return listTables(database).contains(table);
+ }
+
+ public boolean databaseExists(String database) {
+ return listDatabases().contains(database);
+ }
+
+ public Set<String> listDatabases() {
+ return new HashSet<>(
+ extractColumnValuesBySQL(
+ "SELECT `SCHEMA_NAME` FROM `INFORMATION_SCHEMA`.`SCHEMATA`;",
+ 1,
+ dbName -> !builtinDatabases.contains(dbName)));
+ }
+
+ public Set<String> listTables(String databaseName) {
+ if (!databaseExists(databaseName)) {
+ throw new DorisException("database" + databaseName + " is not exists");
+ }
+ return new HashSet<>(
+ extractColumnValuesBySQL(
+ "SELECT TABLE_NAME FROM information_schema.`TABLES` WHERE TABLE_SCHEMA = ?",
+ 1,
+ null,
+ databaseName));
+ }
+
+ public boolean isColumnExist(String database, String tableName, String columnName) {
+ List<String> columnList =
+ extractColumnValuesBySQL(
+ GET_COLUMN_EXISTS_TEMPLATE, 1, null, database, tableName, columnName);
+ return !columnList.isEmpty();
+ }
+
+ public List<String> extractColumnValuesBySQL(
+ String sql, int columnIndex, Predicate<String> filterFunc, Object... params) {
+
+ List<String> columnValues = Lists.newArrayList();
+
+ try (Connection connection = jdbcConnectionProvider.getOrEstablishConnection();
+ PreparedStatement ps = connection.prepareStatement(sql)) {
+
+ if (Objects.nonNull(params) && params.length > 0) {
+ for (int i = 0; i < params.length; i++) {
+ ps.setObject(i + 1, params[i]);
+ }
+ }
+
+ try (ResultSet rs = ps.executeQuery()) {
+ while (rs.next()) {
+ String columnValue = rs.getString(columnIndex);
+ if (filterFunc == null || filterFunc.test(columnValue)) {
+ columnValues.add(columnValue);
+ }
+ }
+ }
+ return columnValues;
+ } catch (Exception e) {
+ LOG.error("The following SQL query could not be executed: {}", sql, e);
+ throw new DorisException(
+ String.format("The following SQL query could not be executed: %s", sql), e);
+ }
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/service/RestService.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/service/RestService.java
new file mode 100644
index 00000000..b01afe56
--- /dev/null
+++ b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/service/RestService.java
@@ -0,0 +1,361 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.rocketmq.connect.doris.service;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.PrintWriter;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Scanner;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.commons.io.IOUtils;
+import org.apache.http.HttpHeaders;
+import org.apache.http.HttpStatus;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpRequestBase;
+import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.util.EntityUtils;
+import org.apache.rocketmq.connect.doris.cfg.DorisOptions;
+import org.apache.rocketmq.connect.doris.exception.ConnectedFailedException;
+import org.apache.rocketmq.connect.doris.exception.DorisException;
+import org.apache.rocketmq.connect.doris.exception.SchemaChangeException;
+import org.apache.rocketmq.connect.doris.model.BackendV2;
+import org.apache.rocketmq.connect.doris.model.LoadOperation;
+import org.apache.rocketmq.connect.doris.model.doris.Schema;
+import org.apache.rocketmq.connect.doris.utils.BackoffAndRetryUtils;
+import org.slf4j.Logger;
+
+public class RestService {
+
+ private static final String BACKENDS_V2 = "/api/backends?is_alive=true";
+ private static final String TABLE_SCHEMA_API = "http://%s/api/%s/%s/_schema";
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+ private static final String UNIQUE_KEYS_TYPE = "UNIQUE_KEYS";
+
+ /**
+ * get Doris BE nodes to request.
+ *
+ * @param options configuration of request
+ * @param logger slf4j logger
+ * @return the chosen one Doris BE node
+ * @throws IllegalArgumentException BE nodes is illegal
+ */
+ public static List<BackendV2.BackendRowV2> getBackendsV2(DorisOptions options, Logger logger) {
+ List<String> feNodeList = options.getHttpUrls();
+ if (options.isAutoRedirect()) {
+ return convert(feNodeList);
+ }
+
+ for (String feNode : feNodeList) {
+ try {
+ String beUrl = "http://" + feNode + BACKENDS_V2;
+ HttpGet httpGet = new HttpGet(beUrl);
+ String response = send(options, httpGet, logger);
+ logger.info("Backend Info:{}", response);
+ List<BackendV2.BackendRowV2> backends = parseBackendV2(response, logger);
+ return backends;
+ } catch (ConnectedFailedException e) {
+ logger.info(
+ "Doris FE node {} is unavailable: {}, Request the next Doris FE node",
+ feNode,
+ e.getMessage());
+ }
+ }
+ String errMsg = "No Doris FE is available, please check configuration";
+ logger.error(errMsg);
+ throw new DorisException(errMsg);
+ }
+
+ /**
+ * When the user turns on redirection, there is no need to explicitly obtain the be list, just
+ * treat the fe list as the be list.
+ */
+ private static List<BackendV2.BackendRowV2> convert(List<String> feNodeList) {
+ List<BackendV2.BackendRowV2> nodeList = new ArrayList<>();
+ for (String node : feNodeList) {
+ String[] split = node.split(":");
+ nodeList.add(BackendV2.BackendRowV2.of(split[0], Integer.parseInt(split[1]), true));
+ }
+ return nodeList;
+ }
+
+ public static boolean isUniqueKeyType(
+ DorisOptions dorisOptions, String tableName, Logger logger) {
+ try {
+ return UNIQUE_KEYS_TYPE.equals(
+ getSchema(dorisOptions, dorisOptions.getDatabase(), tableName, logger)
+ .getKeysType());
+ } catch (Exception e) {
+ logger.error("Failed to match table unique key types", e);
+ throw new DorisException(e);
+ }
+ }
+
+ /**
+ * send request to Doris FE and get response json string.
+ *
+ * @param options configuration of request
+ * @param request {@link HttpRequestBase} real request
+ * @param logger {@link Logger}
+ * @return Doris FE response in json string
+ * @throws ConnectedFailedException throw when cannot connect to Doris FE
+ */
+ private static String send(DorisOptions options, HttpRequestBase request, Logger logger)
+ throws ConnectedFailedException {
+ int connectTimeout = options.getRequestConnectTimeoutMs();
+ int socketTimeout = options.getRequestReadTimeoutMs();
+ logger.trace(
+ "connect timeout set to '{}'. socket timeout set to '{}'.",
+ connectTimeout,
+ socketTimeout);
+
+ RequestConfig requestConfig =
+ RequestConfig.custom()
+ .setConnectTimeout(connectTimeout)
+ .setSocketTimeout(socketTimeout)
+ .build();
+
+ request.setConfig(requestConfig);
+ logger.info(
+ "Send request to Doris FE '{}' with user '{}'.",
+ request.getURI(),
+ options.getUser());
+ int statusCode = -1;
+ AtomicReference<String> result = new AtomicReference<>();
+ try {
+ BackoffAndRetryUtils.backoffAndRetry(
+ LoadOperation.SEND_REQUEST_TO_DORIS,
+ () -> {
+ logger.debug("doris request {}.", request.getURI());
+ try {
+ String response = null;
+ if (request instanceof HttpGet) {
+ response =
+ getConnectionGet(
+ request,
+ options.getUser(),
+ options.getPassword(),
+ logger);
+ } else {
+ response =
+ getConnectionPost(
+ request,
+ options.getUser(),
+ options.getPassword(),
+ logger);
+ }
+ if (Objects.isNull(response)) {
+ logger.warn(
+ "Failed to get response from Doris FE {}, http code is {}",
+ request.getURI(),
+ statusCode);
+ throw new ConnectedFailedException(
+ "Failed to get response from Doris FE {"
+ + request.getURI()
+ + "}, http code is {"
+ + statusCode
+ + "}");
+ }
+ logger.trace(
+ "Success get response from Doris FE: {}, response is: {}.",
+ request.getURI(),
+ response);
+ // Handle the problem of inconsistent data format returned by http v1
+ // and v2
+ Map map = OBJECT_MAPPER.readValue(response, Map.class);
+ if (map.containsKey("code") && map.containsKey("msg")) {
+ Object data = map.get("data");
+ result.set(OBJECT_MAPPER.writeValueAsString(data));
+ } else {
+ result.set(response);
+ }
+ return true;
+ } catch (IOException e) {
+ logger.warn(
+ "Failed to connect doris, requestUri={}", request.getURI(), e);
+ throw new ConnectedFailedException(
+ "Failed to connect doris, requestUri=" + request.getURI(), e);
+ }
+ });
+ } catch (Exception e) {
+ logger.error("Connect to doris {} failed.", request.getURI(), e);
+ throw new ConnectedFailedException(
+ "Failed to connect doris request uri=" + request.getURI(), statusCode, e);
+ }
+ return result.get();
+ }
+
+ private static String getConnectionGet(
+ HttpRequestBase request, String user, String passwd, Logger logger) throws IOException {
+ URL realUrl = new URL(request.getURI().toString());
+ // open connection
+ HttpURLConnection connection = (HttpURLConnection) realUrl.openConnection();
+ String authEncoding =
+ Base64.getEncoder()
+ .encodeToString(
+ String.format("%s:%s", user, passwd)
+ .getBytes(StandardCharsets.UTF_8));
+ connection.setRequestProperty("Authorization", "Basic " + authEncoding);
+
+ connection.connect();
+ connection.setConnectTimeout(request.getConfig().getConnectTimeout());
+ connection.setReadTimeout(request.getConfig().getSocketTimeout());
+ return parseResponse(connection, logger);
+ }
+
+ private static String getConnectionPost(
+ HttpRequestBase request, String user, String passwd, Logger logger) throws IOException {
+ URL url = new URL(request.getURI().toString());
+ HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+ conn.setInstanceFollowRedirects(false);
+ conn.setRequestMethod(request.getMethod());
+ String authEncoding =
+ Base64.getEncoder()
+ .encodeToString(
+ String.format("%s:%s", user, passwd)
+ .getBytes(StandardCharsets.UTF_8));
+ conn.setRequestProperty("Authorization", "Basic " + authEncoding);
+ InputStream content = ((HttpPost) request).getEntity().getContent();
+ String res = IOUtils.toString(content);
+ conn.setDoOutput(true);
+ conn.setDoInput(true);
+ conn.setConnectTimeout(request.getConfig().getConnectTimeout());
+ conn.setReadTimeout(request.getConfig().getSocketTimeout());
+ PrintWriter out = new PrintWriter(conn.getOutputStream());
+ // send request params
+ out.print(res);
+ // flush
+ out.flush();
+ // read response
+ return parseResponse(conn, logger);
+ }
+
+ private static String parseResponse(HttpURLConnection connection, Logger logger)
+ throws IOException {
+ if (connection.getResponseCode() != HttpStatus.SC_OK) {
+ logger.warn(
+ "Failed to get response from Doris {}, http code is {}",
+ connection.getURL(),
+ connection.getResponseCode());
+ throw new IOException("Failed to get response from Doris");
+ }
+ StringBuilder result = new StringBuilder();
+ try (Scanner scanner = new Scanner(connection.getInputStream(), "utf-8")) {
+ while (scanner.hasNext()) {
+ result.append(scanner.next());
+ }
+ return result.toString();
+ }
+ }
+
+ private static List<BackendV2.BackendRowV2> parseBackendV2(String response, Logger logger) {
+ ObjectMapper mapper = new ObjectMapper();
+ BackendV2 backend;
+ try {
+ backend = mapper.readValue(response, BackendV2.class);
+ } catch (JsonParseException e) {
+ String errMsg = "Doris BE's response is not a json. res: " + response;
+ logger.error(errMsg, e);
+ throw new DorisException(errMsg, e);
+ } catch (JsonMappingException e) {
+ String errMsg = "Doris BE's response cannot map to schema. res: " + response;
+ logger.error(errMsg, e);
+ throw new DorisException(errMsg, e);
+ } catch (IOException e) {
+ String errMsg = "Parse Doris BE's response to json failed. res: " + response;
+ logger.error(errMsg, e);
+ throw new DorisException(errMsg, e);
+ }
+
+ if (backend == null) {
+ logger.error("Should not come here.");
+ throw new DorisException();
+ }
+ List<BackendV2.BackendRowV2> backendRows = backend.getBackends();
+ logger.debug("Parsing schema result is '{}'.", backendRows);
+ return backendRows;
+ }
+
+ /**
+ * Get table schema from doris.
+ */
+ public static Schema getSchema(
+ DorisOptions dorisOptions, String db, String table, Logger logger) {
+ logger.trace("start get " + db + "." + table + " schema from doris.");
+ Object responseData = null;
+ try {
+ String tableSchemaUri =
+ String.format(TABLE_SCHEMA_API, dorisOptions.getHttpUrl(), db, table);
+ HttpGet httpGet = new HttpGet(tableSchemaUri);
+ httpGet.setHeader(HttpHeaders.AUTHORIZATION, authHeader(dorisOptions));
+ Map<String, Object> responseMap = handleResponse(httpGet, logger);
+ responseData = responseMap.get("data");
+ String schemaStr = OBJECT_MAPPER.writeValueAsString(responseData);
+ return OBJECT_MAPPER.readValue(schemaStr, Schema.class);
+ } catch (JsonProcessingException | IllegalArgumentException e) {
+ throw new SchemaChangeException("can not parse response schema " + responseData, e);
+ }
+ }
+
+ private static String authHeader(DorisOptions dorisOptions) {
+ return "Basic "
+ + new String(
+ org.apache.commons.codec.binary.Base64.encodeBase64(
+ (dorisOptions.getUser() + ":" + dorisOptions.getPassword())
+ .getBytes(StandardCharsets.UTF_8)));
+ }
+
+ private static Map handleResponse(HttpUriRequest request, Logger logger) {
+ try (CloseableHttpClient httpclient = HttpClients.createDefault()) {
+ CloseableHttpResponse response = httpclient.execute(request);
+ final int statusCode = response.getStatusLine().getStatusCode();
+ final String reasonPhrase = response.getStatusLine().getReasonPhrase();
+ if (statusCode == 200 && response.getEntity() != null) {
+ String responseEntity = EntityUtils.toString(response.getEntity());
+ return OBJECT_MAPPER.readValue(responseEntity, Map.class);
+ } else {
+ throw new SchemaChangeException(
+ "Failed to schemaChange, status: "
+ + statusCode
+ + ", reason: "
+ + reasonPhrase);
+ }
+ } catch (Exception e) {
+ logger.trace("SchemaChange request error,", e);
+ throw new SchemaChangeException("SchemaChange request error with " + e.getMessage());
+ }
+ }
+}
diff --git a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/sink/BufferedRecords.java b/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/sink/BufferedRecords.java
deleted file mode 100644
index 3058a894..00000000
--- a/connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/sink/BufferedRecords.java
+++ /dev/null
@@ -1,234 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.connect.doris.sink;
-
-import io.openmessaging.connector.api.data.ConnectRecord;
-import io.openmessaging.connector.api.data.Schema;
-import io.openmessaging.connector.api.errors.ConnectException;
-import org.apache.rocketmq.connect.doris.exception.DorisException;
-import org.apache.rocketmq.connect.doris.schema.table.TableId;
-import org.apache.rocketmq.connect.doris.connector.DorisSinkConfig;
-import org.apache.rocketmq.connect.doris.schema.db.DbStructure;
-import org.apache.rocketmq.connect.doris.sink.metadata.FieldsMetadata;
-import org.apache.rocketmq.connect.doris.sink.metadata.SchemaPair;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Objects;
-import java.util.Optional;
-import static java.util.Objects.isNull;
-import static java.util.Objects.nonNull;
-
-/**
- * buffered records
- */
-public class BufferedRecords {
- private static final Logger log = LoggerFactory.getLogger(BufferedRecords.class);
-
- private final TableId tableId;
- private final DorisSinkConfig config;
- private final DbStructure dbStructure;
- private List<ConnectRecord> records = new ArrayList<>();
- private Schema keySchema;
- private Schema schema;
- private FieldsMetadata fieldsMetadata;
- private RecordValidator recordValidator;
- private List<ConnectRecord> updatePreparedRecords = new ArrayList<>();
- private List<ConnectRecord> deletePreparedRecords = new ArrayList<>();
- private boolean deletesInBatch = false;
- private DorisStreamLoader loader;
-
- public BufferedRecords(
- DorisSinkConfig config,
- TableId tableId,
- DbStructure dbStructure
- ) {
- this.tableId = tableId;
- this.config = config;
- this.dbStructure = dbStructure;
- this.recordValidator = RecordValidator.create(config);
- this.loader = DorisStreamLoader.create(config);
- }
-
- /**
- * add record
- *
- * @param record
- * @return
- * @throws SQLException
- */
- public List<ConnectRecord> add(ConnectRecord record) throws SQLException {
- recordValidator.validate(record);
- final List<ConnectRecord> flushed = new ArrayList<>();
- boolean schemaChanged = false;
- if (!Objects.equals(keySchema, record.getKeySchema())) {
- keySchema = record.getKeySchema();
- schemaChanged = true;
- }
- if (isNull(record.getSchema())) {
- // For deletes, value and optionally value schema come in as null.
- // We don't want to treat this as a schema change if key schemas is the same
- // otherwise we flush unnecessarily.
- if (config.isDeleteEnabled()) {
- deletesInBatch = true;
- }
- } else if (Objects.equals(schema, record.getSchema())) {
- if (config.isDeleteEnabled() && deletesInBatch) {
- // flush so an insert after a delete of same record isn't lost
- flushed.addAll(flush());
- }
- } else {
- // value schema is not null and has changed. This is a real schema change.
- schema = record.getSchema();
- schemaChanged = true;
- }
-
- if (schemaChanged) {
- // Each batch needs to have the same schemas, so get the buffered records out
- flushed.addAll(flush());
- // re-initialize everything that depends on the record schema
- final SchemaPair schemaPair = new SchemaPair(
- record.getKeySchema(),
- record.getSchema(),
- record.getExtensions()
- );
- // extract field
- fieldsMetadata = FieldsMetadata.extract(
- tableId.tableName(),
- config.pkMode,
- config.getPkFields(),
- config.getFieldsWhitelist(),
- schemaPair
- );
- }
-
- // set deletesInBatch if schema value is not null
- if (isNull(record.getData()) && config.isDeleteEnabled()) {
- deletesInBatch = true;
- }
-
- records.add(record);
- if (records.size() >= config.getBatchSize()) {
- flushed.addAll(flush());
- }
- return flushed;
- }
-
- public List<ConnectRecord> flush() throws SQLException {
- if (records.isEmpty()) {
- log.debug("Records is empty");
- return new ArrayList<>();
- }
- log.debug("Flushing {} buffered records", records.size());
- for (ConnectRecord record : records) {
- if (isNull(record.getData())) {
- deletePreparedRecords.add(record);
- } else {
- updatePreparedRecords.add(record);
- }
- }
- Optional<Long> totalUpdateCount = executeUpdates();
- Optional<Long> totalDeleteCount = executeDeletes();
- final long expectedCount = updateRecordCount();
- log.trace("{} records:{} resulting in totalUpdateCount:{} totalDeleteCount:{}",
- config.getInsertMode(),
- records.size(),
- totalUpdateCount,
- totalDeleteCount
- );
- if (totalUpdateCount.filter(total -> total != expectedCount).isPresent()
- && config.getInsertMode() == DorisSinkConfig.InsertMode.INSERT) {
- throw new ConnectException(
- String.format(
- "Update count (%d) did not sum up to total number of records inserted (%d)",
- totalUpdateCount.get(),
- expectedCount
- )
- );
- }
- if (!totalUpdateCount.isPresent()) {
- log.info(
- "{} records:{} , but no count of the number of rows it affected is available",
- config.getInsertMode(),
- records.size()
- );
- }
-
- final List<ConnectRecord> flushedRecords = records;
- records = new ArrayList<>();
- return flushedRecords;
- }
-
- /**
- * @return an optional count of all updated rows or an empty optional if no info is available
- */
- private Optional<Long> executeUpdates() throws DorisException {
- Optional<Long> count = Optional.empty();
- if (updatePreparedRecords.isEmpty()) {
- return count;
- }
- for (ConnectRecord record : updatePreparedRecords) {
- String jsonData = DorisDialect.convertToUpdateJsonString(record);
- try {
- log.info("[executeUpdates]" + jsonData);
- loader.loadJson(jsonData, record.getSchema().getName());
- } catch (DorisException e) {
- log.error("executeUpdates failed");
- throw e;
- } catch (Exception e) {
- throw new DorisException("doris error");
- }
- count = count.isPresent()
- ? count.map(total -> total + 1)
- : Optional.of(1L);
- }
- return count;
- }
-
- private Optional<Long> executeDeletes() throws SQLException {
- Optional<Long> totalDeleteCount = Optional.empty();
- if (deletePreparedRecords.isEmpty()) {
- return totalDeleteCount;
- }
- for (ConnectRecord record : updatePreparedRecords) {
- String jsonData = DorisDialect.convertToDeleteJsonString(record);
- try {
- log.info("[executeDelete]" + jsonData);
- loader.loadJson(jsonData, record.getSchema().getName());
- } catch (DorisException e) {
- log.error("executeDelete failed");
- throw e;
- } catch (Exception e) {
- throw new DorisException("doris error");
- }
- totalDeleteCount = totalDeleteCount.isPresent()
- ? totalDeleteCount.map(total -> total + 1)
- : Optional.of(1L);
- }
- return totalDeleteCount;
- }
... 5341 lines suppressed ...