You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by di...@apache.org on 2026/04/07 02:31:06 UTC
(doris-kafka-connector) branch master updated: [fix] fix jdbc connection surge caused by improper connection reuse in schema evolution (#94)
This is an automated email from the ASF dual-hosted git repository.
diwu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris-kafka-connector.git
The following commit(s) were added to refs/heads/master by this push:
new e2ee176 [fix] fix jdbc connection surge caused by improper connection reuse in schema evolution (#94)
e2ee176 is described below
commit e2ee17616282134e833df2f28d4d5bef813d7702
Author: wudi <67...@qq.com>
AuthorDate: Tue Apr 7 10:31:01 2026 +0800
[fix] fix jdbc connection surge caused by improper connection reuse in schema evolution (#94)
---
.../doris/kafka/connector/DorisSinkTask.java | 3 ++
.../connection/JdbcConnectionProvider.java | 5 +--
.../kafka/connector/converter/RecordService.java | 8 +++--
.../converter/schema/SchemaChangeManager.java | 5 +++
.../service/DorisCombinedSinkService.java | 8 ++++-
.../connector/service/DorisDefaultSinkService.java | 17 +++++++++-
.../kafka/connector/service/DorisSinkService.java | 3 ++
.../connector/service/DorisSystemService.java | 9 ++++--
.../connector/writer/AsyncStreamLoadWriter.java | 11 ++++++-
.../kafka/connector/writer/CopyIntoWriter.java | 11 ++++++-
.../doris/kafka/connector/writer/DorisWriter.java | 4 ++-
.../kafka/connector/writer/StreamLoadWriter.java | 16 +++++++---
.../e2e/sink/stringconverter/StringMsgE2ETest.java | 37 ++++++++++++++++++++++
.../kafka/connector/writer/TestCopyIntoWriter.java | 4 +++
.../connector/writer/TestStreamLoadWriter.java | 4 +++
.../schema_evolution_connector.json | 25 +++++++++++++++
.../e2e/string_converter/schema_evolution_tab.sql | 11 +++++++
17 files changed, 165 insertions(+), 16 deletions(-)
diff --git a/src/main/java/org/apache/doris/kafka/connector/DorisSinkTask.java b/src/main/java/org/apache/doris/kafka/connector/DorisSinkTask.java
index 9e553c5..521c6ba 100644
--- a/src/main/java/org/apache/doris/kafka/connector/DorisSinkTask.java
+++ b/src/main/java/org/apache/doris/kafka/connector/DorisSinkTask.java
@@ -65,6 +65,9 @@ public class DorisSinkTask extends SinkTask {
@Override
public void stop() {
LOG.info("kafka doris sink task stopped");
+ if (sink != null) {
+ sink.close();
+ }
}
/**
diff --git a/src/main/java/org/apache/doris/kafka/connector/connection/JdbcConnectionProvider.java b/src/main/java/org/apache/doris/kafka/connector/connection/JdbcConnectionProvider.java
index 47da2d9..a0afc23 100644
--- a/src/main/java/org/apache/doris/kafka/connector/connection/JdbcConnectionProvider.java
+++ b/src/main/java/org/apache/doris/kafka/connector/connection/JdbcConnectionProvider.java
@@ -46,7 +46,8 @@ public class JdbcConnectionProvider implements ConnectionProvider, Serializable
}
@Override
- public Connection getOrEstablishConnection() throws ClassNotFoundException, SQLException {
+ public synchronized Connection getOrEstablishConnection()
+ throws ClassNotFoundException, SQLException {
if (connection != null && !connection.isClosed() && connection.isValid(10)) {
return connection;
}
@@ -68,7 +69,7 @@ public class JdbcConnectionProvider implements ConnectionProvider, Serializable
}
@Override
- public void closeConnection() {
+ public synchronized void closeConnection() {
if (connection != null) {
try {
connection.close();
diff --git a/src/main/java/org/apache/doris/kafka/connector/converter/RecordService.java b/src/main/java/org/apache/doris/kafka/connector/converter/RecordService.java
index 51b3aab..9611886 100644
--- a/src/main/java/org/apache/doris/kafka/connector/converter/RecordService.java
+++ b/src/main/java/org/apache/doris/kafka/connector/converter/RecordService.java
@@ -80,11 +80,15 @@ public class RecordService {
}
public RecordService(DorisOptions dorisOptions) {
+ this(dorisOptions, new DorisSystemService(dorisOptions));
+ }
+
+ public RecordService(DorisOptions dorisOptions, DorisSystemService dorisSystemService) {
this();
this.dorisOptions = dorisOptions;
this.recordTypeRegister = new RecordTypeRegister(dorisOptions);
- this.dorisSystemService = new DorisSystemService(dorisOptions);
- this.schemaChangeManager = new SchemaChangeManager(dorisOptions);
+ this.dorisSystemService = dorisSystemService;
+ this.schemaChangeManager = new SchemaChangeManager(dorisOptions, dorisSystemService);
this.dorisTableDescriptorCache = new HashMap<>();
}
diff --git a/src/main/java/org/apache/doris/kafka/connector/converter/schema/SchemaChangeManager.java b/src/main/java/org/apache/doris/kafka/connector/converter/schema/SchemaChangeManager.java
index 3086adb..2f6a750 100644
--- a/src/main/java/org/apache/doris/kafka/connector/converter/schema/SchemaChangeManager.java
+++ b/src/main/java/org/apache/doris/kafka/connector/converter/schema/SchemaChangeManager.java
@@ -58,6 +58,11 @@ public class SchemaChangeManager implements Serializable {
this.dorisSystemService = new DorisSystemService(dorisOptions);
}
+ public SchemaChangeManager(DorisOptions dorisOptions, DorisSystemService dorisSystemService) {
+ this.dorisOptions = dorisOptions;
+ this.dorisSystemService = dorisSystemService;
+ }
+
private boolean handleSchemaChange(Map<String, Object> responseMap, String responseEntity) {
String code = responseMap.getOrDefault("code", "-1").toString();
if (code.equals("0")) {
diff --git a/src/main/java/org/apache/doris/kafka/connector/service/DorisCombinedSinkService.java b/src/main/java/org/apache/doris/kafka/connector/service/DorisCombinedSinkService.java
index f0d48f4..277907c 100644
--- a/src/main/java/org/apache/doris/kafka/connector/service/DorisCombinedSinkService.java
+++ b/src/main/java/org/apache/doris/kafka/connector/service/DorisCombinedSinkService.java
@@ -71,7 +71,13 @@ public class DorisCombinedSinkService extends DorisDefaultSinkService {
int partition = -1;
DorisWriter dorisWriter =
new AsyncStreamLoadWriter(
- tableName, topic, partition, dorisOptions, conn, connectMonitor);
+ tableName,
+ topic,
+ partition,
+ dorisOptions,
+ conn,
+ dorisSystemService,
+ connectMonitor);
writer.put(writerKey, dorisWriter);
metricsJmxReporter.start();
diff --git a/src/main/java/org/apache/doris/kafka/connector/service/DorisDefaultSinkService.java b/src/main/java/org/apache/doris/kafka/connector/service/DorisDefaultSinkService.java
index 6871cc1..50c7ef6 100644
--- a/src/main/java/org/apache/doris/kafka/connector/service/DorisDefaultSinkService.java
+++ b/src/main/java/org/apache/doris/kafka/connector/service/DorisDefaultSinkService.java
@@ -67,6 +67,7 @@ public class DorisDefaultSinkService implements DorisSinkService {
protected final ConnectionProvider conn;
protected final Map<String, DorisWriter> writer;
protected final DorisOptions dorisOptions;
+ protected final DorisSystemService dorisSystemService;
protected final MetricsJmxReporter metricsJmxReporter;
protected final DorisConnectMonitor connectMonitor;
protected final ObjectMapper objectMapper;
@@ -78,6 +79,7 @@ public class DorisDefaultSinkService implements DorisSinkService {
this.objectMapper = new ObjectMapper();
this.writer = new HashMap<>();
this.conn = new JdbcConnectionProvider(dorisOptions);
+ this.dorisSystemService = new DorisSystemService(dorisOptions);
MetricRegistry metricRegistry = new MetricRegistry();
this.metricsJmxReporter = new MetricsJmxReporter(metricRegistry, dorisOptions.getName());
this.connectMonitor =
@@ -90,6 +92,12 @@ public class DorisDefaultSinkService implements DorisSinkService {
@Override
public void init() {}
+ @Override
+ public void close() {
+ dorisSystemService.close();
+ conn.closeConnection();
+ }
+
@Override
public void startTask(TopicPartition topicPartition) {
startTask(dorisOptions.getTopicMapTable(topicPartition.topic()), topicPartition);
@@ -115,13 +123,20 @@ public class DorisDefaultSinkService implements DorisSinkService {
DorisWriter dorisWriter =
LoadModel.COPY_INTO.equals(loadModel)
? new CopyIntoWriter(
- tableName, topic, partition, dorisOptions, conn, connectMonitor)
+ tableName,
+ topic,
+ partition,
+ dorisOptions,
+ conn,
+ dorisSystemService,
+ connectMonitor)
: new StreamLoadWriter(
tableName,
topic,
partition,
dorisOptions,
conn,
+ dorisSystemService,
connectMonitor);
writer.put(writerKey, dorisWriter);
metricsJmxReporter.start();
diff --git a/src/main/java/org/apache/doris/kafka/connector/service/DorisSinkService.java b/src/main/java/org/apache/doris/kafka/connector/service/DorisSinkService.java
index a3d72e9..64903cb 100644
--- a/src/main/java/org/apache/doris/kafka/connector/service/DorisSinkService.java
+++ b/src/main/java/org/apache/doris/kafka/connector/service/DorisSinkService.java
@@ -77,4 +77,7 @@ public interface DorisSinkService {
/** commit data to doris. */
void commit(Map<TopicPartition, OffsetAndMetadata> offsets);
+
+ /** close the service and release resources. */
+ void close();
}
diff --git a/src/main/java/org/apache/doris/kafka/connector/service/DorisSystemService.java b/src/main/java/org/apache/doris/kafka/connector/service/DorisSystemService.java
index 9096aef..03d5ba3 100644
--- a/src/main/java/org/apache/doris/kafka/connector/service/DorisSystemService.java
+++ b/src/main/java/org/apache/doris/kafka/connector/service/DorisSystemService.java
@@ -19,7 +19,6 @@
package org.apache.doris.kafka.connector.service;
-import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.util.Collections;
@@ -84,13 +83,17 @@ public class DorisSystemService {
return !columnList.isEmpty();
}
+ public void close() {
+ jdbcConnectionProvider.closeConnection();
+ }
+
public List<String> extractColumnValuesBySQL(
String sql, int columnIndex, Predicate<String> filterFunc, Object... params) {
List<String> columnValues = Lists.newArrayList();
- try (Connection connection = jdbcConnectionProvider.getOrEstablishConnection();
- PreparedStatement ps = connection.prepareStatement(sql)) {
+ try (PreparedStatement ps =
+ jdbcConnectionProvider.getOrEstablishConnection().prepareStatement(sql)) {
if (Objects.nonNull(params) && params.length > 0) {
for (int i = 0; i < params.length; i++) {
diff --git a/src/main/java/org/apache/doris/kafka/connector/writer/AsyncStreamLoadWriter.java b/src/main/java/org/apache/doris/kafka/connector/writer/AsyncStreamLoadWriter.java
index 7003629..65258c5 100644
--- a/src/main/java/org/apache/doris/kafka/connector/writer/AsyncStreamLoadWriter.java
+++ b/src/main/java/org/apache/doris/kafka/connector/writer/AsyncStreamLoadWriter.java
@@ -25,6 +25,7 @@ import java.util.concurrent.TimeUnit;
import org.apache.doris.kafka.connector.cfg.DorisOptions;
import org.apache.doris.kafka.connector.connection.ConnectionProvider;
import org.apache.doris.kafka.connector.metrics.DorisConnectMonitor;
+import org.apache.doris.kafka.connector.service.DorisSystemService;
import org.apache.doris.kafka.connector.utils.BackendUtils;
import org.apache.doris.kafka.connector.writer.load.AsyncDorisStreamLoad;
import org.apache.doris.kafka.connector.writer.load.DefaultThreadFactory;
@@ -44,8 +45,16 @@ public class AsyncStreamLoadWriter extends DorisWriter {
int partition,
DorisOptions dorisOptions,
ConnectionProvider connectionProvider,
+ DorisSystemService dorisSystemService,
DorisConnectMonitor connectMonitor) {
- super(tableName, topic, partition, dorisOptions, connectionProvider, connectMonitor);
+ super(
+ tableName,
+ topic,
+ partition,
+ dorisOptions,
+ connectionProvider,
+ dorisSystemService,
+ connectMonitor);
this.labelGenerator = new LabelGenerator(topic, partition, tableIdentifier);
BackendUtils backendUtils = BackendUtils.getInstance(dorisOptions, LOG);
this.dorisStreamLoad =
diff --git a/src/main/java/org/apache/doris/kafka/connector/writer/CopyIntoWriter.java b/src/main/java/org/apache/doris/kafka/connector/writer/CopyIntoWriter.java
index 0abfefc..188a6ab 100644
--- a/src/main/java/org/apache/doris/kafka/connector/writer/CopyIntoWriter.java
+++ b/src/main/java/org/apache/doris/kafka/connector/writer/CopyIntoWriter.java
@@ -31,6 +31,7 @@ import org.apache.doris.kafka.connector.cfg.DorisOptions;
import org.apache.doris.kafka.connector.connection.ConnectionProvider;
import org.apache.doris.kafka.connector.exception.CopyLoadException;
import org.apache.doris.kafka.connector.metrics.DorisConnectMonitor;
+import org.apache.doris.kafka.connector.service.DorisSystemService;
import org.apache.doris.kafka.connector.utils.FileNameUtils;
import org.apache.doris.kafka.connector.writer.load.CopyLoad;
import org.apache.kafka.connect.sink.SinkRecord;
@@ -50,8 +51,16 @@ public class CopyIntoWriter extends DorisWriter {
int partition,
DorisOptions dorisOptions,
ConnectionProvider connectionProvider,
+ DorisSystemService dorisSystemService,
DorisConnectMonitor connectMonitor) {
- super(tableName, topic, partition, dorisOptions, connectionProvider, connectMonitor);
+ super(
+ tableName,
+ topic,
+ partition,
+ dorisOptions,
+ connectionProvider,
+ dorisSystemService,
+ connectMonitor);
this.taskId = dorisOptions.getTaskId();
this.prefix = FileNameUtils.filePrefix(dorisOptions.getName(), topic, partition);
this.copyLoad = new CopyLoad(dbName, this.tableName, dorisOptions);
diff --git a/src/main/java/org/apache/doris/kafka/connector/writer/DorisWriter.java b/src/main/java/org/apache/doris/kafka/connector/writer/DorisWriter.java
index cfa3ef9..7116c83 100644
--- a/src/main/java/org/apache/doris/kafka/connector/writer/DorisWriter.java
+++ b/src/main/java/org/apache/doris/kafka/connector/writer/DorisWriter.java
@@ -31,6 +31,7 @@ import org.apache.doris.kafka.connector.connection.ConnectionProvider;
import org.apache.doris.kafka.connector.converter.RecordService;
import org.apache.doris.kafka.connector.exception.ArgumentsException;
import org.apache.doris.kafka.connector.metrics.DorisConnectMonitor;
+import org.apache.doris.kafka.connector.service.DorisSystemService;
import org.apache.kafka.connect.sink.SinkRecord;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -67,6 +68,7 @@ public abstract class DorisWriter {
int partition,
DorisOptions dorisOptions,
ConnectionProvider connectionProvider,
+ DorisSystemService dorisSystemService,
DorisConnectMonitor connectMonitor) {
this.topic = topic;
this.partition = partition;
@@ -94,7 +96,7 @@ public abstract class DorisWriter {
this.dorisOptions = dorisOptions;
this.connectionProvider = connectionProvider;
- this.recordService = new RecordService(dorisOptions);
+ this.recordService = new RecordService(dorisOptions, dorisSystemService);
this.connectMonitor = connectMonitor;
}
diff --git a/src/main/java/org/apache/doris/kafka/connector/writer/StreamLoadWriter.java b/src/main/java/org/apache/doris/kafka/connector/writer/StreamLoadWriter.java
index 67d63e3..93f29f6 100644
--- a/src/main/java/org/apache/doris/kafka/connector/writer/StreamLoadWriter.java
+++ b/src/main/java/org/apache/doris/kafka/connector/writer/StreamLoadWriter.java
@@ -21,7 +21,6 @@ package org.apache.doris.kafka.connector.writer;
import com.google.common.annotations.VisibleForTesting;
import java.io.IOException;
-import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.util.HashMap;
@@ -34,6 +33,7 @@ import org.apache.doris.kafka.connector.connection.ConnectionProvider;
import org.apache.doris.kafka.connector.exception.StreamLoadException;
import org.apache.doris.kafka.connector.metrics.DorisConnectMonitor;
import org.apache.doris.kafka.connector.model.KafkaRespContent;
+import org.apache.doris.kafka.connector.service.DorisSystemService;
import org.apache.doris.kafka.connector.service.RestService;
import org.apache.doris.kafka.connector.utils.BackendUtils;
import org.apache.doris.kafka.connector.utils.FileNameUtils;
@@ -61,8 +61,16 @@ public class StreamLoadWriter extends DorisWriter {
int partition,
DorisOptions dorisOptions,
ConnectionProvider connectionProvider,
+ DorisSystemService dorisSystemService,
DorisConnectMonitor connectMonitor) {
- super(tableName, topic, partition, dorisOptions, connectionProvider, connectMonitor);
+ super(
+ tableName,
+ topic,
+ partition,
+ dorisOptions,
+ connectionProvider,
+ dorisSystemService,
+ connectMonitor);
this.taskId = dorisOptions.getTaskId();
this.labelGenerator = new LabelGenerator(topic, partition, tableIdentifier);
BackendUtils backendUtils = BackendUtils.getInstance(dorisOptions, LOG);
@@ -117,8 +125,8 @@ public class StreamLoadWriter extends DorisWriter {
labelGenerator.buildLabelPrefix());
LOG.info("query doris offset by sql: {}", querySQL);
Map<String, String> label2Status = new HashMap<>();
- try (Connection connection = connectionProvider.getOrEstablishConnection();
- PreparedStatement ps = connection.prepareStatement(querySQL);
+ try (PreparedStatement ps =
+ connectionProvider.getOrEstablishConnection().prepareStatement(querySQL);
ResultSet rs = ps.executeQuery()) {
while (rs.next()) {
String label = rs.getString("Label");
diff --git a/src/test/java/org/apache/doris/kafka/connector/e2e/sink/stringconverter/StringMsgE2ETest.java b/src/test/java/org/apache/doris/kafka/connector/e2e/sink/stringconverter/StringMsgE2ETest.java
index 280ddf3..60abfde 100644
--- a/src/test/java/org/apache/doris/kafka/connector/e2e/sink/stringconverter/StringMsgE2ETest.java
+++ b/src/test/java/org/apache/doris/kafka/connector/e2e/sink/stringconverter/StringMsgE2ETest.java
@@ -481,6 +481,43 @@ public class StringMsgE2ETest extends AbstractStringE2ESinkTest {
}
}
+ @Test
+ public void testDebeziumSchemaEvolution() throws Exception {
+ initialize("src/test/resources/e2e/string_converter/schema_evolution_connector.json");
+ String topic = "schema_evolution_test";
+
+ // First message: schema has id, name, age — matches the table
+ String insert1 =
+ "{\"schema\":{\"type\":\"struct\",\"fields\":[{\"type\":\"struct\",\"fields\":[{\"type\":\"int32\",\"optional\":false,\"field\":\"id\"},{\"type\":\"string\",\"optional\":true,\"field\":\"name\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"age\"}],\"optional\":true,\"name\":\"test.schema_evolution.Value\",\"field\":\"before\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"int32\",\"optional\":false,\"field\":\"id\"},{\"type\":\"string\",\"optional\":true,\"field\":\"n [...]
+
+ String tableSql =
+ loadContent("src/test/resources/e2e/string_converter/schema_evolution_tab.sql");
+ createTable(tableSql);
+ Thread.sleep(2000);
+ kafkaContainerService.registerKafkaConnector(connectorName, jsonMsgConnectorContent);
+ produceMsg2Kafka(topic, insert1);
+
+ String table = dorisOptions.getTopicMapTable(topic);
+ List<String> expected1 = Arrays.asList("1,doris,18");
+ Thread.sleep(20000);
+ String query = String.format("select id,name,age from %s.%s order by id", database, table);
+ checkResult(expected1, query, 3);
+
+ // Second message: schema adds a new column 'new_col' — should trigger schema evolution
+ String insert2 =
+ "{\"schema\":{\"type\":\"struct\",\"fields\":[{\"type\":\"struct\",\"fields\":[{\"type\":\"int32\",\"optional\":false,\"field\":\"id\"},{\"type\":\"string\",\"optional\":true,\"field\":\"name\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"age\"},{\"type\":\"string\",\"optional\":true,\"field\":\"new_col\"}],\"optional\":true,\"name\":\"test.schema_evolution.Value\",\"field\":\"before\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"int32\",\"optional\":false,\"field\ [...]
+
+ produceMsg2Kafka(topic, insert2);
+ Thread.sleep(30000);
+
+ // Verify new_col was added and both rows are visible
+ List<String> expected2 =
+ Arrays.asList("1,doris,18,null", "2,kafka,20,schema_evolution_test");
+ String queryWithNewCol =
+ String.format("select id,name,age,new_col from %s.%s order by id", database, table);
+ checkResult(expected2, queryWithNewCol, 4);
+ }
+
@AfterClass
public static void closeInstance() {
kafkaContainerService.deleteKafkaConnector(connectorName);
diff --git a/src/test/java/org/apache/doris/kafka/connector/writer/TestCopyIntoWriter.java b/src/test/java/org/apache/doris/kafka/connector/writer/TestCopyIntoWriter.java
index d3261f7..1b56e25 100644
--- a/src/test/java/org/apache/doris/kafka/connector/writer/TestCopyIntoWriter.java
+++ b/src/test/java/org/apache/doris/kafka/connector/writer/TestCopyIntoWriter.java
@@ -33,6 +33,7 @@ import org.apache.doris.kafka.connector.cfg.DorisOptions;
import org.apache.doris.kafka.connector.cfg.DorisSinkConnectorConfig;
import org.apache.doris.kafka.connector.connection.JdbcConnectionProvider;
import org.apache.doris.kafka.connector.metrics.DorisConnectMonitor;
+import org.apache.doris.kafka.connector.service.DorisSystemService;
import org.apache.doris.kafka.connector.writer.load.CopyLoad;
import org.apache.kafka.connect.sink.SinkRecord;
import org.junit.Assert;
@@ -83,6 +84,7 @@ public class TestCopyIntoWriter {
5,
dorisOptions,
new JdbcConnectionProvider(dorisOptions),
+ mock(DorisSystemService.class),
dorisConnectMonitor));
doReturn(Arrays.asList(listLoadFiles)).when(copyIntoWriter).listLoadFiles();
return copyIntoWriter;
@@ -108,6 +110,7 @@ public class TestCopyIntoWriter {
0,
dorisOptions,
new JdbcConnectionProvider(dorisOptions),
+ mock(DorisSystemService.class),
dorisConnectMonitor);
copyIntoWriter.setCopyLoad(mockCopyLoad);
@@ -129,6 +132,7 @@ public class TestCopyIntoWriter {
0,
dorisOptions,
new JdbcConnectionProvider(dorisOptions),
+ mock(DorisSystemService.class),
dorisConnectMonitor);
SinkRecord record = TestRecordBuffer.newSinkRecord("doris-1", 2);
dorisWriter.putBuffer(record);
diff --git a/src/test/java/org/apache/doris/kafka/connector/writer/TestStreamLoadWriter.java b/src/test/java/org/apache/doris/kafka/connector/writer/TestStreamLoadWriter.java
index 18014ad..a236314 100644
--- a/src/test/java/org/apache/doris/kafka/connector/writer/TestStreamLoadWriter.java
+++ b/src/test/java/org/apache/doris/kafka/connector/writer/TestStreamLoadWriter.java
@@ -38,6 +38,7 @@ import org.apache.doris.kafka.connector.cfg.DorisOptions;
import org.apache.doris.kafka.connector.cfg.DorisSinkConnectorConfig;
import org.apache.doris.kafka.connector.connection.JdbcConnectionProvider;
import org.apache.doris.kafka.connector.metrics.DorisConnectMonitor;
+import org.apache.doris.kafka.connector.service.DorisSystemService;
import org.apache.doris.kafka.connector.service.RestService;
import org.apache.doris.kafka.connector.writer.commit.DorisCommittable;
import org.apache.doris.kafka.connector.writer.load.DorisStreamLoad;
@@ -104,6 +105,7 @@ public class TestStreamLoadWriter {
2,
dorisOptions,
new JdbcConnectionProvider(dorisOptions),
+ mock(DorisSystemService.class),
dorisConnectMonitor));
mockRestService
@@ -136,6 +138,7 @@ public class TestStreamLoadWriter {
0,
dorisOptions,
new JdbcConnectionProvider(dorisOptions),
+ mock(DorisSystemService.class),
dorisConnectMonitor);
streamLoadWriter.setDorisStreamLoad(streamLoad);
@@ -162,6 +165,7 @@ public class TestStreamLoadWriter {
0,
dorisOptions,
new JdbcConnectionProvider(dorisOptions),
+ mock(DorisSystemService.class),
dorisConnectMonitor);
mockRestService
diff --git a/src/test/resources/e2e/string_converter/schema_evolution_connector.json b/src/test/resources/e2e/string_converter/schema_evolution_connector.json
new file mode 100644
index 0000000..851dc1e
--- /dev/null
+++ b/src/test/resources/e2e/string_converter/schema_evolution_connector.json
@@ -0,0 +1,25 @@
+{
+ "name":"schema_evolution_test",
+ "config": {
+ "connector.class":"org.apache.doris.kafka.connector.DorisSinkConnector",
+ "topics":"schema_evolution_test",
+ "tasks.max":"1",
+ "doris.topic2table.map": "schema_evolution_test:schema_evolution_tab",
+ "buffer.count.records":"1",
+ "buffer.flush.time":"10",
+ "buffer.size.bytes":"10000000",
+ "doris.urls":"127.0.0.1",
+ "doris.user":"root",
+ "doris.password":"",
+ "doris.http.port":"8030",
+ "doris.query.port":"9030",
+ "doris.database":"debezium_ingestion_msg",
+ "converter.mode": "debezium_ingestion",
+ "debezium.schema.evolution": "basic",
+ "enable.delete": "true",
+ "enable.2pc": "false",
+ "load.model":"stream_load",
+ "key.converter":"org.apache.kafka.connect.json.JsonConverter",
+ "value.converter":"org.apache.kafka.connect.json.JsonConverter"
+ }
+}
diff --git a/src/test/resources/e2e/string_converter/schema_evolution_tab.sql b/src/test/resources/e2e/string_converter/schema_evolution_tab.sql
new file mode 100644
index 0000000..c70e752
--- /dev/null
+++ b/src/test/resources/e2e/string_converter/schema_evolution_tab.sql
@@ -0,0 +1,11 @@
+CREATE TABLE debezium_ingestion_msg.schema_evolution_tab
+(
+ `id` INT,
+ `name` VARCHAR(256),
+ `age` INT
+)UNIQUE KEY(`id`)
+DISTRIBUTED BY HASH(`id`) BUCKETS 1
+PROPERTIES (
+"replication_allocation" = "tag.location.default: 1",
+"light_schema_change" = "true"
+);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org