You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by di...@apache.org on 2026/04/07 02:31:06 UTC

(doris-kafka-connector) branch master updated: [fix] fix jdbc connection surge caused by improper connection reuse in schema evolution (#94)

This is an automated email from the ASF dual-hosted git repository.

diwu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris-kafka-connector.git


The following commit(s) were added to refs/heads/master by this push:
     new e2ee176  [fix] fix jdbc connection surge caused by improper connection reuse in schema evolution (#94)
e2ee176 is described below

commit e2ee17616282134e833df2f28d4d5bef813d7702
Author: wudi <67...@qq.com>
AuthorDate: Tue Apr 7 10:31:01 2026 +0800

    [fix] fix jdbc connection surge caused by improper connection reuse in schema evolution (#94)
---
 .../doris/kafka/connector/DorisSinkTask.java       |  3 ++
 .../connection/JdbcConnectionProvider.java         |  5 +--
 .../kafka/connector/converter/RecordService.java   |  8 +++--
 .../converter/schema/SchemaChangeManager.java      |  5 +++
 .../service/DorisCombinedSinkService.java          |  8 ++++-
 .../connector/service/DorisDefaultSinkService.java | 17 +++++++++-
 .../kafka/connector/service/DorisSinkService.java  |  3 ++
 .../connector/service/DorisSystemService.java      |  9 ++++--
 .../connector/writer/AsyncStreamLoadWriter.java    | 11 ++++++-
 .../kafka/connector/writer/CopyIntoWriter.java     | 11 ++++++-
 .../doris/kafka/connector/writer/DorisWriter.java  |  4 ++-
 .../kafka/connector/writer/StreamLoadWriter.java   | 16 +++++++---
 .../e2e/sink/stringconverter/StringMsgE2ETest.java | 37 ++++++++++++++++++++++
 .../kafka/connector/writer/TestCopyIntoWriter.java |  4 +++
 .../connector/writer/TestStreamLoadWriter.java     |  4 +++
 .../schema_evolution_connector.json                | 25 +++++++++++++++
 .../e2e/string_converter/schema_evolution_tab.sql  | 11 +++++++
 17 files changed, 165 insertions(+), 16 deletions(-)

diff --git a/src/main/java/org/apache/doris/kafka/connector/DorisSinkTask.java b/src/main/java/org/apache/doris/kafka/connector/DorisSinkTask.java
index 9e553c5..521c6ba 100644
--- a/src/main/java/org/apache/doris/kafka/connector/DorisSinkTask.java
+++ b/src/main/java/org/apache/doris/kafka/connector/DorisSinkTask.java
@@ -65,6 +65,9 @@ public class DorisSinkTask extends SinkTask {
     @Override
     public void stop() {
         LOG.info("kafka doris sink task stopped");
+        if (sink != null) {
+            sink.close();
+        }
     }
 
     /**
diff --git a/src/main/java/org/apache/doris/kafka/connector/connection/JdbcConnectionProvider.java b/src/main/java/org/apache/doris/kafka/connector/connection/JdbcConnectionProvider.java
index 47da2d9..a0afc23 100644
--- a/src/main/java/org/apache/doris/kafka/connector/connection/JdbcConnectionProvider.java
+++ b/src/main/java/org/apache/doris/kafka/connector/connection/JdbcConnectionProvider.java
@@ -46,7 +46,8 @@ public class JdbcConnectionProvider implements ConnectionProvider, Serializable
     }
 
     @Override
-    public Connection getOrEstablishConnection() throws ClassNotFoundException, SQLException {
+    public synchronized Connection getOrEstablishConnection()
+            throws ClassNotFoundException, SQLException {
         if (connection != null && !connection.isClosed() && connection.isValid(10)) {
             return connection;
         }
@@ -68,7 +69,7 @@ public class JdbcConnectionProvider implements ConnectionProvider, Serializable
     }
 
     @Override
-    public void closeConnection() {
+    public synchronized void closeConnection() {
         if (connection != null) {
             try {
                 connection.close();
diff --git a/src/main/java/org/apache/doris/kafka/connector/converter/RecordService.java b/src/main/java/org/apache/doris/kafka/connector/converter/RecordService.java
index 51b3aab..9611886 100644
--- a/src/main/java/org/apache/doris/kafka/connector/converter/RecordService.java
+++ b/src/main/java/org/apache/doris/kafka/connector/converter/RecordService.java
@@ -80,11 +80,15 @@ public class RecordService {
     }
 
     public RecordService(DorisOptions dorisOptions) {
+        this(dorisOptions, new DorisSystemService(dorisOptions));
+    }
+
+    public RecordService(DorisOptions dorisOptions, DorisSystemService dorisSystemService) {
         this();
         this.dorisOptions = dorisOptions;
         this.recordTypeRegister = new RecordTypeRegister(dorisOptions);
-        this.dorisSystemService = new DorisSystemService(dorisOptions);
-        this.schemaChangeManager = new SchemaChangeManager(dorisOptions);
+        this.dorisSystemService = dorisSystemService;
+        this.schemaChangeManager = new SchemaChangeManager(dorisOptions, dorisSystemService);
         this.dorisTableDescriptorCache = new HashMap<>();
     }
 
diff --git a/src/main/java/org/apache/doris/kafka/connector/converter/schema/SchemaChangeManager.java b/src/main/java/org/apache/doris/kafka/connector/converter/schema/SchemaChangeManager.java
index 3086adb..2f6a750 100644
--- a/src/main/java/org/apache/doris/kafka/connector/converter/schema/SchemaChangeManager.java
+++ b/src/main/java/org/apache/doris/kafka/connector/converter/schema/SchemaChangeManager.java
@@ -58,6 +58,11 @@ public class SchemaChangeManager implements Serializable {
         this.dorisSystemService = new DorisSystemService(dorisOptions);
     }
 
+    public SchemaChangeManager(DorisOptions dorisOptions, DorisSystemService dorisSystemService) {
+        this.dorisOptions = dorisOptions;
+        this.dorisSystemService = dorisSystemService;
+    }
+
     private boolean handleSchemaChange(Map<String, Object> responseMap, String responseEntity) {
         String code = responseMap.getOrDefault("code", "-1").toString();
         if (code.equals("0")) {
diff --git a/src/main/java/org/apache/doris/kafka/connector/service/DorisCombinedSinkService.java b/src/main/java/org/apache/doris/kafka/connector/service/DorisCombinedSinkService.java
index f0d48f4..277907c 100644
--- a/src/main/java/org/apache/doris/kafka/connector/service/DorisCombinedSinkService.java
+++ b/src/main/java/org/apache/doris/kafka/connector/service/DorisCombinedSinkService.java
@@ -71,7 +71,13 @@ public class DorisCombinedSinkService extends DorisDefaultSinkService {
             int partition = -1;
             DorisWriter dorisWriter =
                     new AsyncStreamLoadWriter(
-                            tableName, topic, partition, dorisOptions, conn, connectMonitor);
+                            tableName,
+                            topic,
+                            partition,
+                            dorisOptions,
+                            conn,
+                            dorisSystemService,
+                            connectMonitor);
 
             writer.put(writerKey, dorisWriter);
             metricsJmxReporter.start();
diff --git a/src/main/java/org/apache/doris/kafka/connector/service/DorisDefaultSinkService.java b/src/main/java/org/apache/doris/kafka/connector/service/DorisDefaultSinkService.java
index 6871cc1..50c7ef6 100644
--- a/src/main/java/org/apache/doris/kafka/connector/service/DorisDefaultSinkService.java
+++ b/src/main/java/org/apache/doris/kafka/connector/service/DorisDefaultSinkService.java
@@ -67,6 +67,7 @@ public class DorisDefaultSinkService implements DorisSinkService {
     protected final ConnectionProvider conn;
     protected final Map<String, DorisWriter> writer;
     protected final DorisOptions dorisOptions;
+    protected final DorisSystemService dorisSystemService;
     protected final MetricsJmxReporter metricsJmxReporter;
     protected final DorisConnectMonitor connectMonitor;
     protected final ObjectMapper objectMapper;
@@ -78,6 +79,7 @@ public class DorisDefaultSinkService implements DorisSinkService {
         this.objectMapper = new ObjectMapper();
         this.writer = new HashMap<>();
         this.conn = new JdbcConnectionProvider(dorisOptions);
+        this.dorisSystemService = new DorisSystemService(dorisOptions);
         MetricRegistry metricRegistry = new MetricRegistry();
         this.metricsJmxReporter = new MetricsJmxReporter(metricRegistry, dorisOptions.getName());
         this.connectMonitor =
@@ -90,6 +92,12 @@ public class DorisDefaultSinkService implements DorisSinkService {
     @Override
     public void init() {}
 
+    @Override
+    public void close() {
+        dorisSystemService.close();
+        conn.closeConnection();
+    }
+
     @Override
     public void startTask(TopicPartition topicPartition) {
         startTask(dorisOptions.getTopicMapTable(topicPartition.topic()), topicPartition);
@@ -115,13 +123,20 @@ public class DorisDefaultSinkService implements DorisSinkService {
             DorisWriter dorisWriter =
                     LoadModel.COPY_INTO.equals(loadModel)
                             ? new CopyIntoWriter(
-                                    tableName, topic, partition, dorisOptions, conn, connectMonitor)
+                                    tableName,
+                                    topic,
+                                    partition,
+                                    dorisOptions,
+                                    conn,
+                                    dorisSystemService,
+                                    connectMonitor)
                             : new StreamLoadWriter(
                                     tableName,
                                     topic,
                                     partition,
                                     dorisOptions,
                                     conn,
+                                    dorisSystemService,
                                     connectMonitor);
             writer.put(writerKey, dorisWriter);
             metricsJmxReporter.start();
diff --git a/src/main/java/org/apache/doris/kafka/connector/service/DorisSinkService.java b/src/main/java/org/apache/doris/kafka/connector/service/DorisSinkService.java
index a3d72e9..64903cb 100644
--- a/src/main/java/org/apache/doris/kafka/connector/service/DorisSinkService.java
+++ b/src/main/java/org/apache/doris/kafka/connector/service/DorisSinkService.java
@@ -77,4 +77,7 @@ public interface DorisSinkService {
 
     /** commit data to doris. */
     void commit(Map<TopicPartition, OffsetAndMetadata> offsets);
+
+    /** close the service and release resources. */
+    void close();
 }
diff --git a/src/main/java/org/apache/doris/kafka/connector/service/DorisSystemService.java b/src/main/java/org/apache/doris/kafka/connector/service/DorisSystemService.java
index 9096aef..03d5ba3 100644
--- a/src/main/java/org/apache/doris/kafka/connector/service/DorisSystemService.java
+++ b/src/main/java/org/apache/doris/kafka/connector/service/DorisSystemService.java
@@ -19,7 +19,6 @@
 
 package org.apache.doris.kafka.connector.service;
 
-import java.sql.Connection;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.util.Collections;
@@ -84,13 +83,17 @@ public class DorisSystemService {
         return !columnList.isEmpty();
     }
 
+    public void close() {
+        jdbcConnectionProvider.closeConnection();
+    }
+
     public List<String> extractColumnValuesBySQL(
             String sql, int columnIndex, Predicate<String> filterFunc, Object... params) {
 
         List<String> columnValues = Lists.newArrayList();
 
-        try (Connection connection = jdbcConnectionProvider.getOrEstablishConnection();
-                PreparedStatement ps = connection.prepareStatement(sql)) {
+        try (PreparedStatement ps =
+                jdbcConnectionProvider.getOrEstablishConnection().prepareStatement(sql)) {
 
             if (Objects.nonNull(params) && params.length > 0) {
                 for (int i = 0; i < params.length; i++) {
diff --git a/src/main/java/org/apache/doris/kafka/connector/writer/AsyncStreamLoadWriter.java b/src/main/java/org/apache/doris/kafka/connector/writer/AsyncStreamLoadWriter.java
index 7003629..65258c5 100644
--- a/src/main/java/org/apache/doris/kafka/connector/writer/AsyncStreamLoadWriter.java
+++ b/src/main/java/org/apache/doris/kafka/connector/writer/AsyncStreamLoadWriter.java
@@ -25,6 +25,7 @@ import java.util.concurrent.TimeUnit;
 import org.apache.doris.kafka.connector.cfg.DorisOptions;
 import org.apache.doris.kafka.connector.connection.ConnectionProvider;
 import org.apache.doris.kafka.connector.metrics.DorisConnectMonitor;
+import org.apache.doris.kafka.connector.service.DorisSystemService;
 import org.apache.doris.kafka.connector.utils.BackendUtils;
 import org.apache.doris.kafka.connector.writer.load.AsyncDorisStreamLoad;
 import org.apache.doris.kafka.connector.writer.load.DefaultThreadFactory;
@@ -44,8 +45,16 @@ public class AsyncStreamLoadWriter extends DorisWriter {
             int partition,
             DorisOptions dorisOptions,
             ConnectionProvider connectionProvider,
+            DorisSystemService dorisSystemService,
             DorisConnectMonitor connectMonitor) {
-        super(tableName, topic, partition, dorisOptions, connectionProvider, connectMonitor);
+        super(
+                tableName,
+                topic,
+                partition,
+                dorisOptions,
+                connectionProvider,
+                dorisSystemService,
+                connectMonitor);
         this.labelGenerator = new LabelGenerator(topic, partition, tableIdentifier);
         BackendUtils backendUtils = BackendUtils.getInstance(dorisOptions, LOG);
         this.dorisStreamLoad =
diff --git a/src/main/java/org/apache/doris/kafka/connector/writer/CopyIntoWriter.java b/src/main/java/org/apache/doris/kafka/connector/writer/CopyIntoWriter.java
index 0abfefc..188a6ab 100644
--- a/src/main/java/org/apache/doris/kafka/connector/writer/CopyIntoWriter.java
+++ b/src/main/java/org/apache/doris/kafka/connector/writer/CopyIntoWriter.java
@@ -31,6 +31,7 @@ import org.apache.doris.kafka.connector.cfg.DorisOptions;
 import org.apache.doris.kafka.connector.connection.ConnectionProvider;
 import org.apache.doris.kafka.connector.exception.CopyLoadException;
 import org.apache.doris.kafka.connector.metrics.DorisConnectMonitor;
+import org.apache.doris.kafka.connector.service.DorisSystemService;
 import org.apache.doris.kafka.connector.utils.FileNameUtils;
 import org.apache.doris.kafka.connector.writer.load.CopyLoad;
 import org.apache.kafka.connect.sink.SinkRecord;
@@ -50,8 +51,16 @@ public class CopyIntoWriter extends DorisWriter {
             int partition,
             DorisOptions dorisOptions,
             ConnectionProvider connectionProvider,
+            DorisSystemService dorisSystemService,
             DorisConnectMonitor connectMonitor) {
-        super(tableName, topic, partition, dorisOptions, connectionProvider, connectMonitor);
+        super(
+                tableName,
+                topic,
+                partition,
+                dorisOptions,
+                connectionProvider,
+                dorisSystemService,
+                connectMonitor);
         this.taskId = dorisOptions.getTaskId();
         this.prefix = FileNameUtils.filePrefix(dorisOptions.getName(), topic, partition);
         this.copyLoad = new CopyLoad(dbName, this.tableName, dorisOptions);
diff --git a/src/main/java/org/apache/doris/kafka/connector/writer/DorisWriter.java b/src/main/java/org/apache/doris/kafka/connector/writer/DorisWriter.java
index cfa3ef9..7116c83 100644
--- a/src/main/java/org/apache/doris/kafka/connector/writer/DorisWriter.java
+++ b/src/main/java/org/apache/doris/kafka/connector/writer/DorisWriter.java
@@ -31,6 +31,7 @@ import org.apache.doris.kafka.connector.connection.ConnectionProvider;
 import org.apache.doris.kafka.connector.converter.RecordService;
 import org.apache.doris.kafka.connector.exception.ArgumentsException;
 import org.apache.doris.kafka.connector.metrics.DorisConnectMonitor;
+import org.apache.doris.kafka.connector.service.DorisSystemService;
 import org.apache.kafka.connect.sink.SinkRecord;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -67,6 +68,7 @@ public abstract class DorisWriter {
             int partition,
             DorisOptions dorisOptions,
             ConnectionProvider connectionProvider,
+            DorisSystemService dorisSystemService,
             DorisConnectMonitor connectMonitor) {
         this.topic = topic;
         this.partition = partition;
@@ -94,7 +96,7 @@ public abstract class DorisWriter {
 
         this.dorisOptions = dorisOptions;
         this.connectionProvider = connectionProvider;
-        this.recordService = new RecordService(dorisOptions);
+        this.recordService = new RecordService(dorisOptions, dorisSystemService);
         this.connectMonitor = connectMonitor;
     }
 
diff --git a/src/main/java/org/apache/doris/kafka/connector/writer/StreamLoadWriter.java b/src/main/java/org/apache/doris/kafka/connector/writer/StreamLoadWriter.java
index 67d63e3..93f29f6 100644
--- a/src/main/java/org/apache/doris/kafka/connector/writer/StreamLoadWriter.java
+++ b/src/main/java/org/apache/doris/kafka/connector/writer/StreamLoadWriter.java
@@ -21,7 +21,6 @@ package org.apache.doris.kafka.connector.writer;
 
 import com.google.common.annotations.VisibleForTesting;
 import java.io.IOException;
-import java.sql.Connection;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.util.HashMap;
@@ -34,6 +33,7 @@ import org.apache.doris.kafka.connector.connection.ConnectionProvider;
 import org.apache.doris.kafka.connector.exception.StreamLoadException;
 import org.apache.doris.kafka.connector.metrics.DorisConnectMonitor;
 import org.apache.doris.kafka.connector.model.KafkaRespContent;
+import org.apache.doris.kafka.connector.service.DorisSystemService;
 import org.apache.doris.kafka.connector.service.RestService;
 import org.apache.doris.kafka.connector.utils.BackendUtils;
 import org.apache.doris.kafka.connector.utils.FileNameUtils;
@@ -61,8 +61,16 @@ public class StreamLoadWriter extends DorisWriter {
             int partition,
             DorisOptions dorisOptions,
             ConnectionProvider connectionProvider,
+            DorisSystemService dorisSystemService,
             DorisConnectMonitor connectMonitor) {
-        super(tableName, topic, partition, dorisOptions, connectionProvider, connectMonitor);
+        super(
+                tableName,
+                topic,
+                partition,
+                dorisOptions,
+                connectionProvider,
+                dorisSystemService,
+                connectMonitor);
         this.taskId = dorisOptions.getTaskId();
         this.labelGenerator = new LabelGenerator(topic, partition, tableIdentifier);
         BackendUtils backendUtils = BackendUtils.getInstance(dorisOptions, LOG);
@@ -117,8 +125,8 @@ public class StreamLoadWriter extends DorisWriter {
                         labelGenerator.buildLabelPrefix());
         LOG.info("query doris offset by sql: {}", querySQL);
         Map<String, String> label2Status = new HashMap<>();
-        try (Connection connection = connectionProvider.getOrEstablishConnection();
-                PreparedStatement ps = connection.prepareStatement(querySQL);
+        try (PreparedStatement ps =
+                        connectionProvider.getOrEstablishConnection().prepareStatement(querySQL);
                 ResultSet rs = ps.executeQuery()) {
             while (rs.next()) {
                 String label = rs.getString("Label");
diff --git a/src/test/java/org/apache/doris/kafka/connector/e2e/sink/stringconverter/StringMsgE2ETest.java b/src/test/java/org/apache/doris/kafka/connector/e2e/sink/stringconverter/StringMsgE2ETest.java
index 280ddf3..60abfde 100644
--- a/src/test/java/org/apache/doris/kafka/connector/e2e/sink/stringconverter/StringMsgE2ETest.java
+++ b/src/test/java/org/apache/doris/kafka/connector/e2e/sink/stringconverter/StringMsgE2ETest.java
@@ -481,6 +481,43 @@ public class StringMsgE2ETest extends AbstractStringE2ESinkTest {
         }
     }
 
+    @Test
+    public void testDebeziumSchemaEvolution() throws Exception {
+        initialize("src/test/resources/e2e/string_converter/schema_evolution_connector.json");
+        String topic = "schema_evolution_test";
+
+        // First message: schema has id, name, age — matches the table
+        String insert1 =
+                "{\"schema\":{\"type\":\"struct\",\"fields\":[{\"type\":\"struct\",\"fields\":[{\"type\":\"int32\",\"optional\":false,\"field\":\"id\"},{\"type\":\"string\",\"optional\":true,\"field\":\"name\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"age\"}],\"optional\":true,\"name\":\"test.schema_evolution.Value\",\"field\":\"before\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"int32\",\"optional\":false,\"field\":\"id\"},{\"type\":\"string\",\"optional\":true,\"field\":\"n [...]
+
+        String tableSql =
+                loadContent("src/test/resources/e2e/string_converter/schema_evolution_tab.sql");
+        createTable(tableSql);
+        Thread.sleep(2000);
+        kafkaContainerService.registerKafkaConnector(connectorName, jsonMsgConnectorContent);
+        produceMsg2Kafka(topic, insert1);
+
+        String table = dorisOptions.getTopicMapTable(topic);
+        List<String> expected1 = Arrays.asList("1,doris,18");
+        Thread.sleep(20000);
+        String query = String.format("select id,name,age from %s.%s order by id", database, table);
+        checkResult(expected1, query, 3);
+
+        // Second message: schema adds a new column 'new_col' — should trigger schema evolution
+        String insert2 =
+                "{\"schema\":{\"type\":\"struct\",\"fields\":[{\"type\":\"struct\",\"fields\":[{\"type\":\"int32\",\"optional\":false,\"field\":\"id\"},{\"type\":\"string\",\"optional\":true,\"field\":\"name\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"age\"},{\"type\":\"string\",\"optional\":true,\"field\":\"new_col\"}],\"optional\":true,\"name\":\"test.schema_evolution.Value\",\"field\":\"before\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"int32\",\"optional\":false,\"field\ [...]
+
+        produceMsg2Kafka(topic, insert2);
+        Thread.sleep(30000);
+
+        // Verify new_col was added and both rows are visible
+        List<String> expected2 =
+                Arrays.asList("1,doris,18,null", "2,kafka,20,schema_evolution_test");
+        String queryWithNewCol =
+                String.format("select id,name,age,new_col from %s.%s order by id", database, table);
+        checkResult(expected2, queryWithNewCol, 4);
+    }
+
     @AfterClass
     public static void closeInstance() {
         kafkaContainerService.deleteKafkaConnector(connectorName);
diff --git a/src/test/java/org/apache/doris/kafka/connector/writer/TestCopyIntoWriter.java b/src/test/java/org/apache/doris/kafka/connector/writer/TestCopyIntoWriter.java
index d3261f7..1b56e25 100644
--- a/src/test/java/org/apache/doris/kafka/connector/writer/TestCopyIntoWriter.java
+++ b/src/test/java/org/apache/doris/kafka/connector/writer/TestCopyIntoWriter.java
@@ -33,6 +33,7 @@ import org.apache.doris.kafka.connector.cfg.DorisOptions;
 import org.apache.doris.kafka.connector.cfg.DorisSinkConnectorConfig;
 import org.apache.doris.kafka.connector.connection.JdbcConnectionProvider;
 import org.apache.doris.kafka.connector.metrics.DorisConnectMonitor;
+import org.apache.doris.kafka.connector.service.DorisSystemService;
 import org.apache.doris.kafka.connector.writer.load.CopyLoad;
 import org.apache.kafka.connect.sink.SinkRecord;
 import org.junit.Assert;
@@ -83,6 +84,7 @@ public class TestCopyIntoWriter {
                                 5,
                                 dorisOptions,
                                 new JdbcConnectionProvider(dorisOptions),
+                                mock(DorisSystemService.class),
                                 dorisConnectMonitor));
         doReturn(Arrays.asList(listLoadFiles)).when(copyIntoWriter).listLoadFiles();
         return copyIntoWriter;
@@ -108,6 +110,7 @@ public class TestCopyIntoWriter {
                         0,
                         dorisOptions,
                         new JdbcConnectionProvider(dorisOptions),
+                        mock(DorisSystemService.class),
                         dorisConnectMonitor);
         copyIntoWriter.setCopyLoad(mockCopyLoad);
 
@@ -129,6 +132,7 @@ public class TestCopyIntoWriter {
                         0,
                         dorisOptions,
                         new JdbcConnectionProvider(dorisOptions),
+                        mock(DorisSystemService.class),
                         dorisConnectMonitor);
         SinkRecord record = TestRecordBuffer.newSinkRecord("doris-1", 2);
         dorisWriter.putBuffer(record);
diff --git a/src/test/java/org/apache/doris/kafka/connector/writer/TestStreamLoadWriter.java b/src/test/java/org/apache/doris/kafka/connector/writer/TestStreamLoadWriter.java
index 18014ad..a236314 100644
--- a/src/test/java/org/apache/doris/kafka/connector/writer/TestStreamLoadWriter.java
+++ b/src/test/java/org/apache/doris/kafka/connector/writer/TestStreamLoadWriter.java
@@ -38,6 +38,7 @@ import org.apache.doris.kafka.connector.cfg.DorisOptions;
 import org.apache.doris.kafka.connector.cfg.DorisSinkConnectorConfig;
 import org.apache.doris.kafka.connector.connection.JdbcConnectionProvider;
 import org.apache.doris.kafka.connector.metrics.DorisConnectMonitor;
+import org.apache.doris.kafka.connector.service.DorisSystemService;
 import org.apache.doris.kafka.connector.service.RestService;
 import org.apache.doris.kafka.connector.writer.commit.DorisCommittable;
 import org.apache.doris.kafka.connector.writer.load.DorisStreamLoad;
@@ -104,6 +105,7 @@ public class TestStreamLoadWriter {
                                 2,
                                 dorisOptions,
                                 new JdbcConnectionProvider(dorisOptions),
+                                mock(DorisSystemService.class),
                                 dorisConnectMonitor));
 
         mockRestService
@@ -136,6 +138,7 @@ public class TestStreamLoadWriter {
                         0,
                         dorisOptions,
                         new JdbcConnectionProvider(dorisOptions),
+                        mock(DorisSystemService.class),
                         dorisConnectMonitor);
         streamLoadWriter.setDorisStreamLoad(streamLoad);
 
@@ -162,6 +165,7 @@ public class TestStreamLoadWriter {
                         0,
                         dorisOptions,
                         new JdbcConnectionProvider(dorisOptions),
+                        mock(DorisSystemService.class),
                         dorisConnectMonitor);
 
         mockRestService
diff --git a/src/test/resources/e2e/string_converter/schema_evolution_connector.json b/src/test/resources/e2e/string_converter/schema_evolution_connector.json
new file mode 100644
index 0000000..851dc1e
--- /dev/null
+++ b/src/test/resources/e2e/string_converter/schema_evolution_connector.json
@@ -0,0 +1,25 @@
+{
+  "name":"schema_evolution_test",
+  "config": {
+    "connector.class":"org.apache.doris.kafka.connector.DorisSinkConnector",
+    "topics":"schema_evolution_test",
+    "tasks.max":"1",
+    "doris.topic2table.map": "schema_evolution_test:schema_evolution_tab",
+    "buffer.count.records":"1",
+    "buffer.flush.time":"10",
+    "buffer.size.bytes":"10000000",
+    "doris.urls":"127.0.0.1",
+    "doris.user":"root",
+    "doris.password":"",
+    "doris.http.port":"8030",
+    "doris.query.port":"9030",
+    "doris.database":"debezium_ingestion_msg",
+    "converter.mode": "debezium_ingestion",
+    "debezium.schema.evolution": "basic",
+    "enable.delete": "true",
+    "enable.2pc": "false",
+    "load.model":"stream_load",
+    "key.converter":"org.apache.kafka.connect.json.JsonConverter",
+    "value.converter":"org.apache.kafka.connect.json.JsonConverter"
+  }
+}
diff --git a/src/test/resources/e2e/string_converter/schema_evolution_tab.sql b/src/test/resources/e2e/string_converter/schema_evolution_tab.sql
new file mode 100644
index 0000000..c70e752
--- /dev/null
+++ b/src/test/resources/e2e/string_converter/schema_evolution_tab.sql
@@ -0,0 +1,11 @@
+CREATE TABLE debezium_ingestion_msg.schema_evolution_tab
+(
+    `id`    INT,
+    `name`  VARCHAR(256),
+    `age`   INT
+)UNIQUE KEY(`id`)
+DISTRIBUTED BY HASH(`id`) BUCKETS 1
+PROPERTIES (
+"replication_allocation" = "tag.location.default: 1",
+"light_schema_change" = "true"
+);


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org