You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by zi...@apache.org on 2026/04/30 06:45:36 UTC

(dolphinscheduler) branch dev updated: [Improvement-16754][DataX] Support DataX writer parameter batchSize (#18192)

This is an automated email from the ASF dual-hosted git repository.

SbloodyS pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git


The following commit(s) were added to refs/heads/dev by this push:
     new 16dbda94c8 [Improvement-16754][DataX] Support DataX writer parameter batchSize (#18192)
16dbda94c8 is described below

commit 16dbda94c8eb2d2c904096608d9979d5db7a822f
Author: leocook <le...@163.com>
AuthorDate: Thu Apr 30 14:45:30 2026 +0800

    [Improvement-16754][DataX] Support DataX writer parameter batchSize (#18192)
---
 .../plugin/task/datax/DataxParameters.java         |  6 ++
 .../plugin/task/datax/DataxTask.java               |  4 ++
 .../plugin/task/datax/DataxParametersTest.java     | 14 +++++
 .../plugin/task/datax/DataxTaskTest.java           | 70 ++++++++++++++++++++++
 dolphinscheduler-ui/src/locales/en_US/project.ts   |  2 +
 dolphinscheduler-ui/src/locales/zh_CN/project.ts   |  2 +
 .../task/components/node/fields/use-datax.ts       | 12 ++++
 7 files changed, 110 insertions(+)

diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParameters.java
index 9d9722bc6f..07c20c3203 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParameters.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParameters.java
@@ -101,6 +101,11 @@ public class DataxParameters extends AbstractParameters {
      */
     private int xmx;
 
+    /**
+     * writer batch size for DataX
+     */
+    private int batchSize;
+
     private List<ResourceInfo> resourceList;
 
     @Override
@@ -138,6 +143,7 @@ public class DataxParameters extends AbstractParameters {
                 ", jobChannel=" + jobChannel +
                 ", xms=" + xms +
                 ", xmx=" + xmx +
+                ", batchSize=" + batchSize +
                 ", resourceList=" + JSONUtils.toJsonString(resourceList) +
                 '}';
     }
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java
index a8c39a3a50..e82666d530 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java
@@ -282,6 +282,10 @@ public class DataxTask extends AbstractTask {
             }
         }
 
+        if (dataXParameters.getBatchSize() > 0) {
+            writerParam.put("batchSize", dataXParameters.getBatchSize());
+        }
+
         ObjectNode writer = JSONUtils.createObjectNode();
         writer.put("name", DataxUtils.getWriterPluginName(dataxTaskExecutionContext.getTargetType()));
         writer.set("parameter", writerParam);
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParametersTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParametersTest.java
index a20b5a1695..736d0aab94 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParametersTest.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParametersTest.java
@@ -91,12 +91,26 @@ public class DataxParametersTest {
                 + "jobChannel=1, "
                 + "xms=0, "
                 + "xmx=-100, "
+                + "batchSize=0, "
                 + "resourceList=[{\"id\":null,\"resourceName\":\"/hdfs.keytab\",\"res\":null}]"
                 + "}";
 
         Assertions.assertEquals(expected, dataxParameters.toString());
     }
 
+    @Test
+    public void testBatchSize() {
+        DataxParameters dataxParameters = new DataxParameters();
+        dataxParameters.setBatchSize(0);
+        Assertions.assertEquals(0, dataxParameters.getBatchSize());
+
+        dataxParameters.setBatchSize(2048);
+        Assertions.assertEquals(2048, dataxParameters.getBatchSize());
+
+        dataxParameters.setBatchSize(65536);
+        Assertions.assertEquals(65536, dataxParameters.getBatchSize());
+    }
+
     public String loadJvmEnvTest(DataxParameters dataXParameters) {
         int xms = dataXParameters.getXms() < 1 ? 1 : dataXParameters.getXms();
         int xmx = dataXParameters.getXmx() < 1 ? 1 : dataXParameters.getXmx();
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskTest.java
index 3b1dd82c92..17687e15c1 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskTest.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskTest.java
@@ -27,6 +27,7 @@ import static org.mockito.Mockito.when;
 import org.apache.dolphinscheduler.common.utils.FileUtils;
 import org.apache.dolphinscheduler.common.utils.JSONUtils;
 import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider;
+import org.apache.dolphinscheduler.plugin.datasource.api.utils.DataSourceUtils;
 import org.apache.dolphinscheduler.plugin.task.api.ShellCommandExecutor;
 import org.apache.dolphinscheduler.plugin.task.api.TaskCallBack;
 import org.apache.dolphinscheduler.plugin.task.api.TaskException;
@@ -45,6 +46,7 @@ import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.lang.reflect.Field;
+import java.lang.reflect.Method;
 import java.nio.file.Files;
 import java.sql.Connection;
 import java.sql.PreparedStatement;
@@ -52,6 +54,7 @@ import java.sql.ResultSet;
 import java.sql.ResultSetMetaData;
 import java.sql.SQLException;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import org.junit.jupiter.api.Assertions;
@@ -62,6 +65,8 @@ import org.mockito.MockedStatic;
 import org.mockito.Mockito;
 import org.mockito.junit.jupiter.MockitoExtension;
 
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
 @ExtendWith(MockitoExtension.class)
 public class DataxTaskTest {
 
@@ -273,6 +278,71 @@ public class DataxTaskTest {
         return taskExecutionContext;
     }
 
+    @Test
+    public void testBuildDataxJobContentJsonWithBatchSize() throws Exception {
+        // set batchSize > 0 via reflection
+        Field dataXParametersField = DataxTask.class.getDeclaredField("dataXParameters");
+        dataXParametersField.setAccessible(true);
+        DataxParameters params = (DataxParameters) dataXParametersField.get(dataxTask);
+        params.setBatchSize(1024);
+        params.setDsType("MYSQL");
+        params.setDtType("MYSQL");
+
+        // set dataxTaskExecutionContext via reflection
+        DataxTaskExecutionContext ctx = new DataxTaskExecutionContext();
+        ctx.setSourcetype(DbType.MYSQL);
+        ctx.setTargetType(DbType.MYSQL);
+        ctx.setSourceConnectionParams(
+                "{\"user\":\"root\",\"password\":\"123456\",\"address\":\"jdbc:mysql://localhost:3306\"}");
+        ctx.setTargetConnectionParams(
+                "{\"user\":\"root\",\"password\":\"123456\",\"address\":\"jdbc:mysql://localhost:3306\"}");
+
+        Field ctxField = DataxTask.class.getDeclaredField("dataxTaskExecutionContext");
+        ctxField.setAccessible(true);
+        ctxField.set(dataxTask, ctx);
+
+        BaseConnectionParam mockConnParam = mock(BaseConnectionParam.class);
+        when(mockConnParam.getUser()).thenReturn("root");
+        when(mockConnParam.getPassword()).thenReturn("123456");
+        when(mockConnParam.getCompatibleMode()).thenReturn(null);
+
+        try (
+                MockedStatic<DataSourceUtils> mockedDataSourceUtils = mockStatic(DataSourceUtils.class);
+                MockedStatic<DataSourceClientProvider> mockedProvider = mockStatic(DataSourceClientProvider.class)) {
+
+            mockedDataSourceUtils
+                    .when(() -> DataSourceUtils.buildConnectionParams(Mockito.any(DbType.class), Mockito.anyString()))
+                    .thenReturn(mockConnParam);
+            mockedDataSourceUtils.when(() -> DataSourceUtils.getJdbcUrl(Mockito.any(DbType.class), Mockito.any()))
+                    .thenReturn("jdbc:mysql://localhost:3306/test");
+
+            Connection connection = mock(Connection.class);
+            mockedProvider.when(() -> DataSourceClientProvider.getAdHocConnection(Mockito.any(), Mockito.any()))
+                    .thenReturn(connection);
+
+            PreparedStatement stmt = mock(PreparedStatement.class);
+            when(connection.prepareStatement(anyString())).thenReturn(stmt);
+            ResultSetMetaData md = mock(ResultSetMetaData.class);
+            when(md.getColumnCount()).thenReturn(1);
+            when(md.getColumnLabel(eq(1))).thenReturn("col1");
+            ResultSet resultSet = mock(ResultSet.class);
+            when(resultSet.getMetaData()).thenReturn(md);
+            when(stmt.executeQuery()).thenReturn(resultSet);
+
+            Method method = DataxTask.class.getDeclaredMethod("buildDataxJobContentJson");
+            method.setAccessible(true);
+            Object invokeResult = method.invoke(dataxTask);
+            Assertions.assertNotNull(invokeResult);
+            List<?> result = (List<?>) invokeResult;
+
+            Assertions.assertEquals(1, result.size());
+            ObjectNode contentNode = (ObjectNode) result.get(0);
+            ObjectNode writerParam = (ObjectNode) contentNode.get("writer").get("parameter");
+            Assertions.assertTrue(writerParam.has("batchSize"));
+            Assertions.assertEquals(1024, writerParam.get("batchSize").asInt());
+        }
+    }
+
     private String getJsonString() {
         return "{\n" +
                 "  \"job\": {\n" +
diff --git a/dolphinscheduler-ui/src/locales/en_US/project.ts b/dolphinscheduler-ui/src/locales/en_US/project.ts
index 1e8f23bba5..40e553d4dd 100644
--- a/dolphinscheduler-ui/src/locales/en_US/project.ts
+++ b/dolphinscheduler-ui/src/locales/en_US/project.ts
@@ -648,6 +648,8 @@ export default {
     datax_target_database_pre_sql: 'Pre SQL Statement',
     datax_target_database_post_sql: 'Post SQL Statement',
     datax_non_query_sql_tips: 'Please enter the non-query sql statement',
+    datax_writer_batch_size: 'Writer Batch Size',
+    datax_writer_batch_size_tips: '0 or empty = default 2048',
     datax_job_speed_byte: 'Speed(Byte count)',
     datax_job_speed_byte_info: '(0 means unlimited)',
     datax_job_speed_record: 'Speed(Record count)',
diff --git a/dolphinscheduler-ui/src/locales/zh_CN/project.ts b/dolphinscheduler-ui/src/locales/zh_CN/project.ts
index 8876969f72..abaa84a34b 100644
--- a/dolphinscheduler-ui/src/locales/zh_CN/project.ts
+++ b/dolphinscheduler-ui/src/locales/zh_CN/project.ts
@@ -629,6 +629,8 @@ export default {
     datax_target_database_pre_sql: '目标库前置SQL',
     datax_target_database_post_sql: '目标库后置SQL',
     datax_non_query_sql_tips: '请输入非查询SQL语句',
+    datax_writer_batch_size: 'Writer 批量大小',
+    datax_writer_batch_size_tips: '0 或留空为默认值 2048',
     datax_job_speed_byte: '限流(字节数)',
     datax_job_speed_byte_info: '(KB,0代表不限制)',
     datax_job_speed_record: '限流(记录数)',
diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datax.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datax.ts
index cc38bca9b4..8b5c3ae934 100644
--- a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datax.ts
+++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datax.ts
@@ -254,6 +254,18 @@ export function useDataX(model: { [field: string]: any }): IJsonItem[] {
         autosize: { minRows: 1 }
       }
     },
+    {
+      type: 'input-number',
+      field: 'batchSize',
+      name: t('project.node.datax_writer_batch_size'),
+      span: otherStatementSpan,
+      props: {
+        min: 0,
+        step: 1024,
+        placeholder: t('project.node.datax_writer_batch_size_tips')
+      },
+      value: null
+    },
     {
       type: 'select',
       field: 'jobSpeedByte',