You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by zi...@apache.org on 2026/04/30 06:45:36 UTC
(dolphinscheduler) branch dev updated: [Improvement-16754][DataX] Support DataX writer parameter batchSize (#18192)
This is an automated email from the ASF dual-hosted git repository.
SbloodyS pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new 16dbda94c8 [Improvement-16754][DataX] Support DataX writer parameter batchSize (#18192)
16dbda94c8 is described below
commit 16dbda94c8eb2d2c904096608d9979d5db7a822f
Author: leocook <le...@163.com>
AuthorDate: Thu Apr 30 14:45:30 2026 +0800
[Improvement-16754][DataX] Support DataX writer parameter batchSize (#18192)
---
.../plugin/task/datax/DataxParameters.java | 6 ++
.../plugin/task/datax/DataxTask.java | 4 ++
.../plugin/task/datax/DataxParametersTest.java | 14 +++++
.../plugin/task/datax/DataxTaskTest.java | 70 ++++++++++++++++++++++
dolphinscheduler-ui/src/locales/en_US/project.ts | 2 +
dolphinscheduler-ui/src/locales/zh_CN/project.ts | 2 +
.../task/components/node/fields/use-datax.ts | 12 ++++
7 files changed, 110 insertions(+)
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParameters.java
index 9d9722bc6f..07c20c3203 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParameters.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParameters.java
@@ -101,6 +101,11 @@ public class DataxParameters extends AbstractParameters {
*/
private int xmx;
+ /**
+ * writer batch size for DataX
+ */
+ private int batchSize;
+
private List<ResourceInfo> resourceList;
@Override
@@ -138,6 +143,7 @@ public class DataxParameters extends AbstractParameters {
", jobChannel=" + jobChannel +
", xms=" + xms +
", xmx=" + xmx +
+ ", batchSize=" + batchSize +
", resourceList=" + JSONUtils.toJsonString(resourceList) +
'}';
}
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java
index a8c39a3a50..e82666d530 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java
@@ -282,6 +282,10 @@ public class DataxTask extends AbstractTask {
}
}
+ if (dataXParameters.getBatchSize() > 0) {
+ writerParam.put("batchSize", dataXParameters.getBatchSize());
+ }
+
ObjectNode writer = JSONUtils.createObjectNode();
writer.put("name", DataxUtils.getWriterPluginName(dataxTaskExecutionContext.getTargetType()));
writer.set("parameter", writerParam);
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParametersTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParametersTest.java
index a20b5a1695..736d0aab94 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParametersTest.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParametersTest.java
@@ -91,12 +91,26 @@ public class DataxParametersTest {
+ "jobChannel=1, "
+ "xms=0, "
+ "xmx=-100, "
+ + "batchSize=0, "
+ "resourceList=[{\"id\":null,\"resourceName\":\"/hdfs.keytab\",\"res\":null}]"
+ "}";
Assertions.assertEquals(expected, dataxParameters.toString());
}
+ @Test
+ public void testBatchSize() {
+ DataxParameters dataxParameters = new DataxParameters();
+ dataxParameters.setBatchSize(0);
+ Assertions.assertEquals(0, dataxParameters.getBatchSize());
+
+ dataxParameters.setBatchSize(2048);
+ Assertions.assertEquals(2048, dataxParameters.getBatchSize());
+
+ dataxParameters.setBatchSize(65536);
+ Assertions.assertEquals(65536, dataxParameters.getBatchSize());
+ }
+
public String loadJvmEnvTest(DataxParameters dataXParameters) {
int xms = dataXParameters.getXms() < 1 ? 1 : dataXParameters.getXms();
int xmx = dataXParameters.getXmx() < 1 ? 1 : dataXParameters.getXmx();
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskTest.java
index 3b1dd82c92..17687e15c1 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskTest.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskTest.java
@@ -27,6 +27,7 @@ import static org.mockito.Mockito.when;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider;
+import org.apache.dolphinscheduler.plugin.datasource.api.utils.DataSourceUtils;
import org.apache.dolphinscheduler.plugin.task.api.ShellCommandExecutor;
import org.apache.dolphinscheduler.plugin.task.api.TaskCallBack;
import org.apache.dolphinscheduler.plugin.task.api.TaskException;
@@ -45,6 +46,7 @@ import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.lang.reflect.Field;
+import java.lang.reflect.Method;
import java.nio.file.Files;
import java.sql.Connection;
import java.sql.PreparedStatement;
@@ -52,6 +54,7 @@ import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.Assertions;
@@ -62,6 +65,8 @@ import org.mockito.MockedStatic;
import org.mockito.Mockito;
import org.mockito.junit.jupiter.MockitoExtension;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
@ExtendWith(MockitoExtension.class)
public class DataxTaskTest {
@@ -273,6 +278,71 @@ public class DataxTaskTest {
return taskExecutionContext;
}
+ @Test
+ public void testBuildDataxJobContentJsonWithBatchSize() throws Exception {
+ // set batchSize > 0 via reflection
+ Field dataXParametersField = DataxTask.class.getDeclaredField("dataXParameters");
+ dataXParametersField.setAccessible(true);
+ DataxParameters params = (DataxParameters) dataXParametersField.get(dataxTask);
+ params.setBatchSize(1024);
+ params.setDsType("MYSQL");
+ params.setDtType("MYSQL");
+
+ // set dataxTaskExecutionContext via reflection
+ DataxTaskExecutionContext ctx = new DataxTaskExecutionContext();
+ ctx.setSourcetype(DbType.MYSQL);
+ ctx.setTargetType(DbType.MYSQL);
+ ctx.setSourceConnectionParams(
+ "{\"user\":\"root\",\"password\":\"123456\",\"address\":\"jdbc:mysql://localhost:3306\"}");
+ ctx.setTargetConnectionParams(
+ "{\"user\":\"root\",\"password\":\"123456\",\"address\":\"jdbc:mysql://localhost:3306\"}");
+
+ Field ctxField = DataxTask.class.getDeclaredField("dataxTaskExecutionContext");
+ ctxField.setAccessible(true);
+ ctxField.set(dataxTask, ctx);
+
+ BaseConnectionParam mockConnParam = mock(BaseConnectionParam.class);
+ when(mockConnParam.getUser()).thenReturn("root");
+ when(mockConnParam.getPassword()).thenReturn("123456");
+ when(mockConnParam.getCompatibleMode()).thenReturn(null);
+
+ try (
+ MockedStatic<DataSourceUtils> mockedDataSourceUtils = mockStatic(DataSourceUtils.class);
+ MockedStatic<DataSourceClientProvider> mockedProvider = mockStatic(DataSourceClientProvider.class)) {
+
+ mockedDataSourceUtils
+ .when(() -> DataSourceUtils.buildConnectionParams(Mockito.any(DbType.class), Mockito.anyString()))
+ .thenReturn(mockConnParam);
+ mockedDataSourceUtils.when(() -> DataSourceUtils.getJdbcUrl(Mockito.any(DbType.class), Mockito.any()))
+ .thenReturn("jdbc:mysql://localhost:3306/test");
+
+ Connection connection = mock(Connection.class);
+ mockedProvider.when(() -> DataSourceClientProvider.getAdHocConnection(Mockito.any(), Mockito.any()))
+ .thenReturn(connection);
+
+ PreparedStatement stmt = mock(PreparedStatement.class);
+ when(connection.prepareStatement(anyString())).thenReturn(stmt);
+ ResultSetMetaData md = mock(ResultSetMetaData.class);
+ when(md.getColumnCount()).thenReturn(1);
+ when(md.getColumnLabel(eq(1))).thenReturn("col1");
+ ResultSet resultSet = mock(ResultSet.class);
+ when(resultSet.getMetaData()).thenReturn(md);
+ when(stmt.executeQuery()).thenReturn(resultSet);
+
+ Method method = DataxTask.class.getDeclaredMethod("buildDataxJobContentJson");
+ method.setAccessible(true);
+ Object invokeResult = method.invoke(dataxTask);
+ Assertions.assertNotNull(invokeResult);
+ List<?> result = (List<?>) invokeResult;
+
+ Assertions.assertEquals(1, result.size());
+ ObjectNode contentNode = (ObjectNode) result.get(0);
+ ObjectNode writerParam = (ObjectNode) contentNode.get("writer").get("parameter");
+ Assertions.assertTrue(writerParam.has("batchSize"));
+ Assertions.assertEquals(1024, writerParam.get("batchSize").asInt());
+ }
+ }
+
private String getJsonString() {
return "{\n" +
" \"job\": {\n" +
diff --git a/dolphinscheduler-ui/src/locales/en_US/project.ts b/dolphinscheduler-ui/src/locales/en_US/project.ts
index 1e8f23bba5..40e553d4dd 100644
--- a/dolphinscheduler-ui/src/locales/en_US/project.ts
+++ b/dolphinscheduler-ui/src/locales/en_US/project.ts
@@ -648,6 +648,8 @@ export default {
datax_target_database_pre_sql: 'Pre SQL Statement',
datax_target_database_post_sql: 'Post SQL Statement',
datax_non_query_sql_tips: 'Please enter the non-query sql statement',
+ datax_writer_batch_size: 'Writer Batch Size',
+ datax_writer_batch_size_tips: '0 or empty = default 2048',
datax_job_speed_byte: 'Speed(Byte count)',
datax_job_speed_byte_info: '(0 means unlimited)',
datax_job_speed_record: 'Speed(Record count)',
diff --git a/dolphinscheduler-ui/src/locales/zh_CN/project.ts b/dolphinscheduler-ui/src/locales/zh_CN/project.ts
index 8876969f72..abaa84a34b 100644
--- a/dolphinscheduler-ui/src/locales/zh_CN/project.ts
+++ b/dolphinscheduler-ui/src/locales/zh_CN/project.ts
@@ -629,6 +629,8 @@ export default {
datax_target_database_pre_sql: '目标库前置SQL',
datax_target_database_post_sql: '目标库后置SQL',
datax_non_query_sql_tips: '请输入非查询SQL语句',
+ datax_writer_batch_size: 'Writer 批量大小',
+ datax_writer_batch_size_tips: '0 或留空为默认值 2048',
datax_job_speed_byte: '限流(字节数)',
datax_job_speed_byte_info: '(KB,0代表不限制)',
datax_job_speed_record: '限流(记录数)',
diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datax.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datax.ts
index cc38bca9b4..8b5c3ae934 100644
--- a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datax.ts
+++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datax.ts
@@ -254,6 +254,18 @@ export function useDataX(model: { [field: string]: any }): IJsonItem[] {
autosize: { minRows: 1 }
}
},
+ {
+ type: 'input-number',
+ field: 'batchSize',
+ name: t('project.node.datax_writer_batch_size'),
+ span: otherStatementSpan,
+ props: {
+ min: 0,
+ step: 1024,
+ placeholder: t('project.node.datax_writer_batch_size_tips')
+ },
+ value: null
+ },
{
type: 'select',
field: 'jobSpeedByte',