You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by zi...@apache.org on 2022/07/26 09:41:25 UTC
[dolphinscheduler] branch dev updated: [Feature-10837][Task Plugin] Sqoop Task Add --split-by Parameter (#10960)
This is an automated email from the ASF dual-hosted git repository.
zihaoxiang pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new cdfe115247 [Feature-10837][Task Plugin] Sqoop Task Add --split-by Parameter (#10960)
cdfe115247 is described below
commit cdfe115247c9cfba2c263fc743be50cf9e768b93
Author: Assert <42...@users.noreply.github.com>
AuthorDate: Tue Jul 26 17:41:13 2022 +0800
[Feature-10837][Task Plugin] Sqoop Task Add --split-by Parameter (#10960)
* [Feature-10837][Task Plugin] Sqoop Task Add --split-by Parameter
* checkstyle
---
.../plugin/task/sqoop/SqoopConstants.java | 2 ++
.../task/sqoop/generator/CommonGenerator.java | 4 +++
.../task/sqoop/parameter/SqoopParameters.java | 30 +++++++++++++++-------
dolphinscheduler-ui/src/locales/en_US/project.ts | 2 ++
dolphinscheduler-ui/src/locales/zh_CN/project.ts | 2 ++
.../task/components/node/fields/use-sqoop.ts | 13 +++++++++-
.../projects/task/components/node/format-data.ts | 1 +
.../task/components/node/tasks/use-sqoop.ts | 1 +
.../views/projects/task/components/node/types.ts | 1 +
9 files changed, 46 insertions(+), 10 deletions(-)
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopConstants.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopConstants.java
index e5ed8a1434..002f06bf9c 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopConstants.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopConstants.java
@@ -30,6 +30,8 @@ public final class SqoopConstants {
public static final String LINES_TERMINATED_BY = "--lines-terminated-by";
public static final String FIELD_NULL_PLACEHOLDER = "--null-non-string 'NULL' --null-string 'NULL'";
+ public static final String SPLIT_BY = "--split-by";
+
//sqoop db
public static final String DB_CONNECT = "--connect";
public static final String DB_USERNAME = "--username";
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/CommonGenerator.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/CommonGenerator.java
index d1e13ae096..5cae89e7a4 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/CommonGenerator.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/CommonGenerator.java
@@ -79,6 +79,10 @@ public class CommonGenerator {
if (sqoopParameters.getConcurrency() > 0) {
commonSb.append(SPACE).append(SqoopConstants.SQOOP_PARALLELISM)
.append(SPACE).append(sqoopParameters.getConcurrency());
+ if (sqoopParameters.getConcurrency() > 1) {
+ commonSb.append(SPACE).append(SqoopConstants.SPLIT_BY)
+ .append(SPACE).append(sqoopParameters.getSplitBy());
+ }
}
} catch (Exception e) {
logger.error(String.format("Sqoop task general param build failed: [%s]", e.getMessage()));
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/parameter/SqoopParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/parameter/SqoopParameters.java
index b6019aa8ce..3d88b288f0 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/parameter/SqoopParameters.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/parameter/SqoopParameters.java
@@ -35,7 +35,7 @@ import java.util.Objects;
/**
* sqoop parameters
*/
-public class SqoopParameters extends AbstractParameters {
+public class SqoopParameters extends AbstractParameters {
/**
* sqoop job type:
@@ -62,6 +62,10 @@ public class SqoopParameters extends AbstractParameters {
* concurrency
*/
private int concurrency;
+ /**
+ * split by
+ */
+ private String splitBy;
/**
* source type
*/
@@ -105,6 +109,14 @@ public class SqoopParameters extends AbstractParameters {
this.concurrency = concurrency;
}
+ public String getSplitBy() {
+ return splitBy;
+ }
+
+ public void setSplitBy(String splitBy) {
+ this.splitBy = splitBy;
+ }
+
public String getSourceType() {
return sourceType;
}
@@ -188,16 +200,16 @@ public class SqoopParameters extends AbstractParameters {
if (SqoopJobType.TEMPLATE.getDescp().equals(jobType)) {
sqoopParamsCheck = StringUtils.isEmpty(customShell)
- && StringUtils.isNotEmpty(modelType)
- && StringUtils.isNotEmpty(jobName)
- && concurrency != 0
- && StringUtils.isNotEmpty(sourceType)
- && StringUtils.isNotEmpty(targetType)
- && StringUtils.isNotEmpty(sourceParams)
- && StringUtils.isNotEmpty(targetParams);
+ && StringUtils.isNotEmpty(modelType)
+ && StringUtils.isNotEmpty(jobName)
+ && concurrency != 0
+ && StringUtils.isNotEmpty(sourceType)
+ && StringUtils.isNotEmpty(targetType)
+ && StringUtils.isNotEmpty(sourceParams)
+ && StringUtils.isNotEmpty(targetParams);
} else if (SqoopJobType.CUSTOM.getDescp().equals(jobType)) {
sqoopParamsCheck = StringUtils.isNotEmpty(customShell)
- && StringUtils.isEmpty(jobName);
+ && StringUtils.isEmpty(jobName);
}
return sqoopParamsCheck;
diff --git a/dolphinscheduler-ui/src/locales/en_US/project.ts b/dolphinscheduler-ui/src/locales/en_US/project.ts
index dc7257420c..7f468e13d5 100644
--- a/dolphinscheduler-ui/src/locales/en_US/project.ts
+++ b/dolphinscheduler-ui/src/locales/en_US/project.ts
@@ -497,6 +497,8 @@ export default {
allow_insert: 'AllowInsert',
concurrency: 'Concurrency',
concurrency_tips: 'Please enter Concurrency',
+ concurrency_column: 'Concurrency Column',
+ concurrency_column_tips: 'Please enter Concurrency Column',
sea_tunnel_master: 'Master',
sea_tunnel_master_url: 'Master URL',
sea_tunnel_queue: 'Queue',
diff --git a/dolphinscheduler-ui/src/locales/zh_CN/project.ts b/dolphinscheduler-ui/src/locales/zh_CN/project.ts
index 5be44bf3c2..6723442962 100644
--- a/dolphinscheduler-ui/src/locales/zh_CN/project.ts
+++ b/dolphinscheduler-ui/src/locales/zh_CN/project.ts
@@ -491,6 +491,8 @@ export default {
allow_insert: '无更新便插入',
concurrency: '并发度',
concurrency_tips: '请输入并发度',
+ concurrency_column: '并发键',
+ concurrency_column_tips: '请输入并发键',
sea_tunnel_master: 'Master',
sea_tunnel_master_url: 'Master URL',
sea_tunnel_queue: '队列',
diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-sqoop.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-sqoop.ts
index cb95fe40a0..20db90747c 100644
--- a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-sqoop.ts
+++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-sqoop.ts
@@ -24,6 +24,8 @@ export function useSqoop(model: { [field: string]: any }): IJsonItem[] {
const customSpan = computed(() => (model.isCustomTask ? 24 : 0))
const unCustomSpan = computed(() => (model.isCustomTask ? 0 : 24))
+ const unCustomHalfSpan = computed(() => (model.isCustomTask ? 0 : 12))
+
return [
{
type: 'switch',
@@ -75,12 +77,21 @@ export function useSqoop(model: { [field: string]: any }): IJsonItem[] {
type: 'input-number',
field: 'concurrency',
name: t('project.node.concurrency'),
- span: unCustomSpan,
+ span: unCustomHalfSpan,
props: {
placeholder: t('project.node.concurrency_tips'),
min: 1
}
},
+ {
+ type: 'input',
+ field: 'splitBy',
+ name: t('project.node.concurrency_column'),
+ span: unCustomHalfSpan,
+ props: {
+ placeholder: t('project.node.concurrency_column_tips')
+ }
+ },
{
type: 'editor',
field: 'customShell',
diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts
index 1d719a8e37..7feda203fd 100644
--- a/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts
+++ b/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts
@@ -88,6 +88,7 @@ export function formatParams(data: INodeData): {
taskParams.hadoopCustomParams = data.hadoopCustomParams
taskParams.sqoopAdvancedParams = data.sqoopAdvancedParams
taskParams.concurrency = data.concurrency
+ taskParams.splitBy = data.splitBy
taskParams.modelType = data.modelType
taskParams.sourceType = data.sourceType
taskParams.targetType = data.targetType
diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-sqoop.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-sqoop.ts
index 50220de931..5c8cd68705 100644
--- a/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-sqoop.ts
+++ b/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-sqoop.ts
@@ -65,6 +65,7 @@ export function useSqoop({
targetHiveDropDelimiter: false,
targetHiveOverWrite: true,
concurrency: 1,
+ splitBy: '',
timeoutNotifyStrategy: ['WARN']
} as INodeData)
diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts
index 9878258338..912f4b2132 100644
--- a/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts
+++ b/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts
@@ -265,6 +265,7 @@ interface ITaskParams {
hadoopCustomParams?: ILocalParam[]
sqoopAdvancedParams?: ILocalParam[]
concurrency?: number
+ splitBy?: string
modelType?: ModelType
sourceType?: SourceType
targetType?: SourceType