You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by so...@apache.org on 2022/03/25 11:46:06 UTC
[dolphinscheduler] branch dev updated: [Fix][UI Next][V1.0.0-Alpha] Fix several problems in SQOOP. (#9194)
This is an automated email from the ASF dual-hosted git repository.
songjian pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new ff93dda [Fix][UI Next][V1.0.0-Alpha] Fix several problems in SQOOP. (#9194)
ff93dda is described below
commit ff93dda6c5d5673a02e8104658f81ea7a060f74b
Author: Amy0104 <97...@users.noreply.github.com>
AuthorDate: Fri Mar 25 19:45:59 2022 +0800
[Fix][UI Next][V1.0.0-Alpha] Fix several problems in SQOOP. (#9194)
---
.../src/components/form/get-elements-by-json.ts | 4 +-
.../node/fields/use-sqoop-source-type.ts | 103 ++++++++++-----------
.../node/fields/use-sqoop-target-type.ts | 85 +++++++++--------
.../task/components/node/fields/use-sqoop.ts | 12 +--
.../projects/task/components/node/format-data.ts | 30 ++++--
5 files changed, 129 insertions(+), 105 deletions(-)
diff --git a/dolphinscheduler-ui-next/src/components/form/get-elements-by-json.ts b/dolphinscheduler-ui-next/src/components/form/get-elements-by-json.ts
index 753bdb1..b25ed5f 100644
--- a/dolphinscheduler-ui-next/src/components/form/get-elements-by-json.ts
+++ b/dolphinscheduler-ui-next/src/components/form/get-elements-by-json.ts
@@ -31,7 +31,7 @@ export default function getElementByJson(
const elements: IFormItem[] = []
for (const item of json) {
const mergedItem = isFunction(item) ? item() : item
- const { name, value, field, children, validate, ...rest } = mergedItem
+ const { name, value, field, validate, ...rest } = mergedItem
if (value || value === 0) {
fields[field] = value
initialValues[field] = value
@@ -41,7 +41,7 @@ export default function getElementByJson(
showLabel: !!name,
...omit(rest, ['type', 'props', 'options']),
label: name,
- path: !children ? field : '',
+ path: field,
widget: () => getField(item, fields, rules),
span: toRef(mergedItem, 'span') as Ref<number>
}
diff --git a/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-sqoop-source-type.ts b/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-sqoop-source-type.ts
index 5ac413a..f742c77 100644
--- a/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-sqoop-source-type.ts
+++ b/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-sqoop-source-type.ts
@@ -15,50 +15,36 @@
* limitations under the License.
*/
-import { ref, h, watch, computed, unref } from 'vue'
+import { ref, h, watch, Ref } from 'vue'
import { useI18n } from 'vue-i18n'
import { useDatasource } from './use-sqoop-datasource'
import { useCustomParams } from '.'
import styles from '../index.module.scss'
import type { IJsonItem, IOption, ModelType } from '../types'
-export function useSourceType(model: { [field: string]: any }): IJsonItem[] {
+export function useSourceType(
+ model: { [field: string]: any },
+ unCustomSpan: Ref<number>
+): IJsonItem[] {
const { t } = useI18n()
- const unCustomSpan = computed(() => (model.isCustomTask ? 0 : 24))
- const tableSpan = computed(() =>
- !model.isCustomTask &&
- model.sourceType === 'MYSQL' &&
- model.srcQueryType === '0'
- ? 24
- : 0
- )
- const editorSpan = computed(() =>
- !model.isCustomTask &&
- model.sourceType === 'MYSQL' &&
- model.srcQueryType === '1'
- ? 24
- : 0
- )
- const columnSpan = computed(() =>
- !model.isCustomTask &&
- model.sourceType === 'MYSQL' &&
- model.srcColumnType === '1'
- ? 24
- : 0
- )
- const mysqlSpan = computed(() =>
- !model.isCustomTask && model.sourceType === 'MYSQL' ? 24 : 0
- )
- const hiveSpan = computed(() =>
- !model.isCustomTask && model.sourceType === 'HIVE' ? 24 : 0
- )
- const hdfsSpan = computed(() =>
- !model.isCustomTask && model.sourceType === 'HDFS' ? 24 : 0
- )
- const datasourceSpan = computed(() =>
- !model.isCustomTask && model.sourceType === 'MYSQL' ? 12 : 0
- )
-
+ const mysqlSpan = ref(24)
+ const tableSpan = ref(0)
+ const editorSpan = ref(24)
+ const columnSpan = ref(0)
+ const hiveSpan = ref(0)
+ const hdfsSpan = ref(0)
+ const datasourceSpan = ref(0)
+ const resetSpan = () => {
+ mysqlSpan.value =
+ unCustomSpan.value && model.sourceType === 'MYSQL' ? 24 : 0
+ tableSpan.value = mysqlSpan.value && model.srcQueryType === '0' ? 24 : 0
+ editorSpan.value = mysqlSpan.value && model.srcQueryType === '1' ? 24 : 0
+ columnSpan.value = tableSpan.value && model.srcColumnType === '1' ? 24 : 0
+ hiveSpan.value = unCustomSpan.value && model.sourceType === 'HIVE' ? 24 : 0
+ hdfsSpan.value = unCustomSpan.value && model.sourceType === 'HDFS' ? 24 : 0
+ datasourceSpan.value =
+ unCustomSpan.value && model.sourceType === 'MYSQL' ? 12 : 0
+ }
const sourceTypes = ref([
{
label: 'MYSQL',
@@ -108,7 +94,20 @@ export function useSourceType(model: { [field: string]: any }): IJsonItem[] {
() => model.modelType,
(modelType: ModelType) => {
sourceTypes.value = getSourceTypesByModelType(modelType)
- model.sourceType = sourceTypes.value[0].value
+ if (!sourceTypes.value.find((type) => model.sourceType === type.value)) {
+ model.sourceType = sourceTypes.value[0].value
+ }
+ }
+ )
+ watch(
+ () => [
+ unCustomSpan.value,
+ model.sourceType,
+ model.srcQueryType,
+ model.srcColumnType
+ ],
+ () => {
+ resetSpan()
}
)
@@ -162,9 +161,9 @@ export function useSourceType(model: { [field: string]: any }): IJsonItem[] {
},
validate: {
trigger: ['input', 'blur'],
- required: !!unref(tableSpan),
+ required: true,
validator(validate, value) {
- if (!!unref(tableSpan) && !value) {
+ if (tableSpan.value && !value) {
return new Error(t('project.node.table_tips'))
}
}
@@ -190,9 +189,9 @@ export function useSourceType(model: { [field: string]: any }): IJsonItem[] {
},
validate: {
trigger: ['input', 'blur'],
- required: !!unref(columnSpan),
+ required: true,
validator(validate, value) {
- if (!!unref(columnSpan) && !value) {
+ if (!!columnSpan.value && !value) {
return new Error(t('project.node.column_tips'))
}
}
@@ -208,9 +207,9 @@ export function useSourceType(model: { [field: string]: any }): IJsonItem[] {
},
validate: {
trigger: ['blur', 'input'],
- required: !!unref(hiveSpan),
+ required: true,
validator(validate, value) {
- if (!!unref(hiveSpan) && !value) {
+ if (hiveSpan.value && !value) {
return new Error(t('project.node.database_tips'))
}
}
@@ -226,9 +225,9 @@ export function useSourceType(model: { [field: string]: any }): IJsonItem[] {
},
validate: {
trigger: ['blur', 'input'],
- required: !!unref(hiveSpan),
+ required: true,
validator(validate, value) {
- if (!!unref(hiveSpan) && !value) {
+ if (hiveSpan.value && !value) {
return new Error(t('project.node.hive_table_tips'))
}
}
@@ -262,9 +261,9 @@ export function useSourceType(model: { [field: string]: any }): IJsonItem[] {
},
validate: {
trigger: ['blur', 'input'],
- required: !!unref(hdfsSpan),
+ required: true,
validator(validate, value) {
- if (!!unref(hdfsSpan) && !value) {
+ if (hdfsSpan.value && !value) {
return new Error(t('project.node.export_dir_tips'))
}
}
@@ -277,9 +276,9 @@ export function useSourceType(model: { [field: string]: any }): IJsonItem[] {
span: editorSpan,
validate: {
trigger: ['blur', 'input'],
- required: !!unref(editorSpan),
+ required: true,
validator(validate, value) {
- if (!!unref(editorSpan) && !value) {
+ if (editorSpan.value && !value) {
return new Error(t('project.node.sql_statement_tips'))
}
}
@@ -290,14 +289,14 @@ export function useSourceType(model: { [field: string]: any }): IJsonItem[] {
field: 'mapColumnHive',
name: 'map_column_hive',
isSimple: true,
- span: editorSpan
+ span: mysqlSpan
}),
...useCustomParams({
model,
field: 'mapColumnJava',
name: 'map_column_java',
isSimple: true,
- span: editorSpan
+ span: mysqlSpan
})
]
}
diff --git a/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-sqoop-target-type.ts b/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-sqoop-target-type.ts
index a995d8f..114db2b 100644
--- a/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-sqoop-target-type.ts
+++ b/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-sqoop-target-type.ts
@@ -15,32 +15,32 @@
* limitations under the License.
*/
-import { ref, h, watch, computed, unref } from 'vue'
+import { ref, h, watch, Ref } from 'vue'
import { useI18n } from 'vue-i18n'
import { useDatasource } from './use-sqoop-datasource'
import styles from '../index.module.scss'
import type { IJsonItem, IOption, SourceType } from '../types'
-export function useTargetType(model: { [field: string]: any }): IJsonItem[] {
+export function useTargetType(
+ model: { [field: string]: any },
+ unCustomSpan: Ref<number>
+): IJsonItem[] {
const { t } = useI18n()
- const unCustomSpan = computed(() => (model.isCustomTask ? 0 : 24))
- const hiveSpan = computed(() =>
- !model.isCustomTask && model.targetType === 'HIVE' ? 24 : 0
- )
- const hdfsSpan = computed(() =>
- !model.isCustomTask && model.targetType === 'HDFS' ? 24 : 0
- )
- const mysqlSpan = computed(() =>
- !model.isCustomTask && model.targetType === 'MYSQL' ? 24 : 0
- )
- const dataSourceSpan = computed(() =>
- !model.isCustomTask && model.targetType === 'MYSQL' ? 12 : 0
- )
- const updateSpan = computed(() =>
- !model.isCustomTask && model.targetType === 'MYSQL' && model.isUpdate
- ? 24
- : 0
- )
+ const hiveSpan = ref(24)
+ const hdfsSpan = ref(0)
+ const mysqlSpan = ref(0)
+ const dataSourceSpan = ref(0)
+ const updateSpan = ref(0)
+
+ const resetSpan = () => {
+ hiveSpan.value = unCustomSpan.value && model.targetType === 'HIVE' ? 24 : 0
+ hdfsSpan.value = unCustomSpan.value && model.targetType === 'HDFS' ? 24 : 0
+ mysqlSpan.value =
+ unCustomSpan.value && model.targetType === 'MYSQL' ? 24 : 0
+ dataSourceSpan.value =
+ unCustomSpan.value && model.targetType === 'MYSQL' ? 12 : 0
+ updateSpan.value = mysqlSpan.value && model.targetMysqlIsUpdate ? 24 : 0
+ }
const targetTypes = ref([
{
@@ -69,12 +69,12 @@ export function useTargetType(model: { [field: string]: any }): IJsonItem[] {
}
return [
{
- label: 'HDFS',
- value: 'HDFS'
- },
- {
label: 'HIVE',
value: 'HIVE'
+ },
+ {
+ label: 'HDFS',
+ value: 'HDFS'
}
]
case 'HDFS':
@@ -88,12 +88,12 @@ export function useTargetType(model: { [field: string]: any }): IJsonItem[] {
default:
return [
{
- label: 'HDFS',
- value: 'HDFS'
- },
- {
label: 'HIVE',
value: 'HIVE'
+ },
+ {
+ label: 'HDFS',
+ value: 'HDFS'
}
]
}
@@ -103,7 +103,16 @@ export function useTargetType(model: { [field: string]: any }): IJsonItem[] {
() => [model.sourceType, model.srcQueryType],
([sourceType, srcQueryType]) => {
targetTypes.value = getTargetTypesBySourceType(sourceType, srcQueryType)
- model.targetType = targetTypes.value[0].value
+ if (!targetTypes.value.find((type) => model.targetType === type.value)) {
+ model.targetType = targetTypes.value[0].value
+ }
+ }
+ )
+
+ watch(
+ () => [unCustomSpan.value, model.targetType, model.targetMysqlIsUpdate],
+ () => {
+ resetSpan()
}
)
@@ -135,9 +144,9 @@ export function useTargetType(model: { [field: string]: any }): IJsonItem[] {
},
validate: {
trigger: ['blur', 'input'],
- required: !!unref(hiveSpan),
+ required: true,
validator(validate, value) {
- if (!!unref(hiveSpan) && !value) {
+ if (hiveSpan.value && !value) {
return new Error(t('project.node.database_tips'))
}
}
@@ -146,16 +155,16 @@ export function useTargetType(model: { [field: string]: any }): IJsonItem[] {
{
type: 'input',
field: 'targetHiveTable',
- name: t('project.node.database'),
+ name: t('project.node.table'),
span: hiveSpan,
props: {
placeholder: t('project.node.table')
},
validate: {
trigger: ['blur', 'input'],
- required: !!unref(hiveSpan),
+ required: true,
validator(rule, value) {
- if (!!unref(hiveSpan) && !value) {
+ if (hiveSpan.value && !value) {
return new Error(t('project.node.hive_table_tips'))
}
}
@@ -225,9 +234,9 @@ export function useTargetType(model: { [field: string]: any }): IJsonItem[] {
},
validate: {
trigger: ['blur', 'input'],
- required: !!unref(hdfsSpan),
+ required: true,
validator(rule, value) {
- if (!!unref(hdfsSpan) && !value) {
+ if (hdfsSpan.value && !value) {
return new Error(t('project.node.target_dir_tips'))
}
}
@@ -287,9 +296,9 @@ export function useTargetType(model: { [field: string]: any }): IJsonItem[] {
},
validate: {
trigger: ['blur', 'input'],
- required: !!unref(mysqlSpan),
+ required: true,
validator(validate, value) {
- if (!!unref(mysqlSpan) && !value) {
+ if (mysqlSpan.value && !value) {
return new Error(t('project.node.table_tips'))
}
}
diff --git a/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-sqoop.ts b/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-sqoop.ts
index 39040f6..44125bf 100644
--- a/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-sqoop.ts
+++ b/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-sqoop.ts
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-import { computed, unref } from 'vue'
+import { computed } from 'vue'
import { useI18n } from 'vue-i18n'
import { useCustomParams, useSourceType, useTargetType } from '.'
import type { IJsonItem, ModelType } from '../types'
@@ -40,7 +40,7 @@ export function useSqoop(model: { [field: string]: any }): IJsonItem[] {
},
validate: {
trigger: ['input', 'blur'],
- required: !model.isCustomTask,
+ required: true,
validator(validate, value) {
if (!model.isCustomTask && !value) {
return new Error(t('project.node.sqoop_job_name_tips'))
@@ -69,8 +69,8 @@ export function useSqoop(model: { [field: string]: any }): IJsonItem[] {
isSimple: true,
span: unCustomSpan
}),
- ...useSourceType(model),
- ...useTargetType(model),
+ ...useSourceType(model, unCustomSpan),
+ ...useTargetType(model, unCustomSpan),
{
type: 'input-number',
field: 'concurrency',
@@ -87,9 +87,9 @@ export function useSqoop(model: { [field: string]: any }): IJsonItem[] {
span: customSpan,
validate: {
trigger: ['input', 'trigger'],
- required: !!unref(customSpan),
+ required: true,
validator(rule, value) {
- if (!!unref(customSpan) && !value) {
+ if (customSpan.value && !value) {
return new Error(t('project.node.custom_script'))
}
}
diff --git a/dolphinscheduler-ui-next/src/views/projects/task/components/node/format-data.ts b/dolphinscheduler-ui-next/src/views/projects/task/components/node/format-data.ts
index edc4337..4c2f770 100644
--- a/dolphinscheduler-ui-next/src/views/projects/task/components/node/format-data.ts
+++ b/dolphinscheduler-ui-next/src/views/projects/task/components/node/format-data.ts
@@ -390,15 +390,25 @@ export function formatModel(data: ITaskData) {
params.targetHiveTable = targetParams.hiveTable
params.targetHiveCreateTable = targetParams.createHiveTable
params.targetHiveDropDelimiter = targetParams.dropDelimiter
- params.targetHiveOverWrite = targetParams.hiveOverWrite
+ params.targetHiveOverWrite =
+ targetParams.hiveOverWrite === void 0 ? true : targetParams.hiveOverWrite
params.targetHiveTargetDir = targetParams.hiveTargetDir
params.targetHiveReplaceDelimiter = targetParams.replaceDelimiter
params.targetHivePartitionKey = targetParams.hivePartitionKey
params.targetHivePartitionValue = targetParams.hivePartitionValue
params.targetHdfsTargetPath = targetParams.targetPath
- params.targetHdfsDeleteTargetDir = targetParams.deleteTargetDir
- params.targetHdfsCompressionCodec = targetParams.compressionCodec
- params.targetHdfsFileType = targetParams.fileType
+ params.targetHdfsDeleteTargetDir =
+ targetParams.deleteTargetDir === void 0
+ ? true
+ : targetParams.deleteTargetDir
+ params.targetHdfsCompressionCodec =
+ targetParams.compressionCodec === void 0
+ ? 'snappy'
+ : targetParams.compressionCodec
+ params.targetHdfsFileType =
+ targetParams.fileType === void 0
+ ? '--as-avrodatafile'
+ : targetParams.fileType
params.targetHdfsFieldsTerminated = targetParams.fieldsTerminated
params.targetHdfsLinesTerminated = targetParams.linesTerminated
params.targetMysqlType = targetParams.targetType
@@ -409,7 +419,10 @@ export function formatModel(data: ITaskData) {
params.targetMysqlLinesTerminated = targetParams.linesTerminated
params.targetMysqlIsUpdate = targetParams.isUpdate
params.targetMysqlTargetUpdateKey = targetParams.targetUpdateKey
- params.targetMysqlUpdateMode = targetParams.targetUpdateMode
+ params.targetMysqlUpdateMode =
+ targetParams.targetUpdateMode === void 0
+ ? 'allowinsert'
+ : targetParams.targetUpdateMode
}
if (data.taskParams?.sourceParams) {
const sourceParams: ISqoopSourceParams = JSON.parse(
@@ -422,8 +435,8 @@ export function formatModel(data: ITaskData) {
params.srcQueryType = sourceParams.srcQueryType
params.sourceMysqlType = sourceParams.srcType
params.sourceMysqlDatasource = sourceParams.srcDatasource
- params.mapColumnHive = sourceParams.mapColumnHive
- params.mapColumnJava = sourceParams.mapColumnJava
+ params.mapColumnHive = sourceParams.mapColumnHive || []
+ params.mapColumnJava = sourceParams.mapColumnJava || []
params.sourceHdfsExportDir = sourceParams.exportDir
params.sourceHiveDatabase = sourceParams.hiveDatabase
params.sourceHiveTable = sourceParams.hiveTable
@@ -493,6 +506,9 @@ export function formatModel(data: ITaskData) {
if (data.taskParams?.customConfig !== void 0) {
params.customConfig = data.taskParams.customConfig === 1 ? true : false
}
+ if (data.taskParams?.jobType) {
+ params.isCustomTask = data.taskParams.jobType === 'CUSTOM'
+ }
return params
}