You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by GitBox <gi...@apache.org> on 2022/09/08 09:52:45 UTC
[GitHub] [dolphinscheduler] wen-hemin commented on a diff in pull request #11670: [Feature-11473][Task]Support test task
wen-hemin commented on code in PR #11670:
URL: https://github.com/apache/dolphinscheduler/pull/11670#discussion_r964744217
##########
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java:
##########
@@ -148,23 +148,25 @@ public Result queryDataSource(@ApiIgnore @RequestAttribute(value = Constants.SES
}
/**
- * query datasource by type
+ * query online/testDatasource by type
*
* @param loginUser login user
* @param type data source type
* @return data source list page
*/
@ApiOperation(value = "queryDataSourceList", notes = "QUERY_DATA_SOURCE_LIST_BY_TYPE_NOTES")
@ApiImplicitParams({
- @ApiImplicitParam(name = "type", value = "DB_TYPE", required = true, dataType = "DbType")
+ @ApiImplicitParam(name = "type", value = "DB_TYPE", required = true, dataType = "DbType"),
+ @ApiImplicitParam(name = "testFlag", value = "DB_TEST_FLAG", required = true, dataType = "DbTestFlag")
})
@GetMapping(value = "/list")
@ResponseStatus(HttpStatus.OK)
@ApiException(QUERY_DATASOURCE_ERROR)
@AccessLogAnnotation(ignoreRequestArgs = "loginUser")
public Result queryDataSourceList(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
- @RequestParam("type") DbType type) {
- Map<String, Object> result = dataSourceService.queryDataSourceList(loginUser, type.ordinal());
+ @RequestParam("type") DbType type,
+ @RequestParam("testFlag")int testFlag) {
Review Comment:
missing spaces
@RequestParam("testFlag") int testFlag
##########
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java:
##########
@@ -148,23 +148,25 @@ public Result queryDataSource(@ApiIgnore @RequestAttribute(value = Constants.SES
}
/**
- * query datasource by type
+ * query online/testDatasource by type
*
* @param loginUser login user
* @param type data source type
* @return data source list page
*/
@ApiOperation(value = "queryDataSourceList", notes = "QUERY_DATA_SOURCE_LIST_BY_TYPE_NOTES")
@ApiImplicitParams({
- @ApiImplicitParam(name = "type", value = "DB_TYPE", required = true, dataType = "DbType")
+ @ApiImplicitParam(name = "type", value = "DB_TYPE", required = true, dataType = "DbType"),
+ @ApiImplicitParam(name = "testFlag", value = "DB_TEST_FLAG", required = true, dataType = "DbTestFlag")
})
@GetMapping(value = "/list")
@ResponseStatus(HttpStatus.OK)
@ApiException(QUERY_DATASOURCE_ERROR)
@AccessLogAnnotation(ignoreRequestArgs = "loginUser")
public Result queryDataSourceList(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
- @RequestParam("type") DbType type) {
- Map<String, Object> result = dataSourceService.queryDataSourceList(loginUser, type.ordinal());
+ @RequestParam("type") DbType type,
+ @RequestParam("testFlag")int testFlag) {
+ Map<String, Object> result = dataSourceService.queryDataSourceList(loginUser, type.ordinal(),testFlag);
Review Comment:
missing spaces
type.ordinal(), testFlag
##########
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/builder/TaskExecutionContextBuilder.java:
##########
@@ -65,14 +65,15 @@ public TaskExecutionContextBuilder buildTaskInstanceRelatedInfo(TaskInstance tas
taskExecutionContext.setDelayTime(taskInstance.getDelayTime());
taskExecutionContext.setVarPool(taskInstance.getVarPool());
taskExecutionContext.setDryRun(taskInstance.getDryRun());
+ taskExecutionContext.setTestFlag(taskInstance.getTestFlag());
taskExecutionContext.setCurrentExecutionStatus(TaskExecutionStatus.SUBMITTED_SUCCESS);
taskExecutionContext.setCpuQuota(taskInstance.getCpuQuota());
taskExecutionContext.setMemoryMax(taskInstance.getMemoryMax());
taskExecutionContext.setAppIds(taskInstance.getAppLink());
return this;
}
- public TaskExecutionContextBuilder buildTaskDefinitionRelatedInfo(TaskDefinition taskDefinition) {
+ public TaskExecutionContextBuilder buildTaskDefinitionRelatedInfo(TaskDefinition taskDefinition, TaskInstance taskInstance) {
Review Comment:
Why not use taskDefinition ?
##########
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/BaseDataSourceParamDTO.java:
##########
@@ -117,6 +121,22 @@ public void setOther(Map<String, String> other) {
this.other = other;
}
+ public int getTestFlag() {
+ return testFlag;
+ }
+
+ public void setTestFlag(Integer testFlag) {
Review Comment:
Uniform Field Type
##########
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DataSource.java:
##########
@@ -75,6 +71,17 @@ public class DataSource {
*/
private Date updateTime;
+ /**
+ * test flag
+ */
+ protected int testFlag;
+
+ /**
+ * bind test data source id
+ */
+ @TableField(fill = FieldFill.INSERT_UPDATE)
Review Comment:
What is the function of annotation?
##########
dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/postgresql/dolphinscheduler_ddl.sql:
##########
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+delimiter d//
+
+
+
+CREATE OR REPLACE FUNCTION public.dolphin_update_metadata(
+ )
+ RETURNS character varying
+ LANGUAGE 'plpgsql'
+ COST 100
+ VOLATILE PARALLEL UNSAFE
+AS $BODY$
+DECLARE
+v_schema varchar;
+BEGIN
+ ---get schema name
+ v_schema =current_schema();
+
+
+
+--- add column
+EXECUTE 'ALTER TABLE ' || quote_ident(v_schema) ||'.t_ds_datasource ADD COLUMN IF NOT EXISTS test_flag int DEFAULT null ';
Review Comment:
Please refer to the 1.3.0_schema directory to upgrade the ddl
##########
dolphinscheduler-api/src/main/resources/i18n/messages_zh_CN.properties:
##########
@@ -14,347 +14,348 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-QUERY_SCHEDULE_LIST_NOTES=查询定时列表
Review Comment:
text encoding error
##########
dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecuteRunnable.java:
##########
@@ -274,4 +296,9 @@ protected void clearTaskExecPathIfNeeded() {
return task;
}
+ public boolean checkTaskHaveDataSourceInstance(String taskType) {
+ List<String> testableTaskTypeList = Arrays.asList("PROCEDURE", "SQL", "DATAX", "SQOOP", "DATA_QUALITY");
Review Comment:
This time only sql tasks are supported, other tasks are temporarily closed.
##########
dolphinscheduler-api/src/main/resources/i18n/messages.properties:
##########
@@ -21,7 +21,7 @@ PROCESS_INSTANCE_EXECUTOR_TAG=process instance executor related operation
RUN_PROCESS_INSTANCE_NOTES=run process instance
BATCH_RUN_PROCESS_INSTANCE_NOTES=batch run process instance
BATCH_EXECUTE_ACTION_TO_PROCESS_INSTANCE_NOTES=batch change execute state for process instance
-START_NODE_LIST=start node list（node name）
+START_NODE_LIST=start node list\uFF08node name\uFF09
Review Comment:
text encoding error
##########
dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/mysql/dolphinscheduler_ddl.sql:
##########
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+ALTER TABLE `t_ds_datasource` ADD COLUMN `test_flag` tinyint(4) DEFAULT NULL COMMENT 'test flag：0 normal, 1 testDataSource';
Review Comment:
Please refer to the 1.3.0_schema directory to upgrade the ddl
##########
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessor.java:
##########
@@ -47,6 +52,7 @@ public class CommonTaskProcessor extends BaseTaskProcessor {
@Override
protected boolean submitTask() {
+ checkAndReplaceTestDataSource();
Review Comment:
return is not handled
##########
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java:
##########
@@ -65,7 +65,7 @@ Map<String, Object> execProcessInstance(User loginUser, long projectCode,
RunMode runMode,
Priority processInstancePriority, String workerGroup, Long environmentCode, Integer timeout,
Map<String, String> startParams, Integer expectedParallelismNumber,
- int dryRun,
+ int dryRun,int testFlag,
Review Comment:
optimized format
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@dolphinscheduler.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org