You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by ke...@apache.org on 2023/10/09 08:21:56 UTC
[dolphinscheduler] 01/02: [Bug][Worker] Fix sql/shell task doesn't replace parameter (#14876)
This is an automated email from the ASF dual-hosted git repository.
kerwin pushed a commit to branch 3.2.0-release
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
commit b9adaf58e0c49648efb0f5a3fcd97c50ce6633fd
Author: Jay Chung <zh...@gmail.com>
AuthorDate: Fri Sep 8 10:09:16 2023 +0800
[Bug][Worker] Fix sql/shell task doesn't replace parameter (#14876)
Fix parameter assign error, fix #14869 which including
* shell task param not work, botch local and global
* others task chain assign not work
---
.github/ISSUE_TEMPLATE/bug-report.yml | 2 +-
.../service/expand/CuringParamsService.java | 2 +-
.../service/expand/CuringParamsServiceImpl.java | 14 ++++++++++++--
.../service/expand/CuringParamsServiceTest.java | 6 ++++--
.../dolphinscheduler/plugin/task/shell/ShellTask.java | 2 +-
5 files changed, 19 insertions(+), 7 deletions(-)
diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml
index 259f0022e5..1bd1e44c98 100644
--- a/.github/ISSUE_TEMPLATE/bug-report.yml
+++ b/.github/ISSUE_TEMPLATE/bug-report.yml
@@ -100,7 +100,7 @@ body:
options:
- dev
- 3.1.x
- - 3.2.0-prepare
+ - 3.2.x
validations:
required: true
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringParamsService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringParamsService.java
index 75b6097b21..d6d50ace80 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringParamsService.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringParamsService.java
@@ -53,7 +53,7 @@ public interface CuringParamsService {
* @param allParamMap
* @return
*/
- String convertParameterPlaceholders(String val, Map<String, String> allParamMap);
+ String convertParameterPlaceholders(String val, Map<String, Property> allParamMap);
/**
* curing global params
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringParamsServiceImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringParamsServiceImpl.java
index 903e642c09..e23c07b1b7 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringParamsServiceImpl.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringParamsServiceImpl.java
@@ -17,6 +17,7 @@
package org.apache.dolphinscheduler.service.expand;
+import static java.util.Objects.nonNull;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.PARAMETER_PROJECT_CODE;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.PARAMETER_PROJECT_NAME;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.PARAMETER_TASK_DEFINITION_CODE;
@@ -69,8 +70,13 @@ public class CuringParamsServiceImpl implements CuringParamsService {
private ProjectParameterMapper projectParameterMapper;
@Override
- public String convertParameterPlaceholders(String val, Map<String, String> allParamMap) {
- return ParameterUtils.convertParameterPlaceholders(val, allParamMap);
+ public String convertParameterPlaceholders(String val, Map<String, Property> allParamMap) {
+ Map<String, String> paramMap = allParamMap
+ .entrySet()
+ .stream()
+ .filter(entry -> nonNull(entry.getValue().getValue()))
+ .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().getValue()));
+ return ParameterUtils.convertParameterPlaceholders(val, paramMap);
}
@Override
@@ -212,6 +218,10 @@ public class CuringParamsServiceImpl implements CuringParamsService {
// whether external scaling calculation is required
if (timeFunctionNeedExpand(val)) {
val = timeFunctionExtension(taskInstance.getProcessInstanceId(), timeZone, val);
+ } else {
+ // handle some chain parameter assign, such as `{"var1": "${var2}", "var2": 1}` should be convert to
+ // `{"var1": 1, "var2": 1}`
+ val = convertParameterPlaceholders(val, prepareParamsMap);
}
property.setValue(val);
}
diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/expand/CuringParamsServiceTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/expand/CuringParamsServiceTest.java
index 1bb153bbea..29b3ed58c1 100644
--- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/expand/CuringParamsServiceTest.java
+++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/expand/CuringParamsServiceTest.java
@@ -72,17 +72,19 @@ public class CuringParamsServiceTest {
private TimePlaceholderResolverExpandServiceImpl timePlaceholderResolverExpandServiceImpl;
private final Map<String, String> globalParamMap = new HashMap<>();
+ private final Map<String, Property> paramMap = new HashMap<>();
@BeforeEach
public void init() {
globalParamMap.put("globalParams1", "Params1");
+ paramMap.put("globalParams1", new Property("globalParams1", Direct.IN, DataType.VARCHAR, "Params1"));
}
@Test
public void testConvertParameterPlaceholders() {
- Mockito.when(curingGlobalParamsService.convertParameterPlaceholders(placeHolderName, globalParamMap))
+ Mockito.when(curingGlobalParamsService.convertParameterPlaceholders(placeHolderName, paramMap))
.thenReturn("2022-06-26");
- String result = curingGlobalParamsService.convertParameterPlaceholders(placeHolderName, globalParamMap);
+ String result = curingGlobalParamsService.convertParameterPlaceholders(placeHolderName, paramMap);
Assertions.assertNotNull(result);
}
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTask.java
index c3c2bf62a1..b657f9a9ef 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTask.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTask.java
@@ -81,7 +81,7 @@ public class ShellTask extends AbstractTask {
public void handle(TaskCallBack taskCallBack) throws TaskException {
try {
IShellInterceptorBuilder<?, ?> shellActuatorBuilder = ShellInterceptorBuilderFactory.newBuilder()
- .properties(ParameterUtils.convert(shellParameters.getLocalParametersMap()))
+ .properties(ParameterUtils.convert(taskExecutionContext.getPrepareParamsMap()))
.appendScript(shellParameters.getRawScript());
TaskResponse commandExecuteResult = shellCommandExecutor.run(shellActuatorBuilder, taskCallBack);