You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by ke...@apache.org on 2023/10/09 08:21:55 UTC
[dolphinscheduler] branch 3.2.0-release updated (ab306cedd5 -> a1cf67e19c)
This is an automated email from the ASF dual-hosted git repository.
kerwin pushed a change to branch 3.2.0-release
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
from ab306cedd5 [Fix-14149][master] task finished so not dispatch (#14161)
new b9adaf58e0 [Bug][Worker] Fix sql/shell task doesn't replace parameter (#14876)
new a1cf67e19c [Bug][Worker] Global and local parameters are not replaced in SQL components (#14926)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/ISSUE_TEMPLATE/bug-report.yml | 2 +-
.../service/expand/CuringParamsService.java | 2 +-
.../service/expand/CuringParamsServiceImpl.java | 14 ++++++-
.../service/expand/CuringParamsServiceTest.java | 6 ++-
.../plugin/task/api/AbstractTask.java | 10 ++++-
.../plugin/task/shell/ShellTask.java | 2 +-
.../plugin/task/sql/SqlTaskTest.java | 45 ++++++++++++++++++++++
7 files changed, 72 insertions(+), 9 deletions(-)
[dolphinscheduler] 02/02: [Bug][Worker] Global and local parameters are not replaced in SQL components (#14926)
Posted by ke...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kerwin pushed a commit to branch 3.2.0-release
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
commit a1cf67e19ce78510b69fbbb2379d72a8953986b9
Author: Gallardot <ga...@apache.org>
AuthorDate: Fri Sep 15 10:31:14 2023 +0800
[Bug][Worker] Global and local parameters are not replaced in SQL components (#14926)
Signed-off-by: Gallardot <ga...@apache.org>
---
.../plugin/task/api/AbstractTask.java | 10 ++++-
.../plugin/task/sql/SqlTaskTest.java | 45 ++++++++++++++++++++++
2 files changed, 53 insertions(+), 2 deletions(-)
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractTask.java
index 10f24b7845..95ae22f272 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractTask.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractTask.java
@@ -38,7 +38,9 @@ public abstract class AbstractTask {
protected final Logger log = LoggerFactory.getLogger(AbstractTask.class);
- public String rgex = "['\"]\\$\\{(.*?)}['\"]|\\$\\{(.*?)}";
+ private static String groupName1 = "paramName1";
+ private static String groupName2 = "paramName2";
+ public String rgex = String.format("['\"]\\$\\{(?<%s>.*?)}['\"]|\\$\\{(?<%s>.*?)}", groupName1, groupName2);
/**
* varPool string
@@ -198,7 +200,11 @@ public abstract class AbstractTask {
int index = 1;
while (m.find()) {
- String paramName = m.group(1);
+ String paramName = m.group(groupName1);
+ if (paramName == null) {
+ paramName = m.group(groupName2);
+ }
+
Property prop = paramsPropsMap.get(paramName);
if (prop == null) {
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java
index a13ef4ecc1..a46670f227 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java
@@ -18,11 +18,17 @@
package org.apache.dolphinscheduler.plugin.task.sql;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
+import org.apache.dolphinscheduler.plugin.task.api.enums.DataType;
+import org.apache.dolphinscheduler.plugin.task.api.enums.Direct;
import org.apache.dolphinscheduler.plugin.task.api.enums.ResourceType;
+import org.apache.dolphinscheduler.plugin.task.api.model.Property;
import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.DataSourceParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper;
import org.apache.dolphinscheduler.spi.enums.DbType;
+import java.util.HashMap;
+import java.util.Map;
+
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -66,6 +72,15 @@ class SqlTaskTest {
String hiveLoadSql = "load inpath '/tmp/test_table/dt=${dt}' into table test_table partition(dt=${dt})";
String expected = "load inpath '/tmp/test_table/dt=?' into table test_table partition(dt=?)";
Assertions.assertEquals(expected, hiveLoadSql.replaceAll(sqlTask.rgex, "?"));
+
+ Map<Integer, Property> sqlParamsMap = new HashMap<>();
+ Map<Integer, Property> expectedSQLParamsMap = new HashMap<>();
+ expectedSQLParamsMap.put(1, new Property("dt", Direct.IN, DataType.VARCHAR, "1970"));
+ expectedSQLParamsMap.put(2, new Property("dt", Direct.IN, DataType.VARCHAR, "1970"));
+ Map<String, Property> paramsMap = new HashMap<>();
+ paramsMap.put("dt", new Property("dt", Direct.IN, DataType.VARCHAR, "1970"));
+ sqlTask.setSqlParamsMap(hiveLoadSql, sqlTask.rgex, sqlParamsMap, paramsMap, 1);
+ Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap);
}
@Test
@@ -74,17 +89,37 @@ class SqlTaskTest {
String expected = "select id from student where dt=?";
Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?"));
+ Map<Integer, Property> sqlParamsMap = new HashMap<>();
+ Map<Integer, Property> expectedSQLParamsMap = new HashMap<>();
+ expectedSQLParamsMap.put(1, new Property("dt", Direct.IN, DataType.VARCHAR, "1970"));
+ Map<String, Property> paramsMap = new HashMap<>();
+ paramsMap.put("dt", new Property("dt", Direct.IN, DataType.VARCHAR, "1970"));
+ sqlTask.setSqlParamsMap(querySql, sqlTask.rgex, sqlParamsMap, paramsMap, 1);
+ Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap);
+
querySql = "select id from student where dt=\"${dt}\"";
expected = "select id from student where dt=?";
Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?"));
+ sqlParamsMap.clear();
+ sqlTask.setSqlParamsMap(querySql, sqlTask.rgex, sqlParamsMap, paramsMap, 1);
+ Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap);
+
querySql = "select id from student where dt=${dt}";
expected = "select id from student where dt=?";
Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?"));
+ sqlParamsMap.clear();
+ sqlTask.setSqlParamsMap(querySql, sqlTask.rgex, sqlParamsMap, paramsMap, 1);
+ Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap);
+
querySql = "select id from student where dt=${dt} and gender=1";
expected = "select id from student where dt=? and gender=1";
Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?"));
+
+ sqlParamsMap.clear();
+ sqlTask.setSqlParamsMap(querySql, sqlTask.rgex, sqlParamsMap, paramsMap, 1);
+ Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap);
}
@Test
@@ -92,5 +127,15 @@ class SqlTaskTest {
String querySql = "select id from student where year=${year} and month=${month} and gender=1";
String expected = "select id from student where year=? and month=? and gender=1";
Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?"));
+
+ Map<Integer, Property> sqlParamsMap = new HashMap<>();
+ Map<Integer, Property> expectedSQLParamsMap = new HashMap<>();
+ expectedSQLParamsMap.put(1, new Property("year", Direct.IN, DataType.VARCHAR, "1970"));
+ expectedSQLParamsMap.put(2, new Property("month", Direct.IN, DataType.VARCHAR, "12"));
+ Map<String, Property> paramsMap = new HashMap<>();
+ paramsMap.put("year", new Property("year", Direct.IN, DataType.VARCHAR, "1970"));
+ paramsMap.put("month", new Property("month", Direct.IN, DataType.VARCHAR, "12"));
+ sqlTask.setSqlParamsMap(querySql, sqlTask.rgex, sqlParamsMap, paramsMap, 1);
+ Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap);
}
}
[dolphinscheduler] 01/02: [Bug][Worker] Fix sql/shell task doesn't replace parameter (#14876)
Posted by ke...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kerwin pushed a commit to branch 3.2.0-release
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
commit b9adaf58e0c49648efb0f5a3fcd97c50ce6633fd
Author: Jay Chung <zh...@gmail.com>
AuthorDate: Fri Sep 8 10:09:16 2023 +0800
[Bug][Worker] Fix sql/shell task doesn't replace parameter (#14876)
Fix parameter assign error, fix #14869 which including
* shell task param not work, botch local and global
* others task chain assign not work
---
.github/ISSUE_TEMPLATE/bug-report.yml | 2 +-
.../service/expand/CuringParamsService.java | 2 +-
.../service/expand/CuringParamsServiceImpl.java | 14 ++++++++++++--
.../service/expand/CuringParamsServiceTest.java | 6 ++++--
.../dolphinscheduler/plugin/task/shell/ShellTask.java | 2 +-
5 files changed, 19 insertions(+), 7 deletions(-)
diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml
index 259f0022e5..1bd1e44c98 100644
--- a/.github/ISSUE_TEMPLATE/bug-report.yml
+++ b/.github/ISSUE_TEMPLATE/bug-report.yml
@@ -100,7 +100,7 @@ body:
options:
- dev
- 3.1.x
- - 3.2.0-prepare
+ - 3.2.x
validations:
required: true
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringParamsService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringParamsService.java
index 75b6097b21..d6d50ace80 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringParamsService.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringParamsService.java
@@ -53,7 +53,7 @@ public interface CuringParamsService {
* @param allParamMap
* @return
*/
- String convertParameterPlaceholders(String val, Map<String, String> allParamMap);
+ String convertParameterPlaceholders(String val, Map<String, Property> allParamMap);
/**
* curing global params
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringParamsServiceImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringParamsServiceImpl.java
index 903e642c09..e23c07b1b7 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringParamsServiceImpl.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringParamsServiceImpl.java
@@ -17,6 +17,7 @@
package org.apache.dolphinscheduler.service.expand;
+import static java.util.Objects.nonNull;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.PARAMETER_PROJECT_CODE;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.PARAMETER_PROJECT_NAME;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.PARAMETER_TASK_DEFINITION_CODE;
@@ -69,8 +70,13 @@ public class CuringParamsServiceImpl implements CuringParamsService {
private ProjectParameterMapper projectParameterMapper;
@Override
- public String convertParameterPlaceholders(String val, Map<String, String> allParamMap) {
- return ParameterUtils.convertParameterPlaceholders(val, allParamMap);
+ public String convertParameterPlaceholders(String val, Map<String, Property> allParamMap) {
+ Map<String, String> paramMap = allParamMap
+ .entrySet()
+ .stream()
+ .filter(entry -> nonNull(entry.getValue().getValue()))
+ .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().getValue()));
+ return ParameterUtils.convertParameterPlaceholders(val, paramMap);
}
@Override
@@ -212,6 +218,10 @@ public class CuringParamsServiceImpl implements CuringParamsService {
// whether external scaling calculation is required
if (timeFunctionNeedExpand(val)) {
val = timeFunctionExtension(taskInstance.getProcessInstanceId(), timeZone, val);
+ } else {
+ // handle some chain parameter assign, such as `{"var1": "${var2}", "var2": 1}` should be convert to
+ // `{"var1": 1, "var2": 1}`
+ val = convertParameterPlaceholders(val, prepareParamsMap);
}
property.setValue(val);
}
diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/expand/CuringParamsServiceTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/expand/CuringParamsServiceTest.java
index 1bb153bbea..29b3ed58c1 100644
--- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/expand/CuringParamsServiceTest.java
+++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/expand/CuringParamsServiceTest.java
@@ -72,17 +72,19 @@ public class CuringParamsServiceTest {
private TimePlaceholderResolverExpandServiceImpl timePlaceholderResolverExpandServiceImpl;
private final Map<String, String> globalParamMap = new HashMap<>();
+ private final Map<String, Property> paramMap = new HashMap<>();
@BeforeEach
public void init() {
globalParamMap.put("globalParams1", "Params1");
+ paramMap.put("globalParams1", new Property("globalParams1", Direct.IN, DataType.VARCHAR, "Params1"));
}
@Test
public void testConvertParameterPlaceholders() {
- Mockito.when(curingGlobalParamsService.convertParameterPlaceholders(placeHolderName, globalParamMap))
+ Mockito.when(curingGlobalParamsService.convertParameterPlaceholders(placeHolderName, paramMap))
.thenReturn("2022-06-26");
- String result = curingGlobalParamsService.convertParameterPlaceholders(placeHolderName, globalParamMap);
+ String result = curingGlobalParamsService.convertParameterPlaceholders(placeHolderName, paramMap);
Assertions.assertNotNull(result);
}
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTask.java
index c3c2bf62a1..b657f9a9ef 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTask.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTask.java
@@ -81,7 +81,7 @@ public class ShellTask extends AbstractTask {
public void handle(TaskCallBack taskCallBack) throws TaskException {
try {
IShellInterceptorBuilder<?, ?> shellActuatorBuilder = ShellInterceptorBuilderFactory.newBuilder()
- .properties(ParameterUtils.convert(shellParameters.getLocalParametersMap()))
+ .properties(ParameterUtils.convert(taskExecutionContext.getPrepareParamsMap()))
.appendScript(shellParameters.getRawScript());
TaskResponse commandExecuteResult = shellCommandExecutor.run(shellActuatorBuilder, taskCallBack);