You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by ca...@apache.org on 2022/07/13 03:44:19 UTC
[dolphinscheduler] branch dev updated: [fix][plugin]: fix spark sql use custom parma. (#10821)
This is an automated email from the ASF dual-hosted git repository.
caishunfeng pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new 427c58546c [fix][plugin]: fix spark sql use custom parma. (#10821)
427c58546c is described below
commit 427c58546c93869e0bee7fb853319083c35c8bc4
Author: chuxing <92...@qq.com>
AuthorDate: Wed Jul 13 11:44:13 2022 +0800
[fix][plugin]: fix spark sql use custom parma. (#10821)
* [fix][plugin]: fix spark sql use custom parma.
* [fix][plugin] roll back part of the code
Co-authored-by: huaangcg <hu...@getui.com>
---
.../plugin/task/api/parameters/AbstractParameters.java | 4 ++--
.../apache/dolphinscheduler/plugin/task/spark/SparkTask.java | 10 +++++++++-
.../views/projects/task/components/node/fields/use-spark.ts | 2 +-
3 files changed, 12 insertions(+), 4 deletions(-)
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/AbstractParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/AbstractParameters.java
index b575ece49b..f01198d3b4 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/AbstractParameters.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/AbstractParameters.java
@@ -30,6 +30,7 @@ import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ArrayNode;
@@ -90,9 +91,8 @@ public abstract class AbstractParameters implements IParameters {
public Map<String, Property> getInputLocalParametersMap() {
Map<String, Property> localParametersMaps = new LinkedHashMap<>();
if (localParams != null) {
-
for (Property property : localParams) {
- if (property.getDirect().equals(Direct.IN)) {
+ if (Objects.equals(Direct.IN, property.getDirect())) {
localParametersMaps.put(property.getProp(), property);
}
}
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java
index b30ac64ae6..01db342661 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java
@@ -229,7 +229,7 @@ public class SparkTask extends AbstractYarnTask {
Path path = file.toPath();
if (!Files.exists(path)) {
- String script = sparkParameters.getRawScript().replaceAll("\\r\\n", "\n");
+ String script = replaceParam(sparkParameters.getRawScript());
sparkParameters.setRawScript(script);
logger.info("raw script : {}", sparkParameters.getRawScript());
@@ -255,6 +255,14 @@ public class SparkTask extends AbstractYarnTask {
return scriptFileName;
}
+ private String replaceParam(String script) {
+ script = script.replaceAll("\\r\\n", "\n");
+ // replace placeholder, and combining local and global parameters
+ Map<String, Property> paramsMap = taskExecutionContext.getPrepareParamsMap();
+ script = ParameterUtils.convertParameterPlaceholders(script, ParamUtils.convert(paramsMap));
+ return script;
+ }
+
@Override
protected void setMainJarName() {
// main jar
diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-spark.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-spark.ts
index fdce98a16e..89ac7800d7 100644
--- a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-spark.ts
+++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-spark.ts
@@ -130,7 +130,7 @@ export function useSpark(model: { [field: string]: any }): IJsonItem[] {
}
},
useResources(),
- ...useCustomParams({ model, field: 'localParams', isSimple: true })
+ ...useCustomParams({ model, field: 'localParams', isSimple: false })
]
}