You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by lg...@apache.org on 2020/12/15 02:30:46 UTC
[incubator-dolphinscheduler] branch dev updated: [FIX-#3177]Task
time parameter parsing error (#4224)
This is an automated email from the ASF dual-hosted git repository.
lgcareer pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new 8919e38 [FIX-#3177]Task time parameter parsing error (#4224)
8919e38 is described below
commit 8919e3838f7caf3f1254389b2faed8a6a90dadae
Author: Kirs <ac...@163.com>
AuthorDate: Tue Dec 15 10:30:36 2020 +0800
[FIX-#3177]Task time parameter parsing error (#4224)
* [FIX-#3177]Task time parameter parsing error
rerun schedule time error
this closes #3177
* [FIX-#3177]Task time parameter parsing error
rerun schedule time error
this closes #3177
---
.../common/utils/ParameterUtils.java | 4 +-
.../server/worker/task/shell/ShellTask.java | 211 ++++++++++-----------
2 files changed, 108 insertions(+), 107 deletions(-)
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java
index 1227b4e..b6362c0 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java
@@ -111,7 +111,9 @@ public class ParameterUtils {
}
// replace variable ${} form,refers to the replacement of system variables and custom variables
- parameterString = PlaceholderUtils.replacePlaceholders(parameterString, parameterMap, true);
+ if (!parameterMap.isEmpty()) {
+ parameterString = PlaceholderUtils.replacePlaceholders(parameterString, parameterMap, true);
+ }
// replace time $[...] form, eg. $[yyyyMMdd]
if (cronTime != null) {
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java
index b805939..d1428b4 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java
@@ -14,8 +14,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.server.worker.task.shell;
+package org.apache.dolphinscheduler.server.worker.task.shell;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.CommandType;
@@ -23,7 +23,7 @@ import org.apache.dolphinscheduler.common.process.Property;
import org.apache.dolphinscheduler.common.task.AbstractParameters;
import org.apache.dolphinscheduler.common.task.shell.ShellParameters;
import org.apache.dolphinscheduler.common.utils.DateUtils;
-import org.apache.dolphinscheduler.common.utils.*;
+import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;
@@ -31,7 +31,6 @@ import org.apache.dolphinscheduler.server.utils.ParamUtils;
import org.apache.dolphinscheduler.server.worker.task.AbstractTask;
import org.apache.dolphinscheduler.server.worker.task.CommandExecuteResult;
import org.apache.dolphinscheduler.server.worker.task.ShellCommandExecutor;
-import org.slf4j.Logger;
import java.io.File;
import java.nio.file.Files;
@@ -43,136 +42,136 @@ import java.nio.file.attribute.PosixFilePermissions;
import java.util.Map;
import java.util.Set;
+import org.slf4j.Logger;
+
/**
* shell task
*/
public class ShellTask extends AbstractTask {
- /**
- * shell parameters
- */
- private ShellParameters shellParameters;
-
- /**
- * shell command executor
- */
- private ShellCommandExecutor shellCommandExecutor;
-
- /**
- * taskExecutionContext
- */
- private TaskExecutionContext taskExecutionContext;
-
- /**
- * constructor
- * @param taskExecutionContext taskExecutionContext
- * @param logger logger
- */
- public ShellTask(TaskExecutionContext taskExecutionContext, Logger logger) {
- super(taskExecutionContext, logger);
-
- this.taskExecutionContext = taskExecutionContext;
- this.shellCommandExecutor = new ShellCommandExecutor(this::logHandle,
+ /**
+ * shell parameters
+ */
+ private ShellParameters shellParameters;
+
+ /**
+ * shell command executor
+ */
+ private ShellCommandExecutor shellCommandExecutor;
+
+ /**
+ * taskExecutionContext
+ */
+ private TaskExecutionContext taskExecutionContext;
+
+ /**
+ * constructor
+ *
+ * @param taskExecutionContext taskExecutionContext
+ * @param logger logger
+ */
+ public ShellTask(TaskExecutionContext taskExecutionContext, Logger logger) {
+ super(taskExecutionContext, logger);
+
+ this.taskExecutionContext = taskExecutionContext;
+ this.shellCommandExecutor = new ShellCommandExecutor(this::logHandle,
taskExecutionContext,
logger);
- }
+ }
- @Override
- public void init() {
- logger.info("shell task params {}", taskExecutionContext.getTaskParams());
+ @Override
+ public void init() {
+ logger.info("shell task params {}", taskExecutionContext.getTaskParams());
- shellParameters = JSONUtils.parseObject(taskExecutionContext.getTaskParams(), ShellParameters.class);
+ shellParameters = JSONUtils.parseObject(taskExecutionContext.getTaskParams(), ShellParameters.class);
- if (!shellParameters.checkParameters()) {
- throw new RuntimeException("shell task params is not valid");
- }
- }
-
- @Override
- public void handle() throws Exception {
- try {
- // construct process
- CommandExecuteResult commandExecuteResult = shellCommandExecutor.run(buildCommand());
- setExitStatusCode(commandExecuteResult.getExitStatusCode());
- setAppIds(commandExecuteResult.getAppIds());
- setProcessId(commandExecuteResult.getProcessId());
- } catch (Exception e) {
- logger.error("shell task error", e);
- setExitStatusCode(Constants.EXIT_CODE_FAILURE);
- throw e;
+ if (!shellParameters.checkParameters()) {
+ throw new RuntimeException("shell task params is not valid");
+ }
}
- }
-
- @Override
- public void cancelApplication(boolean cancelApplication) throws Exception {
- // cancel process
- shellCommandExecutor.cancelApplication();
- }
-
- /**
- * create command
- * @return file name
- * @throws Exception exception
- */
- private String buildCommand() throws Exception {
- // generate scripts
- String fileName = String.format("%s/%s_node.%s",
- taskExecutionContext.getExecutePath(),
- taskExecutionContext.getTaskAppId(), OSUtils.isWindows() ? "bat" : "sh");
- Path path = new File(fileName).toPath();
+ @Override
+ public void handle() throws Exception {
+ try {
+ // construct process
+ CommandExecuteResult commandExecuteResult = shellCommandExecutor.run(buildCommand());
+ setExitStatusCode(commandExecuteResult.getExitStatusCode());
+ setAppIds(commandExecuteResult.getAppIds());
+ setProcessId(commandExecuteResult.getProcessId());
+ } catch (Exception e) {
+ logger.error("shell task error", e);
+ setExitStatusCode(Constants.EXIT_CODE_FAILURE);
+ throw e;
+ }
+ }
- if (Files.exists(path)) {
- return fileName;
+ @Override
+ public void cancelApplication(boolean cancelApplication) throws Exception {
+ // cancel process
+ shellCommandExecutor.cancelApplication();
}
- String script = shellParameters.getRawScript().replaceAll("\\r\\n", "\n");
/**
- * combining local and global parameters
+ * create command
+ *
+ * @return file name
+ * @throws Exception exception
*/
- Map<String, Property> paramsMap = ParamUtils.convert(ParamUtils.getUserDefParamsMap(taskExecutionContext.getDefinedParams()),
+ private String buildCommand() throws Exception {
+ // generate scripts
+ String fileName = String.format("%s/%s_node.%s",
+ taskExecutionContext.getExecutePath(),
+ taskExecutionContext.getTaskAppId(), OSUtils.isWindows() ? "bat" : "sh");
+
+ Path path = new File(fileName).toPath();
+
+ if (Files.exists(path)) {
+ return fileName;
+ }
+
+ String script = shellParameters.getRawScript().replaceAll("\\r\\n", "\n");
+ // combining local and global parameters
+ Map<String, Property> paramsMap = ParamUtils.convert(ParamUtils.getUserDefParamsMap(taskExecutionContext.getDefinedParams()),
taskExecutionContext.getDefinedParams(),
shellParameters.getLocalParametersMap(),
CommandType.of(taskExecutionContext.getCmdTypeIfComplement()),
taskExecutionContext.getScheduleTime());
- if (paramsMap != null){
- script = ParameterUtils.convertParameterPlaceholders(script, ParamUtils.convert(paramsMap));
- }
- // new
- // replace variable TIME with $[YYYYmmddd...] in shell file when history run job and batch complement job
- if (paramsMap != null) {
- if (taskExecutionContext.getScheduleTime() != null) {
- String dateTime = DateUtils.format(taskExecutionContext.getScheduleTime(), Constants.PARAMETER_FORMAT_TIME);
- Property p = new Property();
- p.setValue(dateTime);
- p.setProp(Constants.PARAMETER_SHECDULE_TIME);
- paramsMap.put(Constants.PARAMETER_SHECDULE_TIME, p);
- }
- script = ParameterUtils.convertParameterPlaceholders2(script, ParamUtils.convert(paramsMap));
- }
- shellParameters.setRawScript(script);
+ // replace variable TIME with $[YYYYmmddd...] in shell file when history run job and batch complement job
+ if (paramsMap != null) {
+ if (taskExecutionContext.getScheduleTime() != null) {
+ String dateTime = DateUtils.format(taskExecutionContext.getScheduleTime(), Constants.PARAMETER_FORMAT_TIME);
+ Property p = new Property();
+ p.setValue(dateTime);
+ p.setProp(Constants.PARAMETER_SHECDULE_TIME);
+ paramsMap.put(Constants.PARAMETER_SHECDULE_TIME, p);
+ }
+ }
- logger.info("raw script : {}", shellParameters.getRawScript());
- logger.info("task execute path : {}", taskExecutionContext.getExecutePath());
+ script = ParameterUtils.convertParameterPlaceholders2(script, ParamUtils.convert(paramsMap));
- Set<PosixFilePermission> perms = PosixFilePermissions.fromString(Constants.RWXR_XR_X);
- FileAttribute<Set<PosixFilePermission>> attr = PosixFilePermissions.asFileAttribute(perms);
+ shellParameters.setRawScript(script);
- if (OSUtils.isWindows()) {
- Files.createFile(path);
- } else {
- Files.createFile(path, attr);
- }
+ logger.info("raw script : {}", shellParameters.getRawScript());
+ logger.info("task execute path : {}", taskExecutionContext.getExecutePath());
- Files.write(path, shellParameters.getRawScript().getBytes(), StandardOpenOption.APPEND);
+ Set<PosixFilePermission> perms = PosixFilePermissions.fromString(Constants.RWXR_XR_X);
+ FileAttribute<Set<PosixFilePermission>> attr = PosixFilePermissions.asFileAttribute(perms);
- return fileName;
- }
+ if (OSUtils.isWindows()) {
+ Files.createFile(path);
+ } else {
+ Files.createFile(path, attr);
+ }
- @Override
- public AbstractParameters getParameters() {
- return shellParameters;
- }
+ Files.write(path, shellParameters.getRawScript().getBytes(), StandardOpenOption.APPEND);
+
+ return fileName;
+ }
+
+ @Override
+ public AbstractParameters getParameters() {
+ return shellParameters;
+ }
}