You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by ki...@apache.org on 2021/06/11 17:23:27 UTC
[dolphinscheduler] branch dev updated: [Fix-5596][Python] Fix
conflict between python_home and datax_home configuration in
dolphinscheduler_env.sh (#5612)
This is an automated email from the ASF dual-hosted git repository.
kirs pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new 8bf042a [Fix-5596][Python] Fix conflict between python_home and datax_home configuration in dolphinscheduler_env.sh (#5612)
8bf042a is described below
commit 8bf042ae6ef7576209a0489e784684f4960ae6e0
Author: Shiwen Cheng <ch...@gmail.com>
AuthorDate: Sat Jun 12 01:23:18 2021 +0800
[Fix-5596][Python] Fix conflict between python_home and datax_home configuration in dolphinscheduler_env.sh (#5612)
---
.../server/worker/task/PythonCommandExecutor.java | 2 +-
.../server/worker/task/datax/DataxTask.java | 24 +++++++++++++++++++++-
.../server/worker/task/datax/DataxTaskTest.java | 16 +++++++++++++++
3 files changed, 40 insertions(+), 2 deletions(-)
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutor.java
index 7e96196..edf102b 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutor.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutor.java
@@ -141,7 +141,7 @@ public class PythonCommandExecutor extends AbstractCommandExecutor {
if (PYTHON_PATH_PATTERN.matcher(pythonHome).find()) {
return pythonHome;
}
- return pythonHome + "/bin/python";
+ return Paths.get(pythonHome, "/bin/python").toString();
}
/**
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java
index a088a49..a8aa132 100755
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java
@@ -31,6 +31,7 @@ import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
+import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.server.entity.DataxTaskExecutionContext;
import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;
import org.apache.dolphinscheduler.server.utils.DataxUtils;
@@ -45,6 +46,7 @@ import java.io.File;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.nio.file.attribute.FileAttribute;
import java.nio.file.attribute.PosixFilePermission;
@@ -58,6 +60,8 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import org.slf4j.Logger;
@@ -86,6 +90,7 @@ public class DataxTask extends AbstractTask {
* python process(datax only supports version 2.7 by default)
*/
private static final String DATAX_PYTHON = "python2.7";
+ private static final Pattern PYTHON_PATH_PATTERN = Pattern.compile("/bin/python[\\d.]*$");
/**
* datax path
*/
@@ -392,7 +397,7 @@ public class DataxTask extends AbstractTask {
// datax python command
StringBuilder sbr = new StringBuilder();
- sbr.append(DATAX_PYTHON);
+ sbr.append(getPythonCommand());
sbr.append(" ");
sbr.append(DATAX_PATH);
sbr.append(" ");
@@ -419,6 +424,23 @@ public class DataxTask extends AbstractTask {
return fileName;
}
+ public String getPythonCommand() {
+ String pythonHome = System.getenv("PYTHON_HOME");
+ return getPythonCommand(pythonHome);
+ }
+
+ public String getPythonCommand(String pythonHome) {
+ if (StringUtils.isEmpty(pythonHome)) {
+ return DATAX_PYTHON;
+ }
+ String pythonBinPath = "/bin/" + DATAX_PYTHON;
+ Matcher matcher = PYTHON_PATH_PATTERN.matcher(pythonHome);
+ if (matcher.find()) {
+ return matcher.replaceAll(pythonBinPath);
+ }
+ return Paths.get(pythonHome, pythonBinPath).toString();
+ }
+
public String loadJvmEnv(DataxParameters dataXParameters) {
int xms = dataXParameters.getXms() < 1 ? 1 : dataXParameters.getXms();
int xmx = dataXParameters.getXmx() < 1 ? 1 : dataXParameters.getXmx();
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java
index ed49185..ea0cb75 100644
--- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java
+++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java
@@ -456,6 +456,22 @@ public class DataxTaskTest {
}
@Test
+ public void testGetPythonCommand() {
+ String pythonCommand = dataxTask.getPythonCommand();
+ Assert.assertEquals("python2.7", pythonCommand);
+ pythonCommand = dataxTask.getPythonCommand("");
+ Assert.assertEquals("python2.7", pythonCommand);
+ pythonCommand = dataxTask.getPythonCommand("/usr/bin/python");
+ Assert.assertEquals("/usr/bin/python2.7", pythonCommand);
+ pythonCommand = dataxTask.getPythonCommand("/usr/local/bin/python2");
+ Assert.assertEquals("/usr/local/bin/python2.7", pythonCommand);
+ pythonCommand = dataxTask.getPythonCommand("/opt/python/bin/python3.8");
+ Assert.assertEquals("/opt/python/bin/python2.7", pythonCommand);
+ pythonCommand = dataxTask.getPythonCommand("/opt/soft/python");
+ Assert.assertEquals("/opt/soft/python/bin/python2.7", pythonCommand);
+ }
+
+ @Test
public void testLoadJvmEnv() {
DataxTask dataxTask = new DataxTask(null,null);
DataxParameters dataxParameters = new DataxParameters();