You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by zh...@apache.org on 2022/09/23 11:47:20 UTC
[dolphinscheduler] 01/03: fix python home conflict (#12112)
This is an automated email from the ASF dual-hosted git repository.
zhongjiajie pushed a commit to branch 3.1.0-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
commit f871a4a41f05af95f3ac8c4799d520b61c2584c1
Author: JieguangZhou <ji...@163.com>
AuthorDate: Fri Sep 23 09:40:18 2022 +0800
fix python home conflict (#12112)
(cherry picked from commit 6eb1eb722af06b94027994b2ef8955e84fd97d7f)
---
.../plugin/task/pytorch/PythonEnvManager.java | 4 ++--
.../plugin/task/pytorch/PytorchTaskTest.java | 12 ++++++------
2 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PythonEnvManager.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PythonEnvManager.java
index d13e234c33..99c24947d3 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PythonEnvManager.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PythonEnvManager.java
@@ -40,7 +40,7 @@ public class PythonEnvManager {
private static final String VIRTUALENV_BUILD = "virtualenv -p ${PYTHON_HOME} %s";
- private static final String INSTALL_COMMAND = "python -m pip install -r %s";
+ private static final String INSTALL_COMMAND = "%s -m pip install -r %s";
private String pythonEnvTool = ENV_TOOL_VENV;
@@ -54,7 +54,7 @@ public class PythonEnvManager {
} else if (pythonEnvTool.equals(ENV_TOOL_CONDA)) {
buildCommand = String.format(CONDA_BUILD, condaPythonVersion, CREATE_ENV_NAME);
}
- String installCommand = String.format(INSTALL_COMMAND, requirementPath);
+ String installCommand = String.format(INSTALL_COMMAND, getPythonCommand(), requirementPath);
return buildCommand + " && " + sourceCommand + " && " + installCommand;
}
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/test/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/test/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTaskTest.java
index 835d31fe72..86c2d91910 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/test/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTaskTest.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/test/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTaskTest.java
@@ -72,16 +72,16 @@ public class PytorchTaskTest {
envManager.setPythonEnvTool(PythonEnvManager.ENV_TOOL_CONDA);
envManager.setCondaPythonVersion("3.9");
String condaEnvCommand39 = envManager.getBuildEnvCommand(requirementPath);
- Assert.assertEquals(condaEnvCommand39, "conda create -y python=3.9 -p ./venv && source activate ./venv && python -m pip install -r " + requirementPath);
+ Assert.assertEquals(condaEnvCommand39, "conda create -y python=3.9 -p ./venv && source activate ./venv && ./venv/bin/python -m pip install -r " + requirementPath);
envManager.setCondaPythonVersion("3.8");
String condaEnvCommand38 = envManager.getBuildEnvCommand(requirementPath);
- Assert.assertEquals(condaEnvCommand38, "conda create -y python=3.8 -p ./venv && source activate ./venv && python -m pip install -r " + requirementPath);
+ Assert.assertEquals(condaEnvCommand38, "conda create -y python=3.8 -p ./venv && source activate ./venv && ./venv/bin/python -m pip install -r " + requirementPath);
envManager.setPythonEnvTool(PythonEnvManager.ENV_TOOL_VENV);
String venvEnvCommand = envManager.getBuildEnvCommand(requirementPath);
- Assert.assertEquals(venvEnvCommand, "virtualenv -p ${PYTHON_HOME} ./venv && source ./venv/bin/activate && python -m pip install -r " + requirementPath);
+ Assert.assertEquals(venvEnvCommand, "virtualenv -p ${PYTHON_HOME} ./venv && source ./venv/bin/activate && ./venv/bin/python -m pip install -r " + requirementPath);
}
@@ -146,7 +146,7 @@ public class PytorchTaskTest {
PytorchTask task = initTask(parameters);
Assert.assertEquals(task.buildPythonExecuteCommand(),
"export PYTHONPATH=.\n" +
- "conda create -y python=3.6 -p ./venv && source activate ./venv && python -m pip install -r requirements.txt\n" +
+ "conda create -y python=3.6 -p ./venv && source activate ./venv && ./venv/bin/python -m pip install -r requirements.txt\n" +
"./venv/bin/python main.py --epochs=1 --dry-run");
}
@@ -163,7 +163,7 @@ public class PytorchTaskTest {
PytorchTask task = initTask(parameters);
Assert.assertEquals(task.buildPythonExecuteCommand(),
"export PYTHONPATH=.\n" +
- "virtualenv -p ${PYTHON_HOME} ./venv && source ./venv/bin/activate && python -m pip install -r requirements.txt\n" +
+ "virtualenv -p ${PYTHON_HOME} ./venv && source ./venv/bin/activate && ./venv/bin/python -m pip install -r requirements.txt\n" +
"./venv/bin/python main.py --epochs=1 --dry-run");
}
@@ -189,7 +189,7 @@ public class PytorchTaskTest {
createFile(scriptFile);
String expected = "export PYTHONPATH=%s\n" +
- "virtualenv -p ${PYTHON_HOME} ./venv && source ./venv/bin/activate && python -m pip install -r %s\n" +
+ "virtualenv -p ${PYTHON_HOME} ./venv && source ./venv/bin/activate && ./venv/bin/python -m pip install -r %s\n" +
"./venv/bin/python %s";
System.out.println(task.buildPythonExecuteCommand());
Assert.assertEquals(String.format(expected, pythonPath, requirementFile, scriptFile), task.buildPythonExecuteCommand());