You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by zh...@apache.org on 2022/09/23 11:47:19 UTC
[dolphinscheduler] branch 3.1.0-prepare updated (c47e088b73 -> b0b29ed8e1)
This is an automated email from the ASF dual-hosted git repository.
zhongjiajie pushed a change to branch 3.1.0-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
from c47e088b73 [Fix][UI] Fix workflow name overlaps bug (#11862) (#12125)
new f871a4a41f fix python home conflict (#12112)
new 71c51c3c3d [Fix] Also auto formatter workflow instance if location is null (#12080)
new b0b29ed8e1 [fix][python] Task switch branch not show in webui (#12120)
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../src/pydolphinscheduler/core/task.py | 33 ++++++---
.../src/pydolphinscheduler/tasks/switch.py | 5 ++
.../pydolphinscheduler/tests/core/test_task.py | 82 ++++++++++++++++++++--
.../pydolphinscheduler/tests/tasks/test_switch.py | 2 -
.../plugin/task/pytorch/PythonEnvManager.java | 4 +-
.../plugin/task/pytorch/PytorchTaskTest.java | 12 ++--
.../projects/workflow/instance/detail/index.tsx | 10 +++
7 files changed, 122 insertions(+), 26 deletions(-)
[dolphinscheduler] 01/03: fix python home conflict (#12112)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhongjiajie pushed a commit to branch 3.1.0-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
commit f871a4a41f05af95f3ac8c4799d520b61c2584c1
Author: JieguangZhou <ji...@163.com>
AuthorDate: Fri Sep 23 09:40:18 2022 +0800
fix python home conflict (#12112)
(cherry picked from commit 6eb1eb722af06b94027994b2ef8955e84fd97d7f)
---
.../plugin/task/pytorch/PythonEnvManager.java | 4 ++--
.../plugin/task/pytorch/PytorchTaskTest.java | 12 ++++++------
2 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PythonEnvManager.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PythonEnvManager.java
index d13e234c33..99c24947d3 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PythonEnvManager.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PythonEnvManager.java
@@ -40,7 +40,7 @@ public class PythonEnvManager {
private static final String VIRTUALENV_BUILD = "virtualenv -p ${PYTHON_HOME} %s";
- private static final String INSTALL_COMMAND = "python -m pip install -r %s";
+ private static final String INSTALL_COMMAND = "%s -m pip install -r %s";
private String pythonEnvTool = ENV_TOOL_VENV;
@@ -54,7 +54,7 @@ public class PythonEnvManager {
} else if (pythonEnvTool.equals(ENV_TOOL_CONDA)) {
buildCommand = String.format(CONDA_BUILD, condaPythonVersion, CREATE_ENV_NAME);
}
- String installCommand = String.format(INSTALL_COMMAND, requirementPath);
+ String installCommand = String.format(INSTALL_COMMAND, getPythonCommand(), requirementPath);
return buildCommand + " && " + sourceCommand + " && " + installCommand;
}
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/test/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/test/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTaskTest.java
index 835d31fe72..86c2d91910 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/test/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTaskTest.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/test/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTaskTest.java
@@ -72,16 +72,16 @@ public class PytorchTaskTest {
envManager.setPythonEnvTool(PythonEnvManager.ENV_TOOL_CONDA);
envManager.setCondaPythonVersion("3.9");
String condaEnvCommand39 = envManager.getBuildEnvCommand(requirementPath);
- Assert.assertEquals(condaEnvCommand39, "conda create -y python=3.9 -p ./venv && source activate ./venv && python -m pip install -r " + requirementPath);
+ Assert.assertEquals(condaEnvCommand39, "conda create -y python=3.9 -p ./venv && source activate ./venv && ./venv/bin/python -m pip install -r " + requirementPath);
envManager.setCondaPythonVersion("3.8");
String condaEnvCommand38 = envManager.getBuildEnvCommand(requirementPath);
- Assert.assertEquals(condaEnvCommand38, "conda create -y python=3.8 -p ./venv && source activate ./venv && python -m pip install -r " + requirementPath);
+ Assert.assertEquals(condaEnvCommand38, "conda create -y python=3.8 -p ./venv && source activate ./venv && ./venv/bin/python -m pip install -r " + requirementPath);
envManager.setPythonEnvTool(PythonEnvManager.ENV_TOOL_VENV);
String venvEnvCommand = envManager.getBuildEnvCommand(requirementPath);
- Assert.assertEquals(venvEnvCommand, "virtualenv -p ${PYTHON_HOME} ./venv && source ./venv/bin/activate && python -m pip install -r " + requirementPath);
+ Assert.assertEquals(venvEnvCommand, "virtualenv -p ${PYTHON_HOME} ./venv && source ./venv/bin/activate && ./venv/bin/python -m pip install -r " + requirementPath);
}
@@ -146,7 +146,7 @@ public class PytorchTaskTest {
PytorchTask task = initTask(parameters);
Assert.assertEquals(task.buildPythonExecuteCommand(),
"export PYTHONPATH=.\n" +
- "conda create -y python=3.6 -p ./venv && source activate ./venv && python -m pip install -r requirements.txt\n" +
+ "conda create -y python=3.6 -p ./venv && source activate ./venv && ./venv/bin/python -m pip install -r requirements.txt\n" +
"./venv/bin/python main.py --epochs=1 --dry-run");
}
@@ -163,7 +163,7 @@ public class PytorchTaskTest {
PytorchTask task = initTask(parameters);
Assert.assertEquals(task.buildPythonExecuteCommand(),
"export PYTHONPATH=.\n" +
- "virtualenv -p ${PYTHON_HOME} ./venv && source ./venv/bin/activate && python -m pip install -r requirements.txt\n" +
+ "virtualenv -p ${PYTHON_HOME} ./venv && source ./venv/bin/activate && ./venv/bin/python -m pip install -r requirements.txt\n" +
"./venv/bin/python main.py --epochs=1 --dry-run");
}
@@ -189,7 +189,7 @@ public class PytorchTaskTest {
createFile(scriptFile);
String expected = "export PYTHONPATH=%s\n" +
- "virtualenv -p ${PYTHON_HOME} ./venv && source ./venv/bin/activate && python -m pip install -r %s\n" +
+ "virtualenv -p ${PYTHON_HOME} ./venv && source ./venv/bin/activate && ./venv/bin/python -m pip install -r %s\n" +
"./venv/bin/python %s";
System.out.println(task.buildPythonExecuteCommand());
Assert.assertEquals(String.format(expected, pythonPath, requirementFile, scriptFile), task.buildPythonExecuteCommand());
[dolphinscheduler] 03/03: [fix][python] Task switch branch not show in webui (#12120)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhongjiajie pushed a commit to branch 3.1.0-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
commit b0b29ed8e1ff0b9d8c9d44074cb8778d9ee33af1
Author: Jiajie Zhong <zh...@gmail.com>
AuthorDate: Fri Sep 23 19:41:05 2022 +0800
[fix][python] Task switch branch not show in webui (#12120)
when create task switch in python api, switch branch not show correctly,
due to we add some unnecessary attribute to switch task, this patch we
add `_task_ignore_attr` in class `Task` to ignore those unnecessary
attribute from `_task_default_attr`
(cherry picked from commit a86f4e2693b38503e8942cfe80e406a096b954c2)
---
.../src/pydolphinscheduler/core/task.py | 33 ++++++---
.../src/pydolphinscheduler/tasks/switch.py | 5 ++
.../pydolphinscheduler/tests/core/test_task.py | 82 ++++++++++++++++++++--
.../pydolphinscheduler/tests/tasks/test_switch.py | 2 -
4 files changed, 104 insertions(+), 18 deletions(-)
diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py
index d1a2eae9dd..5cbd21da1f 100644
--- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py
+++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py
@@ -16,7 +16,7 @@
# under the License.
"""DolphinScheduler Task and TaskRelation object."""
-
+import copy
from logging import getLogger
from typing import Dict, List, Optional, Sequence, Set, Tuple, Union
@@ -99,6 +99,17 @@ class Task(Base):
"timeout",
}
+ # task default attribute will into `task_params` property
+ _task_default_attr = {
+ "local_params",
+ "resource_list",
+ "dependence",
+ "wait_start_timeout",
+ "condition_result",
+ }
+ # task attribute ignore from _task_default_attr and will not into `task_params` property
+ _task_ignore_attr: set = set()
+ # task custom attribute define in sub class and will append to `task_params` property
_task_custom_attr: set = set()
DEFAULT_CONDITION_RESULT = {"successNode": [""], "failedNode": [""]}
@@ -213,20 +224,24 @@ class Task(Base):
"""Set attribute condition_result."""
self._condition_result = condition_result
+ def _get_attr(self) -> Set[str]:
+ """Get final task task_params attribute.
+
+ Base on `_task_default_attr`, append attribute from `_task_custom_attr` and subtract attribute from
+ `_task_ignore_attr`.
+ """
+ attr = copy.deepcopy(self._task_default_attr)
+ attr -= self._task_ignore_attr
+ attr |= self._task_custom_attr
+ return attr
+
@property
def task_params(self) -> Optional[Dict]:
"""Get task parameter object.
Will get result to combine _task_custom_attr and custom_attr.
"""
- custom_attr = {
- "local_params",
- "resource_list",
- "dependence",
- "wait_start_timeout",
- "condition_result",
- }
- custom_attr |= self._task_custom_attr
+ custom_attr = self._get_attr()
return self.get_define_custom(custom_attr=custom_attr)
def __hash__(self):
diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/tasks/switch.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/tasks/switch.py
index 35eece89b6..45edaa9aac 100644
--- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/tasks/switch.py
+++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/tasks/switch.py
@@ -134,6 +134,11 @@ class Switch(Task):
if task `switch` in this workflow.
"""
+ _task_ignore_attr = {
+ "condition_result",
+ "dependence",
+ }
+
def __init__(self, name: str, condition: SwitchCondition, *args, **kwargs):
super().__init__(name, TaskType.SWITCH, *args, **kwargs)
self.condition = condition
diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py
index 3909077b98..87ebc997cf 100644
--- a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py
+++ b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py
@@ -19,18 +19,86 @@
import logging
import re
from unittest.mock import patch
+from typing import Set
+from unittest.mock import patch
import pytest
from pydolphinscheduler.core.process_definition import ProcessDefinition
from pydolphinscheduler.core.task import Task, TaskRelation
-from tests.testing.task import Task as testTask
+from tests.testing.task import Task as TestTask
from tests.testing.task import TaskWithCode
TEST_TASK_RELATION_SET = set()
TEST_TASK_RELATION_SIZE = 0
+@pytest.mark.parametrize(
+ "addition, ignore, expect",
+ [
+ (
+ set(),
+ set(),
+ {
+ "local_params",
+ "resource_list",
+ "dependence",
+ "wait_start_timeout",
+ "condition_result",
+ },
+ ),
+ (
+ set(),
+ {"dependence", "condition_result", "not_exists"},
+ {
+ "local_params",
+ "resource_list",
+ "wait_start_timeout",
+ },
+ ),
+ (
+ {
+ "not_exists_1",
+ "not_exists_2",
+ },
+ set(),
+ {
+ "not_exists_1",
+ "not_exists_2",
+ "local_params",
+ "resource_list",
+ "dependence",
+ "wait_start_timeout",
+ "condition_result",
+ },
+ ),
+ # test addition and ignore conflict to see behavior
+ (
+ {
+ "not_exists",
+ },
+ {"condition_result", "not_exists"},
+ {
+ "not_exists",
+ "local_params",
+ "resource_list",
+ "dependence",
+ "wait_start_timeout",
+ },
+ ),
+ ],
+)
+def test__get_attr(addition: Set, ignore: Set, expect: Set):
+ """Test task function `_get_attr`."""
+ task = TestTask(
+ name="test-get-attr",
+ task_type="test",
+ )
+ task._task_custom_attr = addition
+ task._task_ignore_attr = ignore
+ assert task._get_attr() == expect
+
+
@pytest.mark.parametrize(
"attr, expect",
[
@@ -72,7 +140,7 @@ TEST_TASK_RELATION_SIZE = 0
)
def test_property_task_params(mock_resource, mock_user_name, attr, expect):
"""Test class task property."""
- task = testTask(
+ task = TestTask(
"test-property-task-params",
"test-task",
**attr,
@@ -182,8 +250,8 @@ def test_two_tasks_shift(shift: str):
Here we test both `>>` and `<<` bit operator.
"""
- upstream = testTask(name="upstream", task_type=shift)
- downstream = testTask(name="downstream", task_type=shift)
+ upstream = TestTask(name="upstream", task_type=shift)
+ downstream = TestTask(name="downstream", task_type=shift)
if shift == "<<":
downstream << upstream
elif shift == ">>":
@@ -219,10 +287,10 @@ def test_tasks_list_shift(dep_expr: str, flag: str):
"downstream": "upstream",
}
task_type = "dep_task_and_tasks"
- task = testTask(name="upstream", task_type=task_type)
+ task = TestTask(name="upstream", task_type=task_type)
tasks = [
- testTask(name="downstream1", task_type=task_type),
- testTask(name="downstream2", task_type=task_type),
+ TestTask(name="downstream1", task_type=task_type),
+ TestTask(name="downstream2", task_type=task_type),
]
# Use build-in function eval to simply test case and reduce duplicate code
diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_switch.py b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_switch.py
index 3206b12f7e..6f9222cec0 100644
--- a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_switch.py
+++ b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_switch.py
@@ -236,8 +236,6 @@ def test_switch_get_define(mock_task_code_version):
"taskParams": {
"resourceList": [],
"localParams": [],
- "dependence": {},
- "conditionResult": {"successNode": [""], "failedNode": [""]},
"waitStartTimeout": {},
"switchResult": {
"dependTaskList": [
[dolphinscheduler] 02/03: [Fix] Also auto formatter workflow instance if location is null (#12080)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhongjiajie pushed a commit to branch 3.1.0-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
commit 71c51c3c3daaa9b6eb3034fa0c37e4b0ec3ac4ae
Author: Jiajie Zhong <zh...@hotmail.com>
AuthorDate: Fri Sep 23 18:20:39 2022 +0800
[Fix] Also auto formatter workflow instance if location is null (#12080)
* [fix] Also auto formatter workflow instance if location is null
In #11535(096fae77) and #11681(4dca488c), we already formatter
workflow definition, but I find out we forget formatter workflow
instance with definition's location is null, this patch also
formatter workflow instance.
* remove console log
(cherry picked from commit 2531c222de5d3707147d9372d391b87b6c1848cd)
---
.../src/views/projects/workflow/instance/detail/index.tsx | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/dolphinscheduler-ui/src/views/projects/workflow/instance/detail/index.tsx b/dolphinscheduler-ui/src/views/projects/workflow/instance/detail/index.tsx
index 88992c3c41..95f895c211 100644
--- a/dolphinscheduler-ui/src/views/projects/workflow/instance/detail/index.tsx
+++ b/dolphinscheduler-ui/src/views/projects/workflow/instance/detail/index.tsx
@@ -33,6 +33,7 @@ import {
Location
} from '../../components/dag/types'
import Styles from './index.module.scss'
+import { useGraphAutoLayout } from '../../components/dag/use-graph-auto-layout'
interface SaveData {
saveForm: SaveForm
@@ -53,10 +54,18 @@ export default defineComponent({
const definition = ref<WorkflowDefinition>()
const instance = ref<WorkflowInstance>()
+ const dagInstanceRef = ref()
const refresh = () => {
queryProcessInstanceById(id, projectCode).then((res: any) => {
instance.value = res
+ if (!res.dagData.processDefinition.locations) {
+ setTimeout(() => {
+ const graph = dagInstanceRef.value
+ const { submit } = useGraphAutoLayout({ graph })
+ submit()
+ }, 1000)
+ }
if (res.dagData) {
definition.value = res.dagData
}
@@ -109,6 +118,7 @@ export default defineComponent({
]}
>
<Dag
+ ref={dagInstanceRef}
instance={instance.value}
definition={definition.value}
onRefresh={refresh}