You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by ca...@apache.org on 2022/05/24 01:42:51 UTC
[dolphinscheduler] branch dev updated: [Feature][MLops] Add MLflow Projects (custom project) component (#10175)
This is an automated email from the ASF dual-hosted git repository.
caishunfeng pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new 54ab2651bd [Feature][MLops] Add MLflow Projects (custom project) component (#10175)
54ab2651bd is described below
commit 54ab2651bdb1f3a9865bb9732ad736ca3fbc900f
Author: JieguangZhou <ji...@163.com>
AuthorDate: Tue May 24 09:42:46 2022 +0800
[Feature][MLops] Add MLflow Projects (custom project) component (#10175)
* Add MLflow Projects (custom project) component
* fix something
* update document
* update document
* Supplementary test cases
---
docs/docs/en/guide/task/mlflow.md | 32 +++++---
docs/docs/zh/guide/task/mlflow.md | 32 ++++++--
.../tasks/demo/mlflow-custom-project-template.png | Bin 0 -> 66296 bytes
docs/img/tasks/demo/mlflow-custom-project.png | Bin 0 -> 84087 bytes
.../plugin/task/mlflow/MlflowConstants.java | 9 ++-
.../plugin/task/mlflow/MlflowParameters.java | 27 ++++++-
.../plugin/task/mlflow/MlflowTask.java | 10 ++-
.../plugin/task/mlflow/MlflowTaskTest.java | 85 +++++++++++++++++----
dolphinscheduler-ui/src/locales/modules/en_US.ts | 4 +
dolphinscheduler-ui/src/locales/modules/zh_CN.ts | 4 +
.../components/node/fields/use-mlflow-projects.ts | 77 +++++++++++++++++--
.../projects/task/components/node/format-data.ts | 2 +
.../task/components/node/tasks/use-mlflow.ts | 6 +-
.../views/projects/task/components/node/types.ts | 2 +
14 files changed, 248 insertions(+), 42 deletions(-)
diff --git a/docs/docs/en/guide/task/mlflow.md b/docs/docs/en/guide/task/mlflow.md
index 486bf601e8..8e15dd753e 100644
--- a/docs/docs/en/guide/task/mlflow.md
+++ b/docs/docs/en/guide/task/mlflow.md
@@ -13,10 +13,10 @@ MLflow task plugin used to execute MLflow tasks，Currently contains Mlflow Proj
The Mlflow plugin currently supports and will support the following:
-- [ ] MLflow Projects
- - [x] BasicAlgorithm: contains lr, svm, lightgbm, xgboost
+- [x] MLflow Projects
+ - [x] BasicAlgorithm: contains LogisticRegression, svm, lightgbm, xgboost
- [x] AutoML: AutoML tool，contains autosklean, flaml
- - [ ] Custom projects: Support for running your own MLflow projects
+ - [x] Custom projects: Support for running your own MLflow projects
- [ ] MLflow Models
- [x] MLFLOW: Use `MLflow models serve` to deploy a model service
- [x] Docker: Run the container after packaging the docker image
@@ -65,9 +65,7 @@ First, introduce some general parameters of DolphinScheduler
**Task Parameter**
- **mlflow server tracking uri** ：MLflow server uri, default http://localhost:5000.
-- **job type** : The type of task to run, currently including the underlying algorithm and AutoML. (User-defined
- MLFlow project task execution will be supported in the near future)
-- **experiment name** ：The experiment in which the task is running, if none, is created.
+- **experiment name** ：Create the experiment where the task is running, if the experiment does not exist. If the name is empty, it is set to ` Default `, the same as MLflow.
- **register model** ：Register the model or not. If register is selected, the following parameters are expanded.
- **model name** : The registered model name is added to the original model version and registered as
Production.
@@ -95,9 +93,7 @@ First, introduce some general parameters of DolphinScheduler
**Task Parameter**
- **mlflow server tracking uri** ：MLflow server uri, default http://localhost:5000.
-- **job type** : The type of task to run, currently including the underlying algorithm and AutoML. (User-defined
- MLFlow project task execution will be supported in the near future)
-- **experiment name** ：The experiment in which the task is running, if none, is created.
+- **experiment name** ：Create the experiment where the task is running, if the experiment does not exist. If the name is empty, it is set to ` Default `, the same as MLflow.
- **register model** ：Register the model or not. If register is selected, the following parameters are expanded.
- **model name** : The registered model name is added to the original model version and registered as
Production.
@@ -114,6 +110,24 @@ First, introduce some general parameters of DolphinScheduler
and [flaml](https://github.com/microsoft/FLAML)
+#### Custom projects
+
+
+
+**Task Parameter**
+
+- **mlflow server tracking uri** ：MLflow server uri, default http://localhost:5000.
+- **experiment name** ：Create the experiment where the task is running, if the experiment does not exist. If the name is empty, it is set to ` Default `, the same as MLflow.
+- **parameters** : `--param-list` in `mlflow run`. For example `-P learning_rate=0.2 -P colsample_bytree=0.8 -P subsample=0.9`
+- **Repository** : Repository url of MLflow Project，Support git address and directory on worker. If it's in a subdirectory，We add `#` to support this (same as `mlflow run`) , for example `https://github.com/mlflow/mlflow#examples/xgboost/xgboost_native`
+- **Project Version** : Version of the project，default master
+
+You can now use this feature to run all mlFlow projects on Github (For example [MLflow examples](https://github.com/mlflow/mlflow/tree/master/examples) )了。You can also create your own machine learning library to reuse your work, and then use DolphinScheduler to use your library with one click.
+
+The actual interface is as follows
+
+
+
### MLflow Models
#### MLFLOW
diff --git a/docs/docs/zh/guide/task/mlflow.md b/docs/docs/zh/guide/task/mlflow.md
index 507ef43ce6..52fd4d32ef 100644
--- a/docs/docs/zh/guide/task/mlflow.md
+++ b/docs/docs/zh/guide/task/mlflow.md
@@ -12,10 +12,10 @@ MLflow 组件用于执行 MLflow 任务，目前包含Mlflow Projects, 和MLflow
目前 Mlflow 组件支持的和即将支持的内容如下中：
-- [ ] MLflow Projects
- - [x] BasicAlgorithm: 基础算法，包含lr, svm, lightgbm, xgboost。
+- [x] MLflow Projects
+ - [x] BasicAlgorithm: 基础算法，包含LogisticRegression, svm, lightgbm, xgboost。
- [x] AutoML: AutoML工具，包含autosklean, flaml。
- - [ ] Custom projects: 支持运行自己的MLflow Projects项目
+ - [x] Custom projects: 支持运行自己的MLflow Projects项目
- [ ] MLflow Models
- [x] MLFLOW: 直接使用 `MLflow models serve` 部署模型
- [x] Docker: 打包 DOCKER 镜像后部署模型
@@ -57,13 +57,12 @@ MLflow 组件用于执行 MLflow 任务，目前包含Mlflow Projects, 和MLflow
**任务参数**
- **mlflow server tracking uri** ：MLflow server 的连接, 默认 http://localhost:5000。
-- **任务类型** : 运行的任务类型，目前包括基础算法与AutoML, 后续将会支持用户自定义的MLflow Project。
-- **实验名称** ：任务运行时所在的实验，若无则创建。
+- **实验名称** ：任务运行时所在的实验，若实验不存在，则创建。若实验名称为空，则设置为`Default`, 与 MLflow 一样。
- **注册模型** ：是否注册模型，若选择注册，则会展开以下参数。
- **注册的模型名称** : 注册的模型名称，会在原来的基础上加上一个模型版本，并注册为Production。
- **数据路径** : 文件/文件夹的绝对路径, 若文件需以.csv结尾（自动切分训练集与测试集）, 文件夹需包含train.csv和test.csv（建议方式，用户应自行构建测试集用于模型评估）。
详细的参数列表如下:
- - [lr](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html#sklearn.linear_model.LogisticRegression)
+ - [LogisticRegression](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html#sklearn.linear_model.LogisticRegression)
- [SVM](https://scikit-learn.org/stable/modules/generated/sklearn.svm.SVC.html?highlight=svc#sklearn.svm.SVC)
- [lightgbm](https://lightgbm.readthedocs.io/en/latest/pythonapi/lightgbm.LGBMClassifier.html#lightgbm.LGBMClassifier)
- [xgboost](https://xgboost.readthedocs.io/en/stable/python/python_api.html#xgboost.XGBClassifier)
@@ -77,8 +76,7 @@ MLflow 组件用于执行 MLflow 任务，目前包含Mlflow Projects, 和MLflow
**任务参数**
- **mlflow server tracking uri** ：MLflow server 的连接, 默认 http://localhost:5000。
-- **任务类型** : 运行的任务类型，目前包括基础算法与AutoML, 后续将会支持用户自定义的MLflow Project。
-- **实验名称** ：任务运行时所在的实验，若无则创建。
+- **实验名称** ：任务运行时所在的实验，若实验不存在，则创建。若实验名称为空，则设置为`Default`, 与 MLflow 一样。
- **注册模型** ：是否注册模型，若选择注册，则会展开以下参数。
- **注册的模型名称** : 注册的模型名称，会在原来的基础上加上一个模型版本，并注册为Production。
- **数据路径** : 文件/文件夹的绝对路径, 若文件需以.csv结尾（自动切分训练集与测试集）, 文件夹需包含train.csv和test.csv（建议方式，用户应自行构建测试集用于模型评估）。
@@ -88,6 +86,24 @@ MLflow 组件用于执行 MLflow 任务，目前包含Mlflow Projects, 和MLflow
- **AutoML工具** : 使用的AutoML工具，目前支持 [autosklearn](https://github.com/automl/auto-sklearn)
, [flaml](https://github.com/microsoft/FLAML)
+#### Custom projects
+
+
+
+**任务参数**
+
+- **mlflow server tracking uri** ：MLflow server 的连接, 默认 http://localhost:5000。
+- **实验名称** ：任务运行时所在的实验，若实验不存在，则创建。若实验名称为空，则设置为`Default`, 与 MLflow 一样。
+- **参数** : `mlflow run`中的 --param-list 如 `-P learning_rate=0.2 -P colsample_bytree=0.8 -P subsample=0.9`
+- **运行仓库** : MLflow Project的仓库地址，可以为github地址，或者worker上的目录, 如Mlflow project位于子目录，可以添加 `#` 隔开, 如 `https://github.com/mlflow/mlflow#examples/xgboost/xgboost_native`
+- **项目版本** : 对应项目中git版本管理中的版本，默认 master
+
+现在你可以使用这个功能来运行github上所有的MLflow Projects (如 [MLflow examples](https://github.com/mlflow/mlflow/tree/master/examples) )了。你也可以创建自己的机器学习库，用来复用你的研究成果，以后你就可以使用DolphinScheduler来一键操作使用你的算法库。
+
+实际运行界面如下
+
+
+
### MLflow Models
diff --git a/docs/img/tasks/demo/mlflow-custom-project-template.png b/docs/img/tasks/demo/mlflow-custom-project-template.png
new file mode 100644
index 0000000000..cbb2d6f732
Binary files /dev/null and b/docs/img/tasks/demo/mlflow-custom-project-template.png differ
diff --git a/docs/img/tasks/demo/mlflow-custom-project.png b/docs/img/tasks/demo/mlflow-custom-project.png
new file mode 100644
index 0000000000..54a8cb0ab8
Binary files /dev/null and b/docs/img/tasks/demo/mlflow-custom-project.png differ
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowConstants.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowConstants.java
index f318f50966..92d849e433 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowConstants.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowConstants.java
@@ -26,6 +26,8 @@ public class MlflowConstants {
public static final String JOB_TYPE_BASIC_ALGORITHM = "BasicAlgorithm";
+ public static final String JOB_TYPE_CUSTOM_PROJECT = "CustomProject";
+
public static final String PRESET_REPOSITORY = "https://github.com/apache/dolphinscheduler-mlflow";
public static final String PRESET_REPOSITORY_VERSION = "main";
@@ -75,9 +77,14 @@ public class MlflowConstants {
"--experiment-name=\"%s\" " +
"--version=main ";
+ public static final String MLFLOW_RUN_CUSTOM_PROJECT = "mlflow run $repo " +
+ "%s " +
+ "--experiment-name=\"%s\" " +
+ "--version=\"%s\" ";
+
public static final String MLFLOW_MODELS_SERVE = "mlflow models serve -m %s --port %s -h 0.0.0.0";
- public static final String MLFLOW_BUILD_DOCKER = "mlflow models build-docker -m %s -n %s --enable-mlserver";
+ public static final String MLFLOW_BUILD_DOCKER = "mlflow models build-docker -m %s -n %s --enable-mlserver";
public static final String DOCKER_RREMOVE_CONTAINER = "docker rm -f %s";
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowParameters.java
index 9d107423fd..3557a0698a 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowParameters.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowParameters.java
@@ -31,6 +31,13 @@ public class MlflowParameters extends AbstractParameters {
private String mlflowJobType = "";
+ /**
+ * CustomProject parameters
+ */
+ private String mlflowProjectRepository;
+
+ private String mlflowProjectVersion = "master";
+
/**
* AutoML parameters
*/
@@ -53,7 +60,7 @@ public class MlflowParameters extends AbstractParameters {
private String mlflowTaskType = "";
- private String experimentName;
+ private String experimentName = "Default";
private String modelName = "";
@@ -145,6 +152,22 @@ public class MlflowParameters extends AbstractParameters {
this.automlTool = automlTool;
}
+ public String getMlflowProjectRepository() {
+ return mlflowProjectRepository;
+ }
+
+ public void setMlflowProjectRepository(String mlflowProjectRepository) {
+ this.mlflowProjectRepository = mlflowProjectRepository;
+ }
+
+ public String getMlflowProjectVersion() {
+ return mlflowProjectVersion;
+ }
+
+ public void setMlflowProjectVersion(String mlflowProjectVersion) {
+ this.mlflowProjectVersion = mlflowProjectVersion;
+ }
+
public String getAutomlTool() {
return automlTool;
}
@@ -232,7 +255,7 @@ public class MlflowParameters extends AbstractParameters {
return scriptPath;
}
- public String getModelKeyName(String tag) throws IllegalArgumentException{
+ public String getModelKeyName(String tag) throws IllegalArgumentException {
String imageName;
if (deployModelKey.startsWith("runs:")) {
imageName = deployModelKey.replace("runs:/", "");
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTask.java
index ea87088e6f..226efb21cf 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTask.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTask.java
@@ -141,8 +141,14 @@ public class MlflowTask extends AbstractTaskExecutor {
runCommand = String.format(runCommand, mlflowParameters.getAutomlTool(), mlflowParameters.getParams(), mlflowParameters.getModelName(), mlflowParameters.getExperimentName());
- } else {
- runCommand = String.format("Cant not Support %s", mlflowParameters.getMlflowTaskType());
+ } else if (mlflowParameters.getMlflowJobType().equals(MlflowConstants.JOB_TYPE_CUSTOM_PROJECT)) {
+ args.add(String.format(MlflowConstants.SET_REPOSITORY, mlflowParameters.getMlflowProjectRepository()));
+
+ runCommand = MlflowConstants.MLFLOW_RUN_CUSTOM_PROJECT;
+ runCommand = String.format(runCommand, mlflowParameters.getParams(), mlflowParameters.getExperimentName(), mlflowParameters.getMlflowProjectVersion());
+ }
+ else {
+ runCommand = String.format("Cant not Support %s", mlflowParameters.getMlflowJobType());
}
args.add(runCommand);
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/test/java/org/apache/dolphinler/plugin/task/mlflow/MlflowTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/test/java/org/apache/dolphinler/plugin/task/mlflow/MlflowTaskTest.java
index bf27835d20..24a8dfc601 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/test/java/org/apache/dolphinler/plugin/task/mlflow/MlflowTaskTest.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/test/java/org/apache/dolphinler/plugin/task/mlflow/MlflowTaskTest.java
@@ -78,28 +78,73 @@ public class MlflowTaskTest {
}
@Test
- public void testInitBasicAlgorithmTask() throws Exception {
+ public void testInitBasicAlgorithmTask() {
MlflowTask mlflowTask = initTask(createBasicAlgorithmParameters());
- String command = mlflowTask.buildCommand();
-
+ Assert.assertEquals(mlflowTask.buildCommand(),
+ "export MLFLOW_TRACKING_URI=http://127.0.0.1:5000\n" +
+ "data_path=/data/iris.csv\n" +
+ "repo=https://github.com/apache/dolphinscheduler-mlflow#Project-BasicAlgorithm\n" +
+ "mlflow run $repo " +
+ "-P algorithm=xgboost " +
+ "-P data_path=$data_path " +
+ "-P params=\"n_estimators=100\" " +
+ "-P search_params=\"\" " +
+ "-P model_name=\"BasicAlgorithm\" " +
+ "--experiment-name=\"BasicAlgorithm\" " +
+ "--version=main ");
}
@Test
- public void testInitAutoMLTask() throws Exception {
+ public void testInitAutoMLTask() {
MlflowTask mlflowTask = initTask(createAutoMLParameters());
- String command = mlflowTask.buildCommand();
+ Assert.assertEquals(mlflowTask.buildCommand(),
+ "export MLFLOW_TRACKING_URI=http://127.0.0.1:5000\n" +
+ "data_path=/data/iris.csv\n" +
+ "repo=https://github.com/apache/dolphinscheduler-mlflow#Project-AutoML\n" +
+ "mlflow run $repo " +
+ "-P tool=autosklearn " +
+ "-P data_path=$data_path " +
+ "-P params=\"time_left_for_this_task=30\" " +
+ "-P model_name=\"AutoML\" " +
+ "--experiment-name=\"AutoML\" " +
+ "--version=main ");
+ }
+
+ @Test
+ public void testInitCustomProjectTask() {
+ MlflowTask mlflowTask = initTask(createCustomProjectParameters());
+ Assert.assertEquals(mlflowTask.buildCommand(),
+ "export MLFLOW_TRACKING_URI=http://127.0.0.1:5000\n" +
+ "repo=https://github.com/mlflow/mlflow#examples/xgboost/xgboost_native\n" +
+ "mlflow run $repo " +
+ "-P learning_rate=0.2 " +
+ "-P colsample_bytree=0.8 " +
+ "-P subsample=0.9 " +
+ "--experiment-name=\"custom_project\" " +
+ "--version=\"master\" ");
}
@Test
- public void testModelsDeployMlflow() throws Exception {
+ public void testModelsDeployMlflow() {
MlflowTask mlflowTask = initTask(createModelDeplyMlflowParameters());
- String command = mlflowTask.buildCommand();
+ Assert.assertEquals(mlflowTask.buildCommand(),
+ "export MLFLOW_TRACKING_URI=http://127.0.0.1:5000\n" +
+ "mlflow models serve -m runs:/a272ec279fc34a8995121ae04281585f/model " +
+ "--port 7000 " +
+ "-h 0.0.0.0");
}
@Test
public void testModelsDeployDocker() throws Exception {
MlflowTask mlflowTask = initTask(createModelDeplyDockerParameters());
- String command = mlflowTask.buildCommand();
+ Assert.assertEquals(mlflowTask.buildCommand(),
+ "export MLFLOW_TRACKING_URI=http://127.0.0.1:5000\n" +
+ "mlflow models build-docker -m runs:/a272ec279fc34a8995121ae04281585f/model " +
+ "-n mlflow/a272ec279fc34a8995121ae04281585f:model " +
+ "--enable-mlserver\n" +
+ "docker rm -f mlflow-a272ec279fc34a8995121ae04281585f-model\n" +
+ "docker run --name=mlflow-a272ec279fc34a8995121ae04281585f-model " +
+ "-p=7000:8080 mlflow/a272ec279fc34a8995121ae04281585f:model");
}
private MlflowTask initTask(MlflowParameters mlflowParameters) {
@@ -116,8 +161,10 @@ public class MlflowTaskTest {
mlflowParameters.setMlflowTaskType(MlflowConstants.MLFLOW_TASK_TYPE_PROJECTS);
mlflowParameters.setMlflowJobType(MlflowConstants.JOB_TYPE_BASIC_ALGORITHM);
mlflowParameters.setAlgorithm("xgboost");
- mlflowParameters.setDataPaths("xxxxxxxxxx");
- mlflowParameters.setExperimentNames("asbbb");
+ mlflowParameters.setDataPaths("/data/iris.csv");
+ mlflowParameters.setParams("n_estimators=100");
+ mlflowParameters.setExperimentNames("BasicAlgorithm");
+ mlflowParameters.setModelNames("BasicAlgorithm");
mlflowParameters.setMlflowTrackingUris("http://127.0.0.1:5000");
return mlflowParameters;
}
@@ -128,13 +175,25 @@ public class MlflowTaskTest {
mlflowParameters.setMlflowJobType(MlflowConstants.JOB_TYPE_AUTOML);
mlflowParameters.setAutomlTool("autosklearn");
mlflowParameters.setParams("time_left_for_this_task=30");
- mlflowParameters.setDataPaths("xxxxxxxxxxx");
- mlflowParameters.setExperimentNames("asbbb");
- mlflowParameters.setModelNames("asbbb");
+ mlflowParameters.setDataPaths("/data/iris.csv");
+ mlflowParameters.setExperimentNames("AutoML");
+ mlflowParameters.setModelNames("AutoML");
mlflowParameters.setMlflowTrackingUris("http://127.0.0.1:5000");
return mlflowParameters;
}
+ private MlflowParameters createCustomProjectParameters() {
+ MlflowParameters mlflowParameters = new MlflowParameters();
+ mlflowParameters.setMlflowTaskType(MlflowConstants.MLFLOW_TASK_TYPE_PROJECTS);
+ mlflowParameters.setMlflowJobType(MlflowConstants.JOB_TYPE_CUSTOM_PROJECT);
+ mlflowParameters.setMlflowTrackingUris("http://127.0.0.1:5000");
+ mlflowParameters.setExperimentNames("custom_project");
+ mlflowParameters.setParams("-P learning_rate=0.2 -P colsample_bytree=0.8 -P subsample=0.9");
+ mlflowParameters.setMlflowProjectRepository("https://github.com/mlflow/mlflow#examples/xgboost/xgboost_native");
+
+ return mlflowParameters;
+ }
+
private MlflowParameters createModelDeplyMlflowParameters() {
MlflowParameters mlflowParameters = new MlflowParameters();
mlflowParameters.setMlflowTaskType(MlflowConstants.MLFLOW_TASK_TYPE_MODELS);
diff --git a/dolphinscheduler-ui/src/locales/modules/en_US.ts b/dolphinscheduler-ui/src/locales/modules/en_US.ts
index bef4da1f40..9e082fdea1 100644
--- a/dolphinscheduler-ui/src/locales/modules/en_US.ts
+++ b/dolphinscheduler-ui/src/locales/modules/en_US.ts
@@ -993,6 +993,10 @@ const project = {
mlflow_deployType: 'Deploy Mode',
mlflow_deployModelKey: 'model-uri',
mlflow_deployPort: 'Port',
+ mlflowProjectRepository: 'Repository',
+ mlflowProjectRepository_tips: 'github respository or path on worker',
+ mlflowProjectVersion: 'Project Version',
+ mlflowProjectVersion_tips: 'git version',
send_email: 'Send Email',
log_display: 'Log display',
rows_of_result: 'rows of result',
diff --git a/dolphinscheduler-ui/src/locales/modules/zh_CN.ts b/dolphinscheduler-ui/src/locales/modules/zh_CN.ts
index 45ac7c1b56..eeeab0a0cd 100644
--- a/dolphinscheduler-ui/src/locales/modules/zh_CN.ts
+++ b/dolphinscheduler-ui/src/locales/modules/zh_CN.ts
@@ -976,6 +976,10 @@ const project = {
mlflow_deployType: '部署类型',
mlflow_deployModelKey: '部署的模型uri',
mlflow_deployPort: '监听端口',
+ mlflowProjectRepository: '运行仓库',
+ mlflowProjectRepository_tips: '可以为github仓库或worker上的路径',
+ mlflowProjectVersion: '项目版本',
+ mlflowProjectVersion_tips: '项目git版本',
send_email: '发送邮件',
log_display: '日志显示',
rows_of_result: '行查询结果',
diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-mlflow-projects.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-mlflow-projects.ts
index 65afbcbeb7..ea033cbb83 100644
--- a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-mlflow-projects.ts
+++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-mlflow-projects.ts
@@ -36,14 +36,16 @@ export function useMlflowProjects(model: {
const resetSpan = () => {
experimentNameSpan.value = model.isProjects ? 12 : 0
- registerModelSpan.value = model.isProjects ? 6 : 0
mlflowJobTypeSpan.value = model.isProjects ? 12 : 0
paramsSpan.value = model.isProjects ? 24 : 0
- dataPathSpan.value = model.isProjects ? 24 : 0
+ registerModelSpan.value =
+ model.isProjects && model.mlflowJobType != 'CustomProject' ? 6 : 0
+ dataPathSpan.value =
+ model.isProjects && model.mlflowJobType != 'CustomProject' ? 24 : 0
}
watch(
- () => [model.mlflowTaskType, model.registerModel],
+ () => [model.mlflowTaskType, model.mlflowJobType],
() => {
setFlag()
resetSpan()
@@ -123,7 +125,8 @@ export function useMlflowProjects(model: {
}
},
...useBasicAlgorithm(model),
- ...useAutoML(model)
+ ...useAutoML(model),
+ ...useCustomProject(model)
]
}
@@ -144,7 +147,7 @@ export function useBasicAlgorithm(model: {
}
const resetSpan = () => {
- algorithmSpan.value = model.isBasicAlgorithm ? 12 : 0
+ algorithmSpan.value = model.isBasicAlgorithm ? 24 : 0
searchParamsSpan.value = model.isBasicAlgorithm ? 24 : 0
}
@@ -155,6 +158,8 @@ export function useBasicAlgorithm(model: {
resetSpan()
}
)
+ setFlag()
+ resetSpan()
return [
{
@@ -204,6 +209,10 @@ export function useAutoML(model: { [field: string]: any }): IJsonItem[] {
resetSpan()
}
)
+
+ setFlag()
+ resetSpan()
+
return [
{
type: 'select',
@@ -215,6 +224,60 @@ export function useAutoML(model: { [field: string]: any }): IJsonItem[] {
]
}
+export function useCustomProject(model: { [field: string]: any }): IJsonItem[] {
+ const { t } = useI18n()
+
+ const mlflowProjectRepositorySpan = ref(0)
+ const mlflowProjectVersionSpan = ref(0)
+ const customParamsSpan = ref(0)
+
+ const setFlag = () => {
+ model.isCustomProject =
+ model.mlflowJobType === 'CustomProject' &&
+ model.mlflowTaskType === 'MLflow Projects'
+ ? true
+ : false
+ }
+
+ const resetSpan = () => {
+ mlflowProjectRepositorySpan.value = model.isCustomProject ? 24 : 0
+ mlflowProjectVersionSpan.value = model.isCustomProject ? 12 : 0
+ customParamsSpan.value = model.isCustomProject ? 24 : 0
+ }
+
+ watch(
+ () => [model.mlflowTaskType, model.mlflowJobType],
+ () => {
+ setFlag()
+ resetSpan()
+ }
+ )
+
+ setFlag()
+ resetSpan()
+
+ return [
+ {
+ type: 'input',
+ field: 'mlflowProjectRepository',
+ name: t('project.node.mlflowProjectRepository'),
+ span: mlflowProjectRepositorySpan,
+ props: {
+ placeholder: t('project.node.mlflowProjectRepository_tips')
+ }
+ },
+ {
+ type: 'input',
+ field: 'mlflowProjectVersion',
+ name: t('project.node.mlflowProjectVersion'),
+ span: mlflowProjectVersionSpan,
+ props: {
+ placeholder: t('project.node.mlflowProjectVersion_tips')
+ }
+ }
+ ]
+}
+
export const MLFLOW_JOB_TYPE = [
{
label: 'BasicAlgorithm',
@@ -223,6 +286,10 @@ export const MLFLOW_JOB_TYPE = [
{
label: 'AutoML',
value: 'AutoML'
+ },
+ {
+ label: 'Custom Project',
+ value: 'CustomProject'
}
]
export const ALGORITHM = [
diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts
index 1b831ad7ee..1d03d650b3 100644
--- a/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts
+++ b/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts
@@ -351,6 +351,8 @@ export function formatParams(data: INodeData): {
taskParams.deployType = data.deployType
taskParams.deployPort = data.deployPort
taskParams.deployModelKey = data.deployModelKey
+ taskParams.mlflowProjectRepository = data.mlflowProjectRepository
+ taskParams.mlflowProjectVersion = data.mlflowProjectVersion
}
if (data.taskType === 'PIGEON') {
diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-mlflow.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-mlflow.ts
index e043fe1a8e..10bb1893c9 100644
--- a/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-mlflow.ts
+++ b/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-mlflow.ts
@@ -43,11 +43,13 @@ export function useMlflow({
workerGroup: 'default',
algorithm: 'svm',
mlflowTrackingUri: 'http://127.0.0.1:5000',
- mlflowTaskType: 'MLflow Models',
+ mlflowTaskType: 'MLflow Projects',
deployType: 'MLFLOW',
deployPort: '7000',
- mlflowJobType: 'AutoML',
+ mlflowJobType: 'CustomProject',
+ mlflowProjectVersion: 'master',
automlTool: 'flaml',
+ mlflowCustomProjectParameters: [],
delayTime: 0,
timeout: 30
} as INodeData)
diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts
index bfe7b2bc9f..8d595a5737 100644
--- a/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts
+++ b/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts
@@ -330,6 +330,8 @@ interface ITaskParams {
automlTool?: string
registerModel?: boolean
mlflowTaskType?: string
+ mlflowProjectRepository?: string
+ mlflowProjectVersion?: string
deployType?: string
deployPort?: string
deployModelKey?: string