You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by zh...@apache.org on 2023/09/01 07:22:03 UTC

[dolphinscheduler] branch 3.2.0-release updated (c5bfaeab12 -> ab306cedd5)

This is an automated email from the ASF dual-hosted git repository.

zhongjiajie pushed a change to branch 3.2.0-release
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git


 discard c5bfaeab12 [maven-release-plugin] prepare for next development iteration
 discard 4fca22c830 [maven-release-plugin] prepare release 3.2.0
     new fb792849e1 fix invalid registry path ends with 'master' or 'worker' (#14328)
     new 8c451a0e19 [Fix-14828][master] check task id not null to avoid NPE (#14829)
     new ab306cedd5 [Fix-14149][master] task finished so not dispatch (#14161)

This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version.  This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:

 * -- * -- B -- O -- O -- O   (c5bfaeab12)
            \
             N -- N -- N   refs/heads/3.2.0-release (ab306cedd5)

You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.

Any revisions marked "omit" are not gone; other references still
refer to them.  Any revisions marked "discard" are gone forever.

The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../dolphinscheduler-alert-all/pom.xml             |  5 ++--
 .../dolphinscheduler-alert-api/pom.xml             |  5 ++--
 .../dolphinscheduler-alert-dingtalk/pom.xml        |  5 ++--
 .../dolphinscheduler-alert-email/pom.xml           |  5 ++--
 .../dolphinscheduler-alert-feishu/pom.xml          |  5 ++--
 .../dolphinscheduler-alert-http/pom.xml            |  5 ++--
 .../dolphinscheduler-alert-pagerduty/pom.xml       |  5 ++--
 .../dolphinscheduler-alert-script/pom.xml          |  5 ++--
 .../dolphinscheduler-alert-slack/pom.xml           |  5 ++--
 .../dolphinscheduler-alert-telegram/pom.xml        |  5 ++--
 .../dolphinscheduler-alert-webexteams/pom.xml      |  5 ++--
 .../dolphinscheduler-alert-wechat/pom.xml          |  5 ++--
 .../dolphinscheduler-alert-plugins/pom.xml         |  5 ++--
 .../dolphinscheduler-alert-server/pom.xml          |  5 ++--
 dolphinscheduler-alert/pom.xml                     |  5 ++--
 dolphinscheduler-aop/pom.xml                       |  5 ++--
 dolphinscheduler-api/pom.xml                       |  5 ++--
 dolphinscheduler-bom/pom.xml                       |  5 ++--
 dolphinscheduler-common/pom.xml                    |  5 ++--
 dolphinscheduler-dao/pom.xml                       |  5 ++--
 dolphinscheduler-data-quality/pom.xml              |  5 ++--
 .../dolphinscheduler-datasource-all/pom.xml        |  5 ++--
 .../dolphinscheduler-datasource-api/pom.xml        |  5 ++--
 .../dolphinscheduler-datasource-athena/pom.xml     |  5 ++--
 .../dolphinscheduler-datasource-azure-sql/pom.xml  |  5 ++--
 .../dolphinscheduler-datasource-clickhouse/pom.xml |  5 ++--
 .../dolphinscheduler-datasource-dameng/pom.xml     |  5 ++--
 .../dolphinscheduler-datasource-databend/pom.xml   |  5 ++--
 .../dolphinscheduler-datasource-db2/pom.xml        |  5 ++--
 .../dolphinscheduler-datasource-doris/pom.xml      |  5 ++--
 .../dolphinscheduler-datasource-hive/pom.xml       |  5 ++--
 .../dolphinscheduler-datasource-kyuubi/pom.xml     |  5 ++--
 .../dolphinscheduler-datasource-mysql/pom.xml      |  5 ++--
 .../dolphinscheduler-datasource-oceanbase/pom.xml  |  5 ++--
 .../dolphinscheduler-datasource-oracle/pom.xml     |  5 ++--
 .../dolphinscheduler-datasource-postgresql/pom.xml |  5 ++--
 .../dolphinscheduler-datasource-presto/pom.xml     |  5 ++--
 .../dolphinscheduler-datasource-redshift/pom.xml   |  5 ++--
 .../dolphinscheduler-datasource-snowflake/pom.xml  |  5 ++--
 .../dolphinscheduler-datasource-spark/pom.xml      |  5 ++--
 .../dolphinscheduler-datasource-sqlserver/pom.xml  |  5 ++--
 .../dolphinscheduler-datasource-ssh/pom.xml        |  5 ++--
 .../dolphinscheduler-datasource-starrocks/pom.xml  |  5 ++--
 .../dolphinscheduler-datasource-trino/pom.xml      |  5 ++--
 .../dolphinscheduler-datasource-vertica/pom.xml    |  5 ++--
 dolphinscheduler-datasource-plugin/pom.xml         |  5 ++--
 dolphinscheduler-dist/pom.xml                      |  5 ++--
 dolphinscheduler-master/pom.xml                    |  5 ++--
 .../master/runner/WorkflowExecuteRunnable.java     | 35 ++++++++++++++++++++--
 .../master/runner/WorkflowExecuteRunnableTest.java | 18 +++++++++++
 dolphinscheduler-meter/pom.xml                     |  5 ++--
 dolphinscheduler-microbench/pom.xml                |  5 ++--
 .../dolphinscheduler-registry-all/pom.xml          |  5 ++--
 .../dolphinscheduler-registry-api/pom.xml          |  5 ++--
 .../registry/api/RegistryClient.java               |  4 +--
 .../dolphinscheduler-registry-etcd/pom.xml         |  5 ++--
 .../dolphinscheduler-registry-jdbc/pom.xml         |  5 ++--
 .../dolphinscheduler-registry-zookeeper/pom.xml    |  5 ++--
 .../dolphinscheduler-registry-plugins/pom.xml      |  5 ++--
 dolphinscheduler-registry/pom.xml                  |  5 ++--
 dolphinscheduler-remote/pom.xml                    |  5 ++--
 .../dolphinscheduler-scheduler-all/pom.xml         |  5 ++--
 .../dolphinscheduler-scheduler-api/pom.xml         |  5 ++--
 .../dolphinscheduler-scheduler-quartz/pom.xml      |  5 ++--
 dolphinscheduler-scheduler-plugin/pom.xml          |  5 ++--
 dolphinscheduler-service/pom.xml                   |  5 ++--
 dolphinscheduler-spi/pom.xml                       |  5 ++--
 dolphinscheduler-standalone-server/pom.xml         |  5 ++--
 .../dolphinscheduler-storage-abs/pom.xml           |  5 ++--
 .../dolphinscheduler-storage-all/pom.xml           |  5 ++--
 .../dolphinscheduler-storage-api/pom.xml           |  5 ++--
 .../dolphinscheduler-storage-gcs/pom.xml           |  5 ++--
 .../dolphinscheduler-storage-hdfs/pom.xml          |  5 ++--
 .../dolphinscheduler-storage-obs/pom.xml           |  5 ++--
 .../dolphinscheduler-storage-oss/pom.xml           |  5 ++--
 .../dolphinscheduler-storage-s3/pom.xml            |  5 ++--
 dolphinscheduler-storage-plugin/pom.xml            |  5 ++--
 .../dolphinscheduler-task-all/pom.xml              |  5 ++--
 .../dolphinscheduler-task-api/pom.xml              |  5 ++--
 .../dolphinscheduler-task-chunjun/pom.xml          |  5 ++--
 .../dolphinscheduler-task-datafactory/pom.xml      |  5 ++--
 .../dolphinscheduler-task-dataquality/pom.xml      |  5 ++--
 .../dolphinscheduler-task-datasync/pom.xml         |  5 ++--
 .../dolphinscheduler-task-datax/pom.xml            |  5 ++--
 .../dolphinscheduler-task-dinky/pom.xml            |  5 ++--
 .../dolphinscheduler-task-dms/pom.xml              |  5 ++--
 .../dolphinscheduler-task-dvc/pom.xml              |  5 ++--
 .../dolphinscheduler-task-emr/pom.xml              |  5 ++--
 .../dolphinscheduler-task-flink-stream/pom.xml     |  5 ++--
 .../dolphinscheduler-task-flink/pom.xml            |  5 ++--
 .../dolphinscheduler-task-hivecli/pom.xml          |  5 ++--
 .../dolphinscheduler-task-http/pom.xml             |  5 ++--
 .../dolphinscheduler-task-java/pom.xml             |  5 ++--
 .../dolphinscheduler-task-jupyter/pom.xml          |  5 ++--
 .../dolphinscheduler-task-k8s/pom.xml              |  5 ++--
 .../dolphinscheduler-task-kubeflow/pom.xml         |  5 ++--
 .../dolphinscheduler-task-linkis/pom.xml           |  5 ++--
 .../dolphinscheduler-task-mlflow/pom.xml           |  5 ++--
 .../dolphinscheduler-task-mr/pom.xml               |  5 ++--
 .../dolphinscheduler-task-openmldb/pom.xml         |  5 ++--
 .../dolphinscheduler-task-pigeon/pom.xml           |  5 ++--
 .../dolphinscheduler-task-procedure/pom.xml        |  5 ++--
 .../dolphinscheduler-task-python/pom.xml           |  5 ++--
 .../dolphinscheduler-task-pytorch/pom.xml          |  5 ++--
 .../dolphinscheduler-task-remoteshell/pom.xml      |  5 ++--
 .../dolphinscheduler-task-sagemaker/pom.xml        |  5 ++--
 .../dolphinscheduler-task-seatunnel/pom.xml        |  5 ++--
 .../dolphinscheduler-task-shell/pom.xml            |  5 ++--
 .../dolphinscheduler-task-spark/pom.xml            |  5 ++--
 .../dolphinscheduler-task-sql/pom.xml              |  5 ++--
 .../dolphinscheduler-task-sqoop/pom.xml            |  5 ++--
 .../dolphinscheduler-task-zeppelin/pom.xml         |  5 ++--
 dolphinscheduler-task-plugin/pom.xml               |  5 ++--
 dolphinscheduler-tools/pom.xml                     |  5 ++--
 dolphinscheduler-ui/pom.xml                        |  5 ++--
 dolphinscheduler-worker/pom.xml                    |  5 ++--
 pom.xml                                            |  5 ++--
 117 files changed, 395 insertions(+), 232 deletions(-)


[dolphinscheduler] 01/03: fix invalid registry path ends with 'master' or 'worker' (#14328)

Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

zhongjiajie pushed a commit to branch 3.2.0-release
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git

commit fb792849e19c79fac667899ba03b124c8b1ff211
Author: Zinway <zi...@gmail.com>
AuthorDate: Fri Sep 1 15:11:49 2023 +0800

    fix invalid registry path ends with 'master' or 'worker' (#14328)
    
    Co-authored-by: Zinway Liu <zi...@ucloud.cn>
    Co-authored-by: Aaron Wang <wa...@gmail.com>
    (cherry picked from commit 96041e38766d1eb391812150fa15adc806ba2d4c)
---
 .../java/org/apache/dolphinscheduler/registry/api/RegistryClient.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/RegistryClient.java b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/RegistryClient.java
index ddd80d94a1..f690b6800f 100644
--- a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/RegistryClient.java
+++ b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/RegistryClient.java
@@ -210,11 +210,11 @@ public class RegistryClient {
     }
 
     public boolean isMasterPath(String path) {
-        return path != null && path.startsWith(RegistryNodeType.MASTER.getRegistryPath());
+        return path != null && path.startsWith(RegistryNodeType.MASTER.getRegistryPath() + Constants.SINGLE_SLASH);
     }
 
     public boolean isWorkerPath(String path) {
-        return path != null && path.startsWith(RegistryNodeType.WORKER.getRegistryPath());
+        return path != null && path.startsWith(RegistryNodeType.WORKER.getRegistryPath() + Constants.SINGLE_SLASH);
     }
 
     public Collection<String> getChildrenKeys(final String key) {


[dolphinscheduler] 02/03: [Fix-14828][master] check task id not null to avoid NPE (#14829)

Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

zhongjiajie pushed a commit to branch 3.2.0-release
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git

commit 8c451a0e19ac60a1e882f570d272d52952ff3a43
Author: eye-gu <73...@qq.com>
AuthorDate: Thu Aug 31 18:34:31 2023 +0800

    [Fix-14828][master] check task id not null to avoid NPE (#14829)
    
    Co-authored-by: eye <ey...@aloudata.com>
    Co-authored-by: Aaron Wang <wa...@gmail.com>
    Co-authored-by: caishunfeng <ca...@gmail.com>
    (cherry picked from commit 33525a17c3463d4bde56ae2335ecec38b24a5a6a)
---
 .../dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
index e8e2a6a233..779180aa89 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
@@ -1906,7 +1906,7 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
         TaskInstance task;
         while ((task = readyToSubmitTaskQueue.peek()) != null) {
             // stop tasks which is retrying if forced success happens
-            if (task.taskCanRetry()) {
+            if (task.getId() != null && task.taskCanRetry()) {
                 TaskInstance retryTask = taskInstanceDao.queryById(task.getId());
                 if (retryTask != null && retryTask.getState().isForceSuccess()) {
                     task.setState(retryTask.getState());


[dolphinscheduler] 03/03: [Fix-14149][master] task finished so not dispatch (#14161)

Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

zhongjiajie pushed a commit to branch 3.2.0-release
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git

commit ab306cedd535d14cfee0401800f15239cc7841c2
Author: eye-gu <73...@qq.com>
AuthorDate: Thu Aug 31 20:08:09 2023 +0800

    [Fix-14149][master] task finished so not dispatch (#14161)
    
    (cherry picked from commit 7ae4fb378779077c75cd6411ef8e3b68f232bd84)
---
 .../master/runner/WorkflowExecuteRunnable.java     | 33 +++++++++++++++++++++-
 .../master/runner/WorkflowExecuteRunnableTest.java | 18 ++++++++++++
 2 files changed, 50 insertions(+), 1 deletion(-)

diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
index 779180aa89..b07256c11f 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
@@ -84,6 +84,7 @@ import org.apache.dolphinscheduler.server.master.metrics.TaskMetrics;
 import org.apache.dolphinscheduler.server.master.rpc.MasterRpcClient;
 import org.apache.dolphinscheduler.server.master.runner.execute.DefaultTaskExecuteRunnable;
 import org.apache.dolphinscheduler.server.master.runner.execute.DefaultTaskExecuteRunnableFactory;
+import org.apache.dolphinscheduler.server.master.runner.execute.TaskExecuteRunnable;
 import org.apache.dolphinscheduler.server.master.utils.TaskUtils;
 import org.apache.dolphinscheduler.server.master.utils.WorkflowInstanceUtils;
 import org.apache.dolphinscheduler.service.alert.ProcessAlertManager;
@@ -124,6 +125,7 @@ import lombok.extern.slf4j.Slf4j;
 
 import org.springframework.beans.BeanUtils;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 
@@ -988,7 +990,7 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
                     }
                 }
                 // 4. submit to dispatch queue
-                taskExecuteRunnable.dispatch();
+                tryToDispatchTaskInstance(taskInstance, taskExecuteRunnable);
 
                 stateWheelExecuteThread.addTask4TimeoutCheck(workflowInstance, taskInstance);
                 return true;
@@ -999,6 +1001,35 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
         }
     }
 
+    /**
+     * Sometimes (such as pause), if the task instance status has already been finished,
+     * there is no need to dispatch it
+     */
+    @VisibleForTesting
+    void tryToDispatchTaskInstance(TaskInstance taskInstance, TaskExecuteRunnable taskExecuteRunnable) {
+        if (!taskInstance.getState().isFinished()) {
+            taskExecuteRunnable.dispatch();
+        } else {
+            if (workflowExecuteContext.getWorkflowInstance().isBlocked()) {
+                TaskStateEvent processBlockEvent = TaskStateEvent.builder()
+                        .processInstanceId(workflowExecuteContext.getWorkflowInstance().getId())
+                        .taskInstanceId(taskInstance.getId())
+                        .status(taskInstance.getState())
+                        .type(StateEventType.PROCESS_BLOCKED)
+                        .build();
+                this.stateEvents.add(processBlockEvent);
+            }
+
+            TaskStateEvent taskStateChangeEvent = TaskStateEvent.builder()
+                    .processInstanceId(workflowExecuteContext.getWorkflowInstance().getId())
+                    .taskInstanceId(taskInstance.getId())
+                    .status(taskInstance.getState())
+                    .type(StateEventType.TASK_STATE_CHANGE)
+                    .build();
+            this.stateEvents.add(taskStateChangeEvent);
+        }
+    }
+
     /**
      * find task instance in db.
      * in case submit more than one same name task in the same time.
diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnableTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnableTest.java
index 3aff6d65e8..2306fbc428 100644
--- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnableTest.java
+++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnableTest.java
@@ -35,10 +35,12 @@ import org.apache.dolphinscheduler.dao.entity.TaskInstance;
 import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao;
 import org.apache.dolphinscheduler.dao.repository.TaskDefinitionLogDao;
 import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao;
+import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
 import org.apache.dolphinscheduler.server.master.config.MasterConfig;
 import org.apache.dolphinscheduler.server.master.graph.IWorkflowGraph;
 import org.apache.dolphinscheduler.server.master.rpc.MasterRpcClient;
 import org.apache.dolphinscheduler.server.master.runner.execute.DefaultTaskExecuteRunnableFactory;
+import org.apache.dolphinscheduler.server.master.runner.execute.TaskExecuteRunnable;
 import org.apache.dolphinscheduler.service.alert.ProcessAlertManager;
 import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
 import org.apache.dolphinscheduler.service.command.CommandService;
@@ -374,4 +376,20 @@ public class WorkflowExecuteRunnableTest {
         return schedulerList;
     }
 
+    @Test
+    void testTryToDispatchTaskInstance() {
+        // task instance already finished, not dispatch
+        TaskInstance taskInstance = new TaskInstance();
+        taskInstance.setState(TaskExecutionStatus.PAUSE);
+        Mockito.when(processInstance.isBlocked()).thenReturn(true);
+        TaskExecuteRunnable taskExecuteRunnable = Mockito.mock(TaskExecuteRunnable.class);
+        workflowExecuteThread.tryToDispatchTaskInstance(taskInstance, taskExecuteRunnable);
+        Mockito.verify(taskExecuteRunnable, Mockito.never()).dispatch();
+
+        // submit success should dispatch
+        taskInstance = new TaskInstance();
+        taskInstance.setState(TaskExecutionStatus.SUBMITTED_SUCCESS);
+        workflowExecuteThread.tryToDispatchTaskInstance(taskInstance, taskExecuteRunnable);
+        Mockito.verify(taskExecuteRunnable).dispatch();
+    }
 }