You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by kl...@apache.org on 2024/03/27 10:43:02 UTC

(incubator-devlake) branch kw-resume-pipeline-after-restart created (now e0490d149)

This is an automated email from the ASF dual-hosted git repository.

klesh pushed a change to branch kw-resume-pipeline-after-restart
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git


      at e0490d149 feat: resume pipelines on restart

This branch includes the following new commits:

     new e0490d149 feat: resume pipelines on restart

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.



(incubator-devlake) 01/01: feat: resume pipelines on restart

Posted by kl...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

klesh pushed a commit to branch kw-resume-pipeline-after-restart
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git

commit e0490d149f5b08b2640e4f0d20e811101f282fa4
Author: Klesh Wong <zh...@merico.dev>
AuthorDate: Wed Mar 27 18:42:42 2024 +0800

    feat: resume pipelines on restart
---
 backend/core/config/config_viper.go |  1 +
 backend/core/models/task.go         |  1 +
 backend/core/runner/run_pipeline.go |  5 ++--
 backend/core/runner/run_task.go     | 27 +++++++++++------
 backend/server/services/pipeline.go | 58 +++++++++++++++++++------------------
 5 files changed, 53 insertions(+), 39 deletions(-)

diff --git a/backend/core/config/config_viper.go b/backend/core/config/config_viper.go
index 3e4e4b0c8..ea5ed6da9 100644
--- a/backend/core/config/config_viper.go
+++ b/backend/core/config/config_viper.go
@@ -104,6 +104,7 @@ func setDefaultValue(v *viper.Viper) {
 	v.SetDefault("PLUGIN_DIR", "bin/plugins")
 	v.SetDefault("REMOTE_PLUGIN_DIR", "python/plugins")
 	v.SetDefault("SWAGGER_DOCS_DIR", "resources/swagger")
+	v.SetDefault("RESUME_PIPELINES", true)
 }
 
 func init() {
diff --git a/backend/core/models/task.go b/backend/core/models/task.go
index 6fba0f760..6fe1b7dd1 100644
--- a/backend/core/models/task.go
+++ b/backend/core/models/task.go
@@ -26,6 +26,7 @@ import (
 const (
 	TASK_CREATED   = "TASK_CREATED"
 	TASK_RERUN     = "TASK_RERUN"
+	TASK_RESUME    = "TASK_RESUME"
 	TASK_RUNNING   = "TASK_RUNNING"
 	TASK_COMPLETED = "TASK_COMPLETED"
 	TASK_FAILED    = "TASK_FAILED"
diff --git a/backend/core/runner/run_pipeline.go b/backend/core/runner/run_pipeline.go
index 24a07453b..7df1d994d 100644
--- a/backend/core/runner/run_pipeline.go
+++ b/backend/core/runner/run_pipeline.go
@@ -19,11 +19,12 @@ package runner
 
 import (
 	gocontext "context"
+	"time"
+
 	"github.com/apache/incubator-devlake/core/context"
 	"github.com/apache/incubator-devlake/core/dal"
 	"github.com/apache/incubator-devlake/core/errors"
 	"github.com/apache/incubator-devlake/core/models"
-	"time"
 )
 
 // RunPipeline FIXME ...
@@ -37,7 +38,7 @@ func RunPipeline(
 	var tasks []models.Task
 	err := db.All(
 		&tasks,
-		dal.Where("pipeline_id = ? AND status in ?", pipelineId, []string{models.TASK_CREATED, models.TASK_RERUN}),
+		dal.Where("pipeline_id = ? AND status in ?", pipelineId, []string{models.TASK_CREATED, models.TASK_RERUN, models.TASK_RESUME}),
 		dal.Orderby("pipeline_row, pipeline_col"),
 	)
 	if err != nil {
diff --git a/backend/core/runner/run_task.go b/backend/core/runner/run_task.go
index 6ee04eccb..d6d8d5092 100644
--- a/backend/core/runner/run_task.go
+++ b/backend/core/runner/run_task.go
@@ -47,9 +47,6 @@ func RunTask(
 	if err := db.First(task, dal.Where("id = ?", taskId)); err != nil {
 		return err
 	}
-	if task.Status == models.TASK_COMPLETED {
-		return errors.Default.New("invalid task status")
-	}
 	dbPipeline := &models.Pipeline{}
 	if err := db.First(dbPipeline, dal.Where("id = ? ", task.PipelineId)); err != nil {
 		return err
@@ -60,6 +57,9 @@ func RunTask(
 		return err
 	}
 	beganAt := time.Now()
+	if dbPipeline.BeganAt != nil {
+		beganAt = *dbPipeline.BeganAt
+	}
 	// make sure task status always correct even if it panicked
 	defer func() {
 		if r := recover(); r != nil {
@@ -119,6 +119,10 @@ func RunTask(
 		}
 	}()
 
+	if task.Status == models.TASK_COMPLETED {
+		return nil
+	}
+
 	// start execution
 	logger.Info("start executing task: %d", task.ID)
 	dbe := db.UpdateColumns(task, []dal.DalSet{
@@ -261,7 +265,6 @@ func RunPluginSubTasks(
 		}
 
 		// run subtask
-		logger.Info("executing subtask %s", subtaskMeta.Name)
 		subtaskNumber++
 		if progress != nil {
 			progress <- plugin.RunningProgress{
@@ -270,11 +273,17 @@ func RunPluginSubTasks(
 				SubTaskNumber: subtaskNumber,
 			}
 		}
-		err = runSubtask(basicRes, subtaskCtx, task.ID, subtaskNumber, subtaskMeta.EntryPoint)
-		if err != nil {
-			err = errors.SubtaskErr.Wrap(err, fmt.Sprintf("subtask %s ended unexpectedly", subtaskMeta.Name), errors.WithData(&subtaskMeta))
-			logger.Error(err, "")
-			return err
+		subtaskFinsied := errors.Must1(basicRes.GetDal().Count(dal.From(&models.Subtask{}), dal.Where("task_id = ? AND name = ?", task.ID, subtaskMeta.Name)))
+		if subtaskFinsied > 0 {
+			logger.Info("subtask %s already finished previously", subtaskMeta.Name)
+		} else {
+			logger.Info("executing subtask %s", subtaskMeta.Name)
+			err = runSubtask(basicRes, subtaskCtx, task.ID, subtaskNumber, subtaskMeta.EntryPoint)
+			if err != nil {
+				err = errors.SubtaskErr.Wrap(err, fmt.Sprintf("subtask %s ended unexpectedly", subtaskMeta.Name), errors.WithData(&subtaskMeta))
+				logger.Error(err, "")
+				return err
+			}
 		}
 		taskCtx.IncProgress(1)
 	}
diff --git a/backend/server/services/pipeline.go b/backend/server/services/pipeline.go
index 2a1ac8ccb..80ce47e4e 100644
--- a/backend/server/services/pipeline.go
+++ b/backend/server/services/pipeline.go
@@ -79,34 +79,14 @@ func pipelineServiceInit() {
 	}
 
 	// standalone mode: reset pipeline status
-	errMsg := "The process was terminated unexpectedly"
-	err := db.UpdateColumns(
-		&models.Pipeline{},
-		[]dal.DalSet{
-			{ColumnName: "status", Value: models.TASK_FAILED},
-			{ColumnName: "message", Value: errMsg},
-		},
-		dal.Where("status = ?", models.TASK_RUNNING),
-	)
-	if err != nil {
-		panic(err)
-	}
-	err = db.UpdateColumns(
-		&models.Task{},
-		[]dal.DalSet{
-			{ColumnName: "status", Value: models.TASK_FAILED},
-			{ColumnName: "message", Value: errMsg},
-		},
-		dal.Where("status = ?", models.TASK_RUNNING),
-	)
-	if err != nil {
-		panic(err)
+	if cfg.GetBool("RESUME_PIPELINES") {
+		markInterruptedPipelineAs(models.TASK_RESUME)
+	} else {
+		markInterruptedPipelineAs(models.TASK_FAILED)
 	}
 
-	err = ReloadBlueprints()
-	if err != nil {
-		panic(err)
-	}
+	// load cronjobs for blueprints
+	errors.Must(ReloadBlueprints())
 
 	var pipelineMaxParallel = cfg.GetInt64("PIPELINE_MAX_PARALLEL")
 	if pipelineMaxParallel < 0 {
@@ -120,6 +100,23 @@ func pipelineServiceInit() {
 	go RunPipelineInQueue(pipelineMaxParallel)
 }
 
+func markInterruptedPipelineAs(status string) {
+	errors.Must(db.UpdateColumns(
+		&models.Pipeline{},
+		[]dal.DalSet{
+			{ColumnName: "status", Value: status},
+		},
+		dal.Where("status = ?", models.TASK_RUNNING),
+	))
+	errors.Must(db.UpdateColumns(
+		&models.Task{},
+		[]dal.DalSet{
+			{ColumnName: "status", Value: status},
+		},
+		dal.Where("status = ?", models.TASK_RUNNING),
+	))
+}
+
 // CreatePipeline and return the model
 func CreatePipeline(newPipeline *models.NewPipeline, shouldSanitize bool) (*models.Pipeline, errors.Error) {
 	pipeline, err := CreateDbPipeline(newPipeline)
@@ -238,7 +235,7 @@ func dequeuePipeline(runningParallelLabels []string) (pipeline *models.Pipeline,
 	// prepare query to find an appropriate pipeline to execute
 	pipeline = &models.Pipeline{}
 	err = tx.First(pipeline,
-		dal.Where("status IN ?", []string{models.TASK_CREATED, models.TASK_RERUN}),
+		dal.Where("status IN ?", []string{models.TASK_CREATED, models.TASK_RERUN, models.TASK_RESUME}),
 		dal.Join(
 			`left join _devlake_pipeline_labels ON
 				_devlake_pipeline_labels.pipeline_id = _devlake_pipelines.id AND
@@ -254,11 +251,16 @@ func dequeuePipeline(runningParallelLabels []string) (pipeline *models.Pipeline,
 	)
 	if err == nil {
 		// mark the pipeline running, now we want a write lock
+		if pipeline.BeganAt == nil {
+			now := time.Now()
+			pipeline.BeganAt = &now
+			globalPipelineLog.Info("resumed pipeline #%d", pipeline.ID)
+		}
 		errors.Must(tx.LockTables(dal.LockTables{{Table: "_devlake_pipelines", Exclusive: true}}))
 		err = tx.UpdateColumns(&models.Pipeline{}, []dal.DalSet{
 			{ColumnName: "status", Value: models.TASK_RUNNING},
 			{ColumnName: "message", Value: ""},
-			{ColumnName: "began_at", Value: time.Now()},
+			{ColumnName: "began_at", Value: pipeline.BeganAt},
 		}, dal.Where("id = ?", pipeline.ID))
 		if err != nil {
 			panic(err)