[incubator-devlake] branch release-v0.14 updated: feat: check count after starrocks sync (#3550)

[ab...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

abeizn pushed a commit to branch release-v0.14
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git


The following commit(s) were added to refs/heads/release-v0.14 by this push:
     new a61cf40d feat: check count after starrocks sync (#3550)
a61cf40d is described below

commit a61cf40d9f908c6de6fcd30fd970036138c023e3
Author: long2ice <lo...@gmail.com>
AuthorDate: Mon Oct 24 12:42:29 2022 +0800

    feat: check count after starrocks sync (#3550)
---
 plugins/starrocks/tasks.go | 37 ++++++++++++++++++++++++++++++++-----
 1 file changed, 32 insertions(+), 5 deletions(-)

diff --git a/plugins/starrocks/tasks.go b/plugins/starrocks/tasks.go
index f23cc095..6493ed8f 100644
--- a/plugins/starrocks/tasks.go
+++ b/plugins/starrocks/tasks.go
@@ -22,6 +22,12 @@ import (
 	"database/sql"
 	"encoding/json"
 	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strings"
+
 	"github.com/apache/incubator-devlake/errors"
 	"github.com/apache/incubator-devlake/impl/dalgorm"
 	"github.com/apache/incubator-devlake/plugins/core"
@@ -30,11 +36,6 @@ import (
 	"gorm.io/driver/mysql"
 	"gorm.io/driver/postgres"
 	"gorm.io/gorm"
-	"io"
-	"net/http"
-	"net/url"
-	"regexp"
-	"strings"
 )
 
 type Table struct {
@@ -334,6 +335,32 @@ func loadData(starrocks *sql.DB, c core.SubTaskContext, starrocksTable, starrock
 	if err != nil {
 		return err
 	}
+	// check data count
+	rows, err := db.RawCursor(fmt.Sprintf("select count(*) from %s", table))
+	if err != nil {
+		return err
+	}
+	var sourceCount int
+	for rows.Next() {
+		err = rows.Scan(&sourceCount)
+		if err != nil {
+			return err
+		}
+	}
+	rows, err = starrocks.Query(fmt.Sprintf("select count(*) from %s", starrocksTable))
+	if err != nil {
+		return err
+	}
+	var starrocksCount int
+	for rows.Next() {
+		err = rows.Scan(&starrocksCount)
+		if err != nil {
+			return err
+		}
+	}
+	if sourceCount != starrocksCount {
+		c.GetLogger().Warn(nil, "source count %d not equal to starrocks count %d", sourceCount, starrocksCount)
+	}
 	c.GetLogger().Info("load %s to starrocks success", table)
 	return nil
 }