You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by ab...@apache.org on 2024/04/12 02:34:22 UTC
(incubator-devlake) branch release-v1.0 updated (291014e54 -> 055efd3bf)
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a change to branch release-v1.0
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
from 291014e54 fix(azuredevops): fix _tool_azuredevops_gitrepositories.id field (#7301) (#7303)
new 7fe4ee546 fix: dora details deployment frequency (#7298)
new fad6de12f fix: task begin_at replace of pipeline begin_at (#7299)
new 53d884a04 Add more log on gitclone errors (#7300)
new af235a968 pr-type/bug-fix 7114 - wire up the page size event to alter page size (#7124)
new 9c3d8769c fix: more friendly msg when no data with shallow clone (#7308)
new dd0bc6b74 feat: upgrade grafana from 10.2.6 to 10.4.1 (#7309)
new 689c73271 fix: giturls CVE-2023-46402 (#7310)
new b46bcdb1c refactor: prevent concurrent migration (#7311)
new de321b86d fix: db timeafter adjust (#7314)
new 055efd3bf Kw gitext collect blob by default (#7315)
The 10 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.gitignore | 4 ++-
backend/core/runner/run_task.go | 4 +--
backend/go.mod | 6 ++---
backend/go.sum | 4 +--
.../pluginhelper/api/api_collector_with_state.go | 7 ++++-
backend/plugins/gitextractor/impl/impl.go | 4 +--
.../plugins/gitextractor/parser/clone_gitcli.go | 14 +++++++---
.../tasks/{clone.go => repo_cloner.go} | 0
.../{git_repo_collector.go => repo_collector.go} | 0
backend/server/api/api.go | 30 ++++++++++++----------
backend/server/services/init.go | 20 ++++++++++++---
config-ui/src/routes/connection/connection.tsx | 8 ++++--
grafana/Dockerfile | 2 +-
.../DORADetails-DeploymentFrequency.json | 6 ++---
14 files changed, 71 insertions(+), 38 deletions(-)
rename backend/plugins/gitextractor/tasks/{clone.go => repo_cloner.go} (100%)
rename backend/plugins/gitextractor/tasks/{git_repo_collector.go => repo_collector.go} (100%)
(incubator-devlake) 02/10: fix: task begin_at replace of pipeline begin_at (#7299)
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch release-v1.0
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit fad6de12fee6d1be148bc67181c29edd092f81ce
Author: abeizn <zi...@merico.dev>
AuthorDate: Tue Apr 9 14:51:48 2024 +0800
fix: task begin_at replace of pipeline begin_at (#7299)
---
backend/core/runner/run_task.go | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/backend/core/runner/run_task.go b/backend/core/runner/run_task.go
index 0b4e0604c..aeace3940 100644
--- a/backend/core/runner/run_task.go
+++ b/backend/core/runner/run_task.go
@@ -57,8 +57,8 @@ func RunTask(
return err
}
beganAt := time.Now()
- if dbPipeline.BeganAt != nil {
- beganAt = *dbPipeline.BeganAt
+ if task.BeganAt != nil {
+ beganAt = *task.BeganAt
}
// make sure task status always correct even if it panicked
defer func() {
(incubator-devlake) 01/10: fix: dora details deployment frequency (#7298)
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch release-v1.0
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 7fe4ee546e36cfd24ee464b6c093b864303a9aa2
Author: abeizn <zi...@merico.dev>
AuthorDate: Tue Apr 9 10:59:06 2024 +0800
fix: dora details deployment frequency (#7298)
---
grafana/dashboards/DORADetails-DeploymentFrequency.json | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/grafana/dashboards/DORADetails-DeploymentFrequency.json b/grafana/dashboards/DORADetails-DeploymentFrequency.json
index d3e3d5c9e..2712404a9 100644
--- a/grafana/dashboards/DORADetails-DeploymentFrequency.json
+++ b/grafana/dashboards/DORADetails-DeploymentFrequency.json
@@ -457,7 +457,7 @@
"metricColumn": "none",
"queryType": "randomWalk",
"rawQuery": true,
- "rawSql": "-- Metric 1: Deployment Frequency\nwith last_few_calendar_months as(\n-- construct the last few calendar months within the selected time period in the top-right corner\n\tSELECT CAST(($__timeTo()-INTERVAL (H+T+U) DAY) AS date) day\n\tFROM ( SELECT 0 H\n\t\t\tUNION ALL SELECT 100 UNION ALL SELECT 200 UNION ALL SELECT 300\n\t\t) H CROSS JOIN ( SELECT 0 T\n\t\t\tUNION ALL SELECT 10 UNION ALL SELECT 20 UNION ALL SELECT 30\n\t\t\tUNION ALL SELECT 40 UNION ALL SELECT [...]
+ "rawSql": "-- Metric 1: Deployment Frequency\nwith last_few_calendar_months as(\n-- construct the last few calendar months within the selected time period in the top-right corner\n\tSELECT CAST(($__timeTo()-INTERVAL (H+T+U) DAY) AS date) day\n\tFROM ( SELECT 0 H\n\t\t\tUNION ALL SELECT 100 UNION ALL SELECT 200 UNION ALL SELECT 300\n\t\t) H CROSS JOIN ( SELECT 0 T\n\t\t\tUNION ALL SELECT 10 UNION ALL SELECT 20 UNION ALL SELECT 30\n\t\t\tUNION ALL SELECT 40 UNION ALL SELECT [...]
"refId": "A",
"select": [
[
@@ -586,7 +586,7 @@
"metricColumn": "none",
"queryType": "randomWalk",
"rawQuery": true,
- "rawSql": "-- Metric 1: Deployment Frequency\nwith last_few_calendar_months as(\n-- construct the last few calendar months within the selected time period in the top-right corner\n\tSELECT CAST(($__timeTo()-INTERVAL (H+T+U) DAY) AS date) day\n\tFROM ( SELECT 0 H\n\t\t\tUNION ALL SELECT 100 UNION ALL SELECT 200 UNION ALL SELECT 300\n\t\t) H CROSS JOIN ( SELECT 0 T\n\t\t\tUNION ALL SELECT 10 UNION ALL SELECT 20 UNION ALL SELECT 30\n\t\t\tUNION ALL SELECT 40 UNION ALL SELECT [...]
+ "rawSql": "-- Metric 1: Deployment Frequency\nwith last_few_calendar_months as(\n-- construct the last few calendar months within the selected time period in the top-right corner\n\tSELECT CAST(($__timeTo()-INTERVAL (H+T+U) DAY) AS date) day\n\tFROM ( SELECT 0 H\n\t\t\tUNION ALL SELECT 100 UNION ALL SELECT 200 UNION ALL SELECT 300\n\t\t) H CROSS JOIN ( SELECT 0 T\n\t\t\tUNION ALL SELECT 10 UNION ALL SELECT 20 UNION ALL SELECT 30\n\t\t\tUNION ALL SELECT 40 UNION ALL SELECT [...]
"refId": "A",
"select": [
[
@@ -714,7 +714,7 @@
"metricColumn": "none",
"queryType": "randomWalk",
"rawQuery": true,
- "rawSql": "-- Metric 1: Deployment Frequency\nwith last_few_calendar_months as(\n-- construct the last few calendar months within the selected time period in the top-right corner\n\tSELECT CAST(($__timeTo()-INTERVAL (H+T+U) DAY) AS date) day\n\tFROM ( SELECT 0 H\n\t\t\tUNION ALL SELECT 100 UNION ALL SELECT 200 UNION ALL SELECT 300\n\t\t) H CROSS JOIN ( SELECT 0 T\n\t\t\tUNION ALL SELECT 10 UNION ALL SELECT 20 UNION ALL SELECT 30\n\t\t\tUNION ALL SELECT 40 UNION ALL SELECT [...]
+ "rawSql": "-- Metric 1: Deployment Frequency\nwith last_few_calendar_months as(\n-- construct the last few calendar months within the selected time period in the top-right corner\n\tSELECT CAST(($__timeTo()-INTERVAL (H+T+U) DAY) AS date) day\n\tFROM ( SELECT 0 H\n\t\t\tUNION ALL SELECT 100 UNION ALL SELECT 200 UNION ALL SELECT 300\n\t\t) H CROSS JOIN ( SELECT 0 T\n\t\t\tUNION ALL SELECT 10 UNION ALL SELECT 20 UNION ALL SELECT 30\n\t\t\tUNION ALL SELECT 40 UNION ALL SELECT [...]
"refId": "A",
"select": [
[
(incubator-devlake) 10/10: Kw gitext collect blob by default (#7315)
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch release-v1.0
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 055efd3bfe165738372a61acdd01eeb31802b5fa
Author: Klesh Wong <zh...@merico.dev>
AuthorDate: Fri Apr 12 10:22:30 2024 +0800
Kw gitext collect blob by default (#7315)
* feat: collect commit stat by default
* refactor: naming gitext tasks file names
---
backend/plugins/gitextractor/impl/impl.go | 2 +-
backend/plugins/gitextractor/tasks/{clone.go => repo_cloner.go} | 0
.../gitextractor/tasks/{git_repo_collector.go => repo_collector.go} | 0
3 files changed, 1 insertion(+), 1 deletion(-)
diff --git a/backend/plugins/gitextractor/impl/impl.go b/backend/plugins/gitextractor/impl/impl.go
index ae4982551..37d2aaec2 100644
--- a/backend/plugins/gitextractor/impl/impl.go
+++ b/backend/plugins/gitextractor/impl/impl.go
@@ -93,7 +93,7 @@ func (p GitExtractor) PrepareTaskData(taskCtx plugin.TaskContext, options map[st
*optValue = &defValue
}
loadBool(&op.UseGoGit, "UseGoGit", false)
- loadBool(&op.SkipCommitStat, "SKIP_COMMIT_STAT", true)
+ loadBool(&op.SkipCommitStat, "SKIP_COMMIT_STAT", false)
loadBool(&op.SkipCommitFiles, "SKIP_COMMIT_FILES", true)
log.Info("UseGoGit: %v", *op.UseGoGit)
log.Info("SkipCommitStat: %v", *op.SkipCommitStat)
diff --git a/backend/plugins/gitextractor/tasks/clone.go b/backend/plugins/gitextractor/tasks/repo_cloner.go
similarity index 100%
rename from backend/plugins/gitextractor/tasks/clone.go
rename to backend/plugins/gitextractor/tasks/repo_cloner.go
diff --git a/backend/plugins/gitextractor/tasks/git_repo_collector.go b/backend/plugins/gitextractor/tasks/repo_collector.go
similarity index 100%
rename from backend/plugins/gitextractor/tasks/git_repo_collector.go
rename to backend/plugins/gitextractor/tasks/repo_collector.go
(incubator-devlake) 05/10: fix: more friendly msg when no data with shallow clone (#7308)
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch release-v1.0
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 9c3d8769c907a92e32d076ab4aa4eed0f647db12
Author: Klesh Wong <zh...@merico.dev>
AuthorDate: Wed Apr 10 16:19:15 2024 +0800
fix: more friendly msg when no data with shallow clone (#7308)
---
backend/plugins/gitextractor/parser/clone_gitcli.go | 3 +++
1 file changed, 3 insertions(+)
diff --git a/backend/plugins/gitextractor/parser/clone_gitcli.go b/backend/plugins/gitextractor/parser/clone_gitcli.go
index dd81b284f..0fbe0333b 100644
--- a/backend/plugins/gitextractor/parser/clone_gitcli.go
+++ b/backend/plugins/gitextractor/parser/clone_gitcli.go
@@ -157,6 +157,9 @@ func (g *GitcliCloner) CloneRepo(ctx plugin.SubTaskContext, localDir string) err
err = cmd.Wait()
if err != nil {
g.logger.Error(err, "git exited with error\n%s", combinedOutput.String())
+ if strings.Contains(combinedOutput.String(), "stderr: fatal: error processing shallow info: 4") {
+ return errors.BadInput.New("No data found for the selected time range. Please revise the 'Time Range' on your Project/Blueprint/Configuration page or in the API parameter.")
+ }
return errors.Default.New("git exit error")
}
return nil
(incubator-devlake) 09/10: fix: db timeafter adjust (#7314)
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch release-v1.0
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit de321b86d889eed255d50367d4b59b2133089539
Author: abeizn <zi...@merico.dev>
AuthorDate: Fri Apr 12 10:11:25 2024 +0800
fix: db timeafter adjust (#7314)
---
backend/helpers/pluginhelper/api/api_collector_with_state.go | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/backend/helpers/pluginhelper/api/api_collector_with_state.go b/backend/helpers/pluginhelper/api/api_collector_with_state.go
index e60b25033..513c442cb 100644
--- a/backend/helpers/pluginhelper/api/api_collector_with_state.go
+++ b/backend/helpers/pluginhelper/api/api_collector_with_state.go
@@ -135,7 +135,12 @@ func NewStatefulApiCollector(args RawDataSubTaskArgs) (*ApiCollectorStateManager
currentTime := time.Now()
oldState.LatestSuccessStart = ¤tTime
- oldState.TimeAfter = syncPolicy.TimeAfter
+ if syncPolicy != nil {
+ oldState.TimeAfter = syncPolicy.TimeAfter
+ if syncPolicy.TimeAfter != nil && oldTimeAfter != nil && (oldTimeAfter).Before(*syncPolicy.TimeAfter) && !syncPolicy.FullSync {
+ oldState.TimeAfter = oldTimeAfter
+ }
+ }
return &ApiCollectorStateManager{
RawDataSubTaskArgs: args,
(incubator-devlake) 07/10: fix: giturls CVE-2023-46402 (#7310)
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch release-v1.0
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 689c7327165470a113acd38733b6f2af51b2093e
Author: Klesh Wong <zh...@merico.dev>
AuthorDate: Thu Apr 11 11:41:47 2024 +0800
fix: giturls CVE-2023-46402 (#7310)
---
backend/go.mod | 6 +++---
backend/go.sum | 4 ++--
backend/plugins/gitextractor/impl/impl.go | 2 +-
3 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/backend/go.mod b/backend/go.mod
index 0f4aa5fc1..8062a749b 100644
--- a/backend/go.mod
+++ b/backend/go.mod
@@ -30,7 +30,7 @@ require (
github.com/swaggo/swag v1.16.1
github.com/tidwall/gjson v1.14.3
github.com/viant/afs v1.16.0
- golang.org/x/crypto v0.21.0
+ golang.org/x/crypto v0.21.0 // indirect
golang.org/x/exp v0.0.0-20221028150844-83b7d23a625f
golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602
golang.org/x/sync v0.3.0
@@ -93,7 +93,6 @@ require (
github.com/pjbgf/sha1cd v0.3.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
- github.com/rogpeppe/go-internal v1.11.0 // indirect
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
github.com/skeema/knownhosts v1.2.2 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
@@ -104,7 +103,6 @@ require (
github.com/tidwall/pretty v1.2.0 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/ugorji/go/codec v1.2.11 // indirect
- github.com/whilp/git-urls v1.0.0 // indirect
github.com/xanzy/ssh-agent v0.3.3 // indirect
golang.org/x/arch v0.3.0 // indirect
golang.org/x/net v0.22.0 // indirect
@@ -120,8 +118,10 @@ require (
)
require (
+ github.com/chainguard-dev/git-urls v1.0.2
github.com/go-sql-driver/mysql v1.7.1
github.com/golang-jwt/jwt/v5 v5.0.0-rc.1
+ github.com/rogpeppe/go-internal v1.11.0
golang.org/x/mod v0.13.0
)
diff --git a/backend/go.sum b/backend/go.sum
index 3f69d4be1..d48de10e0 100644
--- a/backend/go.sum
+++ b/backend/go.sum
@@ -61,6 +61,8 @@ github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1
github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s=
github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/chainguard-dev/git-urls v1.0.2 h1:pSpT7ifrpc5X55n4aTTm7FFUE+ZQHKiqpiwNkJrVcKQ=
+github.com/chainguard-dev/git-urls v1.0.2/go.mod h1:rbGgj10OS7UgZlbzdUQIQpT0k/D4+An04HJY7Ol+Y/o=
github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams=
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk=
@@ -512,8 +514,6 @@ github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4d
github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
github.com/viant/afs v1.16.0 h1:yb9TQ1gjVVLji9lcXLWaarklqmGWeXTZOwc2fwJevCI=
github.com/viant/afs v1.16.0/go.mod h1:wdiEDffZKJwj1ZSFasy7hHoxLQdSpFZkd3XOWNt1aN0=
-github.com/whilp/git-urls v1.0.0 h1:95f6UMWN5FKW71ECsXRUd3FVYiXdrE7aX4NZKcPmIjU=
-github.com/whilp/git-urls v1.0.0/go.mod h1:J16SAmobsqc3Qcy98brfl5f5+e0clUvg1krgwk/qCfE=
github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
diff --git a/backend/plugins/gitextractor/impl/impl.go b/backend/plugins/gitextractor/impl/impl.go
index 6d63622a5..ae4982551 100644
--- a/backend/plugins/gitextractor/impl/impl.go
+++ b/backend/plugins/gitextractor/impl/impl.go
@@ -26,7 +26,7 @@ import (
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/gitextractor/parser"
"github.com/apache/incubator-devlake/plugins/gitextractor/tasks"
- giturls "github.com/whilp/git-urls"
+ giturls "github.com/chainguard-dev/git-urls"
)
var _ interface {
(incubator-devlake) 03/10: Add more log on gitclone errors (#7300)
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch release-v1.0
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 53d884a043d099891d184a6ee480a074f665fd19
Author: Klesh Wong <zh...@merico.dev>
AuthorDate: Tue Apr 9 16:52:29 2024 +0800
Add more log on gitclone errors (#7300)
* chore: add more log on git clone errors
* fix: ignore debugging binary file generated by vscode
---
.gitignore | 4 +++-
backend/plugins/gitextractor/parser/clone_gitcli.go | 11 +++++++----
2 files changed, 10 insertions(+), 5 deletions(-)
diff --git a/.gitignore b/.gitignore
index 0aa085362..b62fdf26f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -162,4 +162,6 @@ __pycache__
venv
/backend/plugins/gitextractor/parser/output*
-/backend/plugins/gitextractor/parser/demo_repo/
\ No newline at end of file
+/backend/plugins/gitextractor/parser/demo_repo/
+
+__debug_bin*
\ No newline at end of file
diff --git a/backend/plugins/gitextractor/parser/clone_gitcli.go b/backend/plugins/gitextractor/parser/clone_gitcli.go
index 8f085bacd..dd81b284f 100644
--- a/backend/plugins/gitextractor/parser/clone_gitcli.go
+++ b/backend/plugins/gitextractor/parser/clone_gitcli.go
@@ -128,6 +128,7 @@ func (g *GitcliCloner) CloneRepo(ctx plugin.SubTaskContext, localDir string) err
g.logger.Error(err, "stderr pipe error")
return errors.Default.New("stderr pipe error")
}
+ combinedOutput := new(strings.Builder)
stdoutScanner := bufio.NewScanner(stdout)
stdoutScanner.Split(bufio.ScanLines)
stderrScanner := bufio.NewScanner(stderr)
@@ -135,25 +136,27 @@ func (g *GitcliCloner) CloneRepo(ctx plugin.SubTaskContext, localDir string) err
done := make(chan bool)
go func() {
for stdoutScanner.Scan() {
- fmt.Printf("stdout: %s\n", stdoutScanner.Text())
+ // TODO: extract progress?
+ combinedOutput.WriteString(fmt.Sprintf("stdout: %s\n", stdoutScanner.Text()))
}
done <- true
}()
go func() {
+ // TODO: extract progress?
for stderrScanner.Scan() {
- fmt.Printf("stderr: %s\n", stderrScanner.Text())
+ combinedOutput.WriteString(fmt.Sprintf("stderr: %s\n", stderrScanner.Text()))
}
done <- true
}()
if e := cmd.Start(); e != nil {
- g.logger.Error(e, "failed to start")
+ g.logger.Error(e, "failed to start\n%s", combinedOutput.String())
return errors.Default.New("failed to start")
}
<-done
<-done
err = cmd.Wait()
if err != nil {
- g.logger.Error(err, "git exited with error")
+ g.logger.Error(err, "git exited with error\n%s", combinedOutput.String())
return errors.Default.New("git exit error")
}
return nil
(incubator-devlake) 08/10: refactor: prevent concurrent migration (#7311)
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch release-v1.0
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit b46bcdb1c3db1aa75a81f1e28612134d4eb6ca83
Author: Klesh Wong <zh...@merico.dev>
AuthorDate: Thu Apr 11 17:22:13 2024 +0800
refactor: prevent concurrent migration (#7311)
* refactor: prevent concurrent migration
* fix: python test
---
backend/server/api/api.go | 30 ++++++++++++++++--------------
backend/server/services/init.go | 20 +++++++++++++++++---
2 files changed, 33 insertions(+), 17 deletions(-)
diff --git a/backend/server/api/api.go b/backend/server/api/api.go
index 7f17b5dc7..482adb960 100644
--- a/backend/server/api/api.go
+++ b/backend/server/api/api.go
@@ -49,6 +49,7 @@ WARNING: Performing migration may wipe collected data for consistency and re-col
To proceed, please send a request to <config-ui-endpoint>/api/proceed-db-migration (or <devlake-endpoint>/proceed-db-migration).
Alternatively, you may downgrade back to the previous DevLake version.
`
+const DB_MIGRATING = `Database migration is in progress. Please wait until it is completed.`
var basicRes context.BasicRes
@@ -95,12 +96,6 @@ func SetupApiServer(router *gin.Engine) {
// Endpoint to proceed database migration
router.GET("/proceed-db-migration", func(ctx *gin.Context) {
- // Check if migration requires confirmation
- if !services.MigrationRequireConfirmation() {
- // Return success response
- shared.ApiOutputSuccess(ctx, nil, http.StatusOK)
- return
- }
// Execute database migration
err := services.ExecuteMigration()
if err != nil {
@@ -114,15 +109,22 @@ func SetupApiServer(router *gin.Engine) {
// Restrict access if database migration is required
router.Use(func(ctx *gin.Context) {
- if !services.MigrationRequireConfirmation() {
- return
+ serviceStatus := services.CurrentStatus()
+ if serviceStatus == services.SERVICE_STATUS_WAIT_CONFIRM {
+ // Return error response
+ shared.ApiOutputError(
+ ctx,
+ errors.HttpStatus(http.StatusPreconditionRequired).New(DB_MIGRATION_REQUIRED),
+ )
+ ctx.Abort()
+ } else if serviceStatus == services.SERVICE_STATUS_MIGRATING {
+ // Return error response
+ shared.ApiOutputError(
+ ctx,
+ errors.HttpStatus(http.StatusPreconditionRequired).New(DB_MIGRATING),
+ )
+ ctx.Abort()
}
- // Return error response
- shared.ApiOutputError(
- ctx,
- errors.HttpStatus(http.StatusPreconditionRequired).New(DB_MIGRATION_REQUIRED),
- )
- ctx.Abort()
})
// Add swagger handlers
diff --git a/backend/server/services/init.go b/backend/server/services/init.go
index 2d5dad010..07b565fb7 100644
--- a/backend/server/services/init.go
+++ b/backend/server/services/init.go
@@ -18,6 +18,7 @@ limitations under the License.
package services
import (
+ "sync"
"time"
"github.com/apache/incubator-devlake/core/config"
@@ -84,6 +85,7 @@ func GetMigrator() plugin.Migrator {
}
// Init the services module
+// Should not be called concurrently
func Init() {
InitResources()
@@ -115,9 +117,21 @@ func Init() {
}
}
+var statusLock sync.Mutex
+
// ExecuteMigration executes all pending migration scripts and initialize services module
+// This might be called concurrently across multiple API requests
func ExecuteMigration() errors.Error {
+ statusLock.Lock()
+ defer statusLock.Unlock()
+ if serviceStatus == SERVICE_STATUS_MIGRATING {
+ return errors.BadInput.New("already migrating")
+ }
+ if serviceStatus == SERVICE_STATUS_READY {
+ return nil
+ }
serviceStatus = SERVICE_STATUS_MIGRATING
+ statusLock.Unlock() // unlock to allow other API requests to check the status
// apply all pending migration scripts
err := migrator.Execute()
if err != nil {
@@ -130,11 +144,11 @@ func ExecuteMigration() errors.Error {
// initialize pipeline server, mainly to start the pipeline consuming process
pipelineServiceInit()
+ statusLock.Lock()
serviceStatus = SERVICE_STATUS_READY
return nil
}
-// MigrationRequireConfirmation returns if there were migration scripts waiting to be executed
-func MigrationRequireConfirmation() bool {
- return migrator.HasPendingScripts()
+func CurrentStatus() string {
+ return serviceStatus
}
(incubator-devlake) 04/10: pr-type/bug-fix 7114 - wire up the page size event to alter page size (#7124)
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch release-v1.0
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit af235a968840513a07b55653fb5c9f753e2cb0e0
Author: wouldd <13...@users.noreply.github.com>
AuthorDate: Wed Apr 10 09:01:44 2024 +0100
pr-type/bug-fix 7114 - wire up the page size event to alter page size (#7124)
* 7114 - wire up the page size event to alter page size
* rollback bad merge change
---------
Co-authored-by: Daniel Would <da...@oribs.com>
---
config-ui/src/routes/connection/connection.tsx | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/config-ui/src/routes/connection/connection.tsx b/config-ui/src/routes/connection/connection.tsx
index 05dbc5bcc..957a793a7 100644
--- a/config-ui/src/routes/connection/connection.tsx
+++ b/config-ui/src/routes/connection/connection.tsx
@@ -53,7 +53,7 @@ export const Connection = () => {
const [operating, setOperating] = useState(false);
const [version, setVersion] = useState(1);
const [page, setPage] = useState(1);
- const [pageSize] = useState(10);
+ const [pageSize, setPageSize] = useState(10);
const [scopeId, setScopeId] = useState<ID>();
const [scopeIds, setScopeIds] = useState<ID[]>([]);
const [scopeConfigId, setScopeConfigId] = useState<ID>();
@@ -97,7 +97,7 @@ export const Connection = () => {
const handleHideDialog = () => {
setType(undefined);
};
-
+
const handleShowDeleteDialog = () => {
setType('deleteConnection');
};
@@ -332,6 +332,10 @@ export const Connection = () => {
pageSize,
total,
onChange: setPage,
+ onShowSizeChange: (current,size)=> {
+ setPage(1);
+ setPageSize(size);
+ }
}}
rowSelection={{
selectedRowKeys: scopeIds,
(incubator-devlake) 06/10: feat: upgrade grafana from 10.2.6 to 10.4.1 (#7309)
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch release-v1.0
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit dd0bc6b749bca2118f2c7bc59aa5df90facc5ef2
Author: abeizn <zi...@merico.dev>
AuthorDate: Thu Apr 11 11:24:46 2024 +0800
feat: upgrade grafana from 10.2.6 to 10.4.1 (#7309)
---
grafana/Dockerfile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/grafana/Dockerfile b/grafana/Dockerfile
index 5356a9c16..9d62a2168 100644
--- a/grafana/Dockerfile
+++ b/grafana/Dockerfile
@@ -23,7 +23,7 @@
#While incubation status is not necessarily a reflection of the completeness or stability of the code,
#it does indicate that the project has yet to be fully endorsed by the ASF.
-FROM grafana/grafana:10.2.6
+FROM grafana/grafana:10.4.1
COPY ./provisioning/dashboards /etc/grafana/provisioning/dashboards
COPY ./provisioning/datasources /etc/grafana/provisioning/datasources
COPY ./dashboards /etc/grafana/dashboards