You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by pe...@apache.org on 2025/09/22 15:17:16 UTC
(incubator-devlake) branch fix/8538-github-zero-time created (now 7e7baa2c5)
This is an automated email from the ASF dual-hosted git repository.
petkostas pushed a change to branch fix/8538-github-zero-time
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
at 7e7baa2c5 fix(lint): fix linting issue
This branch includes the following new commits:
new b58f6eceb fix(github): fix zerotime issues for GraphQL
new 7e7baa2c5 fix(lint): fix linting issue
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(incubator-devlake) 01/02: fix(github): fix zerotime issues for GraphQL
Posted by pe...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
petkostas pushed a commit to branch fix/8538-github-zero-time
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit b58f6eceb56f4231017e15d54b35706fddd2fffd
Author: Kostas Petrakis <pe...@gmail.com>
AuthorDate: Sat Aug 23 13:30:04 2025 +0200
fix(github): fix zerotime issues for GraphQL
---
.../utils/time.go} | 38 +++++-----------
backend/core/utils/time_test.go | 51 ++++++++++++++++++++++
backend/plugins/github/models/release.go | 38 ++++++++--------
backend/plugins/github/tasks/release_convertor.go | 2 +-
.../github_graphql/tasks/account_collector.go | 15 +++----
.../tasks/account_graphql_pre_extractor.go | 1 -
.../github_graphql/tasks/issue_collector.go | 2 +
.../github_graphql/tasks/issue_extractor.go | 3 ++
.../plugins/github_graphql/tasks/job_collector.go | 14 +++---
.../plugins/github_graphql/tasks/job_extractor.go | 5 ++-
.../github_graphql/tasks/release_collector.go | 4 +-
.../github_graphql/tasks/release_extractor.go | 3 +-
12 files changed, 108 insertions(+), 68 deletions(-)
diff --git a/backend/plugins/github_graphql/tasks/account_graphql_pre_extractor.go b/backend/core/utils/time.go
similarity index 52%
copy from backend/plugins/github_graphql/tasks/account_graphql_pre_extractor.go
copy to backend/core/utils/time.go
index 92e4611cb..f1b95f499 100644
--- a/backend/plugins/github_graphql/tasks/account_graphql_pre_extractor.go
+++ b/backend/core/utils/time.go
@@ -15,34 +15,18 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package tasks
+package utils
-import (
- "github.com/apache/incubator-devlake/plugins/github/models"
-)
+import "time"
-type GithubAccountEdge struct {
- Login string
- Id int `graphql:"databaseId"`
- Name string
- Company string
- Email string
- AvatarUrl string
- HtmlUrl string `graphql:"url"`
- //Type string
-}
-type GraphqlInlineAccountQuery struct {
- GithubAccountEdge `graphql:"... on User"`
-}
-
-func extractGraphqlPreAccount(result *[]interface{}, res *GraphqlInlineAccountQuery, repoId int, connId uint64) {
- if res == nil || res.Id == 0 {
- return
+// NilIfZeroTime returns nil if t is nil or represents the zero time (0001-01-01...).
+// Otherwise, it returns t unchanged.
+func NilIfZeroTime(t *time.Time) *time.Time {
+ if t == nil {
+ return nil
+ }
+ if t.IsZero() {
+ return nil
}
- *result = append(*result, &models.GithubRepoAccount{
- ConnectionId: connId,
- RepoGithubId: repoId,
- Login: res.Login,
- AccountId: res.Id,
- })
+ return t
}
diff --git a/backend/core/utils/time_test.go b/backend/core/utils/time_test.go
new file mode 100644
index 000000000..a0e949f4e
--- /dev/null
+++ b/backend/core/utils/time_test.go
@@ -0,0 +1,51 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package utils
+
+import (
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestNilIfZeroTime(t *testing.T) {
+ type args struct {
+ t *time.Time
+ }
+ tests := []struct {
+ name string
+ args args
+ want *time.Time
+ }{
+ {
+ name: "Empty date should be nil",
+ args: args{nil},
+ want: nil,
+ },
+ {
+ name: "Zero date should be nil",
+ args: args{&time.Time{}},
+ want: nil,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ assert.Equalf(t, tt.want, NilIfZeroTime(tt.args.t), "NilIfZeroTime(%v)", tt.args.t)
+ })
+ }
+}
diff --git a/backend/plugins/github/models/release.go b/backend/plugins/github/models/release.go
index 1e4ae1688..5442f9fda 100644
--- a/backend/plugins/github/models/release.go
+++ b/backend/plugins/github/models/release.go
@@ -25,25 +25,25 @@ import (
type GithubRelease struct {
common.NoPKModel `json:"-" mapstructure:"-"`
- ConnectionId uint64 `json:"connection_id" gorm:"primaryKey"`
- GithubId int `json:"github_id"`
- Id string `json:"id" gorm:"type:varchar(255);primaryKey"`
- AuthorName string `json:"authorName"`
- AuthorID string `json:"authorId"`
- CreatedAt time.Time `json:"createdAt"`
- DatabaseID int `json:"databaseId"`
- Description string `json:"description"`
- DescriptionHTML string `json:"descriptionHTML"`
- IsDraft bool `json:"isDraft"`
- IsLatest bool `json:"isLatest"`
- IsPrerelease bool `json:"isPrerelease"`
- Name string `json:"name"`
- PublishedAt time.Time `json:"publishedAt"`
- ResourcePath string `json:"resourcePath"`
- TagName string `json:"tagName"`
- UpdatedAt time.Time `json:"updatedAt"`
- CommitSha string `json:"commit_sha"`
- URL string `json:"url"`
+ ConnectionId uint64 `json:"connection_id" gorm:"primaryKey"`
+ GithubId int `json:"github_id"`
+ Id string `json:"id" gorm:"type:varchar(255);primaryKey"`
+ AuthorName string `json:"authorName"`
+ AuthorID string `json:"authorId"`
+ CreatedAt time.Time `json:"createdAt"`
+ DatabaseID int `json:"databaseId"`
+ Description string `json:"description"`
+ DescriptionHTML string `json:"descriptionHTML"`
+ IsDraft bool `json:"isDraft"`
+ IsLatest bool `json:"isLatest"`
+ IsPrerelease bool `json:"isPrerelease"`
+ Name string `json:"name"`
+ PublishedAt *time.Time `json:"publishedAt"`
+ ResourcePath string `json:"resourcePath"`
+ TagName string `json:"tagName"`
+ UpdatedAt time.Time `json:"updatedAt"`
+ CommitSha string `json:"commit_sha"`
+ URL string `json:"url"`
}
func (GithubRelease) TableName() string {
diff --git a/backend/plugins/github/tasks/release_convertor.go b/backend/plugins/github/tasks/release_convertor.go
index 1a1389093..d0b5b4bb7 100644
--- a/backend/plugins/github/tasks/release_convertor.go
+++ b/backend/plugins/github/tasks/release_convertor.go
@@ -73,7 +73,7 @@ func ConvertRelease(taskCtx plugin.SubTaskContext) errors.Error {
DomainEntity: domainlayer.DomainEntity{
Id: releaseIdGen.Generate(githubRelease.ConnectionId, githubRelease.Id),
},
- PublishedAt: githubRelease.PublishedAt,
+ PublishedAt: *githubRelease.PublishedAt,
CicdScopeId: releaseScopeIdGen.Generate(githubRelease.ConnectionId, githubRelease.GithubId),
Name: githubRelease.Name,
DisplayTitle: githubRelease.Name,
diff --git a/backend/plugins/github_graphql/tasks/account_collector.go b/backend/plugins/github_graphql/tasks/account_collector.go
index 27d71b65f..34e4faac1 100644
--- a/backend/plugins/github_graphql/tasks/account_collector.go
+++ b/backend/plugins/github_graphql/tasks/account_collector.go
@@ -40,14 +40,13 @@ type GraphqlQueryAccountWrapper struct {
}
type GraphqlQueryAccount struct {
- Login string
- Id int `graphql:"databaseId"`
- Name string
- Company string
- Email string
- AvatarUrl string
- HtmlUrl string `graphql:"url"`
- //Type string
+ Login string
+ Id int `graphql:"databaseId"`
+ Name string
+ Company string
+ Email string
+ AvatarUrl string
+ HtmlUrl string `graphql:"url"`
Organizations struct {
Nodes []struct {
Email string
diff --git a/backend/plugins/github_graphql/tasks/account_graphql_pre_extractor.go b/backend/plugins/github_graphql/tasks/account_graphql_pre_extractor.go
index 92e4611cb..e4b47e475 100644
--- a/backend/plugins/github_graphql/tasks/account_graphql_pre_extractor.go
+++ b/backend/plugins/github_graphql/tasks/account_graphql_pre_extractor.go
@@ -29,7 +29,6 @@ type GithubAccountEdge struct {
Email string
AvatarUrl string
HtmlUrl string `graphql:"url"`
- //Type string
}
type GraphqlInlineAccountQuery struct {
GithubAccountEdge `graphql:"... on User"`
diff --git a/backend/plugins/github_graphql/tasks/issue_collector.go b/backend/plugins/github_graphql/tasks/issue_collector.go
index 0945be007..bc4d7a17b 100644
--- a/backend/plugins/github_graphql/tasks/issue_collector.go
+++ b/backend/plugins/github_graphql/tasks/issue_collector.go
@@ -26,6 +26,7 @@ import (
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/core/utils"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/github/models"
githubTasks "github.com/apache/incubator-devlake/plugins/github/tasks"
@@ -134,6 +135,7 @@ func CollectIssues(taskCtx plugin.SubTaskContext) errors.Error {
query := queryWrapper.(*GraphqlQueryIssueWrapper)
issues := query.Repository.IssueList.Issues
for _, rawL := range issues {
+ rawL.ClosedAt = utils.NilIfZeroTime(rawL.ClosedAt)
if since != nil && since.After(rawL.UpdatedAt) {
return messages, api.ErrFinishCollect
}
diff --git a/backend/plugins/github_graphql/tasks/issue_extractor.go b/backend/plugins/github_graphql/tasks/issue_extractor.go
index 7ebd40822..0ebd28d18 100644
--- a/backend/plugins/github_graphql/tasks/issue_extractor.go
+++ b/backend/plugins/github_graphql/tasks/issue_extractor.go
@@ -25,6 +25,7 @@ import (
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
"github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/core/utils"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/github/models"
githubTasks "github.com/apache/incubator-devlake/plugins/github/tasks"
@@ -68,6 +69,8 @@ func ExtractIssues(taskCtx plugin.SubTaskContext) errors.Error {
if err != nil {
return nil, err
}
+ // Normalize zero-date to nil for closedAt
+ issue.ClosedAt = utils.NilIfZeroTime(issue.ClosedAt)
results := make([]interface{}, 0, 1)
githubIssue, err := convertGithubIssue(milestoneMap, issue, data.Options.ConnectionId, data.Options.GithubId)
if err != nil {
diff --git a/backend/plugins/github_graphql/tasks/job_collector.go b/backend/plugins/github_graphql/tasks/job_collector.go
index b204d55ba..c67d6a879 100644
--- a/backend/plugins/github_graphql/tasks/job_collector.go
+++ b/backend/plugins/github_graphql/tasks/job_collector.go
@@ -19,13 +19,13 @@ package tasks
import (
"encoding/json"
- "fmt"
"reflect"
"time"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/core/utils"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/github/models"
githubTasks "github.com/apache/incubator-devlake/plugins/github/tasks"
@@ -202,14 +202,10 @@ func CollectJobs(taskCtx plugin.SubTaskContext) errors.Error {
RunId: runId,
GraphqlQueryCheckRun: &checkRun,
}
- // A checkRun without a startedAt time is a run that was never started (skipped)
- // akwardly, GitHub assigns a completedAt time to such runs, which is the time when the run was skipped
- // TODO: Decide if we want to skip those runs or should we assign the startedAt time to the completedAt time
- if dbCheckRun.StartedAt == nil || dbCheckRun.StartedAt.IsZero() {
- debug := fmt.Sprintf("collector: checkRun.StartedAt is nil or zero: %s", dbCheckRun.Id)
- taskCtx.GetLogger().Debug(debug, "Collector: CheckRun started at is nil or zero")
- continue
- }
+ // A checkRun without a startedAt time is a run that was never started (skipped), GitHub returns
+ // a ZeroTime (Due to the GO implementation) for startedAt, so we need to check for that here.
+ dbCheckRun.StartedAt = utils.NilIfZeroTime(dbCheckRun.StartedAt)
+ dbCheckRun.CompletedAt = utils.NilIfZeroTime(dbCheckRun.CompletedAt)
updatedAt := dbCheckRun.StartedAt
if dbCheckRun.CompletedAt != nil {
updatedAt = dbCheckRun.CompletedAt
diff --git a/backend/plugins/github_graphql/tasks/job_extractor.go b/backend/plugins/github_graphql/tasks/job_extractor.go
index ac4401e06..ea4b7c7db 100644
--- a/backend/plugins/github_graphql/tasks/job_extractor.go
+++ b/backend/plugins/github_graphql/tasks/job_extractor.go
@@ -24,6 +24,7 @@ import (
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
"github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/core/utils"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/github/models"
githubTasks "github.com/apache/incubator-devlake/plugins/github/tasks"
@@ -57,7 +58,9 @@ func ExtractJobs(taskCtx plugin.SubTaskContext) errors.Error {
return nil, err
}
results := make([]interface{}, 0, 1)
-
+ // Normalize zero-date times to nil
+ checkRun.StartedAt = utils.NilIfZeroTime(checkRun.StartedAt)
+ checkRun.CompletedAt = utils.NilIfZeroTime(checkRun.CompletedAt)
paramsBytes, marshalError := json.Marshal(checkRun.Steps.Nodes)
err = errors.Convert(marshalError)
if err != nil {
diff --git a/backend/plugins/github_graphql/tasks/release_collector.go b/backend/plugins/github_graphql/tasks/release_collector.go
index 0090bd60c..83581c4c1 100644
--- a/backend/plugins/github_graphql/tasks/release_collector.go
+++ b/backend/plugins/github_graphql/tasks/release_collector.go
@@ -24,6 +24,7 @@ import (
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/core/utils"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
githubTasks "github.com/apache/incubator-devlake/plugins/github/tasks"
"github.com/merico-dev/graphql"
@@ -66,7 +67,7 @@ type GraphqlQueryRelease struct {
IsLatest bool `graphql:"isLatest"`
IsPrerelease bool `graphql:"isPrerelease"`
Name string `graphql:"name"`
- PublishedAt time.Time `graphql:"publishedAt"`
+ PublishedAt *time.Time `graphql:"publishedAt"`
ResourcePath string `graphql:"resourcePath"`
TagName string `graphql:"tagName"`
TagCommit GraphqlQueryReleaseTagCommit `graphql:"tagCommit"`
@@ -125,6 +126,7 @@ func CollectRelease(taskCtx plugin.SubTaskContext) errors.Error {
query := queryWrapper.(*GraphqlQueryReleaseWrapper)
releases := query.Repository.Releases.Releases
for _, rawL := range releases {
+ rawL.PublishedAt = utils.NilIfZeroTime(rawL.PublishedAt)
if since != nil && since.After(rawL.UpdatedAt) {
return messages, helper.ErrFinishCollect
}
diff --git a/backend/plugins/github_graphql/tasks/release_extractor.go b/backend/plugins/github_graphql/tasks/release_extractor.go
index 4b819fc23..6fd4f25ac 100644
--- a/backend/plugins/github_graphql/tasks/release_extractor.go
+++ b/backend/plugins/github_graphql/tasks/release_extractor.go
@@ -23,6 +23,7 @@ import (
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/core/utils"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
githubModels "github.com/apache/incubator-devlake/plugins/github/models"
githubTasks "github.com/apache/incubator-devlake/plugins/github/tasks"
@@ -55,7 +56,7 @@ func ExtractReleases(taskCtx plugin.SubTaskContext) errors.Error {
if err != nil {
return nil, err
}
-
+ release.PublishedAt = utils.NilIfZeroTime(release.PublishedAt)
var results []interface{}
githubRelease, err := convertGitHubRelease(release, data.Options.ConnectionId, data.Options.GithubId)
if err != nil {
(incubator-devlake) 02/02: fix(lint): fix linting issue
Posted by pe...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
petkostas pushed a commit to branch fix/8538-github-zero-time
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 7e7baa2c55d482436b0a0ee142c0d275fb885343
Author: Kostas Petrakis <pe...@mail.com>
AuthorDate: Mon Sep 22 17:09:00 2025 +0200
fix(lint): fix linting issue
---
.../plugins/q_dev/models/migrationscripts/20250320_modify_file_meta.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/backend/plugins/q_dev/models/migrationscripts/20250320_modify_file_meta.go b/backend/plugins/q_dev/models/migrationscripts/20250320_modify_file_meta.go
index 9744f6d09..83311cf07 100644
--- a/backend/plugins/q_dev/models/migrationscripts/20250320_modify_file_meta.go
+++ b/backend/plugins/q_dev/models/migrationscripts/20250320_modify_file_meta.go
@@ -51,7 +51,7 @@ func (*modifyFileMetaTable) Up(basicRes context.BasicRes) errors.Error {
return errors.Default.Wrap(err, "failed to load column metadata for _tool_q_dev_s3_file_meta.processed_time")
}
if len(cols) == 0 {
- // If column is not visible in metadata, treat as no processing needed
+ // If column is not visible in metadata, treat as no processing needed
return nil
}
if nullable, ok := cols[0].Nullable(); ok {