You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by wa...@apache.org on 2022/11/10 06:43:55 UTC
[incubator-devlake] branch main updated: feat: add check_run collector in github graphql (#3705)
This is an automated email from the ASF dual-hosted git repository.
warren pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 46fbd483 feat: add check_run collector in github graphql (#3705)
46fbd483 is described below
commit 46fbd48397ce4feef84ec34252e021c461bb1183
Author: Likyh <l...@likyh.com>
AuthorDate: Thu Nov 10 14:43:50 2022 +0800
feat: add check_run collector in github graphql (#3705)
* feat: add check_run collector in github graphql
* fix: fix for review
---
plugins/github_graphql/plugin_main.go | 12 ++
.../github_graphql/tasks/check_run_collector.go | 192 +++++++++++++++++++++
2 files changed, 204 insertions(+)
diff --git a/plugins/github_graphql/plugin_main.go b/plugins/github_graphql/plugin_main.go
index a5c4bb2b..ddce4e94 100644
--- a/plugins/github_graphql/plugin_main.go
+++ b/plugins/github_graphql/plugin_main.go
@@ -42,6 +42,7 @@ var _ core.PluginMeta = (*GithubGraphql)(nil)
var _ core.PluginInit = (*GithubGraphql)(nil)
var _ core.PluginTask = (*GithubGraphql)(nil)
var _ core.PluginApi = (*GithubGraphql)(nil)
+var _ core.CloseablePluginTask = (*GithubGraphql)(nil)
// PluginEntry exports a symbol for Framework to load
var PluginEntry GithubGraphql //nolint
@@ -60,12 +61,20 @@ func (plugin GithubGraphql) SubTaskMetas() []core.SubTaskMeta {
return []core.SubTaskMeta{
tasks.CollectRepoMeta,
+ // collect millstones
githubTasks.CollectMilestonesMeta,
githubTasks.ExtractMilestonesMeta,
+ // collect issue & pr, deps on millstone
tasks.CollectIssueMeta,
tasks.CollectPrMeta,
+ // collect workflow run & job
+ githubTasks.CollectRunsMeta,
+ githubTasks.ExtractRunsMeta,
+ tasks.CollectCheckRunMeta,
+
+ // collect others
githubTasks.CollectApiCommentsMeta,
githubTasks.ExtractApiCommentsMeta,
githubTasks.CollectApiEventsMeta,
@@ -73,8 +82,11 @@ func (plugin GithubGraphql) SubTaskMetas() []core.SubTaskMeta {
githubTasks.CollectApiPrReviewCommentsMeta,
githubTasks.ExtractApiPrReviewCommentsMeta,
+ // collect account, deps on all before
tasks.CollectAccountMeta,
+ // convert to domain layer
+ githubTasks.ConvertRunsMeta,
githubTasks.ConvertJobsMeta,
githubTasks.EnrichPullRequestIssuesMeta,
githubTasks.ConvertRepoMeta,
diff --git a/plugins/github_graphql/tasks/check_run_collector.go b/plugins/github_graphql/tasks/check_run_collector.go
new file mode 100644
index 00000000..85ec1af3
--- /dev/null
+++ b/plugins/github_graphql/tasks/check_run_collector.go
@@ -0,0 +1,192 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "github.com/apache/incubator-devlake/errors"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+ "github.com/apache/incubator-devlake/plugins/github/models"
+ githubTasks "github.com/apache/incubator-devlake/plugins/github/tasks"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/merico-dev/graphql"
+ "reflect"
+ "time"
+)
+
+const RAW_CHECK_RUNS_TABLE = "github_graphql_check_runs"
+
+type GraphqlQueryCheckRunWrapper struct {
+ RateLimit struct {
+ Cost int
+ }
+ Node []GraphqlQueryCheckSuite `graphql:"node(id: $id)" graphql-extend:"true"`
+}
+
+type GraphqlQueryCheckSuite struct {
+ Id string
+ Typename string `graphql:"__typename"`
+ CheckSuite struct {
+ WorkflowRun struct {
+ DatabaseId int
+ }
+ CheckRuns struct {
+ TotalCount int
+ Nodes []struct {
+ Id string
+ Name string
+ DetailsUrl string
+ DatabaseId int
+ Status string
+ StartedAt *time.Time
+ Conclusion string
+ CompletedAt *time.Time
+ //ExternalId string
+ //Url string
+ //Title interface{}
+ //Text interface{}
+ //Summary interface{}
+
+ Steps struct {
+ TotalCount int
+ Nodes []struct {
+ CompletedAt *time.Time `json:"completed_at"`
+ Conclusion string `json:"conclusion"`
+ Name string `json:"name"`
+ Number int `json:"number"`
+ SecondsToCompletion int `json:"seconds_to_completion"`
+ StartedAt *time.Time `json:"started_at"`
+ Status string `json:"status"`
+ }
+ } `graphql:"steps(first: 50)"`
+ }
+ } `graphql:"checkRuns(first: 50)"`
+ } `graphql:"... on CheckSuite"`
+}
+
+type SimpleWorkflowRun struct {
+ CheckSuiteNodeID string
+}
+
+var CollectCheckRunMeta = core.SubTaskMeta{
+ Name: "CollectCheckRun",
+ EntryPoint: CollectCheckRun,
+ EnabledByDefault: true,
+ Description: "Collect CheckRun data from GithubGraphql api",
+}
+
+var _ core.SubTaskEntryPoint = CollectAccount
+
+func CollectCheckRun(taskCtx core.SubTaskContext) errors.Error {
+ logger := taskCtx.GetLogger()
+ db := taskCtx.GetDal()
+ data := taskCtx.GetData().(*githubTasks.GithubTaskData)
+
+ cursor, err := db.Cursor(
+ dal.Select("check_suite_node_id"),
+ dal.From(models.GithubRun{}.TableName()),
+ dal.Where("repo_id = ? and connection_id=?", data.Repo.GithubId, data.Options.ConnectionId),
+ )
+ if err != nil {
+ return err
+ }
+ iterator, err := helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimpleWorkflowRun{}))
+ if err != nil {
+ return err
+ }
+
+ collector, err := helper.NewGraphqlCollector(helper.GraphqlCollectorArgs{
+ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: githubTasks.GithubApiParams{
+ ConnectionId: data.Options.ConnectionId,
+ Owner: data.Options.Owner,
+ Repo: data.Options.Repo,
+ },
+ Table: RAW_CHECK_RUNS_TABLE,
+ },
+ Input: iterator,
+ InputStep: 60,
+ GraphqlClient: data.GraphqlClient,
+ BuildQuery: func(reqData *helper.GraphqlRequestData) (interface{}, map[string]interface{}, error) {
+ workflowRuns := reqData.Input.([]interface{})
+ query := &GraphqlQueryCheckRunWrapper{}
+ checkSuiteIds := []map[string]interface{}{}
+ for _, iWorkflowRuns := range workflowRuns {
+ workflowRun := iWorkflowRuns.(*SimpleWorkflowRun)
+ checkSuiteIds = append(checkSuiteIds, map[string]interface{}{
+ `id`: graphql.ID(workflowRun.CheckSuiteNodeID),
+ })
+ }
+ variables := map[string]interface{}{
+ "node": checkSuiteIds,
+ }
+ return query, variables, nil
+ },
+ ResponseParserWithDataErrors: func(iQuery interface{}, variables map[string]interface{}, dataErrors []graphql.DataError) ([]interface{}, error) {
+ for _, dataError := range dataErrors {
+ // log and ignore
+ taskCtx.GetLogger().Warn(dataError, `query check run get error but ignore`)
+ }
+ query := iQuery.(*GraphqlQueryCheckRunWrapper)
+ nodes := query.Node
+
+ results := make([]interface{}, 0, 1)
+ for _, node := range nodes {
+ for _, checkRun := range node.CheckSuite.CheckRuns.Nodes {
+
+ paramsBytes, err := json.Marshal(checkRun.Steps.Nodes)
+ if err != nil {
+ logger.Error(err, `Marshal checkRun.Steps.Nodes fail and ignore`)
+ }
+ githubJob := &models.GithubJob{
+ ConnectionId: data.Options.ConnectionId,
+ RunID: node.CheckSuite.WorkflowRun.DatabaseId,
+ RepoId: data.Repo.GithubId,
+ ID: checkRun.DatabaseId,
+ NodeID: checkRun.Id,
+ HTMLURL: checkRun.DetailsUrl,
+ Status: checkRun.Status,
+ Conclusion: checkRun.Conclusion,
+ StartedAt: checkRun.StartedAt,
+ CompletedAt: checkRun.CompletedAt,
+ Name: checkRun.Name,
+ Steps: paramsBytes,
+ // these columns can not fill by graphql
+ //HeadSha: ``, // use _tool_github_runs
+ //RunURL: ``,
+ //CheckRunURL: ``,
+ //Labels: ``, // not on use
+ //RunnerID: ``, // not on use
+ //RunnerName: ``, // not on use
+ //RunnerGroupID: ``, // not on use
+ }
+ results = append(results, githubJob)
+ }
+ }
+ return results, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return collector.Execute()
+}