You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by gi...@apache.org on 2024/01/26 02:52:36 UTC
(incubator-devlake) branch release-v0.20-auto-cherry-pick-6876 created (now 28152e5c8)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch release-v0.20-auto-cherry-pick-6876
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
at 28152e5c8 fix: github run api page<35 limit (#6876)
This branch includes the following new commits:
new 28152e5c8 fix: github run api page<35 limit (#6876)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(incubator-devlake) 01/01: fix: github run api page<35 limit (#6876)
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch release-v0.20-auto-cherry-pick-6876
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 28152e5c8a45fe55fbed104f4319b6203c337495
Author: abeizn <zi...@merico.dev>
AuthorDate: Fri Jan 26 10:52:13 2024 +0800
fix: github run api page<35 limit (#6876)
* fix: github run api page<35 limit
* fix: some tips
* fix: update note
---
backend/plugins/github/tasks/cicd_run_collector.go | 270 ++++++++++++++++++++-
1 file changed, 262 insertions(+), 8 deletions(-)
diff --git a/backend/plugins/github/tasks/cicd_run_collector.go b/backend/plugins/github/tasks/cicd_run_collector.go
index c94f194ca..3cd1bd5d5 100644
--- a/backend/plugins/github/tasks/cicd_run_collector.go
+++ b/backend/plugins/github/tasks/cicd_run_collector.go
@@ -41,11 +41,6 @@ const RAW_RUN_TABLE = "github_api_runs"
// https://github.com/apache/incubator-devlake/issues/3199
const PAGE_SIZE = 30
-type GithubRawRunsResult struct {
- TotalCount int64 `json:"total_count"`
- GithubWorkflowRuns []json.RawMessage `json:"workflow_runs"`
-}
-
type SimpleGithubApiJob struct {
ID int64
CreatedAt common.Iso8601Time `json:"created_at"`
@@ -80,7 +75,9 @@ func CollectRuns(taskCtx plugin.SubTaskContext) errors.Error {
UrlTemplate: "repos/{{ .Params.Name }}/actions/runs",
Query: func(reqData *helper.RequestData, createdAfter *time.Time) (url.Values, errors.Error) {
query := url.Values{}
- query.Set("status", "completed")
+ // GitHub API returns only the first 34 pages (with a size of 30) when specifying status=compleleted, try the following API request to verify the problem.
+ // https://api.github.com/repos/apache/incubator-devlake/actions/runs?per_page=30&page=35&status=completed
+ // query.Set("status", "completed")
query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size))
return query, nil
@@ -91,10 +88,21 @@ func CollectRuns(taskCtx plugin.SubTaskContext) errors.Error {
if err != nil {
return nil, err
}
- if len(body.GithubWorkflowRuns) == 0 {
+ if len(body.WorkflowRuns) == 0 {
return nil, nil
}
- return body.GithubWorkflowRuns, nil
+ // filter out the runs that are not completed
+ filteredRuns := make([]json.RawMessage, 0)
+ for _, run := range body.WorkflowRuns {
+ if run.Status == "completed" {
+ runJSON, err := json.Marshal(run)
+ if err != nil {
+ return nil, errors.Convert(err)
+ }
+ filteredRuns = append(filteredRuns, json.RawMessage(runJSON))
+ }
+ }
+ return filteredRuns, nil
},
},
GetCreated: func(item json.RawMessage) (time.Time, errors.Error) {
@@ -115,3 +123,249 @@ func CollectRuns(taskCtx plugin.SubTaskContext) errors.Error {
return collector.Execute()
}
+
+type GithubRawRunsResult struct {
+ TotalCount int `json:"total_count"`
+ WorkflowRuns []struct {
+ ID int64 `json:"id"`
+ Name string `json:"name"`
+ NodeID string `json:"node_id"`
+ HeadBranch string `json:"head_branch"`
+ HeadSha string `json:"head_sha"`
+ Path string `json:"path"`
+ DisplayTitle string `json:"display_title"`
+ RunNumber int `json:"run_number"`
+ Event string `json:"event"`
+ Status string `json:"status"`
+ Conclusion string `json:"conclusion"`
+ WorkflowID int `json:"workflow_id"`
+ CheckSuiteID int64 `json:"check_suite_id"`
+ CheckSuiteNodeID string `json:"check_suite_node_id"`
+ URL string `json:"url"`
+ HTMLURL string `json:"html_url"`
+ PullRequests []struct {
+ URL string `json:"url"`
+ ID int `json:"id"`
+ Number int `json:"number"`
+ Head struct {
+ Ref string `json:"ref"`
+ Sha string `json:"sha"`
+ Repo struct {
+ ID int `json:"id"`
+ URL string `json:"url"`
+ Name string `json:"name"`
+ } `json:"repo"`
+ } `json:"head"`
+ Base struct {
+ Ref string `json:"ref"`
+ Sha string `json:"sha"`
+ Repo struct {
+ ID int `json:"id"`
+ URL string `json:"url"`
+ Name string `json:"name"`
+ } `json:"repo"`
+ } `json:"base"`
+ } `json:"pull_requests"`
+ CreatedAt time.Time `json:"created_at"`
+ UpdatedAt time.Time `json:"updated_at"`
+ Actor struct {
+ Login string `json:"login"`
+ ID int `json:"id"`
+ NodeID string `json:"node_id"`
+ AvatarURL string `json:"avatar_url"`
+ GravatarID string `json:"gravatar_id"`
+ URL string `json:"url"`
+ HTMLURL string `json:"html_url"`
+ FollowersURL string `json:"followers_url"`
+ FollowingURL string `json:"following_url"`
+ GistsURL string `json:"gists_url"`
+ StarredURL string `json:"starred_url"`
+ SubscriptionsURL string `json:"subscriptions_url"`
+ OrganizationsURL string `json:"organizations_url"`
+ ReposURL string `json:"repos_url"`
+ EventsURL string `json:"events_url"`
+ ReceivedEventsURL string `json:"received_events_url"`
+ Type string `json:"type"`
+ SiteAdmin bool `json:"site_admin"`
+ } `json:"actor"`
+ RunAttempt int `json:"run_attempt"`
+ ReferencedWorkflows []any `json:"referenced_workflows"`
+ RunStartedAt time.Time `json:"run_started_at"`
+ TriggeringActor struct {
+ Login string `json:"login"`
+ ID int `json:"id"`
+ NodeID string `json:"node_id"`
+ AvatarURL string `json:"avatar_url"`
+ GravatarID string `json:"gravatar_id"`
+ URL string `json:"url"`
+ HTMLURL string `json:"html_url"`
+ FollowersURL string `json:"followers_url"`
+ FollowingURL string `json:"following_url"`
+ GistsURL string `json:"gists_url"`
+ StarredURL string `json:"starred_url"`
+ SubscriptionsURL string `json:"subscriptions_url"`
+ OrganizationsURL string `json:"organizations_url"`
+ ReposURL string `json:"repos_url"`
+ EventsURL string `json:"events_url"`
+ ReceivedEventsURL string `json:"received_events_url"`
+ Type string `json:"type"`
+ SiteAdmin bool `json:"site_admin"`
+ } `json:"triggering_actor"`
+ JobsURL string `json:"jobs_url"`
+ LogsURL string `json:"logs_url"`
+ CheckSuiteURL string `json:"check_suite_url"`
+ ArtifactsURL string `json:"artifacts_url"`
+ CancelURL string `json:"cancel_url"`
+ RerunURL string `json:"rerun_url"`
+ PreviousAttemptURL any `json:"previous_attempt_url"`
+ WorkflowURL string `json:"workflow_url"`
+ HeadCommit struct {
+ ID string `json:"id"`
+ TreeID string `json:"tree_id"`
+ Message string `json:"message"`
+ Timestamp time.Time `json:"timestamp"`
+ Author struct {
+ Name string `json:"name"`
+ Email string `json:"email"`
+ } `json:"author"`
+ Committer struct {
+ Name string `json:"name"`
+ Email string `json:"email"`
+ } `json:"committer"`
+ } `json:"head_commit"`
+ Repository struct {
+ ID int `json:"id"`
+ NodeID string `json:"node_id"`
+ Name string `json:"name"`
+ FullName string `json:"full_name"`
+ Private bool `json:"private"`
+ Owner struct {
+ Login string `json:"login"`
+ ID int `json:"id"`
+ NodeID string `json:"node_id"`
+ AvatarURL string `json:"avatar_url"`
+ GravatarID string `json:"gravatar_id"`
+ URL string `json:"url"`
+ HTMLURL string `json:"html_url"`
+ FollowersURL string `json:"followers_url"`
+ FollowingURL string `json:"following_url"`
+ GistsURL string `json:"gists_url"`
+ StarredURL string `json:"starred_url"`
+ SubscriptionsURL string `json:"subscriptions_url"`
+ OrganizationsURL string `json:"organizations_url"`
+ ReposURL string `json:"repos_url"`
+ EventsURL string `json:"events_url"`
+ ReceivedEventsURL string `json:"received_events_url"`
+ Type string `json:"type"`
+ SiteAdmin bool `json:"site_admin"`
+ } `json:"owner"`
+ HTMLURL string `json:"html_url"`
+ Description string `json:"description"`
+ Fork bool `json:"fork"`
+ URL string `json:"url"`
+ ForksURL string `json:"forks_url"`
+ KeysURL string `json:"keys_url"`
+ CollaboratorsURL string `json:"collaborators_url"`
+ TeamsURL string `json:"teams_url"`
+ HooksURL string `json:"hooks_url"`
+ IssueEventsURL string `json:"issue_events_url"`
+ EventsURL string `json:"events_url"`
+ AssigneesURL string `json:"assignees_url"`
+ BranchesURL string `json:"branches_url"`
+ TagsURL string `json:"tags_url"`
+ BlobsURL string `json:"blobs_url"`
+ GitTagsURL string `json:"git_tags_url"`
+ GitRefsURL string `json:"git_refs_url"`
+ TreesURL string `json:"trees_url"`
+ StatusesURL string `json:"statuses_url"`
+ LanguagesURL string `json:"languages_url"`
+ StargazersURL string `json:"stargazers_url"`
+ ContributorsURL string `json:"contributors_url"`
+ SubscribersURL string `json:"subscribers_url"`
+ SubscriptionURL string `json:"subscription_url"`
+ CommitsURL string `json:"commits_url"`
+ GitCommitsURL string `json:"git_commits_url"`
+ CommentsURL string `json:"comments_url"`
+ IssueCommentURL string `json:"issue_comment_url"`
+ ContentsURL string `json:"contents_url"`
+ CompareURL string `json:"compare_url"`
+ MergesURL string `json:"merges_url"`
+ ArchiveURL string `json:"archive_url"`
+ DownloadsURL string `json:"downloads_url"`
+ IssuesURL string `json:"issues_url"`
+ PullsURL string `json:"pulls_url"`
+ MilestonesURL string `json:"milestones_url"`
+ NotificationsURL string `json:"notifications_url"`
+ LabelsURL string `json:"labels_url"`
+ ReleasesURL string `json:"releases_url"`
+ DeploymentsURL string `json:"deployments_url"`
+ } `json:"repository"`
+ HeadRepository struct {
+ ID int `json:"id"`
+ NodeID string `json:"node_id"`
+ Name string `json:"name"`
+ FullName string `json:"full_name"`
+ Private bool `json:"private"`
+ Owner struct {
+ Login string `json:"login"`
+ ID int `json:"id"`
+ NodeID string `json:"node_id"`
+ AvatarURL string `json:"avatar_url"`
+ GravatarID string `json:"gravatar_id"`
+ URL string `json:"url"`
+ HTMLURL string `json:"html_url"`
+ FollowersURL string `json:"followers_url"`
+ FollowingURL string `json:"following_url"`
+ GistsURL string `json:"gists_url"`
+ StarredURL string `json:"starred_url"`
+ SubscriptionsURL string `json:"subscriptions_url"`
+ OrganizationsURL string `json:"organizations_url"`
+ ReposURL string `json:"repos_url"`
+ EventsURL string `json:"events_url"`
+ ReceivedEventsURL string `json:"received_events_url"`
+ Type string `json:"type"`
+ SiteAdmin bool `json:"site_admin"`
+ } `json:"owner"`
+ HTMLURL string `json:"html_url"`
+ Description string `json:"description"`
+ Fork bool `json:"fork"`
+ URL string `json:"url"`
+ ForksURL string `json:"forks_url"`
+ KeysURL string `json:"keys_url"`
+ CollaboratorsURL string `json:"collaborators_url"`
+ TeamsURL string `json:"teams_url"`
+ HooksURL string `json:"hooks_url"`
+ IssueEventsURL string `json:"issue_events_url"`
+ EventsURL string `json:"events_url"`
+ AssigneesURL string `json:"assignees_url"`
+ BranchesURL string `json:"branches_url"`
+ TagsURL string `json:"tags_url"`
+ BlobsURL string `json:"blobs_url"`
+ GitTagsURL string `json:"git_tags_url"`
+ GitRefsURL string `json:"git_refs_url"`
+ TreesURL string `json:"trees_url"`
+ StatusesURL string `json:"statuses_url"`
+ LanguagesURL string `json:"languages_url"`
+ StargazersURL string `json:"stargazers_url"`
+ ContributorsURL string `json:"contributors_url"`
+ SubscribersURL string `json:"subscribers_url"`
+ SubscriptionURL string `json:"subscription_url"`
+ CommitsURL string `json:"commits_url"`
+ GitCommitsURL string `json:"git_commits_url"`
+ CommentsURL string `json:"comments_url"`
+ IssueCommentURL string `json:"issue_comment_url"`
+ ContentsURL string `json:"contents_url"`
+ CompareURL string `json:"compare_url"`
+ MergesURL string `json:"merges_url"`
+ ArchiveURL string `json:"archive_url"`
+ DownloadsURL string `json:"downloads_url"`
+ IssuesURL string `json:"issues_url"`
+ PullsURL string `json:"pulls_url"`
+ MilestonesURL string `json:"milestones_url"`
+ NotificationsURL string `json:"notifications_url"`
+ LabelsURL string `json:"labels_url"`
+ ReleasesURL string `json:"releases_url"`
+ DeploymentsURL string `json:"deployments_url"`
+ } `json:"head_repository"`
+ } `json:"workflow_runs"`
+}