You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by wa...@apache.org on 2022/06/16 05:02:37 UTC
[incubator-devlake] 02/05: feat: jenkins add e2e test for jobs and builds
This is an automated email from the ASF dual-hosted git repository.
warren pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 65fa50514d03fcb4e24bfbda7251a374fe9f9a53
Author: abeizn <zi...@merico.dev>
AuthorDate: Thu Jun 16 09:32:47 2022 +0800
feat: jenkins add e2e test for jobs and builds
---
plugins/jenkins/e2e/builds_test.go | 86 ++++++++++++++++++++++
plugins/jenkins/e2e/jobs_test.go | 81 ++++++++++++++++++++
.../jenkins/e2e/tables/_raw_jenkins_api_builds.csv | 21 ++++++
.../jenkins/e2e/tables/_raw_jenkins_api_jobs.csv | 12 +++
.../jenkins/e2e/tables/_tool_jenkins_builds.csv | 21 ++++++
plugins/jenkins/e2e/tables/_tool_jenkins_jobs.csv | 12 +++
plugins/jenkins/e2e/tables/builds.csv | 12 +++
plugins/jenkins/e2e/tables/jobs.csv | 12 +++
plugins/jenkins/tasks/build_collector.go | 2 +-
plugins/jenkins/tasks/build_convertor.go | 2 +-
plugins/jenkins/tasks/build_extractor.go | 4 +-
plugins/jenkins/tasks/job_collector.go | 2 +-
plugins/jenkins/tasks/job_convertor.go | 2 +-
plugins/jenkins/tasks/job_extractor.go | 4 +-
14 files changed, 265 insertions(+), 8 deletions(-)
diff --git a/plugins/jenkins/e2e/builds_test.go b/plugins/jenkins/e2e/builds_test.go
new file mode 100644
index 00000000..24c92b5b
--- /dev/null
+++ b/plugins/jenkins/e2e/builds_test.go
@@ -0,0 +1,86 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+ "testing"
+
+ "github.com/apache/incubator-devlake/models/domainlayer/devops"
+
+ "github.com/apache/incubator-devlake/helpers/e2ehelper"
+ "github.com/apache/incubator-devlake/plugins/jenkins/impl"
+ "github.com/apache/incubator-devlake/plugins/jenkins/models"
+ "github.com/apache/incubator-devlake/plugins/jenkins/tasks"
+)
+
+func TestJenkinsBuildsDataFlow(t *testing.T) {
+
+ var jenkins impl.Jenkins
+ dataflowTester := e2ehelper.NewDataFlowTester(t, "jenkins", jenkins)
+
+ taskData := &tasks.JenkinsTaskData{
+ Options: &tasks.JenkinsOptions{
+ ConnectionId: 1,
+ },
+ }
+
+ // import raw data table
+ dataflowTester.ImportCsvIntoRawTable("./tables/_raw_jenkins_api_builds.csv", "_raw_jenkins_api_builds")
+
+ // verify extraction
+ dataflowTester.FlushTabler(&models.JenkinsBuild{})
+ dataflowTester.Subtask(tasks.ExtractApiBuildsMeta, taskData)
+ dataflowTester.VerifyTable(
+ models.JenkinsBuild{},
+ "tables/_tool_jenkins_builds.csv",
+ []string{"connection_id", "job_name", "number"},
+ []string{
+ "_raw_data_params",
+ "_raw_data_table",
+ "_raw_data_id",
+ "_raw_data_remark",
+ "connection_id",
+ "job_name",
+ "duration",
+ "display_name",
+ "estimated_duration",
+ "number",
+ "result",
+ "timestamp",
+ "start_time",
+ "commit_sha",
+ },
+ )
+
+ // verify conversion
+ dataflowTester.FlushTabler(&devops.Build{})
+ dataflowTester.Subtask(tasks.ConvertBuildsMeta, taskData)
+ dataflowTester.VerifyTable(
+ devops.Job{},
+ "tables/builds.csv",
+ []string{"id"},
+ []string{
+ "id",
+ "_raw_data_params",
+ "_raw_data_table",
+ "_raw_data_id",
+ "_raw_data_remark",
+ "name",
+ },
+ )
+}
diff --git a/plugins/jenkins/e2e/jobs_test.go b/plugins/jenkins/e2e/jobs_test.go
new file mode 100644
index 00000000..267e71ad
--- /dev/null
+++ b/plugins/jenkins/e2e/jobs_test.go
@@ -0,0 +1,81 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+ "testing"
+
+ "github.com/apache/incubator-devlake/models/domainlayer/devops"
+
+ "github.com/apache/incubator-devlake/helpers/e2ehelper"
+ "github.com/apache/incubator-devlake/plugins/jenkins/impl"
+ "github.com/apache/incubator-devlake/plugins/jenkins/models"
+ "github.com/apache/incubator-devlake/plugins/jenkins/tasks"
+)
+
+func TestJenkinsJobsDataFlow(t *testing.T) {
+
+ var jenkins impl.Jenkins
+ dataflowTester := e2ehelper.NewDataFlowTester(t, "jenkins", jenkins)
+
+ taskData := &tasks.JenkinsTaskData{
+ Options: &tasks.JenkinsOptions{
+ ConnectionId: 1,
+ },
+ }
+
+ // import raw data table
+ dataflowTester.ImportCsvIntoRawTable("./tables/_raw_jenkins_api_jobs.csv", "_raw_jenkins_api_jobs")
+
+ // verify extraction
+ dataflowTester.FlushTabler(&models.JenkinsJob{})
+ dataflowTester.Subtask(tasks.ExtractApiJobsMeta, taskData)
+ dataflowTester.VerifyTable(
+ models.JenkinsJob{},
+ "tables/_tool_jenkins_jobs.csv",
+ []string{"connection_id", "name"},
+ []string{
+ "connection_id",
+ "name",
+ "class",
+ "color",
+ "base",
+ "_raw_data_params",
+ "_raw_data_table",
+ "_raw_data_id",
+ "_raw_data_remark",
+ },
+ )
+
+ // verify conversion
+ dataflowTester.FlushTabler(&devops.Job{})
+ dataflowTester.Subtask(tasks.ConvertJobsMeta, taskData)
+ dataflowTester.VerifyTable(
+ devops.Job{},
+ "tables/jobs.csv",
+ []string{"id"},
+ []string{
+ "name",
+ "id",
+ "_raw_data_params",
+ "_raw_data_table",
+ "_raw_data_id",
+ "_raw_data_remark",
+ },
+ )
+}
diff --git a/plugins/jenkins/e2e/tables/_raw_jenkins_api_builds.csv b/plugins/jenkins/e2e/tables/_raw_jenkins_api_builds.csv
new file mode 100644
index 00000000..f5399bd1
--- /dev/null
+++ b/plugins/jenkins/e2e/tables/_raw_jenkins_api_builds.csv
@@ -0,0 +1,21 @@
+"id","params","data","url","input","created_at"
+"1","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleBuild"",""actions"":[{""_class"":""hudson.model.ParametersAction""},{""_class"":""hudson.model.CauseAction""},{},{},{}],""displayName"":""#205"",""duration"":843,""estimatedDuration"":845,""number"":205,""result"":""SUCCESS"",""timestamp"":1647841830657,""changeSet"":{""_class"":""hudson.scm.EmptyChangeLogSet"",""kind"":null}}","https://jenkins.merico.cn/job/Create_License/api/json?tree=allBuilds%5Bnumber%2Ctimestamp%2Cdurat [...]
+"2","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleBuild"",""actions"":[{""_class"":""hudson.model.ParametersAction""},{""_class"":""hudson.model.CauseAction""},{},{},{}],""displayName"":""#204"",""duration"":849,""estimatedDuration"":845,""number"":204,""result"":""SUCCESS"",""timestamp"":1645152359034,""changeSet"":{""_class"":""hudson.scm.EmptyChangeLogSet"",""kind"":null}}","https://jenkins.merico.cn/job/Create_License/api/json?tree=allBuilds%5Bnumber%2Ctimestamp%2Cdurat [...]
+"3","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleBuild"",""actions"":[{""_class"":""hudson.model.ParametersAction""},{""_class"":""hudson.model.CauseAction""},{},{},{}],""displayName"":""#203"",""duration"":823,""estimatedDuration"":845,""number"":203,""result"":""SUCCESS"",""timestamp"":1642560020873,""changeSet"":{""_class"":""hudson.scm.EmptyChangeLogSet"",""kind"":null}}","https://jenkins.merico.cn/job/Create_License/api/json?tree=allBuilds%5Bnumber%2Ctimestamp%2Cdurat [...]
+"4","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleBuild"",""actions"":[{""_class"":""hudson.model.ParametersAction""},{""_class"":""hudson.model.CauseAction""},{},{},{}],""displayName"":""#202"",""duration"":814,""estimatedDuration"":845,""number"":202,""result"":""SUCCESS"",""timestamp"":1641524412998,""changeSet"":{""_class"":""hudson.scm.EmptyChangeLogSet"",""kind"":null}}","https://jenkins.merico.cn/job/Create_License/api/json?tree=allBuilds%5Bnumber%2Ctimestamp%2Cdurat [...]
+"5","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleBuild"",""actions"":[{""_class"":""hudson.model.ParametersAction""},{""_class"":""hudson.model.CauseAction""},{},{},{}],""displayName"":""#201"",""duration"":872,""estimatedDuration"":845,""number"":201,""result"":""SUCCESS"",""timestamp"":1641286809958,""changeSet"":{""_class"":""hudson.scm.EmptyChangeLogSet"",""kind"":null}}","https://jenkins.merico.cn/job/Create_License/api/json?tree=allBuilds%5Bnumber%2Ctimestamp%2Cdurat [...]
+"6","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleBuild"",""actions"":[{""_class"":""hudson.model.ParametersAction""},{""_class"":""hudson.model.CauseAction""},{},{},{}],""displayName"":""#200"",""duration"":826,""estimatedDuration"":845,""number"":200,""result"":""SUCCESS"",""timestamp"":1638425697887,""changeSet"":{""_class"":""hudson.scm.EmptyChangeLogSet"",""kind"":null}}","https://jenkins.merico.cn/job/Create_License/api/json?tree=allBuilds%5Bnumber%2Ctimestamp%2Cdurat [...]
+"7","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleBuild"",""actions"":[{""_class"":""hudson.model.ParametersAction""},{""_class"":""hudson.model.CauseAction""},{},{},{}],""displayName"":""#199"",""duration"":711,""estimatedDuration"":845,""number"":199,""result"":""FAILURE"",""timestamp"":1637636904142,""changeSet"":{""_class"":""hudson.scm.EmptyChangeLogSet"",""kind"":null}}","https://jenkins.merico.cn/job/Create_License/api/json?tree=allBuilds%5Bnumber%2Ctimestamp%2Cdurat [...]
+"8","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleBuild"",""actions"":[{""_class"":""hudson.model.ParametersAction""},{""_class"":""hudson.model.CauseAction""},{},{},{}],""displayName"":""#198"",""duration"":855,""estimatedDuration"":845,""number"":198,""result"":""SUCCESS"",""timestamp"":1635842717468,""changeSet"":{""_class"":""hudson.scm.EmptyChangeLogSet"",""kind"":null}}","https://jenkins.merico.cn/job/Create_License/api/json?tree=allBuilds%5Bnumber%2Ctimestamp%2Cdurat [...]
+"9","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleBuild"",""actions"":[{""_class"":""hudson.model.ParametersAction""},{""_class"":""hudson.model.CauseAction""},{},{},{}],""displayName"":""#197"",""duration"":806,""estimatedDuration"":845,""number"":197,""result"":""SUCCESS"",""timestamp"":1635143830616,""changeSet"":{""_class"":""hudson.scm.EmptyChangeLogSet"",""kind"":null}}","https://jenkins.merico.cn/job/Create_License/api/json?tree=allBuilds%5Bnumber%2Ctimestamp%2Cdurat [...]
+"10","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleBuild"",""actions"":[{""_class"":""hudson.model.ParametersAction""},{""_class"":""hudson.model.CauseAction""},{},{},{}],""displayName"":""#196"",""duration"":840,""estimatedDuration"":845,""number"":196,""result"":""SUCCESS"",""timestamp"":1634713846944,""changeSet"":{""_class"":""hudson.scm.EmptyChangeLogSet"",""kind"":null}}","https://jenkins.merico.cn/job/Create_License/api/json?tree=allBuilds%5Bnumber%2Ctimestamp%2Cdura [...]
+"11","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleBuild"",""actions"":[{""_class"":""hudson.model.ParametersAction""},{""_class"":""hudson.model.CauseAction""},{},{},{}],""displayName"":""#195"",""duration"":756,""estimatedDuration"":845,""number"":195,""result"":""SUCCESS"",""timestamp"":1634289009179,""changeSet"":{""_class"":""hudson.scm.EmptyChangeLogSet"",""kind"":null}}","https://jenkins.merico.cn/job/Create_License/api/json?tree=allBuilds%5Bnumber%2Ctimestamp%2Cdura [...]
+"12","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleBuild"",""actions"":[{""_class"":""hudson.model.ParametersAction""},{""_class"":""hudson.model.CauseAction""},{},{},{}],""displayName"":""#194"",""duration"":735,""estimatedDuration"":845,""number"":194,""result"":""SUCCESS"",""timestamp"":1634111680194,""changeSet"":{""_class"":""hudson.scm.EmptyChangeLogSet"",""kind"":null}}","https://jenkins.merico.cn/job/Create_License/api/json?tree=allBuilds%5Bnumber%2Ctimestamp%2Cdura [...]
+"13","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleBuild"",""actions"":[{""_class"":""hudson.model.ParametersAction""},{""_class"":""hudson.model.CauseAction""},{},{},{}],""displayName"":""#193"",""duration"":805,""estimatedDuration"":845,""number"":193,""result"":""SUCCESS"",""timestamp"":1634108932844,""changeSet"":{""_class"":""hudson.scm.EmptyChangeLogSet"",""kind"":null}}","https://jenkins.merico.cn/job/Create_License/api/json?tree=allBuilds%5Bnumber%2Ctimestamp%2Cdura [...]
+"14","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleBuild"",""actions"":[{""_class"":""hudson.model.ParametersAction""},{""_class"":""hudson.model.CauseAction""},{},{},{}],""displayName"":""#192"",""duration"":776,""estimatedDuration"":845,""number"":192,""result"":""SUCCESS"",""timestamp"":1631935845537,""changeSet"":{""_class"":""hudson.scm.EmptyChangeLogSet"",""kind"":null}}","https://jenkins.merico.cn/job/Create_License/api/json?tree=allBuilds%5Bnumber%2Ctimestamp%2Cdura [...]
+"15","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleBuild"",""actions"":[{""_class"":""hudson.model.ParametersAction""},{""_class"":""hudson.model.CauseAction""},{},{},{}],""displayName"":""#191"",""duration"":833,""estimatedDuration"":845,""number"":191,""result"":""SUCCESS"",""timestamp"":1631237966948,""changeSet"":{""_class"":""hudson.scm.EmptyChangeLogSet"",""kind"":null}}","https://jenkins.merico.cn/job/Create_License/api/json?tree=allBuilds%5Bnumber%2Ctimestamp%2Cdura [...]
+"16","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleBuild"",""actions"":[{""_class"":""hudson.model.ParametersAction""},{""_class"":""hudson.model.CauseAction""},{},{},{}],""displayName"":""#190"",""duration"":819,""estimatedDuration"":845,""number"":190,""result"":""SUCCESS"",""timestamp"":1631000546032,""changeSet"":{""_class"":""hudson.scm.EmptyChangeLogSet"",""kind"":null}}","https://jenkins.merico.cn/job/Create_License/api/json?tree=allBuilds%5Bnumber%2Ctimestamp%2Cdura [...]
+"17","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleBuild"",""actions"":[{""_class"":""hudson.model.ParametersAction""},{""_class"":""hudson.model.CauseAction""},{},{},{}],""displayName"":""#189"",""duration"":785,""estimatedDuration"":845,""number"":189,""result"":""SUCCESS"",""timestamp"":1630632234339,""changeSet"":{""_class"":""hudson.scm.EmptyChangeLogSet"",""kind"":null}}","https://jenkins.merico.cn/job/Create_License/api/json?tree=allBuilds%5Bnumber%2Ctimestamp%2Cdura [...]
+"18","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleBuild"",""actions"":[{""_class"":""hudson.model.ParametersAction""},{""_class"":""hudson.model.CauseAction""},{},{},{}],""displayName"":""#188"",""duration"":752,""estimatedDuration"":845,""number"":188,""result"":""SUCCESS"",""timestamp"":1629169239734,""changeSet"":{""_class"":""hudson.scm.EmptyChangeLogSet"",""kind"":null}}","https://jenkins.merico.cn/job/Create_License/api/json?tree=allBuilds%5Bnumber%2Ctimestamp%2Cdura [...]
+"19","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleBuild"",""actions"":[{""_class"":""hudson.model.ParametersAction""},{""_class"":""hudson.model.CauseAction""},{},{},{}],""displayName"":""#187"",""duration"":822,""estimatedDuration"":845,""number"":187,""result"":""SUCCESS"",""timestamp"":1628838712486,""changeSet"":{""_class"":""hudson.scm.EmptyChangeLogSet"",""kind"":null}}","https://jenkins.merico.cn/job/Create_License/api/json?tree=allBuilds%5Bnumber%2Ctimestamp%2Cdura [...]
+"20","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleBuild"",""actions"":[{""_class"":""hudson.model.ParametersAction""},{""_class"":""hudson.model.CauseAction""},{},{},{}],""displayName"":""#186"",""duration"":725,""estimatedDuration"":845,""number"":186,""result"":""SUCCESS"",""timestamp"":1628674732761,""changeSet"":{""_class"":""hudson.scm.EmptyChangeLogSet"",""kind"":null}}","https://jenkins.merico.cn/job/Create_License/api/json?tree=allBuilds%5Bnumber%2Ctimestamp%2Cdura [...]
\ No newline at end of file
diff --git a/plugins/jenkins/e2e/tables/_raw_jenkins_api_jobs.csv b/plugins/jenkins/e2e/tables/_raw_jenkins_api_jobs.csv
new file mode 100644
index 00000000..b4e45e6a
--- /dev/null
+++ b/plugins/jenkins/e2e/tables/_raw_jenkins_api_jobs.csv
@@ -0,0 +1,12 @@
+"id","params","data","url","input","created_at"
+"1","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleProject"",""name"":""auto_test"",""color"":""blue""}","https://jenkins.merico.cn/api/json?tree=jobs%5Bname%2Cclass%2Ccolor%2Cbase%5D%7B1%2C100%7D","null","2022-06-15 15:24:40.697"
+"2","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleProject"",""name"":""auto_test_framework"",""color"":""disabled""}","https://jenkins.merico.cn/api/json?tree=jobs%5Bname%2Cclass%2Ccolor%2Cbase%5D%7B1%2C100%7D","null","2022-06-15 15:24:40.697"
+"3","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleProject"",""name"":""build_devlake"",""color"":""blue""}","https://jenkins.merico.cn/api/json?tree=jobs%5Bname%2Cclass%2Ccolor%2Cbase%5D%7B1%2C100%7D","null","2022-06-15 15:24:40.697"
+"4","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleProject"",""name"":""Create_License"",""color"":""blue""}","https://jenkins.merico.cn/api/json?tree=jobs%5Bname%2Cclass%2Ccolor%2Cbase%5D%7B1%2C100%7D","null","2022-06-15 15:24:40.697"
+"5","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleProject"",""name"":""Create_License_new"",""color"":""blue""}","https://jenkins.merico.cn/api/json?tree=jobs%5Bname%2Cclass%2Ccolor%2Cbase%5D%7B1%2C100%7D","null","2022-06-15 15:24:40.697"
+"6","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleProject"",""name"":""Deploy k8s single_data"",""color"":""blue""}","https://jenkins.merico.cn/api/json?tree=jobs%5Bname%2Cclass%2Ccolor%2Cbase%5D%7B1%2C100%7D","null","2022-06-15 15:24:40.697"
+"7","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleProject"",""name"":""Deploy k8s single_new"",""color"":""blue""}","https://jenkins.merico.cn/api/json?tree=jobs%5Bname%2Cclass%2Ccolor%2Cbase%5D%7B1%2C100%7D","null","2022-06-15 15:24:40.697"
+"8","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleProject"",""name"":""Deploy k8s test"",""color"":""blue""}","https://jenkins.merico.cn/api/json?tree=jobs%5Bname%2Cclass%2Ccolor%2Cbase%5D%7B1%2C100%7D","null","2022-06-15 15:24:40.697"
+"9","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleProject"",""name"":""devlake_empty_build"",""color"":""blue""}","https://jenkins.merico.cn/api/json?tree=jobs%5Bname%2Cclass%2Ccolor%2Cbase%5D%7B1%2C100%7D","null","2022-06-15 15:24:40.697"
+"10","{""ConnectionId"":1}","{""_class"":""org.jenkinsci.plugins.workflow.job.WorkflowJob"",""name"":""test-platform-backend"",""color"":""blue""}","https://jenkins.merico.cn/api/json?tree=jobs%5Bname%2Cclass%2Ccolor%2Cbase%5D%7B1%2C100%7D","null","2022-06-15 15:24:40.697"
+"11","{""ConnectionId"":1}","{""_class"":""org.jenkinsci.plugins.workflow.job.WorkflowJob"",""name"":""test-platform-frontend"",""color"":""blue""}","https://jenkins.merico.cn/api/json?tree=jobs%5Bname%2Cclass%2Ccolor%2Cbase%5D%7B1%2C100%7D","null","2022-06-15 15:24:40.697"
diff --git a/plugins/jenkins/e2e/tables/_tool_jenkins_builds.csv b/plugins/jenkins/e2e/tables/_tool_jenkins_builds.csv
new file mode 100644
index 00000000..9a5c5a04
--- /dev/null
+++ b/plugins/jenkins/e2e/tables/_tool_jenkins_builds.csv
@@ -0,0 +1,21 @@
+connection_id,job_name,number,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark,connection_id,job_name,duration,display_name,estimated_duration,number,result,timestamp,start_time,commit_sha
+1,Create_License,186,"{""ConnectionId"":1}",_raw_jenkins_api_builds,20,,1,Create_License,725,#186,845,186,SUCCESS,1628674732761,2021-08-11T09:38:52.000+00:00,
+1,Create_License,187,"{""ConnectionId"":1}",_raw_jenkins_api_builds,19,,1,Create_License,822,#187,845,187,SUCCESS,1628838712486,2021-08-13T07:11:52.000+00:00,
+1,Create_License,188,"{""ConnectionId"":1}",_raw_jenkins_api_builds,18,,1,Create_License,752,#188,845,188,SUCCESS,1629169239734,2021-08-17T03:00:39.000+00:00,
+1,Create_License,189,"{""ConnectionId"":1}",_raw_jenkins_api_builds,17,,1,Create_License,785,#189,845,189,SUCCESS,1630632234339,2021-09-03T01:23:54.000+00:00,
+1,Create_License,190,"{""ConnectionId"":1}",_raw_jenkins_api_builds,16,,1,Create_License,819,#190,845,190,SUCCESS,1631000546032,2021-09-07T07:42:26.000+00:00,
+1,Create_License,191,"{""ConnectionId"":1}",_raw_jenkins_api_builds,15,,1,Create_License,833,#191,845,191,SUCCESS,1631237966948,2021-09-10T01:39:26.000+00:00,
+1,Create_License,192,"{""ConnectionId"":1}",_raw_jenkins_api_builds,14,,1,Create_License,776,#192,845,192,SUCCESS,1631935845537,2021-09-18T03:30:45.000+00:00,
+1,Create_License,193,"{""ConnectionId"":1}",_raw_jenkins_api_builds,13,,1,Create_License,805,#193,845,193,SUCCESS,1634108932844,2021-10-13T07:08:52.000+00:00,
+1,Create_License,194,"{""ConnectionId"":1}",_raw_jenkins_api_builds,12,,1,Create_License,735,#194,845,194,SUCCESS,1634111680194,2021-10-13T07:54:40.000+00:00,
+1,Create_License,195,"{""ConnectionId"":1}",_raw_jenkins_api_builds,11,,1,Create_License,756,#195,845,195,SUCCESS,1634289009179,2021-10-15T09:10:09.000+00:00,
+1,Create_License,196,"{""ConnectionId"":1}",_raw_jenkins_api_builds,10,,1,Create_License,840,#196,845,196,SUCCESS,1634713846944,2021-10-20T07:10:46.000+00:00,
+1,Create_License,197,"{""ConnectionId"":1}",_raw_jenkins_api_builds,9,,1,Create_License,806,#197,845,197,SUCCESS,1635143830616,2021-10-25T06:37:10.000+00:00,
+1,Create_License,198,"{""ConnectionId"":1}",_raw_jenkins_api_builds,8,,1,Create_License,855,#198,845,198,SUCCESS,1635842717468,2021-11-02T08:45:17.000+00:00,
+1,Create_License,199,"{""ConnectionId"":1}",_raw_jenkins_api_builds,7,,1,Create_License,711,#199,845,199,FAILURE,1637636904142,2021-11-23T03:08:24.000+00:00,
+1,Create_License,200,"{""ConnectionId"":1}",_raw_jenkins_api_builds,6,,1,Create_License,826,#200,845,200,SUCCESS,1638425697887,2021-12-02T06:14:57.000+00:00,
+1,Create_License,201,"{""ConnectionId"":1}",_raw_jenkins_api_builds,5,,1,Create_License,872,#201,845,201,SUCCESS,1641286809958,2022-01-04T09:00:09.000+00:00,
+1,Create_License,202,"{""ConnectionId"":1}",_raw_jenkins_api_builds,4,,1,Create_License,814,#202,845,202,SUCCESS,1641524412998,2022-01-07T03:00:12.000+00:00,
+1,Create_License,203,"{""ConnectionId"":1}",_raw_jenkins_api_builds,3,,1,Create_License,823,#203,845,203,SUCCESS,1642560020873,2022-01-19T02:40:20.000+00:00,
+1,Create_License,204,"{""ConnectionId"":1}",_raw_jenkins_api_builds,2,,1,Create_License,849,#204,845,204,SUCCESS,1645152359034,2022-02-18T02:45:59.000+00:00,
+1,Create_License,205,"{""ConnectionId"":1}",_raw_jenkins_api_builds,1,,1,Create_License,843,#205,845,205,SUCCESS,1647841830657,2022-03-21T05:50:30.000+00:00,
diff --git a/plugins/jenkins/e2e/tables/_tool_jenkins_jobs.csv b/plugins/jenkins/e2e/tables/_tool_jenkins_jobs.csv
new file mode 100644
index 00000000..5d45f725
--- /dev/null
+++ b/plugins/jenkins/e2e/tables/_tool_jenkins_jobs.csv
@@ -0,0 +1,12 @@
+connection_id,name,name,class,color,base,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+1,auto_test,auto_test,hudson.model.FreeStyleProject,blue,,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,1,
+1,auto_test_framework,auto_test_framework,hudson.model.FreeStyleProject,disabled,,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,2,
+1,build_devlake,build_devlake,hudson.model.FreeStyleProject,blue,,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,3,
+1,Create_License,Create_License,hudson.model.FreeStyleProject,blue,,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,4,
+1,Create_License_new,Create_License_new,hudson.model.FreeStyleProject,blue,,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,5,
+1,Deploy k8s single_data,Deploy k8s single_data,hudson.model.FreeStyleProject,blue,,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,6,
+1,Deploy k8s single_new,Deploy k8s single_new,hudson.model.FreeStyleProject,blue,,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,7,
+1,Deploy k8s test,Deploy k8s test,hudson.model.FreeStyleProject,blue,,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,8,
+1,devlake_empty_build,devlake_empty_build,hudson.model.FreeStyleProject,blue,,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,9,
+1,test-platform-backend,test-platform-backend,org.jenkinsci.plugins.workflow.job.WorkflowJob,blue,,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,10,
+1,test-platform-frontend,test-platform-frontend,org.jenkinsci.plugins.workflow.job.WorkflowJob,blue,,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,11,
diff --git a/plugins/jenkins/e2e/tables/builds.csv b/plugins/jenkins/e2e/tables/builds.csv
new file mode 100644
index 00000000..a26be590
--- /dev/null
+++ b/plugins/jenkins/e2e/tables/builds.csv
@@ -0,0 +1,12 @@
+id,id,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark,name
+jenkins:JenkinsJob:1:auto_test,jenkins:JenkinsJob:1:auto_test,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,1,,auto_test
+jenkins:JenkinsJob:1:auto_test_framework,jenkins:JenkinsJob:1:auto_test_framework,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,2,,auto_test_framework
+jenkins:JenkinsJob:1:build_devlake,jenkins:JenkinsJob:1:build_devlake,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,3,,build_devlake
+jenkins:JenkinsJob:1:Create_License,jenkins:JenkinsJob:1:Create_License,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,4,,Create_License
+jenkins:JenkinsJob:1:Create_License_new,jenkins:JenkinsJob:1:Create_License_new,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,5,,Create_License_new
+jenkins:JenkinsJob:1:Deploy k8s single_data,jenkins:JenkinsJob:1:Deploy k8s single_data,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,6,,Deploy k8s single_data
+jenkins:JenkinsJob:1:Deploy k8s single_new,jenkins:JenkinsJob:1:Deploy k8s single_new,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,7,,Deploy k8s single_new
+jenkins:JenkinsJob:1:Deploy k8s test,jenkins:JenkinsJob:1:Deploy k8s test,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,8,,Deploy k8s test
+jenkins:JenkinsJob:1:devlake_empty_build,jenkins:JenkinsJob:1:devlake_empty_build,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,9,,devlake_empty_build
+jenkins:JenkinsJob:1:test-platform-backend,jenkins:JenkinsJob:1:test-platform-backend,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,10,,test-platform-backend
+jenkins:JenkinsJob:1:test-platform-frontend,jenkins:JenkinsJob:1:test-platform-frontend,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,11,,test-platform-frontend
diff --git a/plugins/jenkins/e2e/tables/jobs.csv b/plugins/jenkins/e2e/tables/jobs.csv
new file mode 100644
index 00000000..7348d739
--- /dev/null
+++ b/plugins/jenkins/e2e/tables/jobs.csv
@@ -0,0 +1,12 @@
+id,name,id,created_at,updated_at,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+jenkins:JenkinsJob:1:auto_test,auto_test,jenkins:JenkinsJob:1:auto_test,2022-06-16T00:58:39.909+00:00,2022-06-16T00:58:39.909+00:00,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,1,
+jenkins:JenkinsJob:1:auto_test_framework,auto_test_framework,jenkins:JenkinsJob:1:auto_test_framework,2022-06-16T00:58:39.909+00:00,2022-06-16T00:58:39.909+00:00,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,2,
+jenkins:JenkinsJob:1:build_devlake,build_devlake,jenkins:JenkinsJob:1:build_devlake,2022-06-16T00:58:39.909+00:00,2022-06-16T00:58:39.909+00:00,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,3,
+jenkins:JenkinsJob:1:Create_License,Create_License,jenkins:JenkinsJob:1:Create_License,2022-06-16T00:58:39.909+00:00,2022-06-16T00:58:39.909+00:00,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,4,
+jenkins:JenkinsJob:1:Create_License_new,Create_License_new,jenkins:JenkinsJob:1:Create_License_new,2022-06-16T00:58:39.909+00:00,2022-06-16T00:58:39.909+00:00,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,5,
+jenkins:JenkinsJob:1:Deploy k8s single_data,Deploy k8s single_data,jenkins:JenkinsJob:1:Deploy k8s single_data,2022-06-16T00:58:39.909+00:00,2022-06-16T00:58:39.909+00:00,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,6,
+jenkins:JenkinsJob:1:Deploy k8s single_new,Deploy k8s single_new,jenkins:JenkinsJob:1:Deploy k8s single_new,2022-06-16T00:58:39.909+00:00,2022-06-16T00:58:39.909+00:00,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,7,
+jenkins:JenkinsJob:1:Deploy k8s test,Deploy k8s test,jenkins:JenkinsJob:1:Deploy k8s test,2022-06-16T00:58:39.909+00:00,2022-06-16T00:58:39.909+00:00,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,8,
+jenkins:JenkinsJob:1:devlake_empty_build,devlake_empty_build,jenkins:JenkinsJob:1:devlake_empty_build,2022-06-16T00:58:39.909+00:00,2022-06-16T00:58:39.909+00:00,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,9,
+jenkins:JenkinsJob:1:test-platform-backend,test-platform-backend,jenkins:JenkinsJob:1:test-platform-backend,2022-06-16T00:58:39.909+00:00,2022-06-16T00:58:39.909+00:00,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,10,
+jenkins:JenkinsJob:1:test-platform-frontend,test-platform-frontend,jenkins:JenkinsJob:1:test-platform-frontend,2022-06-16T00:58:39.909+00:00,2022-06-16T00:58:39.909+00:00,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,11,
diff --git a/plugins/jenkins/tasks/build_collector.go b/plugins/jenkins/tasks/build_collector.go
index ca711333..8ce663fa 100644
--- a/plugins/jenkins/tasks/build_collector.go
+++ b/plugins/jenkins/tasks/build_collector.go
@@ -65,7 +65,7 @@ func CollectApiBuilds(taskCtx core.SubTaskContext) error {
collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
Params: JenkinsApiParams{
- ConnectionId: data.Connection.ID,
+ ConnectionId: data.Options.ConnectionId,
},
Ctx: taskCtx,
Table: RAW_BUILD_TABLE,
diff --git a/plugins/jenkins/tasks/build_convertor.go b/plugins/jenkins/tasks/build_convertor.go
index 5f12ef47..cb160f69 100644
--- a/plugins/jenkins/tasks/build_convertor.go
+++ b/plugins/jenkins/tasks/build_convertor.go
@@ -58,7 +58,7 @@ func ConvertBuilds(taskCtx core.SubTaskContext) error {
Input: cursor,
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
Params: JenkinsApiParams{
- ConnectionId: data.Connection.ID,
+ ConnectionId: data.Options.ConnectionId,
},
Ctx: taskCtx,
Table: RAW_BUILD_TABLE,
diff --git a/plugins/jenkins/tasks/build_extractor.go b/plugins/jenkins/tasks/build_extractor.go
index ac57728e..a3df36fb 100644
--- a/plugins/jenkins/tasks/build_extractor.go
+++ b/plugins/jenkins/tasks/build_extractor.go
@@ -41,7 +41,7 @@ func ExtractApiBuilds(taskCtx core.SubTaskContext) error {
extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
Params: JenkinsApiParams{
- ConnectionId: data.Connection.ID,
+ ConnectionId: data.Options.ConnectionId,
},
Ctx: taskCtx,
/*
@@ -68,7 +68,7 @@ func ExtractApiBuilds(taskCtx core.SubTaskContext) error {
results := make([]interface{}, 0, 1)
build := &models.JenkinsBuild{
- ConnectionId: data.Connection.ID,
+ ConnectionId: data.Options.ConnectionId,
JobName: input.Name,
Duration: body.Duration,
DisplayName: body.DisplayName,
diff --git a/plugins/jenkins/tasks/job_collector.go b/plugins/jenkins/tasks/job_collector.go
index a1dcbb36..e608d63d 100644
--- a/plugins/jenkins/tasks/job_collector.go
+++ b/plugins/jenkins/tasks/job_collector.go
@@ -42,7 +42,7 @@ func CollectApiJobs(taskCtx core.SubTaskContext) error {
collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
Params: JenkinsApiParams{
- ConnectionId: data.Connection.ID,
+ ConnectionId: data.Options.ConnectionId,
},
Ctx: taskCtx,
/*
diff --git a/plugins/jenkins/tasks/job_convertor.go b/plugins/jenkins/tasks/job_convertor.go
index 25461a2a..55d08098 100644
--- a/plugins/jenkins/tasks/job_convertor.go
+++ b/plugins/jenkins/tasks/job_convertor.go
@@ -57,7 +57,7 @@ func ConvertJobs(taskCtx core.SubTaskContext) error {
Input: cursor,
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
Params: JenkinsApiParams{
- ConnectionId: data.Connection.ID,
+ ConnectionId: data.Options.ConnectionId,
},
Ctx: taskCtx,
Table: RAW_JOB_TABLE,
diff --git a/plugins/jenkins/tasks/job_extractor.go b/plugins/jenkins/tasks/job_extractor.go
index 762aa37d..18967eea 100644
--- a/plugins/jenkins/tasks/job_extractor.go
+++ b/plugins/jenkins/tasks/job_extractor.go
@@ -39,7 +39,7 @@ func ExtractApiJobs(taskCtx core.SubTaskContext) error {
extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
Params: JenkinsApiParams{
- ConnectionId: data.Connection.ID,
+ ConnectionId: data.Options.ConnectionId,
},
Ctx: taskCtx,
/*
@@ -61,7 +61,7 @@ func ExtractApiJobs(taskCtx core.SubTaskContext) error {
job := &models.JenkinsJob{
JenkinsJobProps: models.JenkinsJobProps{
- ConnectionId: data.Connection.ID,
+ ConnectionId: data.Options.ConnectionId,
Name: body.Name,
Class: body.Class,
Color: body.Color,