You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by kl...@apache.org on 2022/07/06 01:01:07 UTC
[incubator-devlake] branch main updated: [feat-1022][github]: Support GitHub Milestones (#2215)
This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 74226840 [feat-1022][github]: Support GitHub Milestones (#2215)
74226840 is described below
commit 74226840d5dd64200c1b8a3953589dd37efa2c65
Author: Keon Amini <ke...@gmail.com>
AuthorDate: Tue Jul 5 19:01:03 2022 -0600
[feat-1022][github]: Support GitHub Milestones (#2215)
* feat(github): added support for github milestones (#1022)
* test: added unit tests to data_flow_tester
* test: improved the added tests
Co-authored-by: Keon Amini <ke...@merico.dev>
---
config-ui/src/hooks/useConnectionManager.jsx | 2 +-
config/config.go | 8 +-
go.mod | 6 +-
go.sum | 5 +-
helpers/e2ehelper/data_flow_tester.go | 117 ++++++++--
helpers/e2ehelper/data_flow_tester_test.go | 153 +++++++++++++
plugins/github/e2e/issue_test.go | 1 +
plugins/github/e2e/milestone_test.go | 76 +++++++
.../e2e/raw_tables/_raw_github_api_issues.csv | 84 +++----
.../e2e/raw_tables/_raw_github_api_milestones.csv | 2 +
.../e2e/snapshot_tables/_tool_github_issues.csv | 54 ++---
.../snapshot_tables/_tool_github_milestones.csv | 2 +
.../github/e2e/snapshot_tables/board_sprint.csv | 2 +
plugins/github/e2e/snapshot_tables/boards.csv | 2 +-
.../github/e2e/snapshot_tables/sprint_issue.csv | 27 +++
plugins/github/e2e/snapshot_tables/sprints.csv | 2 +
plugins/github/impl/impl.go | 8 +-
plugins/github/models/issue.go | 1 +
plugins/github/models/migrationscripts/register.go | 5 +-
.../migrationscripts/updateSchemas20220620.go | 73 +++++++
.../{migrationscripts/register.go => milestone.go} | 30 ++-
plugins/github/tasks/issue_collector.go | 2 +-
plugins/github/tasks/issue_extractor.go | 241 ++++++++++++---------
plugins/github/tasks/milestone_collector.go | 78 +++++++
plugins/github/tasks/milestone_converter.go | 109 ++++++++++
plugins/github/tasks/milestone_extractor.go | 118 ++++++++++
runner/directrun.go | 56 +++--
27 files changed, 1034 insertions(+), 230 deletions(-)
diff --git a/config-ui/src/hooks/useConnectionManager.jsx b/config-ui/src/hooks/useConnectionManager.jsx
index c4d8eeda..56917613 100644
--- a/config-ui/src/hooks/useConnectionManager.jsx
+++ b/config-ui/src/hooks/useConnectionManager.jsx
@@ -86,7 +86,7 @@ function useConnectionManager ({
connectionPayload = { endpoint: endpointUrl, username: username, password: password, proxy: proxy }
break
case Providers.GITHUB:
- connectionPayload = { endpoint: endpointUrl, auth: token, proxy: proxy }
+ connectionPayload = { endpoint: endpointUrl, token: token, proxy: proxy }
break
case Providers.JENKINS:
connectionPayload = { endpoint: endpointUrl, username: username, password: password }
diff --git a/config/config.go b/config/config.go
index ec2edc07..734db1a7 100644
--- a/config/config.go
+++ b/config/config.go
@@ -46,10 +46,10 @@ func initConfig(v *viper.Viper) {
v.SetConfigType("env")
envPath := getEnvPath()
// AddConfigPath adds a path for Viper to search for the config file in.
- v.AddConfigPath("$PWD/../..")
- v.AddConfigPath("$PWD/../../..")
- v.AddConfigPath("..")
- v.AddConfigPath(".")
+ v.AddConfigPath("./../../..")
+ v.AddConfigPath("./../..")
+ v.AddConfigPath("./../")
+ v.AddConfigPath("./")
v.AddConfigPath(envPath)
}
diff --git a/go.mod b/go.mod
index 38ce2129..710d9139 100644
--- a/go.mod
+++ b/go.mod
@@ -3,13 +3,13 @@ module github.com/apache/incubator-devlake
go 1.17
require (
- github.com/agiledragon/gomonkey/v2 v2.7.0
github.com/gin-contrib/cors v1.3.1
github.com/gin-gonic/gin v1.7.7
github.com/go-git/go-git/v5 v5.4.2
github.com/go-playground/validator/v10 v10.9.0
github.com/libgit2/git2go/v33 v33.0.6
github.com/magiconair/properties v1.8.5
+ github.com/manifoldco/promptui v0.9.0
github.com/mitchellh/mapstructure v1.4.1
github.com/panjf2000/ants/v2 v2.4.6
github.com/robfig/cron/v3 v3.0.0
@@ -17,6 +17,7 @@ require (
github.com/spf13/afero v1.6.0
github.com/spf13/cobra v1.5.0
github.com/spf13/viper v1.8.1
+ github.com/stoewer/go-strcase v1.2.0
github.com/stretchr/testify v1.7.0
github.com/swaggo/gin-swagger v1.4.3
github.com/swaggo/swag v1.8.1
@@ -37,6 +38,7 @@ require (
github.com/PuerkitoBio/purell v1.1.1 // indirect
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
github.com/acomagu/bufpipe v1.0.3 // indirect
+ github.com/agiledragon/gomonkey/v2 v2.7.0 // indirect
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
@@ -81,7 +83,6 @@ require (
github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect
github.com/leodido/go-urn v1.2.1 // indirect
github.com/mailru/easyjson v0.7.6 // indirect
- github.com/manifoldco/promptui v0.9.0 // indirect
github.com/mattn/go-colorable v0.1.6 // indirect
github.com/mattn/go-isatty v0.0.13 // indirect
github.com/mattn/go-sqlite3 v1.14.6 // indirect
@@ -100,7 +101,6 @@ require (
github.com/spf13/cast v1.4.1 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/pflag v1.0.6-0.20200504143853-81378bbcd8a1 // indirect
- github.com/stoewer/go-strcase v1.2.0 // indirect
github.com/stretchr/objx v0.3.0 // indirect
github.com/subosito/gotenv v1.2.0 // indirect
github.com/ugorji/go/codec v1.2.6 // indirect
diff --git a/go.sum b/go.sum
index 217f9ee8..7fc8d5cb 100644
--- a/go.sum
+++ b/go.sum
@@ -70,9 +70,11 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB
github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
@@ -89,7 +91,6 @@ github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7
github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
-github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
@@ -525,8 +526,6 @@ github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z
github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA=
github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw=
-github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk=
github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU=
github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM=
github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
diff --git a/helpers/e2ehelper/data_flow_tester.go b/helpers/e2ehelper/data_flow_tester.go
index 2474b046..28c33875 100644
--- a/helpers/e2ehelper/data_flow_tester.go
+++ b/helpers/e2ehelper/data_flow_tester.go
@@ -38,6 +38,7 @@ import (
"os"
"strconv"
"strings"
+ "sync"
"testing"
"time"
)
@@ -73,6 +74,18 @@ type DataFlowTester struct {
Log core.Logger
}
+type TableOptions struct {
+ // CSVRelPath relative path to the CSV file that contains the seeded data
+ CSVRelPath string
+ // TargetFields the fields (columns) to consider for verification. Leave empty to default to all.
+ TargetFields []string
+ // IgnoreFields the fields (columns) to ignore/skip.
+ IgnoreFields []string
+ // IgnoreTypes similar to IgnoreFields, this will ignore the fields contained in the type. Useful for ignoring embedded
+ // types and their fields in the target model
+ IgnoreTypes []interface{}
+}
+
// NewDataFlowTester create a *DataFlowTester to help developer test their subtasks data flow
func NewDataFlowTester(t *testing.T, pluginName string, pluginMeta core.PluginMeta) *DataFlowTester {
err := core.RegisterPlugin(pluginName, pluginMeta)
@@ -174,24 +187,51 @@ func (t *DataFlowTester) Subtask(subtaskMeta core.SubTaskMeta, taskData interfac
}
func (t *DataFlowTester) getPkFields(dst schema.Tabler) []string {
+ return t.getFields(dst, func(column gorm.ColumnType) bool {
+ isPk, _ := column.PrimaryKey()
+ return isPk
+ })
+}
+
+func filterColumn(column gorm.ColumnType, opts TableOptions) bool {
+ for _, ignore := range opts.IgnoreFields {
+ if column.Name() == ignore {
+ return false
+ }
+ }
+ if len(opts.TargetFields) == 0 {
+ return true
+ }
+ targetFound := false
+ for _, target := range opts.TargetFields {
+ if column.Name() == target {
+ targetFound = true
+ break
+ }
+ }
+ return targetFound
+}
+
+func (t *DataFlowTester) getFields(dst schema.Tabler, filter func(column gorm.ColumnType) bool) []string {
columnTypes, err := t.Db.Migrator().ColumnTypes(dst)
- var pkFields []string
+ var fields []string
if err != nil {
panic(err)
}
for _, columnType := range columnTypes {
- if isPrimaryKey, _ := columnType.PrimaryKey(); isPrimaryKey {
- pkFields = append(pkFields, columnType.Name())
+ if filter == nil || filter(columnType) {
+ fields = append(fields, columnType.Name())
}
}
- return pkFields
+ return fields
}
// CreateSnapshot reads rows from database and write them into .csv file.
-func (t *DataFlowTester) CreateSnapshot(dst schema.Tabler, csvRelPath string, targetfields []string) {
+func (t *DataFlowTester) CreateSnapshot(dst schema.Tabler, opts TableOptions) {
location, _ := time.LoadLocation(`UTC`)
pkFields := t.getPkFields(dst)
- allFields := append(pkFields, targetfields...)
+ targetFields := t.resolveTargetFields(dst, opts)
+ allFields := append(pkFields, targetFields...)
allFields = utils.StringsUniq(allFields)
dbCursor, err := t.Dal.Cursor(
dal.Select(strings.Join(allFields, `,`)),
@@ -199,14 +239,14 @@ func (t *DataFlowTester) CreateSnapshot(dst schema.Tabler, csvRelPath string, ta
dal.Orderby(strings.Join(pkFields, `,`)),
)
if err != nil {
- panic(err)
+ panic(fmt.Errorf("unable to run select query on table %s: %v", dst.TableName(), err))
}
columns, err := dbCursor.Columns()
if err != nil {
- panic(err)
+ panic(fmt.Errorf("unable to get columns from table %s: %v", dst.TableName(), err))
}
- csvWriter := pluginhelper.NewCsvFileWriter(csvRelPath, columns)
+ csvWriter := pluginhelper.NewCsvFileWriter(opts.CSVRelPath, columns)
defer csvWriter.Close()
// define how to scan value
@@ -225,7 +265,7 @@ func (t *DataFlowTester) CreateSnapshot(dst schema.Tabler, csvRelPath string, ta
for dbCursor.Next() {
err = dbCursor.Scan(forScanValues...)
if err != nil {
- panic(err)
+ panic(fmt.Errorf("unable to scan row on table %s: %v", dst.TableName(), err))
}
values := make([]string, len(allFields))
for i := range forScanValues {
@@ -249,6 +289,7 @@ func (t *DataFlowTester) CreateSnapshot(dst schema.Tabler, csvRelPath string, ta
}
csvWriter.Write(values)
}
+ fmt.Printf("created CSV file: %s\n", opts.CSVRelPath)
}
// ExportRawTable reads rows from raw table and write them into .csv file.
@@ -300,20 +341,57 @@ func formatDbValue(value interface{}) string {
return ``
}
-// VerifyTable reads rows from csv file and compare with records from database one by one. You must specified the
+// VerifyTable reads rows from csv file and compare with records from database one by one. You must specify the
// Primary Key Fields with `pkFields` so DataFlowTester could select the exact record from database, as well as which
-// fields to compare with by specifying `targetFields` parameter.
+// fields to compare with by specifying `targetFields` parameter. Leaving `targetFields` empty/nil will compare all fields.
func (t *DataFlowTester) VerifyTable(dst schema.Tabler, csvRelPath string, targetFields []string) {
- _, err := os.Stat(csvRelPath)
+ t.VerifyTableWithOptions(dst, TableOptions{
+ CSVRelPath: csvRelPath,
+ TargetFields: targetFields,
+ })
+}
+
+func (t *DataFlowTester) extractColumns(ifc interface{}) []string {
+ sch, err := schema.Parse(ifc, &sync.Map{}, schema.NamingStrategy{})
+ if err != nil {
+ panic(fmt.Sprintf("error getting object schema: %v", err))
+ }
+ var columns []string
+ for _, f := range sch.Fields {
+ columns = append(columns, f.DBName)
+ }
+ return columns
+}
+
+func (t *DataFlowTester) resolveTargetFields(dst schema.Tabler, opts TableOptions) []string {
+ for _, ignore := range opts.IgnoreTypes {
+ opts.IgnoreFields = append(opts.IgnoreFields, t.extractColumns(ignore)...)
+ }
+ var targetFields []string
+ if len(opts.TargetFields) == 0 || len(opts.IgnoreFields) > 0 {
+ targetFields = append(targetFields, t.getFields(dst, func(column gorm.ColumnType) bool {
+ return filterColumn(column, opts)
+ })...)
+ } else {
+ targetFields = opts.TargetFields
+ }
+ return targetFields
+}
+
+// VerifyTableWithOptions extends VerifyTable and allows for more advanced usages using TableOptions
+func (t *DataFlowTester) VerifyTableWithOptions(dst schema.Tabler, opts TableOptions) {
+ if opts.CSVRelPath == "" {
+ panic("CSV relative path missing")
+ }
+ _, err := os.Stat(opts.CSVRelPath)
if os.IsNotExist(err) {
- t.CreateSnapshot(dst, csvRelPath, targetFields)
+ t.CreateSnapshot(dst, opts)
return
}
+ targetFields := t.resolveTargetFields(dst, opts)
pkFields := t.getPkFields(dst)
-
- csvIter := pluginhelper.NewCsvFileIterator(csvRelPath)
+ csvIter := pluginhelper.NewCsvFileIterator(opts.CSVRelPath)
defer csvIter.Close()
-
expectedTotal := int64(0)
csvMap := map[string]map[string]interface{}{}
for csvIter.HasNext() {
@@ -325,9 +403,11 @@ func (t *DataFlowTester) VerifyTable(dst schema.Tabler, csvRelPath string, targe
pkValueStr := strings.Join(pkValues, `-`)
_, ok := csvMap[pkValueStr]
assert.False(t.T, ok, fmt.Sprintf(`%s duplicated in csv (with params from csv %s)`, dst.TableName(), pkValues))
+ for _, ignore := range opts.IgnoreFields {
+ delete(expected, ignore)
+ }
csvMap[pkValueStr] = expected
}
-
dbRows := &[]map[string]interface{}{}
err = t.Db.Table(dst.TableName()).Find(dbRows).Error
if err != nil {
@@ -348,7 +428,6 @@ func (t *DataFlowTester) VerifyTable(dst schema.Tabler, csvRelPath string, targe
}
expectedTotal++
}
-
var actualTotal int64
err = t.Db.Table(dst.TableName()).Count(&actualTotal).Error
if err != nil {
diff --git a/helpers/e2ehelper/data_flow_tester_test.go b/helpers/e2ehelper/data_flow_tester_test.go
index 8c0b5c8d..e98d792f 100644
--- a/helpers/e2ehelper/data_flow_tester_test.go
+++ b/helpers/e2ehelper/data_flow_tester_test.go
@@ -18,13 +18,27 @@ limitations under the License.
package e2ehelper
import (
+ "github.com/apache/incubator-devlake/models/common"
gitlabModels "github.com/apache/incubator-devlake/plugins/gitlab/models"
+ "github.com/stretchr/testify/assert"
+ "gorm.io/gorm"
"testing"
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/gitlab/tasks"
)
+type TestModel struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+ IssueId int `gorm:"primaryKey;autoIncrement:false"`
+ LabelName string `gorm:"primaryKey;type:varchar(255)"`
+ common.NoPKModel
+}
+
+func (t TestModel) TableName() string {
+ return "_tool_test_model"
+}
+
func ExampleDataFlowTester() {
var t *testing.T // stub
@@ -68,3 +82,142 @@ func ExampleDataFlowTester() {
},
)
}
+
+func TestGetTableMetaData(t *testing.T) {
+ var meta core.PluginMeta
+ dataflowTester := NewDataFlowTester(t, "test_dataflow", meta)
+ dataflowTester.FlushTabler(&TestModel{})
+ t.Run("get_fields", func(t *testing.T) {
+ fields := dataflowTester.getFields(&TestModel{}, func(column gorm.ColumnType) bool {
+ return true
+ })
+ assert.Equal(t, 9, len(fields))
+ for _, e := range []string{
+ "connection_id",
+ "issue_id",
+ "label_name",
+ "created_at",
+ "updated_at",
+ "_raw_data_params",
+ "_raw_data_table",
+ "_raw_data_id",
+ "_raw_data_remark",
+ } {
+ assert.Contains(t, fields, e)
+ }
+ })
+ t.Run("extract_columns", func(t *testing.T) {
+ columns := dataflowTester.extractColumns(&common.RawDataOrigin{})
+ assert.Equal(t, 4, len(columns))
+ for _, e := range []string{
+ "_raw_data_params",
+ "_raw_data_table",
+ "_raw_data_id",
+ "_raw_data_remark",
+ } {
+ assert.Contains(t, columns, e)
+ }
+ })
+ t.Run("get_pk_fields", func(t *testing.T) {
+ fields := dataflowTester.getPkFields(&TestModel{})
+ assert.Equal(t, 3, len(fields))
+ for _, e := range []string{
+ "connection_id",
+ "issue_id",
+ "label_name",
+ } {
+ assert.Contains(t, fields, e)
+ }
+ })
+ t.Run("resolve_fields_targetFieldsOnly", func(t *testing.T) {
+ fields := dataflowTester.resolveTargetFields(&TestModel{}, TableOptions{
+ TargetFields: []string{"connection_id"},
+ IgnoreFields: nil,
+ IgnoreTypes: nil,
+ })
+ assert.Equal(t, 1, len(fields))
+ for _, e := range []string{"connection_id"} {
+ assert.Contains(t, fields, e)
+ }
+ })
+ t.Run("resolve_fields_ignoreFieldsOnly", func(t *testing.T) {
+ fields := dataflowTester.resolveTargetFields(&TestModel{}, TableOptions{
+ TargetFields: nil,
+ IgnoreFields: []string{
+ "label_name",
+ "created_at",
+ "updated_at",
+ "_raw_data_params",
+ "_raw_data_table",
+ "_raw_data_id",
+ "_raw_data_remark",
+ },
+ IgnoreTypes: nil,
+ })
+ assert.Equal(t, 2, len(fields))
+ for _, e := range []string{"connection_id", "issue_id"} {
+ assert.Contains(t, fields, e)
+ }
+ })
+ t.Run("resolve_fields_ignoreFieldsOnly", func(t *testing.T) {
+ fields := dataflowTester.resolveTargetFields(&TestModel{}, TableOptions{
+ TargetFields: nil,
+ IgnoreFields: []string{
+ "label_name",
+ "created_at",
+ "updated_at",
+ "_raw_data_params",
+ "_raw_data_table",
+ "_raw_data_id",
+ "_raw_data_remark",
+ },
+ IgnoreTypes: nil,
+ })
+ assert.Equal(t, 2, len(fields))
+ for _, e := range []string{"connection_id", "issue_id"} {
+ assert.Contains(t, fields, e)
+ }
+ })
+ t.Run("resolve_fields_ignoreType", func(t *testing.T) {
+ fields := dataflowTester.resolveTargetFields(&TestModel{}, TableOptions{
+ TargetFields: nil,
+ IgnoreFields: nil,
+ IgnoreTypes: []interface{}{&common.NoPKModel{}},
+ })
+ assert.Equal(t, 3, len(fields))
+ for _, e := range []string{
+ "connection_id",
+ "issue_id",
+ "label_name",
+ } {
+ assert.Contains(t, fields, e)
+ }
+ })
+ t.Run("resolve_fields_ignoreType_ignoreFields", func(t *testing.T) {
+ fields := dataflowTester.resolveTargetFields(&TestModel{}, TableOptions{
+ TargetFields: nil,
+ IgnoreFields: []string{"label_name"},
+ IgnoreTypes: []interface{}{&common.NoPKModel{}},
+ })
+ assert.Equal(t, 2, len(fields))
+ for _, e := range []string{
+ "connection_id",
+ "issue_id",
+ } {
+ assert.Contains(t, fields, e)
+ }
+ })
+ t.Run("resolve_fields_targetFields_ignoreType_ignoreFields", func(t *testing.T) {
+ fields := dataflowTester.resolveTargetFields(&TestModel{}, TableOptions{
+ TargetFields: []string{"label_name", "createdAt", "connection_id"},
+ IgnoreFields: []string{"label_name"},
+ IgnoreTypes: []interface{}{&common.NoPKModel{}},
+ })
+ assert.Equal(t, 1, len(fields))
+ for _, e := range []string{
+ "connection_id",
+ } {
+ assert.Contains(t, fields, e)
+ }
+ })
+}
diff --git a/plugins/github/e2e/issue_test.go b/plugins/github/e2e/issue_test.go
index 1b33d02c..3cbec5cb 100644
--- a/plugins/github/e2e/issue_test.go
+++ b/plugins/github/e2e/issue_test.go
@@ -81,6 +81,7 @@ func TestIssueDataFlow(t *testing.T) {
"author_name",
"assignee_id",
"assignee_name",
+ "milestone_id",
"lead_time_minutes",
"url",
"closed_at",
diff --git a/plugins/github/e2e/milestone_test.go b/plugins/github/e2e/milestone_test.go
new file mode 100644
index 00000000..465f3c8f
--- /dev/null
+++ b/plugins/github/e2e/milestone_test.go
@@ -0,0 +1,76 @@
+package e2e
+
+import (
+ "github.com/apache/incubator-devlake/helpers/e2ehelper"
+ "github.com/apache/incubator-devlake/models/common"
+ "github.com/apache/incubator-devlake/models/domainlayer/ticket"
+ "github.com/apache/incubator-devlake/plugins/github/impl"
+ "github.com/apache/incubator-devlake/plugins/github/models"
+ "github.com/apache/incubator-devlake/plugins/github/tasks"
+ "testing"
+ "time"
+)
+
+func TestMilestoneDataFlow(t *testing.T) {
+ var plugin impl.Github
+ dataflowTester := e2ehelper.NewDataFlowTester(t, "github", plugin)
+ githubRepository := &models.GithubRepo{
+ ConnectionId: 1,
+ GithubId: 134018330,
+ CreatedDate: func() time.Time {
+ createdTime, _ := time.Parse(time.RFC3339, "2006-01-02T15:04:05Z")
+ return createdTime
+ }(),
+ }
+ taskData := &tasks.GithubTaskData{
+ Options: &tasks.GithubOptions{
+ ConnectionId: 1,
+ Owner: "panjf2000",
+ Repo: "ants",
+ TransformationRules: models.TransformationRules{
+ PrType: "type/(.*)$",
+ PrComponent: "component/(.*)$",
+ PrBodyClosePattern: "(?mi)(fix|close|resolve|fixes|closes|resolves|fixed|closed|resolved)[\\s]*.*(((and )?(#|https:\\/\\/github.com\\/%s\\/%s\\/issues\\/)\\d+[ ]*)+)",
+ IssueSeverity: "severity/(.*)$",
+ IssuePriority: "^(highest|high|medium|low)$",
+ IssueComponent: "component/(.*)$",
+ IssueTypeBug: "^(bug|failure|error)$",
+ IssueTypeIncident: "",
+ IssueTypeRequirement: "^(feat|feature|proposal|requirement)$",
+ },
+ },
+ Repo: githubRepository,
+ }
+
+ dataflowTester.FlushTabler(&models.GithubMilestone{})
+ dataflowTester.FlushTabler(&models.GithubIssue{})
+
+ // import raw data table
+ dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_github_api_milestones.csv", "_raw_"+tasks.RAW_MILESTONE_TABLE)
+ dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_github_api_issues.csv", "_raw_"+tasks.RAW_ISSUE_TABLE)
+
+ dataflowTester.Subtask(tasks.ExtractApiIssuesMeta, taskData)
+ dataflowTester.Subtask(tasks.ExtractMilestonesMeta, taskData)
+ dataflowTester.VerifyTableWithOptions(&models.GithubMilestone{}, e2ehelper.TableOptions{
+ CSVRelPath: "./snapshot_tables/_tool_github_milestones.csv",
+ IgnoreTypes: []interface{}{common.NoPKModel{}},
+ })
+
+ dataflowTester.FlushTabler(&ticket.Sprint{})
+ dataflowTester.FlushTabler(&ticket.BoardSprint{})
+ dataflowTester.FlushTabler(&ticket.SprintIssue{})
+
+ dataflowTester.Subtask(tasks.ConvertMilestonesMeta, taskData)
+ dataflowTester.VerifyTableWithOptions(&ticket.Sprint{}, e2ehelper.TableOptions{
+ CSVRelPath: "./snapshot_tables/sprints.csv",
+ IgnoreTypes: []interface{}{common.NoPKModel{}},
+ })
+ dataflowTester.VerifyTableWithOptions(&ticket.BoardSprint{}, e2ehelper.TableOptions{
+ CSVRelPath: "./snapshot_tables/board_sprint.csv",
+ IgnoreTypes: []interface{}{common.NoPKModel{}},
+ })
+ dataflowTester.VerifyTableWithOptions(&ticket.SprintIssue{}, e2ehelper.TableOptions{
+ CSVRelPath: "./snapshot_tables/sprint_issue.csv",
+ IgnoreTypes: []interface{}{common.NoPKModel{}},
+ })
+}
diff --git a/plugins/github/e2e/raw_tables/_raw_github_api_issues.csv b/plugins/github/e2e/raw_tables/_raw_github_api_issues.csv
index 1c8760f3..00b79cd6 100644
--- a/plugins/github/e2e/raw_tables/_raw_github_api_issues.csv
+++ b/plugins/github/e2e/raw_tables/_raw_github_api_issues.csv
@@ -1,43 +1,43 @@
id,params,data,url,input,created_at
-9,"{""ConnectionId"":2,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/4"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/4/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/4/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/4/events"",""html_url"":""https://github.com/panjf2000 [...]
-10,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/4"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/4/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/4/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/4/events"",""html_url"":""https://github.com/panjf200 [...]
-11,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/5"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/5/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/5/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/5/events"",""html_url"":""https://github.com/panjf200 [...]
-12,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/6"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/6/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/6/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/6/events"",""html_url"":""https://github.com/panjf200 [...]
-13,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/7"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/7/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/7/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/7/events"",""html_url"":""https://github.com/panjf200 [...]
-14,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/8"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/8/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/8/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/8/events"",""html_url"":""https://github.com/panjf200 [...]
-15,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/9"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/9/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/9/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/9/events"",""html_url"":""https://github.com/panjf200 [...]
-16,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/10"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/10/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/10/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/10/events"",""html_url"":""https://github.com/panj [...]
-17,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/11"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/11/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/11/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/11/events"",""html_url"":""https://github.com/panj [...]
-18,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/12"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/12/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/12/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/12/events"",""html_url"":""https://github.com/panj [...]
-19,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/13"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/13/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/13/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/13/events"",""html_url"":""https://github.com/panj [...]
-20,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/14"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/14/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/14/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/14/events"",""html_url"":""https://github.com/panj [...]
-21,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/15"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/15/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/15/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/15/events"",""html_url"":""https://github.com/panj [...]
-22,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/16"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/16/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/16/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/16/events"",""html_url"":""https://github.com/panj [...]
-23,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/17"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/17/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/17/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/17/events"",""html_url"":""https://github.com/panj [...]
-24,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/18"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/18/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/18/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/18/events"",""html_url"":""https://github.com/panj [...]
-25,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/19"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/19/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/19/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/19/events"",""html_url"":""https://github.com/panj [...]
-26,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/20"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/20/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/20/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/20/events"",""html_url"":""https://github.com/panj [...]
-27,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/21"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/21/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/21/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/21/events"",""html_url"":""https://github.com/panj [...]
-28,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/22"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/22/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/22/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/22/events"",""html_url"":""https://github.com/panj [...]
-29,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/23"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/23/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/23/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/23/events"",""html_url"":""https://github.com/panj [...]
-30,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/24"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/24/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/24/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/24/events"",""html_url"":""https://github.com/panj [...]
-31,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/25"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/25/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/25/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/25/events"",""html_url"":""https://github.com/panj [...]
-32,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/26"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/26/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/26/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/26/events"",""html_url"":""https://github.com/panj [...]
-33,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/27"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/27/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/27/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/27/events"",""html_url"":""https://github.com/panj [...]
-34,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/28"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/28/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/28/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/28/events"",""html_url"":""https://github.com/panj [...]
-35,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/29"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/29/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/29/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/29/events"",""html_url"":""https://github.com/panj [...]
-36,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/30"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/30/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/30/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/30/events"",""html_url"":""https://github.com/panj [...]
-37,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/31"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/31/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/31/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/31/events"",""html_url"":""https://github.com/panj [...]
-38,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/32"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/32/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/32/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/32/events"",""html_url"":""https://github.com/panj [...]
-39,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/33"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/33/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/33/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/33/events"",""html_url"":""https://github.com/panj [...]
-40,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/34"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/34/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/34/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/34/events"",""html_url"":""https://github.com/panj [...]
-41,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/35"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/35/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/35/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/35/events"",""html_url"":""https://github.com/panj [...]
-42,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/36"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/36/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/36/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/36/events"",""html_url"":""https://github.com/panj [...]
-43,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/37"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/37/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/37/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/37/events"",""html_url"":""https://github.com/panj [...]
-44,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/38"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/38/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/38/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/38/events"",""html_url"":""https://github.com/panj [...]
-45,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/39"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/39/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/39/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/39/events"",""html_url"":""https://github.com/panj [...]
-46,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/40"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/40/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/40/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/40/events"",""html_url"":""https://github.com/panj [...]
-47,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/41"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/41/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/41/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/41/events"",""html_url"":""https://github.com/panj [...]
-48,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/42"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/42/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/42/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/42/events"",""html_url"":""https://github.com/panj [...]
-49,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/43"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/43/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/43/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/43/events"",""html_url"":""https://github.com/panj [...]
-50,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/44"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/44/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/44/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/44/events"",""html_url"":""https://github.com/panj [...]
+9,"{""ConnectionId"":2,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/4"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/4/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/4/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/4/events"",""html_url"":""https://github.com/panjf2000 [...]
+10,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/4"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/4/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/4/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/4/events"",""html_url"":""https://github.com/panjf200 [...]
+11,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/5"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/5/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/5/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/5/events"",""html_url"":""https://github.com/panjf200 [...]
+12,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/6"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/6/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/6/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/6/events"",""html_url"":""https://github.com/panjf200 [...]
+13,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/7"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/7/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/7/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/7/events"",""html_url"":""https://github.com/panjf200 [...]
+14,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/8"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/8/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/8/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/8/events"",""html_url"":""https://github.com/panjf200 [...]
+15,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/9"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/9/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/9/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/9/events"",""html_url"":""https://github.com/panjf200 [...]
+16,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/10"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/10/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/10/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/10/events"",""html_url"":""https://github.com/panj [...]
+17,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/11"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/11/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/11/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/11/events"",""html_url"":""https://github.com/panj [...]
+18,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/12"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/12/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/12/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/12/events"",""html_url"":""https://github.com/panj [...]
+19,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/13"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/13/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/13/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/13/events"",""html_url"":""https://github.com/panj [...]
+20,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/14"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/14/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/14/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/14/events"",""html_url"":""https://github.com/panj [...]
+21,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/15"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/15/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/15/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/15/events"",""html_url"":""https://github.com/panj [...]
+22,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/16"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/16/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/16/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/16/events"",""html_url"":""https://github.com/panj [...]
+23,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/17"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/17/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/17/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/17/events"",""html_url"":""https://github.com/panj [...]
+24,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/18"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/18/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/18/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/18/events"",""html_url"":""https://github.com/panj [...]
+25,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/19"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/19/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/19/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/19/events"",""html_url"":""https://github.com/panj [...]
+26,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/20"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/20/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/20/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/20/events"",""html_url"":""https://github.com/panj [...]
+27,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/21"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/21/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/21/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/21/events"",""html_url"":""https://github.com/panj [...]
+28,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/22"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/22/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/22/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/22/events"",""html_url"":""https://github.com/panj [...]
+29,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/23"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/23/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/23/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/23/events"",""html_url"":""https://github.com/panj [...]
+30,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/24"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/24/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/24/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/24/events"",""html_url"":""https://github.com/panj [...]
+31,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/25"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/25/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/25/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/25/events"",""html_url"":""https://github.com/panj [...]
+32,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/26"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/26/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/26/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/26/events"",""html_url"":""https://github.com/panj [...]
+33,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/27"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/27/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/27/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/27/events"",""html_url"":""https://github.com/panj [...]
+34,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/28"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/28/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/28/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/28/events"",""html_url"":""https://github.com/panj [...]
+35,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/29"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/29/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/29/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/29/events"",""html_url"":""https://github.com/panj [...]
+36,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/30"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/30/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/30/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/30/events"",""html_url"":""https://github.com/panj [...]
+37,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/31"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/31/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/31/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/31/events"",""html_url"":""https://github.com/panj [...]
+38,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/32"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/32/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/32/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/32/events"",""html_url"":""https://github.com/panj [...]
+39,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/33"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/33/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/33/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/33/events"",""html_url"":""https://github.com/panj [...]
+40,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/34"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/34/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/34/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/34/events"",""html_url"":""https://github.com/panj [...]
+41,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/35"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/35/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/35/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/35/events"",""html_url"":""https://github.com/panj [...]
+42,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/36"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/36/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/36/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/36/events"",""html_url"":""https://github.com/panj [...]
+43,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/37"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/37/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/37/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/37/events"",""html_url"":""https://github.com/panj [...]
+44,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/38"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/38/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/38/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/38/events"",""html_url"":""https://github.com/panj [...]
+45,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/39"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/39/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/39/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/39/events"",""html_url"":""https://github.com/panj [...]
+46,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/40"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/40/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/40/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/40/events"",""html_url"":""https://github.com/panj [...]
+47,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/41"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/41/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/41/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/41/events"",""html_url"":""https://github.com/panj [...]
+48,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/42"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/42/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/42/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/42/events"",""html_url"":""https://github.com/panj [...]
+49,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/43"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/43/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/43/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/43/events"",""html_url"":""https://github.com/panj [...]
+50,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/44"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/44/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/44/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/44/events"",""html_url"":""https://github.com/panj [...]
diff --git a/plugins/github/e2e/raw_tables/_raw_github_api_milestones.csv b/plugins/github/e2e/raw_tables/_raw_github_api_milestones.csv
new file mode 100644
index 00000000..32cacaad
--- /dev/null
+++ b/plugins/github/e2e/raw_tables/_raw_github_api_milestones.csv
@@ -0,0 +1,2 @@
+id,params,data,url,input,created_at
+109,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/apache/incubator-devlake/milestones/7"",""html_url"":""https://github.com/apache/incubator-devlake/milestone/7"",""labels_url"":""https://api.github.com/repos/apache/incubator-devlake/milestones/7/labels"",""id"":7856149,""node_id"":""MI_kwDOFuUSzs4Ad-AV"",""number"":7,""title"":""v0.11.0"",""description"":null,""creator"":{""login"":""Startrekzky"",""id"":14050754,""node_id"":"" [...]
diff --git a/plugins/github/e2e/snapshot_tables/_tool_github_issues.csv b/plugins/github/e2e/snapshot_tables/_tool_github_issues.csv
index 801fc2dd..071ad042 100644
--- a/plugins/github/e2e/snapshot_tables/_tool_github_issues.csv
+++ b/plugins/github/e2e/snapshot_tables/_tool_github_issues.csv
@@ -1,27 +1,27 @@
-connection_id,github_id,repo_id,number,state,title,body,priority,type,status,author_id,author_name,assignee_id,assignee_name,lead_time_minutes,url,closed_at,github_created_at,github_updated_at,severity,component,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-1,346842831,134018330,5,closed,关于 <-p.freeSignal 的疑惑,"""Hi,\r\n 我阅读了源码，对 `<-p.freeSignal` 这句代码有疑惑。 这句代码出现在了多个地方，freeSignal 的作用英文注释我是理解的，并且知道在 `putWorker` 中才进行 `p.freeSignal <- sig{}`\r\n\r\n对于下面的代码\r\n```\r\nfunc (p *Pool) getWorker() *Worker {\r\n\tvar w *Worker\r\n\twaiting := false\r\n\r\n\tp.lock.Lock()\r\n\tidleWorkers := p.workers\r\n\tn := len(idleWorkers) - 1\r\n\tif n < 0 { // 说明 pool中没有worker了\r\n\t\twaiting = p.Running() >= p.Cap()\r\n\t} else { // 说明pool中有worker\r\n\t\t<-p [...]
-1,347255859,134018330,6,closed,死锁bug,"""func (p *Pool) getWorker() *Worker 这个函数的 199行 \r\n必须先解锁在加锁， 要不然会产生死锁\r\n\r\n\tp.lock.Unlock()\r\n\t\t<-p.freeSignal\r\n\t\tp.lock.Lock()""",,BUG,,13118848,lovelly,0,,1786,https://github.com/panjf2000/ants/issues/6,2018-08-04T10:18:41.000+00:00,2018-08-03T04:32:28.000+00:00,2018-08-04T10:18:41.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,12,
-1,348630179,134018330,7,closed,清理过期协程报错,"""你好，非常感谢提供这么好用的工具包。我在使用ants时，发现报异常。结果见下图\r\n\r\n日志是我在periodicallyPurge里加的调试信息\r\n\r\n\r\n### 原因分析\r\n\r\n我认为可能原因是没有处理n==0的情况\r\n```\r\nif n > 0 {\r\n\tn++\r\n\tp.workers = idleWorkers[n:]\r\n}\r\n```\r\n\r\n\r\n### 测试代码\r\n```\r\npa [...]
-1,356703393,134018330,10,closed,高并发下设定较小的worker数量问题,"""会存在cpu飚升的问题吧？""",,,,11763614,Moonlight-Zhao,0,,36198,https://github.com/panjf2000/ants/issues/10,2018-09-29T11:45:00.000+00:00,2018-09-04T08:26:55.000+00:00,2018-09-29T11:45:00.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,16,
-1,364361014,134018330,12,closed,潘少，更新下tag吧,"""鄙人现在在弄dep依赖管理，有用到你写的ants项目，可是你好像忘记打最新的tag了。最新的tag 3.6是指向ed55924这个提交，git上的最新代码是af376f1b这次提交，两次提交都隔了快5个月了，看到的话，麻烦打一个最新的tag吧。（手动可怜）""",,,,29452204,edcismybrother,0,,1293,https://github.com/panjf2000/ants/issues/12,2018-09-28T06:05:58.000+00:00,2018-09-27T08:32:25.000+00:00,2019-04-21T08:19:58.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,18,
-1,381941219,134018330,17,closed,关于优雅退出的问题,"""关于这个package优雅退出的问题，我看了一下Release的代码：\r\n\r\n`\r\n\t// Release Closed this pool.\r\n\tfunc (p *PoolWithFunc) Release() error {\r\n\t\tp.once.Do(func() {\r\n\t\t\tp.release <- sig{}\r\n\t\t\tp.lock.Lock()\r\n\t\t\tidleWorkers := p.workers\r\n\t\t\tfor i, w := range idleWorkers {\r\n\t\t\t\tw.args <- nil\r\n\t\t\t\tidleWorkers[i] = nil\r\n\t\t\t}\r\n\t\t\tp.workers = nil\r\n\t\t\tp.lock.Unlock()\r\n\t\t})\r\n\t\treturn nil\r\n\t}\r\n`\r\n\r\nrelea [...]
-1,382039050,134018330,18,closed,go协程的理解,"""你好楼主，向您请教一个协程和线程的问题，协程基于go进程调度，线程基于系统内核调度，调度协程的过程是先调度线程后获得资源再去调度协程。\""官方解释： GOMAXPROCS sets the maximum number of CPUs that can be executing simultaneously。限制cpu数，本质上是什么，限制并行数?，并行数即同时执行数量?，执行单元即线程?，即限制最大并行线程数量?\""""",,,,13944100,LinuxForYQH,0,,20213,https://github.com/panjf2000/ants/issues/18,2018-12-03T03:53:50.000+00:00,2018-11-19T02:59:53.000+00:00,2018-12-03T03:53:50.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""} [...]
-1,382574800,134018330,20,closed,是否考虑任务支持回调函数处理失败的逻辑和任务依赖,"""#""",,,,5668717,kklinan,0,,95398,https://github.com/panjf2000/ants/issues/20,2019-01-25T15:34:03.000+00:00,2018-11-20T09:36:02.000+00:00,2019-01-25T15:34:03.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,26,
-1,388907811,134018330,21,closed,Benchmark 下直接使用 Semaphore 似乎更快呢？,"""简单跑了一下 benchmark，Semaphore 更快且很简单\r\n\r\n```bash\r\n$ go test -bench .\r\ngoos: darwin\r\ngoarch: amd64\r\npkg: github.com/panjf2000/ants\r\nBenchmarkGoroutineWithFunc-4 \t 1\t3445631705 ns/op\r\nBenchmarkSemaphoreWithFunc-4 \t 1\t1037219073 ns/op\r\nBenchmarkAntsPoolWithFunc-4 \t 1\t1138053222 ns/op\r\nBenchmarkGoroutine-4 \t 2\t 731850771 ns/op\r\nBenchmarkSemaphore-4 [...]
-1,401277739,134018330,22,closed,是否考虑 worker 中添加 PanicHandler ？,"""比方说在创建 Pool 的时候传入一个 PanicHandler，然后在每个 worker 创建的时候 recover 之后传给 PanicHandler 处理。否则池子里如果发生 panic 会直接挂掉整个进程。""",,,,8923413,choleraehyq,0,,1174,https://github.com/panjf2000/ants/issues/22,2019-01-22T05:41:34.000+00:00,2019-01-21T10:06:56.000+00:00,2019-01-22T05:41:34.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,28,
-1,402513849,134018330,24,closed,提交任务不阻塞,"""`Pool.Submit`和`PoolWithFunc.Server`提交任务，如果没有空的worker，会一直阻塞。建议增加不阻塞的接口，当前失败时直接返回错误。""",,,,5044825,tenfyzhong,0,,300032,https://github.com/panjf2000/ants/issues/24,2019-08-20T10:56:30.000+00:00,2019-01-24T02:24:13.000+00:00,2019-08-20T10:56:30.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,30,
-1,405951301,134018330,25,closed,use example errors,"""./antstest.go:37:14: cannot use syncCalculateSum (type func()) as type ants.f in argument to ants.Submit\r\n./antstest.go:45:35: cannot use func literal (type func(interface {})) as type ants.pf in argument to ants.NewPoolWithFunc\r\n""",,,,5244267,jiashiwen,0,,3088,https://github.com/panjf2000/ants/issues/25,2019-02-04T09:11:52.000+00:00,2019-02-02T05:43:38.000+00:00,2019-02-04T09:11:52.000+00:00,,,"{""ConnectionId"":1,""Owner"":""pa [...]
-1,413968505,134018330,26,closed,running可能大于cap的问题,"""running与cap的比较判断与incRuning分开执行的, 可能会出现running大于cap的问题?\r\n`func (p *Pool) retrieveWorker() *Worker {\r\n\tvar w *Worker\r\n\r\n\tp.lock.Lock()\r\n\tidleWorkers := p.workers\r\n\tn := len(idleWorkers) - 1\r\n\tif n >= 0 {\r\n\t\tw = idleWorkers[n]\r\n\t\tidleWorkers[n] = nil\r\n\t\tp.workers = idleWorkers[:n]\r\n\t\tp.lock.Unlock()\r\n\t} else if p.Running() < p.Cap() {\r\n\t\tp.lock.Unlock()\r\n\t\tif cacheWorker := p.workerCache.Get() [...]
-1,419183961,134018330,27,closed,为何goroutine一直上不去，用户量也打不上去,"""为何goroutine一直上不去，用户量也打不上去\r\n是我用的有问题吗？\r\n\r\nwebsocket server\r\nhttps://github.com/im-ai/pushm/blob/master/learn/goroutine/goroutinepoolwebsocket.go\r\n\r\nwebsocket cient\r\nhttps://github.com/im-ai/pushm/blob/master/learn/goroutine/goroutinepoolwebsocketclient.go\r\n""",,,,38367404,liliang8858,0,,37496,https://github.com/panjf2000/ants/issues/27,2019-04-05T14:05:20.000+00:00,2019-03-10T13:08:52.000+00:00,2019-04-05T14:05:20 [...]
-1,419268851,134018330,28,closed,cap 和 running 比较的问题,"""这是我在 Playground 上面的代码 https://play.golang.org/p/D94YUU3FnX6\r\natomic 只能保证自增自减时的原子操作，在比较过程中，其他线程对变量进行了操作 比较过程并无感知，所以这个比较结果 不是完全正确的，想要实现 比较的数量完全正确，只能在修改和比较两个值的地方加锁\r\n像 #26 说的是对的""",,,,29243953,naiba,0,,237002,https://github.com/panjf2000/ants/issues/28,2019-08-22T16:27:37.000+00:00,2019-03-11T02:24:41.000+00:00,2019-08-22T16:27:37.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,34,
-1,424634533,134018330,29,closed,任务传参,"""你好，你的项目太酷了👍\r\n\r\nhttps://github.com/panjf2000/ants/blob/master/pool.go#L124 貌似不支持带参数的任务, 请问传参是用闭包的方式吗?\r\n""",,,,8509898,prprprus,0,,999,https://github.com/panjf2000/ants/issues/29,2019-03-25T09:32:11.000+00:00,2019-03-24T16:52:21.000+00:00,2019-03-25T09:45:05.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,35,
-1,429972115,134018330,31,closed,Add go.mod,"""""",,,,48135919,tsatke,0,,3474,https://github.com/panjf2000/ants/issues/31,2019-04-08T09:45:31.000+00:00,2019-04-05T23:50:36.000+00:00,2019-10-17T03:12:19.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,37,
-1,433564955,134018330,32,closed,关于版本问题，我发现小版本(0.0.x)这种更新就会不向下兼容？,"""如题，我感觉这样不好。\r\n\r\n功能版本号不向下兼容能理解\r\n\r\n修复问题的版本号也不向下兼容，难以理解。""",,,,7931755,zplzpl,0,,7440,https://github.com/panjf2000/ants/issues/32,2019-04-21T07:16:26.000+00:00,2019-04-16T03:16:02.000+00:00,2019-04-21T07:16:26.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,38,
-1,434069015,134018330,33,closed,support semantic versioning.,"""建议将发布的tag兼容为semantic versioning，vX.Y.Z。go modules对此支持比较良好。\r\nhttps://semver.org/\r\nhttps://research.swtch.com/vgo-import""",,,,1284892,jjeffcaii,0,,6090,https://github.com/panjf2000/ants/issues/33,2019-04-21T08:25:20.000+00:00,2019-04-17T02:55:11.000+00:00,2019-04-21T08:25:20.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,39,
-1,435486645,134018330,34,closed,Important announcement about <ants> from author !!!,"""**Dear users of `ants`:**\r\nI am apologetically telling you that I have to dump all tags which already presents in `ants` repository.\r\n\r\nThe reason why I'm doing so is to standardize the version management with `Semantic Versioning`, which will make a formal and clear dependency management in go, for go modules, godep, or glide, etc. So I decide to start over the tag sequence, you could find more [...]
-1,461280653,134018330,35,closed,worker exit on panic,"""个人认为PanicHandler设计不妥。\r\n1.无PanicHandler时，抛出给外面的不是panic,外层感受不到。\r\n2.无论有没有PanicHandler，都会导致worker退出，最终pool阻塞住全部任务。""",,,,38849208,king526,0,,74481,https://github.com/panjf2000/ants/issues/35,2019-08-17T20:33:10.000+00:00,2019-06-27T03:11:49.000+00:00,2019-08-17T20:33:10.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,41,
-1,462631417,134018330,37,closed,请不要再随意变更版本号了。。。,"""之前用的是 3.9.9，结果今天构建出了问题，一看发现这个版本没了，变成 1.0.0。这种变更完全不考虑现有用户的情况。希望以后不要随意变更了""",,,,8923413,choleraehyq,0,,140,https://github.com/panjf2000/ants/issues/37,2019-07-01T12:37:55.000+00:00,2019-07-01T10:17:15.000+00:00,2019-07-02T10:17:31.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,43,
-1,472125082,134018330,38,closed,retrieveWorker与revertWorker之间会导致死锁,"""func (p *Pool) retrieveWorker() *Worker {\r\n\tvar w *Worker\r\n\r\n\t**p.lock.Lock()**\r\n\tidleWorkers := p.workers\r\n\tn := len(idleWorkers) - 1\r\n\tif n >= 0 {\r\n\t\tw = idleWorkers[n]\r\n\t\tidleWorkers[n] = nil\r\n\t\tp.workers = idleWorkers[:n]\r\n\t\tp.lock.Unlock()\r\n\t} else if p.Running() < p.Cap() {\r\n\t\tp.lock.Unlock()\r\n\t\tif cacheWorker := p.workerCache.Get(); cacheWorker != nil {\r\n\t\t\tw = ca [...]
-1,483164833,134018330,42,closed,带选项的初始化函数，我觉得用 functional options 更好一点,"""以下是示意代码\r\n如果用 functional options，原来的写法是\r\n```\r\nants.NewPool(10)\r\n```\r\n新的写法，如果不加 option，写法是不变的，因为 options 是作为可变参数传进去的。如果要加 option，只需要改成\r\n```\r\nants.NewPool(10, ants.WithNonblocking(true))\r\n```\r\n这样。\r\n\r\n现在是直接传一个 Option 结构体进去，所有的地方都要改，感觉很不优雅。\r\n具体 functional options 的设计可以看 rob pike 的一篇博客 https://commandcenter.blogspot.com/2014/01/self-referential-functions-and-design.html""",,,,8923413,choleraehyq,0 [...]
-1,483736247,134018330,43,closed,1.3.0 是不兼容更新,"""Pool 里那些暴露出来的字段（PanicHandler 之类的）都没了，这是一个不兼容更新，根据语义化版本的要求要发大版本。""",,,,8923413,choleraehyq,0,,652,https://github.com/panjf2000/ants/issues/43,2019-08-22T13:22:10.000+00:00,2019-08-22T02:29:34.000+00:00,2019-08-22T13:22:10.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,49,
-1,484311063,134018330,44,closed,1.1.1 -> 1.2.0 也是不兼容更新,"""Pool.Release 的返回值没了""",,,,8923413,choleraehyq,0,,3068,https://github.com/panjf2000/ants/issues/44,2019-08-25T06:36:14.000+00:00,2019-08-23T03:27:38.000+00:00,2019-08-25T06:36:14.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,50,
+connection_id,github_id,repo_id,milestone_id,number,state,title,body,priority,type,status,author_id,author_name,assignee_id,assignee_name,lead_time_minutes,url,closed_at,github_created_at,github_updated_at,severity,component,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+1,346842831,134018330,7856149,5,closed,关于 <-p.freeSignal 的疑惑,"""Hi,\r\n 我阅读了源码，对 `<-p.freeSignal` 这句代码有疑惑。 这句代码出现在了多个地方，freeSignal 的作用英文注释我是理解的，并且知道在 `putWorker` 中才进行 `p.freeSignal <- sig{}`\r\n\r\n对于下面的代码\r\n```\r\nfunc (p *Pool) getWorker() *Worker {\r\n\tvar w *Worker\r\n\twaiting := false\r\n\r\n\tp.lock.Lock()\r\n\tidleWorkers := p.workers\r\n\tn := len(idleWorkers) - 1\r\n\tif n < 0 { // 说明 pool中没有worker了\r\n\t\twaiting = p.Running() >= p.Cap()\r\n\t} else { // 说明pool中有worker\r\ [...]
+1,347255859,134018330,7856149,6,closed,死锁bug,"""func (p *Pool) getWorker() *Worker 这个函数的 199行 \r\n必须先解锁在加锁， 要不然会产生死锁\r\n\r\n\tp.lock.Unlock()\r\n\t\t<-p.freeSignal\r\n\t\tp.lock.Lock()""",,BUG,,13118848,lovelly,0,,1786,https://github.com/panjf2000/ants/issues/6,2018-08-04T10:18:41.000+00:00,2018-08-03T04:32:28.000+00:00,2018-08-04T10:18:41.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,12,
+1,348630179,134018330,7856149,7,closed,清理过期协程报错,"""你好，非常感谢提供这么好用的工具包。我在使用ants时，发现报异常。结果见下图\r\n\r\n日志是我在periodicallyPurge里加的调试信息\r\n\r\n\r\n### 原因分析\r\n\r\n我认为可能原因是没有处理n==0的情况\r\n```\r\nif n > 0 {\r\n\tn++\r\n\tp.workers = idleWorkers[n:]\r\n}\r\n```\r\n\r\n\r\n### 测试代码\r\n` [...]
+1,356703393,134018330,7856149,10,closed,高并发下设定较小的worker数量问题,"""会存在cpu飚升的问题吧？""",,,,11763614,Moonlight-Zhao,0,,36198,https://github.com/panjf2000/ants/issues/10,2018-09-29T11:45:00.000+00:00,2018-09-04T08:26:55.000+00:00,2018-09-29T11:45:00.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,16,
+1,364361014,134018330,7856149,12,closed,潘少，更新下tag吧,"""鄙人现在在弄dep依赖管理，有用到你写的ants项目，可是你好像忘记打最新的tag了。最新的tag 3.6是指向ed55924这个提交，git上的最新代码是af376f1b这次提交，两次提交都隔了快5个月了，看到的话，麻烦打一个最新的tag吧。（手动可怜）""",,,,29452204,edcismybrother,0,,1293,https://github.com/panjf2000/ants/issues/12,2018-09-28T06:05:58.000+00:00,2018-09-27T08:32:25.000+00:00,2019-04-21T08:19:58.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,18,
+1,381941219,134018330,7856149,17,closed,关于优雅退出的问题,"""关于这个package优雅退出的问题，我看了一下Release的代码：\r\n\r\n`\r\n\t// Release Closed this pool.\r\n\tfunc (p *PoolWithFunc) Release() error {\r\n\t\tp.once.Do(func() {\r\n\t\t\tp.release <- sig{}\r\n\t\t\tp.lock.Lock()\r\n\t\t\tidleWorkers := p.workers\r\n\t\t\tfor i, w := range idleWorkers {\r\n\t\t\t\tw.args <- nil\r\n\t\t\t\tidleWorkers[i] = nil\r\n\t\t\t}\r\n\t\t\tp.workers = nil\r\n\t\t\tp.lock.Unlock()\r\n\t\t})\r\n\t\treturn nil\r\n\t}\r\n`\r\n\ [...]
+1,382039050,134018330,7856149,18,closed,go协程的理解,"""你好楼主，向您请教一个协程和线程的问题，协程基于go进程调度，线程基于系统内核调度，调度协程的过程是先调度线程后获得资源再去调度协程。\""官方解释： GOMAXPROCS sets the maximum number of CPUs that can be executing simultaneously。限制cpu数，本质上是什么，限制并行数?，并行数即同时执行数量?，执行单元即线程?，即限制最大并行线程数量?\""""",,,,13944100,LinuxForYQH,0,,20213,https://github.com/panjf2000/ants/issues/18,2018-12-03T03:53:50.000+00:00,2018-11-19T02:59:53.000+00:00,2018-12-03T03:53:50.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":" [...]
+1,382574800,134018330,7856149,20,closed,是否考虑任务支持回调函数处理失败的逻辑和任务依赖,"""#""",,,,5668717,kklinan,0,,95398,https://github.com/panjf2000/ants/issues/20,2019-01-25T15:34:03.000+00:00,2018-11-20T09:36:02.000+00:00,2019-01-25T15:34:03.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,26,
+1,388907811,134018330,7856149,21,closed,Benchmark 下直接使用 Semaphore 似乎更快呢？,"""简单跑了一下 benchmark，Semaphore 更快且很简单\r\n\r\n```bash\r\n$ go test -bench .\r\ngoos: darwin\r\ngoarch: amd64\r\npkg: github.com/panjf2000/ants\r\nBenchmarkGoroutineWithFunc-4 \t 1\t3445631705 ns/op\r\nBenchmarkSemaphoreWithFunc-4 \t 1\t1037219073 ns/op\r\nBenchmarkAntsPoolWithFunc-4 \t 1\t1138053222 ns/op\r\nBenchmarkGoroutine-4 \t 2\t 731850771 ns/op\r\nBenchmarkSemaphore-4 [...]
+1,401277739,134018330,7856149,22,closed,是否考虑 worker 中添加 PanicHandler ？,"""比方说在创建 Pool 的时候传入一个 PanicHandler，然后在每个 worker 创建的时候 recover 之后传给 PanicHandler 处理。否则池子里如果发生 panic 会直接挂掉整个进程。""",,,,8923413,choleraehyq,0,,1174,https://github.com/panjf2000/ants/issues/22,2019-01-22T05:41:34.000+00:00,2019-01-21T10:06:56.000+00:00,2019-01-22T05:41:34.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,28,
+1,402513849,134018330,7856149,24,closed,提交任务不阻塞,"""`Pool.Submit`和`PoolWithFunc.Server`提交任务，如果没有空的worker，会一直阻塞。建议增加不阻塞的接口，当前失败时直接返回错误。""",,,,5044825,tenfyzhong,0,,300032,https://github.com/panjf2000/ants/issues/24,2019-08-20T10:56:30.000+00:00,2019-01-24T02:24:13.000+00:00,2019-08-20T10:56:30.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,30,
+1,405951301,134018330,7856149,25,closed,use example errors,"""./antstest.go:37:14: cannot use syncCalculateSum (type func()) as type ants.f in argument to ants.Submit\r\n./antstest.go:45:35: cannot use func literal (type func(interface {})) as type ants.pf in argument to ants.NewPoolWithFunc\r\n""",,,,5244267,jiashiwen,0,,3088,https://github.com/panjf2000/ants/issues/25,2019-02-04T09:11:52.000+00:00,2019-02-02T05:43:38.000+00:00,2019-02-04T09:11:52.000+00:00,,,"{""ConnectionId"":1,""Owne [...]
+1,413968505,134018330,7856149,26,closed,running可能大于cap的问题,"""running与cap的比较判断与incRuning分开执行的, 可能会出现running大于cap的问题?\r\n`func (p *Pool) retrieveWorker() *Worker {\r\n\tvar w *Worker\r\n\r\n\tp.lock.Lock()\r\n\tidleWorkers := p.workers\r\n\tn := len(idleWorkers) - 1\r\n\tif n >= 0 {\r\n\t\tw = idleWorkers[n]\r\n\t\tidleWorkers[n] = nil\r\n\t\tp.workers = idleWorkers[:n]\r\n\t\tp.lock.Unlock()\r\n\t} else if p.Running() < p.Cap() {\r\n\t\tp.lock.Unlock()\r\n\t\tif cacheWorker := p.workerCac [...]
+1,419183961,134018330,7856149,27,closed,为何goroutine一直上不去，用户量也打不上去,"""为何goroutine一直上不去，用户量也打不上去\r\n是我用的有问题吗？\r\n\r\nwebsocket server\r\nhttps://github.com/im-ai/pushm/blob/master/learn/goroutine/goroutinepoolwebsocket.go\r\n\r\nwebsocket cient\r\nhttps://github.com/im-ai/pushm/blob/master/learn/goroutine/goroutinepoolwebsocketclient.go\r\n""",,,,38367404,liliang8858,0,,37496,https://github.com/panjf2000/ants/issues/27,2019-04-05T14:05:20.000+00:00,2019-03-10T13:08:52.000+00:00,2019-04-05T [...]
+1,419268851,134018330,7856149,28,closed,cap 和 running 比较的问题,"""这是我在 Playground 上面的代码 https://play.golang.org/p/D94YUU3FnX6\r\natomic 只能保证自增自减时的原子操作，在比较过程中，其他线程对变量进行了操作 比较过程并无感知，所以这个比较结果 不是完全正确的，想要实现 比较的数量完全正确，只能在修改和比较两个值的地方加锁\r\n像 #26 说的是对的""",,,,29243953,naiba,0,,237002,https://github.com/panjf2000/ants/issues/28,2019-08-22T16:27:37.000+00:00,2019-03-11T02:24:41.000+00:00,2019-08-22T16:27:37.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,34,
+1,424634533,134018330,7856149,29,closed,任务传参,"""你好，你的项目太酷了👍\r\n\r\nhttps://github.com/panjf2000/ants/blob/master/pool.go#L124 貌似不支持带参数的任务, 请问传参是用闭包的方式吗?\r\n""",,,,8509898,prprprus,0,,999,https://github.com/panjf2000/ants/issues/29,2019-03-25T09:32:11.000+00:00,2019-03-24T16:52:21.000+00:00,2019-03-25T09:45:05.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,35,
+1,429972115,134018330,7856149,31,closed,Add go.mod,"""""",,,,48135919,tsatke,0,,3474,https://github.com/panjf2000/ants/issues/31,2019-04-08T09:45:31.000+00:00,2019-04-05T23:50:36.000+00:00,2019-10-17T03:12:19.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,37,
+1,433564955,134018330,7856149,32,closed,关于版本问题，我发现小版本(0.0.x)这种更新就会不向下兼容？,"""如题，我感觉这样不好。\r\n\r\n功能版本号不向下兼容能理解\r\n\r\n修复问题的版本号也不向下兼容，难以理解。""",,,,7931755,zplzpl,0,,7440,https://github.com/panjf2000/ants/issues/32,2019-04-21T07:16:26.000+00:00,2019-04-16T03:16:02.000+00:00,2019-04-21T07:16:26.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,38,
+1,434069015,134018330,7856149,33,closed,support semantic versioning.,"""建议将发布的tag兼容为semantic versioning，vX.Y.Z。go modules对此支持比较良好。\r\nhttps://semver.org/\r\nhttps://research.swtch.com/vgo-import""",,,,1284892,jjeffcaii,0,,6090,https://github.com/panjf2000/ants/issues/33,2019-04-21T08:25:20.000+00:00,2019-04-17T02:55:11.000+00:00,2019-04-21T08:25:20.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,39,
+1,435486645,134018330,7856149,34,closed,Important announcement about <ants> from author !!!,"""**Dear users of `ants`:**\r\nI am apologetically telling you that I have to dump all tags which already presents in `ants` repository.\r\n\r\nThe reason why I'm doing so is to standardize the version management with `Semantic Versioning`, which will make a formal and clear dependency management in go, for go modules, godep, or glide, etc. So I decide to start over the tag sequence, you could fi [...]
+1,461280653,134018330,7856149,35,closed,worker exit on panic,"""个人认为PanicHandler设计不妥。\r\n1.无PanicHandler时，抛出给外面的不是panic,外层感受不到。\r\n2.无论有没有PanicHandler，都会导致worker退出，最终pool阻塞住全部任务。""",,,,38849208,king526,0,,74481,https://github.com/panjf2000/ants/issues/35,2019-08-17T20:33:10.000+00:00,2019-06-27T03:11:49.000+00:00,2019-08-17T20:33:10.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,41,
+1,462631417,134018330,7856149,37,closed,请不要再随意变更版本号了。。。,"""之前用的是 3.9.9，结果今天构建出了问题，一看发现这个版本没了，变成 1.0.0。这种变更完全不考虑现有用户的情况。希望以后不要随意变更了""",,,,8923413,choleraehyq,0,,140,https://github.com/panjf2000/ants/issues/37,2019-07-01T12:37:55.000+00:00,2019-07-01T10:17:15.000+00:00,2019-07-02T10:17:31.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,43,
+1,472125082,134018330,7856149,38,closed,retrieveWorker与revertWorker之间会导致死锁,"""func (p *Pool) retrieveWorker() *Worker {\r\n\tvar w *Worker\r\n\r\n\t**p.lock.Lock()**\r\n\tidleWorkers := p.workers\r\n\tn := len(idleWorkers) - 1\r\n\tif n >= 0 {\r\n\t\tw = idleWorkers[n]\r\n\t\tidleWorkers[n] = nil\r\n\t\tp.workers = idleWorkers[:n]\r\n\t\tp.lock.Unlock()\r\n\t} else if p.Running() < p.Cap() {\r\n\t\tp.lock.Unlock()\r\n\t\tif cacheWorker := p.workerCache.Get(); cacheWorker != nil {\r\n\t\t [...]
+1,483164833,134018330,7856149,42,closed,带选项的初始化函数，我觉得用 functional options 更好一点,"""以下是示意代码\r\n如果用 functional options，原来的写法是\r\n```\r\nants.NewPool(10)\r\n```\r\n新的写法，如果不加 option，写法是不变的，因为 options 是作为可变参数传进去的。如果要加 option，只需要改成\r\n```\r\nants.NewPool(10, ants.WithNonblocking(true))\r\n```\r\n这样。\r\n\r\n现在是直接传一个 Option 结构体进去，所有的地方都要改，感觉很不优雅。\r\n具体 functional options 的设计可以看 rob pike 的一篇博客 https://commandcenter.blogspot.com/2014/01/self-referential-functions-and-design.html""",,,,8923413,chole [...]
+1,483736247,134018330,7856149,43,closed,1.3.0 是不兼容更新,"""Pool 里那些暴露出来的字段（PanicHandler 之类的）都没了，这是一个不兼容更新，根据语义化版本的要求要发大版本。""",,,,8923413,choleraehyq,0,,652,https://github.com/panjf2000/ants/issues/43,2019-08-22T13:22:10.000+00:00,2019-08-22T02:29:34.000+00:00,2019-08-22T13:22:10.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,49,
+1,484311063,134018330,7856149,44,closed,1.1.1 -> 1.2.0 也是不兼容更新,"""Pool.Release 的返回值没了""",,,,8923413,choleraehyq,0,,3068,https://github.com/panjf2000/ants/issues/44,2019-08-25T06:36:14.000+00:00,2019-08-23T03:27:38.000+00:00,2019-08-25T06:36:14.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,50,
diff --git a/plugins/github/e2e/snapshot_tables/_tool_github_milestones.csv b/plugins/github/e2e/snapshot_tables/_tool_github_milestones.csv
new file mode 100644
index 00000000..75dfe942
--- /dev/null
+++ b/plugins/github/e2e/snapshot_tables/_tool_github_milestones.csv
@@ -0,0 +1,2 @@
+connection_id,milestone_id,repo_id,number,url,title,open_issues,closed_issues,state,created_at,updated_at,closed_at,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+1,7856149,134018330,7,https://api.github.com/repos/apache/incubator-devlake/milestones/7,v0.11.0,2,118,open,2022-04-08T02:05:35.000+00:00,2022-06-24T01:34:37.000+00:00,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_milestones,109,
diff --git a/plugins/github/e2e/snapshot_tables/board_sprint.csv b/plugins/github/e2e/snapshot_tables/board_sprint.csv
new file mode 100644
index 00000000..0f45a0ae
--- /dev/null
+++ b/plugins/github/e2e/snapshot_tables/board_sprint.csv
@@ -0,0 +1,2 @@
+board_id,sprint_id
+github:GithubRepo:1:134018330,github:GithubMilestone:1:7856149
diff --git a/plugins/github/e2e/snapshot_tables/boards.csv b/plugins/github/e2e/snapshot_tables/boards.csv
index 7201be87..e98f6bfa 100644
--- a/plugins/github/e2e/snapshot_tables/boards.csv
+++ b/plugins/github/e2e/snapshot_tables/boards.csv
@@ -1,2 +1,2 @@
id,name,description,url,created_date
-github:GithubRepo:1:134018330,panjf2000/ants,"🐜🐜🐜 ants is a high-performance and low-cost goroutine pool in Go, inspired by fasthttp./ ants 是一个高性能且低损耗的 goroutine 池。",https://github.com/panjf2000/ants/issues,2018-05-19T01:13:38.000+00:00
+github:GithubRepo:1:134018330,panjf2000/ants,"🐜🐜🐜 ants is a high-performance and low-cost goroutine pool in Go, inspired by fasthttp./ ants 是一个高性能且低损耗的 goroutine 池。",https://github.com/panjf2000/ants/issues,2018-05-19T01:13:38.000+00:00
\ No newline at end of file
diff --git a/plugins/github/e2e/snapshot_tables/sprint_issue.csv b/plugins/github/e2e/snapshot_tables/sprint_issue.csv
new file mode 100644
index 00000000..153ed83d
--- /dev/null
+++ b/plugins/github/e2e/snapshot_tables/sprint_issue.csv
@@ -0,0 +1,27 @@
+sprint_id,issue_id
+github:GithubMilestone:1:7856149,github:GithubIssue:1:346842831
+github:GithubMilestone:1:7856149,github:GithubIssue:1:347255859
+github:GithubMilestone:1:7856149,github:GithubIssue:1:348630179
+github:GithubMilestone:1:7856149,github:GithubIssue:1:356703393
+github:GithubMilestone:1:7856149,github:GithubIssue:1:364361014
+github:GithubMilestone:1:7856149,github:GithubIssue:1:381941219
+github:GithubMilestone:1:7856149,github:GithubIssue:1:382039050
+github:GithubMilestone:1:7856149,github:GithubIssue:1:382574800
+github:GithubMilestone:1:7856149,github:GithubIssue:1:388907811
+github:GithubMilestone:1:7856149,github:GithubIssue:1:401277739
+github:GithubMilestone:1:7856149,github:GithubIssue:1:402513849
+github:GithubMilestone:1:7856149,github:GithubIssue:1:405951301
+github:GithubMilestone:1:7856149,github:GithubIssue:1:413968505
+github:GithubMilestone:1:7856149,github:GithubIssue:1:419183961
+github:GithubMilestone:1:7856149,github:GithubIssue:1:419268851
+github:GithubMilestone:1:7856149,github:GithubIssue:1:424634533
+github:GithubMilestone:1:7856149,github:GithubIssue:1:429972115
+github:GithubMilestone:1:7856149,github:GithubIssue:1:433564955
+github:GithubMilestone:1:7856149,github:GithubIssue:1:434069015
+github:GithubMilestone:1:7856149,github:GithubIssue:1:435486645
+github:GithubMilestone:1:7856149,github:GithubIssue:1:461280653
+github:GithubMilestone:1:7856149,github:GithubIssue:1:462631417
+github:GithubMilestone:1:7856149,github:GithubIssue:1:472125082
+github:GithubMilestone:1:7856149,github:GithubIssue:1:483164833
+github:GithubMilestone:1:7856149,github:GithubIssue:1:483736247
+github:GithubMilestone:1:7856149,github:GithubIssue:1:484311063
diff --git a/plugins/github/e2e/snapshot_tables/sprints.csv b/plugins/github/e2e/snapshot_tables/sprints.csv
new file mode 100644
index 00000000..92f3df97
--- /dev/null
+++ b/plugins/github/e2e/snapshot_tables/sprints.csv
@@ -0,0 +1,2 @@
+id,name,url,status,started_date,ended_date,completed_date,original_board_id
+github:GithubMilestone:1:7856149,v0.11.0,https://api.github.com/repos/apache/incubator-devlake/milestones/7,open,2022-04-08T02:05:35.000+00:00,,,github:GithubRepo:1:134018330
diff --git a/plugins/github/impl/impl.go b/plugins/github/impl/impl.go
index 7f5a5ccd..8ade2967 100644
--- a/plugins/github/impl/impl.go
+++ b/plugins/github/impl/impl.go
@@ -18,6 +18,7 @@ limitations under the License.
package impl
import (
+ "fmt"
"github.com/apache/incubator-devlake/migration"
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/github/api"
@@ -67,6 +68,8 @@ func (plugin Github) SubTaskMetas() []core.SubTaskMeta {
tasks.ExtractApiCommitsMeta,
tasks.CollectApiCommitStatsMeta,
tasks.ExtractApiCommitStatsMeta,
+ tasks.CollectMilestonesMeta,
+ tasks.ExtractMilestonesMeta,
tasks.EnrichPullRequestIssuesMeta,
tasks.ConvertRepoMeta,
tasks.ConvertIssuesMeta,
@@ -79,6 +82,7 @@ func (plugin Github) SubTaskMetas() []core.SubTaskMeta {
tasks.ConvertUsersMeta,
tasks.ConvertIssueCommentsMeta,
tasks.ConvertPullRequestCommentsMeta,
+ tasks.ConvertMilestonesMeta,
}
}
@@ -94,12 +98,12 @@ func (plugin Github) PrepareTaskData(taskCtx core.TaskContext, options map[strin
connection := &models.GithubConnection{}
err = connectionHelper.FirstById(connection, op.ConnectionId)
if err != nil {
- return err, nil
+ return nil, fmt.Errorf("unable to get github connection by the given connection ID: %v", err)
}
apiClient, err := tasks.CreateApiClient(taskCtx, connection)
if err != nil {
- return nil, err
+ return nil, fmt.Errorf("unable to get github API client instance: %v", err)
}
return &tasks.GithubTaskData{
diff --git a/plugins/github/models/issue.go b/plugins/github/models/issue.go
index b508e55d..c658d416 100644
--- a/plugins/github/models/issue.go
+++ b/plugins/github/models/issue.go
@@ -37,6 +37,7 @@ type GithubIssue struct {
AuthorName string `gorm:"type:varchar(255)"`
AssigneeId int
AssigneeName string `gorm:"type:varchar(255)"`
+ MilestoneId int `gorm:"index"`
LeadTimeMinutes uint
Url string `gorm:"type:varchar(255)"`
ClosedAt *time.Time
diff --git a/plugins/github/models/migrationscripts/register.go b/plugins/github/models/migrationscripts/register.go
index 51152508..e0b31a88 100644
--- a/plugins/github/models/migrationscripts/register.go
+++ b/plugins/github/models/migrationscripts/register.go
@@ -17,11 +17,14 @@ limitations under the License.
package migrationscripts
-import "github.com/apache/incubator-devlake/migration"
+import (
+ "github.com/apache/incubator-devlake/migration"
+)
// All return all the migration scripts
func All() []migration.Script {
return []migration.Script{
new(initSchemas),
+ new(UpdateSchemas20220620),
}
}
diff --git a/plugins/github/models/migrationscripts/updateSchemas20220620.go b/plugins/github/models/migrationscripts/updateSchemas20220620.go
new file mode 100644
index 00000000..c5178cfa
--- /dev/null
+++ b/plugins/github/models/migrationscripts/updateSchemas20220620.go
@@ -0,0 +1,73 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "context"
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+ "gorm.io/gorm"
+ "time"
+)
+
+// GithubMilestone20220620 new struct for milestones
+type GithubMilestone20220620 struct {
+ archived.NoPKModel
+ ConnectionId uint64 `gorm:"primaryKey"`
+ MilestoneId int `gorm:"primaryKey;autoIncrement:false"`
+ RepoId int
+ Number int
+ URL string
+ OpenIssues int
+ ClosedIssues int
+ State string
+ Title string
+ CreatedAt time.Time
+ UpdatedAt time.Time
+ ClosedAt time.Time
+}
+
+// GithubIssue20220620 new field for models.GithubIssue
+type GithubIssue20220620 struct {
+ MilestoneId int
+}
+
+type UpdateSchemas20220620 struct{}
+
+func (GithubMilestone20220620) TableName() string {
+ return "_tool_github_milestones"
+}
+
+func (GithubIssue20220620) TableName() string {
+ return "_tool_github_issues"
+}
+
+func (*UpdateSchemas20220620) Up(_ context.Context, db *gorm.DB) error {
+ err := db.Migrator().AddColumn(GithubIssue20220620{}, "milestone_id")
+ if err != nil {
+ return err
+ }
+ return db.Migrator().CreateTable(GithubMilestone20220620{})
+}
+
+func (*UpdateSchemas20220620) Version() uint64 {
+ return 20220620000001
+}
+
+func (*UpdateSchemas20220620) Name() string {
+ return "Add milestone for github"
+}
diff --git a/plugins/github/models/migrationscripts/register.go b/plugins/github/models/milestone.go
similarity index 59%
copy from plugins/github/models/migrationscripts/register.go
copy to plugins/github/models/milestone.go
index 51152508..561cbfa5 100644
--- a/plugins/github/models/migrationscripts/register.go
+++ b/plugins/github/models/milestone.go
@@ -15,13 +15,29 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package migrationscripts
+package models
-import "github.com/apache/incubator-devlake/migration"
+import (
+ "github.com/apache/incubator-devlake/models/common"
+ "time"
+)
-// All return all the migration scripts
-func All() []migration.Script {
- return []migration.Script{
- new(initSchemas),
- }
+type GithubMilestone struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+ MilestoneId int `gorm:"primaryKey;autoIncrement:false"`
+ RepoId int
+ Number int
+ URL string
+ Title string
+ OpenIssues int
+ ClosedIssues int
+ State string
+ CreatedAt time.Time
+ UpdatedAt time.Time
+ ClosedAt *time.Time
+ common.NoPKModel
+}
+
+func (GithubMilestone) TableName() string {
+ return "_tool_github_milestones"
}
diff --git a/plugins/github/tasks/issue_collector.go b/plugins/github/tasks/issue_collector.go
index 35f618d1..ae735f12 100644
--- a/plugins/github/tasks/issue_collector.go
+++ b/plugins/github/tasks/issue_collector.go
@@ -114,7 +114,7 @@ func CollectApiIssues(taskCtx core.SubTaskContext) error {
query.Set("direction", "asc")
query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size))
-
+ query.Set("milestone", "*")
return query, nil
},
/*
diff --git a/plugins/github/tasks/issue_extractor.go b/plugins/github/tasks/issue_extractor.go
index efbbf54a..f5799c8b 100644
--- a/plugins/github/tasks/issue_extractor.go
+++ b/plugins/github/tasks/issue_extractor.go
@@ -51,67 +51,33 @@ type IssuesResponse struct {
Labels []struct {
Name string `json:"name"`
} `json:"labels"`
-
- Assignee *GithubUserResponse
- User *GithubUserResponse
+ Assignee *GithubUserResponse
+ User *GithubUserResponse
+ Milestone *struct {
+ Id int
+ }
ClosedAt *helper.Iso8601Time `json:"closed_at"`
GithubCreatedAt helper.Iso8601Time `json:"created_at"`
GithubUpdatedAt helper.Iso8601Time `json:"updated_at"`
}
+type IssueRegexes struct {
+ SeverityRegex *regexp.Regexp
+ ComponentRegex *regexp.Regexp
+ PriorityRegex *regexp.Regexp
+ TypeBugRegex *regexp.Regexp
+ TypeRequirementRegex *regexp.Regexp
+ TypeIncidentRegex *regexp.Regexp
+}
+
func ExtractApiIssues(taskCtx core.SubTaskContext) error {
data := taskCtx.GetData().(*GithubTaskData)
+
config := data.Options.TransformationRules
- var issueSeverityRegex *regexp.Regexp
- var issueComponentRegex *regexp.Regexp
- var issuePriorityRegex *regexp.Regexp
- var issueTypeBugRegex *regexp.Regexp
- var issueTypeRequirementRegex *regexp.Regexp
- var issueTypeIncidentRegex *regexp.Regexp
- var issueSeverity = config.IssueSeverity
- var err error
- if len(issueSeverity) > 0 {
- issueSeverityRegex, err = regexp.Compile(issueSeverity)
- if err != nil {
- return fmt.Errorf("regexp Compile issueSeverity failed:[%s] stack:[%s]", err.Error(), debug.Stack())
- }
- }
- var issueComponent = config.IssueComponent
- if len(issueComponent) > 0 {
- issueComponentRegex, err = regexp.Compile(issueComponent)
- if err != nil {
- return fmt.Errorf("regexp Compile issueComponent failed:[%s] stack:[%s]", err.Error(), debug.Stack())
- }
- }
- var issuePriority = config.IssuePriority
- if len(issuePriority) > 0 {
- issuePriorityRegex, err = regexp.Compile(issuePriority)
- if err != nil {
- return fmt.Errorf("regexp Compile issuePriority failed:[%s] stack:[%s]", err.Error(), debug.Stack())
- }
- }
- var issueTypeBug = config.IssueTypeBug
- if len(issueTypeBug) > 0 {
- issueTypeBugRegex, err = regexp.Compile(issueTypeBug)
- if err != nil {
- return fmt.Errorf("regexp Compile issueTypeBug failed:[%s] stack:[%s]", err.Error(), debug.Stack())
- }
- }
- var issueTypeRequirement = config.IssueTypeRequirement
- if len(issueTypeRequirement) > 0 {
- issueTypeRequirementRegex, err = regexp.Compile(issueTypeRequirement)
- if err != nil {
- return fmt.Errorf("regexp Compile issueTypeRequirement failed:[%s] stack:[%s]", err.Error(), debug.Stack())
- }
- }
- var issueTypeIncident = config.IssueTypeIncident
- if len(issueTypeIncident) > 0 {
- issueTypeIncidentRegex, err = regexp.Compile(issueTypeIncident)
- if err != nil {
- return fmt.Errorf("regexp Compile issueTypeIncident failed:[%s] stack:[%s]", err.Error(), debug.Stack())
- }
+ issueRegexes, err := NewIssueRegexes(config)
+ if err != nil {
+ return nil
}
-
extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
Ctx: taskCtx,
@@ -144,13 +110,18 @@ func ExtractApiIssues(taskCtx core.SubTaskContext) error {
return nil, nil
}
results := make([]interface{}, 0, 2)
+
githubIssue, err := convertGithubIssue(body, data.Options.ConnectionId, data.Repo.GithubId)
if err != nil {
return nil, err
}
+ githubLabels, err := convertGithubLabels(issueRegexes, body, githubIssue)
+ if err != nil {
+ return nil, err
+ }
+ results = append(results, githubLabels...)
+ results = append(results, githubIssue)
if body.Assignee != nil {
- githubIssue.AssigneeId = body.Assignee.Id
- githubIssue.AssigneeName = body.Assignee.Login
relatedUser, err := convertUser(body.Assignee, data.Options.ConnectionId)
if err != nil {
return nil, err
@@ -158,71 +129,22 @@ func ExtractApiIssues(taskCtx core.SubTaskContext) error {
results = append(results, relatedUser)
}
if body.User != nil {
- githubIssue.AuthorId = body.User.Id
- githubIssue.AuthorName = body.User.Login
relatedUser, err := convertUser(body.User, data.Options.ConnectionId)
if err != nil {
return nil, err
}
results = append(results, relatedUser)
}
- for _, label := range body.Labels {
- results = append(results, &models.GithubIssueLabel{
- ConnectionId: data.Options.ConnectionId,
- IssueId: githubIssue.GithubId,
- LabelName: label.Name,
- })
- if issueSeverityRegex != nil {
- groups := issueSeverityRegex.FindStringSubmatch(label.Name)
- if len(groups) > 0 {
- githubIssue.Severity = groups[1]
- }
- }
-
- if issueComponentRegex != nil {
- groups := issueComponentRegex.FindStringSubmatch(label.Name)
- if len(groups) > 0 {
- githubIssue.Component = groups[1]
- }
- }
-
- if issuePriorityRegex != nil {
- groups := issuePriorityRegex.FindStringSubmatch(label.Name)
- if len(groups) > 0 {
- githubIssue.Priority = groups[1]
- }
- }
-
- if issueTypeBugRegex != nil {
- if ok := issueTypeBugRegex.MatchString(label.Name); ok {
- githubIssue.Type = ticket.BUG
- }
- }
-
- if issueTypeRequirementRegex != nil {
- if ok := issueTypeRequirementRegex.MatchString(label.Name); ok {
- githubIssue.Type = ticket.REQUIREMENT
- }
- }
-
- if issueTypeIncidentRegex != nil {
- if ok := issueTypeIncidentRegex.MatchString(label.Name); ok {
- githubIssue.Type = ticket.INCIDENT
- }
- }
- }
- results = append(results, githubIssue)
-
return results, nil
},
})
-
if err != nil {
return err
}
return extractor.Execute()
}
+
func convertGithubIssue(issue *IssuesResponse, connectionId uint64, repositoryId int) (*models.GithubIssue, error) {
githubIssue := &models.GithubIssue{
ConnectionId: connectionId,
@@ -233,13 +155,122 @@ func convertGithubIssue(issue *IssuesResponse, connectionId uint64, repositoryId
Title: issue.Title,
Body: string(issue.Body),
Url: issue.HtmlUrl,
+ MilestoneId: issue.Milestone.Id,
ClosedAt: helper.Iso8601TimeToTime(issue.ClosedAt),
GithubCreatedAt: issue.GithubCreatedAt.ToTime(),
GithubUpdatedAt: issue.GithubUpdatedAt.ToTime(),
}
+ if issue.Assignee != nil {
+ githubIssue.AssigneeId = issue.Assignee.Id
+ githubIssue.AssigneeName = issue.Assignee.Login
+ }
+ if issue.User != nil {
+ githubIssue.AuthorId = issue.User.Id
+ githubIssue.AuthorName = issue.User.Login
+ }
if issue.ClosedAt != nil {
githubIssue.LeadTimeMinutes = uint(issue.ClosedAt.ToTime().Sub(issue.GithubCreatedAt.ToTime()).Minutes())
}
-
+ if issue.Assignee != nil {
+ githubIssue.AssigneeId = issue.Assignee.Id
+ githubIssue.AssigneeName = issue.Assignee.Login
+ }
+ if issue.User != nil {
+ githubIssue.AuthorId = issue.User.Id
+ githubIssue.AuthorName = issue.User.Login
+ }
return githubIssue, nil
}
+
+func convertGithubLabels(issueRegexes *IssueRegexes, issue *IssuesResponse, githubIssue *models.GithubIssue) ([]interface{}, error) {
+ var results []interface{}
+ for _, label := range issue.Labels {
+ results = append(results, &models.GithubIssueLabel{
+ ConnectionId: githubIssue.ConnectionId,
+ IssueId: githubIssue.GithubId,
+ LabelName: label.Name,
+ })
+ if issueRegexes.SeverityRegex != nil {
+ groups := issueRegexes.SeverityRegex.FindStringSubmatch(label.Name)
+ if len(groups) > 0 {
+ githubIssue.Severity = groups[1]
+ }
+ }
+ if issueRegexes.ComponentRegex != nil {
+ groups := issueRegexes.ComponentRegex.FindStringSubmatch(label.Name)
+ if len(groups) > 0 {
+ githubIssue.Component = groups[1]
+ }
+ }
+ if issueRegexes.PriorityRegex != nil {
+ groups := issueRegexes.PriorityRegex.FindStringSubmatch(label.Name)
+ if len(groups) > 0 {
+ githubIssue.Priority = groups[1]
+ }
+ }
+ if issueRegexes.TypeBugRegex != nil {
+ if ok := issueRegexes.TypeBugRegex.MatchString(label.Name); ok {
+ githubIssue.Type = ticket.BUG
+ }
+ }
+ if issueRegexes.TypeRequirementRegex != nil {
+ if ok := issueRegexes.TypeRequirementRegex.MatchString(label.Name); ok {
+ githubIssue.Type = ticket.REQUIREMENT
+ }
+ }
+ if issueRegexes.TypeIncidentRegex != nil {
+ if ok := issueRegexes.TypeIncidentRegex.MatchString(label.Name); ok {
+ githubIssue.Type = ticket.INCIDENT
+ }
+ }
+ }
+ return results, nil
+}
+
+func NewIssueRegexes(config models.TransformationRules) (*IssueRegexes, error) {
+ var issueRegexes IssueRegexes
+ var issueSeverity = config.IssueSeverity
+ var err error
+ if len(issueSeverity) > 0 {
+ issueRegexes.SeverityRegex, err = regexp.Compile(issueSeverity)
+ if err != nil {
+ return nil, fmt.Errorf("regexp Compile issueSeverity failed:[%s] stack:[%s]", err.Error(), debug.Stack())
+ }
+ }
+ var issueComponent = config.IssueComponent
+ if len(issueComponent) > 0 {
+ issueRegexes.ComponentRegex, err = regexp.Compile(issueComponent)
+ if err != nil {
+ return nil, fmt.Errorf("regexp Compile issueComponent failed:[%s] stack:[%s]", err.Error(), debug.Stack())
+ }
+ }
+ var issuePriority = config.IssuePriority
+ if len(issuePriority) > 0 {
+ issueRegexes.PriorityRegex, err = regexp.Compile(issuePriority)
+ if err != nil {
+ return nil, fmt.Errorf("regexp Compile issuePriority failed:[%s] stack:[%s]", err.Error(), debug.Stack())
+ }
+ }
+ var issueTypeBug = config.IssueTypeBug
+ if len(issueTypeBug) > 0 {
+ issueRegexes.TypeBugRegex, err = regexp.Compile(issueTypeBug)
+ if err != nil {
+ return nil, fmt.Errorf("regexp Compile issueTypeBug failed:[%s] stack:[%s]", err.Error(), debug.Stack())
+ }
+ }
+ var issueTypeRequirement = config.IssueTypeRequirement
+ if len(issueTypeRequirement) > 0 {
+ issueRegexes.TypeRequirementRegex, err = regexp.Compile(issueTypeRequirement)
+ if err != nil {
+ return nil, fmt.Errorf("regexp Compile issueTypeRequirement failed:[%s] stack:[%s]", err.Error(), debug.Stack())
+ }
+ }
+ var issueTypeIncident = config.IssueTypeIncident
+ if len(issueTypeIncident) > 0 {
+ issueRegexes.TypeIncidentRegex, err = regexp.Compile(issueTypeIncident)
+ if err != nil {
+ return nil, fmt.Errorf("regexp Compile issueTypeIncident failed:[%s] stack:[%s]", err.Error(), debug.Stack())
+ }
+ }
+ return &issueRegexes, nil
+}
diff --git a/plugins/github/tasks/milestone_collector.go b/plugins/github/tasks/milestone_collector.go
new file mode 100644
index 00000000..ff43bd10
--- /dev/null
+++ b/plugins/github/tasks/milestone_collector.go
@@ -0,0 +1,78 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/url"
+
+ "github.com/apache/incubator-devlake/plugins/helper"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+)
+
+const RAW_MILESTONE_TABLE = "github_milestones"
+
+var CollectMilestonesMeta = core.SubTaskMeta{
+ Name: "collectApiMilestones",
+ EntryPoint: CollectApiMilestones,
+ EnabledByDefault: true,
+ Description: "Collect milestone data from Github api",
+}
+
+func CollectApiMilestones(taskCtx core.SubTaskContext) error {
+ data := taskCtx.GetData().(*GithubTaskData)
+ collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: GithubApiParams{
+ Owner: data.Options.Owner,
+ Repo: data.Options.Repo,
+ },
+ Table: RAW_MILESTONE_TABLE,
+ },
+ ApiClient: data.ApiClient,
+ PageSize: 100,
+ Incremental: false,
+ UrlTemplate: "repos/{{ .Params.Owner }}/{{ .Params.Repo }}/milestones",
+ Query: func(reqData *helper.RequestData) (url.Values, error) {
+ query := url.Values{}
+ query.Set("state", "all")
+ query.Set("direction", "asc")
+ query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
+ query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size))
+ return query, nil
+ },
+ GetTotalPages: GetTotalPagesFromResponse,
+ ResponseParser: func(res *http.Response) ([]json.RawMessage, error) {
+ var items []json.RawMessage
+ err := helper.UnmarshalResponse(res, &items)
+ if err != nil {
+ return nil, err
+ }
+ return items, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+ return collector.Execute()
+}
diff --git a/plugins/github/tasks/milestone_converter.go b/plugins/github/tasks/milestone_converter.go
new file mode 100644
index 00000000..069f9623
--- /dev/null
+++ b/plugins/github/tasks/milestone_converter.go
@@ -0,0 +1,109 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "github.com/apache/incubator-devlake/models/common"
+ "github.com/apache/incubator-devlake/models/domainlayer"
+ "github.com/apache/incubator-devlake/models/domainlayer/didgen"
+ "github.com/apache/incubator-devlake/models/domainlayer/ticket"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+ githubModels "github.com/apache/incubator-devlake/plugins/github/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "reflect"
+)
+
+var ConvertMilestonesMeta = core.SubTaskMeta{
+ Name: "convertMilestones",
+ EntryPoint: ConvertMilestones,
+ EnabledByDefault: true,
+ Description: "Convert tool layer table github_milestones into domain layer table milestones",
+}
+
+type MilestoneConverterModel struct {
+ common.RawDataOrigin
+ githubModels.GithubMilestone
+ GithubId int
+}
+
+func ConvertMilestones(taskCtx core.SubTaskContext) error {
+ data := taskCtx.GetData().(*GithubTaskData)
+ repoId := data.Repo.GithubId
+ connectionId := data.Options.ConnectionId
+ db := taskCtx.GetDal()
+ clauses := []dal.Clause{
+ dal.Select("gi.github_id, gm.*"),
+ dal.From("_tool_github_issues gi"),
+ dal.Join("JOIN _tool_github_milestones gm ON gm.milestone_id = gi.milestone_id"),
+ dal.Where("gm.repo_id = ?", repoId),
+ }
+ cursor, err := db.Cursor(clauses...)
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ boardIdGen := didgen.NewDomainIdGenerator(&githubModels.GithubRepo{})
+ domainBoardId := boardIdGen.Generate(connectionId, repoId)
+ sprintIdGen := didgen.NewDomainIdGenerator(&githubModels.GithubMilestone{})
+ issueIdGen := didgen.NewDomainIdGenerator(&githubModels.GithubIssue{})
+
+ converter, err := helper.NewDataConverter(helper.DataConverterArgs{
+ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: GithubApiParams{
+ ConnectionId: connectionId,
+ Owner: data.Options.Owner,
+ Repo: data.Options.Repo,
+ },
+ Table: RAW_MILESTONE_TABLE,
+ },
+ InputRowType: reflect.TypeOf(MilestoneConverterModel{}),
+ Input: cursor,
+ Convert: func(inputRow interface{}) ([]interface{}, error) {
+ response := inputRow.(*MilestoneConverterModel)
+ domainSprintId := sprintIdGen.Generate(connectionId, response.GithubMilestone.MilestoneId)
+ domainIssueId := issueIdGen.Generate(connectionId, response.GithubId)
+ sprint := &ticket.Sprint{
+ DomainEntity: domainlayer.DomainEntity{Id: domainSprintId},
+ Name: response.GithubMilestone.Title,
+ Url: response.GithubMilestone.URL,
+ Status: response.GithubMilestone.State,
+ StartedDate: &response.GithubMilestone.CreatedAt, //GitHub doesn't give us a "start date"
+ EndedDate: response.GithubMilestone.ClosedAt,
+ CompletedDate: response.GithubMilestone.ClosedAt,
+ OriginalBoardID: domainBoardId,
+ }
+ boardSprint := &ticket.BoardSprint{
+ BoardId: domainBoardId,
+ SprintId: domainSprintId,
+ }
+ sprintIssue := &ticket.SprintIssue{
+ SprintId: domainSprintId,
+ IssueId: domainIssueId,
+ }
+ return []interface{}{sprint, sprintIssue, boardSprint}, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return converter.Execute()
+}
diff --git a/plugins/github/tasks/milestone_extractor.go b/plugins/github/tasks/milestone_extractor.go
new file mode 100644
index 00000000..fc61aa96
--- /dev/null
+++ b/plugins/github/tasks/milestone_extractor.go
@@ -0,0 +1,118 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/github/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ExtractMilestonesMeta = core.SubTaskMeta{
+ Name: "extractMilestones",
+ EntryPoint: ExtractMilestones,
+ EnabledByDefault: true,
+ Description: "Extract raw milestone data into tool layer table github_milestones",
+}
+
+type MilestonesResponse struct {
+ Url string `json:"url"`
+ HtmlUrl string `json:"html_url"`
+ LabelsUrl string `json:"labels_url"`
+ Id int `json:"id"`
+ NodeId string `json:"node_id"`
+ Number int `json:"number"`
+ Title string `json:"title"`
+ Description string `json:"description"`
+ Creator struct {
+ Login string `json:"login"`
+ Id int `json:"id"`
+ NodeId string `json:"node_id"`
+ AvatarUrl string `json:"avatar_url"`
+ GravatarId string `json:"gravatar_id"`
+ Url string `json:"url"`
+ HtmlUrl string `json:"html_url"`
+ FollowersUrl string `json:"followers_url"`
+ FollowingUrl string `json:"following_url"`
+ GistsUrl string `json:"gists_url"`
+ StarredUrl string `json:"starred_url"`
+ SubscriptionsUrl string `json:"subscriptions_url"`
+ OrganizationsUrl string `json:"organizations_url"`
+ ReposUrl string `json:"repos_url"`
+ EventsUrl string `json:"events_url"`
+ ReceivedEventsUrl string `json:"received_events_url"`
+ Type string `json:"type"`
+ SiteAdmin bool `json:"site_admin"`
+ } `json:"creator"`
+ OpenIssues int `json:"open_issues"`
+ ClosedIssues int `json:"closed_issues"`
+ State string `json:"state"`
+ CreatedAt helper.Iso8601Time `json:"created_at"`
+ UpdatedAt helper.Iso8601Time `json:"updated_at"`
+ DueOn *helper.Iso8601Time `json:"due_on"`
+ ClosedAt *helper.Iso8601Time `json:"closed_at"`
+}
+
+func ExtractMilestones(taskCtx core.SubTaskContext) error {
+ data := taskCtx.GetData().(*GithubTaskData)
+ extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
+ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: GithubApiParams{
+ ConnectionId: data.Options.ConnectionId,
+ Owner: data.Options.Owner,
+ Repo: data.Options.Repo,
+ },
+ Table: RAW_MILESTONE_TABLE,
+ },
+ Extract: func(row *helper.RawData) ([]interface{}, error) {
+ response := &MilestonesResponse{}
+ err := json.Unmarshal(row.Data, response)
+ if err != nil {
+ return nil, err
+ }
+ results := make([]interface{}, 0, 1)
+ results = append(results, convertGithubMilestone(response, data.Options.ConnectionId, data.Repo.GithubId))
+ return results, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return extractor.Execute()
+}
+
+func convertGithubMilestone(response *MilestonesResponse, connectionId uint64, repositoryId int) *models.GithubMilestone {
+ milestone := &models.GithubMilestone{
+ ConnectionId: connectionId,
+ MilestoneId: response.Id,
+ RepoId: repositoryId,
+ Number: response.Number,
+ URL: response.Url,
+ Title: response.Title,
+ OpenIssues: response.OpenIssues,
+ ClosedIssues: response.ClosedIssues,
+ State: response.State,
+ ClosedAt: helper.Iso8601TimeToTime(response.ClosedAt),
+ CreatedAt: response.CreatedAt.ToTime(),
+ UpdatedAt: response.UpdatedAt.ToTime(),
+ }
+ return milestone
+}
diff --git a/runner/directrun.go b/runner/directrun.go
index 4e0fa19a..05199e07 100644
--- a/runner/directrun.go
+++ b/runner/directrun.go
@@ -19,13 +19,18 @@ package runner
import (
"context"
+ "errors"
"fmt"
-
"github.com/apache/incubator-devlake/config"
"github.com/apache/incubator-devlake/logger"
"github.com/apache/incubator-devlake/migration"
"github.com/apache/incubator-devlake/plugins/core"
"github.com/spf13/cobra"
+ "io"
+ "os"
+ "os/signal"
+ "runtime"
+ "syscall"
)
func RunCmd(cmd *cobra.Command) {
@@ -72,13 +77,39 @@ func DirectRun(cmd *cobra.Command, args []string, pluginTask core.PluginTask, op
if err != nil {
panic(err)
}
+ ctx := createContext()
+ err = RunPluginSubTasks(
+ cfg,
+ log,
+ db,
+ ctx,
+ cmd.Use,
+ tasks,
+ options,
+ pluginTask,
+ nil,
+ )
+ if err != nil {
+ panic(err)
+ }
+}
+func createContext() context.Context {
ctx, cancel := context.WithCancel(context.Background())
+ sigc := make(chan os.Signal, 1)
+ signal.Notify(sigc, getStopSignals()...)
+ go func() {
+ <-sigc
+ cancel()
+ }()
go func() {
var buf string
n, err := fmt.Scan(&buf)
if err != nil {
+ if errors.Is(err, io.EOF) {
+ return
+ }
panic(err)
} else if n == 1 && buf == "c" {
cancel()
@@ -88,19 +119,16 @@ func DirectRun(cmd *cobra.Command, args []string, pluginTask core.PluginTask, op
}
}()
println("press `c` and enter to send cancel signal")
+ return ctx
+}
- err = RunPluginSubTasks(
- cfg,
- log,
- db,
- ctx,
- cmd.Use,
- tasks,
- options,
- pluginTask,
- nil,
- )
- if err != nil {
- panic(err)
+func getStopSignals() []os.Signal {
+ if runtime.GOOS == "windows" {
+ return []os.Signal{
+ syscall.Signal(0x6), //syscall.SIGABRT for windows
+ }
+ }
+ return []os.Signal{
+ syscall.Signal(0x14), //syscall.SIGTSTP for posix
}
}