You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by wa...@apache.org on 2022/10/12 06:33:41 UTC
[incubator-devlake] branch feat-plugin-coding updated: feat(coding): add depot extractor
This is an automated email from the ASF dual-hosted git repository.
warren pushed a commit to branch feat-plugin-coding
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/feat-plugin-coding by this push:
new 6f8b44c2 feat(coding): add depot extractor
6f8b44c2 is described below
commit 6f8b44c29cf577fc33e5be6905406bd415f36e7b
Author: Yingchu Chen <yi...@merico.dev>
AuthorDate: Wed Oct 12 14:31:36 2022 +0800
feat(coding): add depot extractor
---
plugins/coding/coding.go | 8 ++--
plugins/coding/impl/impl.go | 1 +
.../20220928_add_init_tables.go => depot.go} | 32 ++++++-------
.../migrationscripts/20220928_add_init_tables.go | 5 ++-
.../depot.go} | 32 ++++++-------
plugins/coding/tasks/depot_collector.go | 4 +-
.../{depot_collector.go => depot_extractor.go} | 52 ++++++++--------------
plugins/coding/tasks/task_data.go | 4 +-
8 files changed, 58 insertions(+), 80 deletions(-)
diff --git a/plugins/coding/coding.go b/plugins/coding/coding.go
index 6f79dec7..f1da5238 100644
--- a/plugins/coding/coding.go
+++ b/plugins/coding/coding.go
@@ -31,13 +31,13 @@ func main() {
cmd := &cobra.Command{Use: "coding"}
// TODO add your cmd flag if necessary
- depotId := cmd.Flags().IntP("depotId", "d", 1, "depotId")
- connectionId := cmd.Flags().Uint64P("connectionId", "c", 0, "coding connection id")
+ //depotId := cmd.Flags().Int64P("depotId", "d", 9748559, "depotId")
+ //connectionId := cmd.Flags().Int64P("connectionId", "c", 1, "coding connection id")
cmd.Run = func(cmd *cobra.Command, args []string) {
runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{
- "connectionId": *connectionId,
- "depotId": *depotId,
+ "connectionId": 1,
+ "depotId": 9748559,
})
}
runner.RunCmd(cmd)
diff --git a/plugins/coding/impl/impl.go b/plugins/coding/impl/impl.go
index e7947574..f398ea38 100644
--- a/plugins/coding/impl/impl.go
+++ b/plugins/coding/impl/impl.go
@@ -54,6 +54,7 @@ func (plugin Coding) SubTaskMetas() []core.SubTaskMeta {
// TODO add your sub task here
return []core.SubTaskMeta{
tasks.CollectDepotMeta,
+ tasks.ExtractDepotMeta,
}
}
diff --git a/plugins/coding/models/migrationscripts/20220928_add_init_tables.go b/plugins/coding/models/depot.go
similarity index 58%
copy from plugins/coding/models/migrationscripts/20220928_add_init_tables.go
copy to plugins/coding/models/depot.go
index 6d365b08..5a56b5b9 100644
--- a/plugins/coding/models/migrationscripts/20220928_add_init_tables.go
+++ b/plugins/coding/models/depot.go
@@ -15,28 +15,24 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package migrationscripts
+package models
import (
- "context"
- "github.com/apache/incubator-devlake/errors"
- "github.com/apache/incubator-devlake/plugins/coding/models/migrationscripts/archived"
- "gorm.io/gorm"
+ "github.com/apache/incubator-devlake/models/common"
)
-type addInitTables struct{}
-
-func (u *addInitTables) Up(ctx context.Context, db *gorm.DB) errors.Error {
- err := db.Migrator().AutoMigrate(
- archived.CodingConnection{},
- )
- return errors.Convert(err)
-}
-
-func (*addInitTables) Version() uint64 {
- return 20220928000001
+type CodingDepot struct {
+ common.NoPKModel
+ ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"`
+ ID int64 `gorm:"primaryKey;type:BIGINT" json:"Id"`
+ Name string `gorm:"type:varchar(255)" json:"Name"`
+ HTTPSURL string `json:"HttpsUrl"`
+ ProjectID int64 `json:"ProjectId"`
+ SSHURL string `json:"SshUrl"`
+ WebURL string `json:"WebUrl"`
+ VcsType string `json:"VcsType"`
}
-func (*addInitTables) Name() string {
- return "coding init schemas"
+func (CodingDepot) TableName() string {
+ return "_tool_coding_depots"
}
diff --git a/plugins/coding/models/migrationscripts/20220928_add_init_tables.go b/plugins/coding/models/migrationscripts/20220928_add_init_tables.go
index 6d365b08..b38b3249 100644
--- a/plugins/coding/models/migrationscripts/20220928_add_init_tables.go
+++ b/plugins/coding/models/migrationscripts/20220928_add_init_tables.go
@@ -28,13 +28,14 @@ type addInitTables struct{}
func (u *addInitTables) Up(ctx context.Context, db *gorm.DB) errors.Error {
err := db.Migrator().AutoMigrate(
- archived.CodingConnection{},
+ //archived.CodingConnection{},
+ archived.CodingDepot{},
)
return errors.Convert(err)
}
func (*addInitTables) Version() uint64 {
- return 20220928000001
+ return 20221012000001
}
func (*addInitTables) Name() string {
diff --git a/plugins/coding/models/migrationscripts/20220928_add_init_tables.go b/plugins/coding/models/migrationscripts/archived/depot.go
similarity index 57%
copy from plugins/coding/models/migrationscripts/20220928_add_init_tables.go
copy to plugins/coding/models/migrationscripts/archived/depot.go
index 6d365b08..80777b27 100644
--- a/plugins/coding/models/migrationscripts/20220928_add_init_tables.go
+++ b/plugins/coding/models/migrationscripts/archived/depot.go
@@ -15,28 +15,24 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package migrationscripts
+package archived
import (
- "context"
- "github.com/apache/incubator-devlake/errors"
- "github.com/apache/incubator-devlake/plugins/coding/models/migrationscripts/archived"
- "gorm.io/gorm"
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
)
-type addInitTables struct{}
-
-func (u *addInitTables) Up(ctx context.Context, db *gorm.DB) errors.Error {
- err := db.Migrator().AutoMigrate(
- archived.CodingConnection{},
- )
- return errors.Convert(err)
-}
-
-func (*addInitTables) Version() uint64 {
- return 20220928000001
+type CodingDepot struct {
+ archived.NoPKModel
+ ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"`
+ ID int64 `gorm:"primaryKey;type:BIGINT" json:"Id"`
+ Name string `gorm:"type:varchar(255)" json:"Name"`
+ HTTPSURL string `json:"HttpsUrl"`
+ ProjectID int64 `json:"ProjectId"`
+ SSHURL string `json:"SshUrl"`
+ WebURL string `json:"WebUrl"`
+ VcsType string `json:"VcsType"`
}
-func (*addInitTables) Name() string {
- return "coding init schemas"
+func (CodingDepot) TableName() string {
+ return "_tool_coding_depots"
}
diff --git a/plugins/coding/tasks/depot_collector.go b/plugins/coding/tasks/depot_collector.go
index 9ad3aa73..defe9ca9 100644
--- a/plugins/coding/tasks/depot_collector.go
+++ b/plugins/coding/tasks/depot_collector.go
@@ -45,14 +45,14 @@ func CollectDepot(taskCtx core.SubTaskContext) errors.Error {
Incremental: false,
//PageSize: 100,
// TODO write which api would you want request
- UrlTemplate: "open-api",
+ UrlTemplate: "",
ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) {
var resData struct {
Response struct {
Depot json.RawMessage `json:"Depot"`
} `json:"Response"`
}
- err := helper.UnmarshalResponse(res, &resData.Response.Depot)
+ err := helper.UnmarshalResponse(res, &resData)
return []json.RawMessage{resData.Response.Depot}, err
},
diff --git a/plugins/coding/tasks/depot_collector.go b/plugins/coding/tasks/depot_extractor.go
similarity index 51%
copy from plugins/coding/tasks/depot_collector.go
copy to plugins/coding/tasks/depot_extractor.go
index 9ad3aa73..eaa52661 100644
--- a/plugins/coding/tasks/depot_collector.go
+++ b/plugins/coding/tasks/depot_extractor.go
@@ -20,19 +20,16 @@ package tasks
import (
"encoding/json"
"github.com/apache/incubator-devlake/errors"
+ "github.com/apache/incubator-devlake/plugins/coding/models"
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/helper"
- "net/http"
)
-const RAW_DEPOT_TABLE = "coding_depot"
+var _ core.SubTaskEntryPoint = ExtractDepot
-var _ core.SubTaskEntryPoint = CollectDepot
-
-func CollectDepot(taskCtx core.SubTaskContext) errors.Error {
+func ExtractDepot(taskCtx core.SubTaskContext) errors.Error {
data := taskCtx.GetData().(*CodingTaskData)
-
- collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+ extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
Ctx: taskCtx,
Params: CodingApiParams{
@@ -41,41 +38,28 @@ func CollectDepot(taskCtx core.SubTaskContext) errors.Error {
},
Table: RAW_DEPOT_TABLE,
},
- ApiClient: data.ApiClient,
- Incremental: false,
- //PageSize: 100,
- // TODO write which api would you want request
- UrlTemplate: "open-api",
- ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) {
- var resData struct {
- Response struct {
- Depot json.RawMessage `json:"Depot"`
- } `json:"Response"`
+ Extract: func(resData *helper.RawData) ([]interface{}, errors.Error) {
+ extractedModels := make([]interface{}, 0)
+ toolL := &models.CodingDepot{}
+ err := errors.Convert(json.Unmarshal(resData.Data, &toolL))
+ if err != nil {
+ return nil, err
}
- err := helper.UnmarshalResponse(res, &resData.Response.Depot)
-
- return []json.RawMessage{resData.Response.Depot}, err
- },
- RequestBody: func(reqData *helper.RequestData) map[string]interface{} {
- body := make(map[string]interface{})
- body["Action"] = "DescribeGitDepot"
- body["DepotId"] = data.Options.DepotId
- //body["PageNumber"] = reqData.Pager.Page
- //body["PageSize"] = reqData.Pager.Size
- return body
+ toolL.ConnectionId = data.Options.ConnectionId
+ extractedModels = append(extractedModels, toolL)
+ return extractedModels, nil
},
- Method: http.MethodPost,
})
if err != nil {
return err
}
- return collector.Execute()
+ return extractor.Execute()
}
-var CollectDepotMeta = core.SubTaskMeta{
- Name: "CollectDepot",
- EntryPoint: CollectDepot,
+var ExtractDepotMeta = core.SubTaskMeta{
+ Name: "ExtractDepot",
+ EntryPoint: ExtractDepot,
EnabledByDefault: true,
- Description: "Collect Depot data from Coding api",
+ Description: "Extract raw data into tool layer table coding_depot",
}
diff --git a/plugins/coding/tasks/task_data.go b/plugins/coding/tasks/task_data.go
index bada0113..377334aa 100644
--- a/plugins/coding/tasks/task_data.go
+++ b/plugins/coding/tasks/task_data.go
@@ -24,7 +24,7 @@ import (
type CodingApiParams struct {
ConnectionId uint64
- DepotId uint64
+ DepotId int64
}
type CodingOptions struct {
@@ -34,7 +34,7 @@ type CodingOptions struct {
// You can use it in sub tasks and you need pass it in main.go and pipelines.
ConnectionId uint64 `json:"connectionId"`
Tasks []string `json:"tasks,omitempty"`
- DepotId uint64
+ DepotId int64
Since string
}