You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by kl...@apache.org on 2022/06/08 08:15:22 UTC
[incubator-devlake] branch main updated: Issues/2087 create template and icla collect (#2088)
This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 735c181d Issues/2087 create template and icla collect (#2088)
735c181d is described below
commit 735c181d62299962afae82bfb1b9bd5916dc6e0f
Author: likyh <l...@likyh.com>
AuthorDate: Wed Jun 8 16:15:18 2022 +0800
Issues/2087 create template and icla collect (#2088)
* add some template
* collect apache committer info
* fix a camel
* append
Co-authored-by: linyh <ya...@meri.co>
---
generator/template/plugin/api_client.go-template | 71 ++++++++++++++
.../template/plugin/api_collector.go-template | 77 +++++++++++++++
generator/template/plugin/extractor.go-template | 57 +++++++++++
generator/template/plugin/plugin_main.go-template | 105 +++++++++++++++++++++
generator/template/plugin/task_data.go-template | 38 ++++++++
plugins/icla/models/committer.go | 32 +++++++
plugins/icla/plugin_main.go | 103 ++++++++++++++++++++
plugins/icla/tasks/api_client.go | 69 ++++++++++++++
plugins/icla/tasks/committer_collector.go | 73 ++++++++++++++
plugins/icla/tasks/committer_extractor.go | 64 +++++++++++++
plugins/icla/tasks/task_data.go | 36 +++++++
11 files changed, 725 insertions(+)
diff --git a/generator/template/plugin/api_client.go-template b/generator/template/plugin/api_client.go-template
new file mode 100644
index 00000000..8da60666
--- /dev/null
+++ b/generator/template/plugin/api_client.go-template
@@ -0,0 +1,71 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package plugin
+
+import (
+ "fmt"
+ "net/http"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/apache/incubator-devlake/utils"
+)
+
+// TODO add what host would want to requist
+const ENDPOINT = "https://open.example.cn/api/v1"
+
+func New{{ .PluginName }}ApiClient(taskCtx core.TaskContext) (*helper.ApiAsyncClient, error) {
+ // load and process configuration
+ token := taskCtx.GetConfig("{{ .PLUGIN_NAME }}_TOKEN")
+ if token == "" {
+ println("invalid {{ .PLUGIN_NAME }}_TOKEN, but ignore this error now")
+ }
+ userRateLimit, err := utils.StrToIntOr(taskCtx.GetConfig("{{ .PLUGIN_NAME }}_API_REQUESTS_PER_HOUR"), 18000)
+ if err != nil {
+ return nil, err
+ }
+ proxy := taskCtx.GetConfig("{{ .PLUGIN_NAME }}_PROXY")
+
+ // real request apiClient
+ apiClient, err := helper.NewApiClient(ENDPOINT, nil, 0, proxy, taskCtx.GetContext())
+ if err != nil {
+ return nil, err
+ }
+ // set token
+ apiClient.SetHeaders(map[string]string{
+ "Authorization": fmt.Sprintf("Bearer %v", token),
+ })
+
+ // TODO add some check after request if necessary
+ // apiClient.SetAfterFunction(func(res *http.Response) error {
+ // if res.StatusCode == http.StatusUnauthorized {
+ // return fmt.Errorf("authentication failed, please check your Bearer Auth Token")
+ // }
+ // return nil
+ // })
+
+ // create async api client
+ asyncApiClient, err := helper.CreateAsyncApiClient(taskCtx, apiClient, &helper.ApiRateLimitCalculator{
+ UserRateLimitPerHour: userRateLimit,
+ })
+ if err != nil {
+ return nil, err
+ }
+
+ return asyncApiClient, nil
+}
diff --git a/generator/template/plugin/api_collector.go-template b/generator/template/plugin/api_collector.go-template
new file mode 100644
index 00000000..c744801d
--- /dev/null
+++ b/generator/template/plugin/api_collector.go-template
@@ -0,0 +1,77 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "net/http"
+ "net/url"
+ "strconv"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+const RAW_{{ .COLLECTOR_DATA_NAME }}_TABLE = "{{ .pluginName }}_{{ .collector_data_name }}"
+
+var _ core.SubTaskEntryPoint = Collect{{ .CollectorDataName }}
+
+func Collect{{ .CollectorDataName }}(taskCtx core.SubTaskContext) error {
+ data := taskCtx.GetData().(*{{ .PluginName }}TaskData)
+ iterator, err := helper.NewDateIterator(365)
+ if err != nil {
+ return err
+ }
+
+ collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: {{ .PluginName }}ApiParams{
+ },
+ Table: RAW_{{ .COLLECTOR_DATA_NAME }}_TABLE,
+ },
+ ApiClient: data.ApiClient,
+ Incremental: false,
+ Input: iterator,
+ // TODO write which api would you want request
+ UrlTemplate: "/example/",
+ Query: func(reqData *helper.RequestData) (url.Values, error) {
+ query := url.Values{}
+ input := reqData.Input.(*helper.DatePair)
+ query.Set("start_time", strconv.FormatInt(input.PairStartTime.Unix(), 10))
+ query.Set("end_time", strconv.FormatInt(input.PairEndTime.Unix(), 10))
+ return query, nil
+ },
+ ResponseParser: func(res *http.Response) ([]json.RawMessage, error) {
+ // TODO decode result from api request
+ return []json.RawMessage{}, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return collector.Execute()
+}
+
+var Collect{{ .CollectorDataName }}Meta = core.SubTaskMeta{
+ Name: "Collect{{ .CollectorDataName }}",
+ EntryPoint: Collect{{ .CollectorDataName }},
+ EnabledByDefault: true,
+ Description: "Collect {{ .CollectorDataName }} data from {{ .PluginName }} api",
+}
diff --git a/generator/template/plugin/extractor.go-template b/generator/template/plugin/extractor.go-template
new file mode 100644
index 00000000..eceb661c
--- /dev/null
+++ b/generator/template/plugin/extractor.go-template
@@ -0,0 +1,57 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var _ core.SubTaskEntryPoint = Extract{{ .ExtractorDataName }}
+
+func Extract{{ .ExtractorDataName }}(taskCtx core.SubTaskContext) error {
+ extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
+ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: {{ .PluginName }}ApiParams{
+ },
+ Table: RAW_{{ .COLLECTOR_DATA_NAME }}_TABLE,
+ },
+ Extract: func(resData *helper.RawData) ([]interface{}, error) {
+ extractedModels := make([]interface{}, 0)
+ println(resData.Data)
+ println(resData.Input)
+ // TODO decode some db models from api result
+ // extractedModels = append(extractedModels, &models.XXXXXX)
+ return extractedModels, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return extractor.Execute()
+}
+
+var Extract{{ .ExtractorDataName }}Meta = core.SubTaskMeta{
+ Name: "Extract{{ .ExtractorDataName }}",
+ EntryPoint: Extract{{ .ExtractorDataName }},
+ EnabledByDefault: true,
+ Description: "Extract raw data into tool layer table {{ .plugin_name }}_{{ .extractor_data_name }}",
+}
diff --git a/generator/template/plugin/plugin_main.go-template b/generator/template/plugin/plugin_main.go-template
new file mode 100644
index 00000000..ba5b3feb
--- /dev/null
+++ b/generator/template/plugin/plugin_main.go-template
@@ -0,0 +1,105 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "context"
+ "github.com/apache/incubator-devlake/migration"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/{{ .pluginName }}/tasks"
+ "github.com/apache/incubator-devlake/runner"
+ "github.com/mitchellh/mapstructure"
+ "github.com/spf13/cobra"
+ "github.com/spf13/viper"
+ "gorm.io/gorm"
+ "time"
+)
+
+// make sure interface is implemented
+var _ core.PluginMeta = (*{{ .PluginName }})(nil)
+var _ core.PluginInit = (*{{ .PluginName }})(nil)
+var _ core.PluginTask = (*{{ .PluginName }})(nil)
+var _ core.PluginApi = (*{{ .PluginName }})(nil)
+
+// Export a variable named PluginEntry for Framework to search and load
+var PluginEntry {{ .PluginName }} //nolint
+
+type {{ .PluginName }} struct{}
+
+func (plugin {{ .PluginName }}) Description() string {
+ return "collect some {{ .PluginName }} data"
+}
+
+func (plugin {{ .PluginName }}) Init(config *viper.Viper, logger core.Logger, db *gorm.DB) error {
+ // AutoSchemas is a **develop** script to auto migrate models easily.
+ // FIXME Don't submit it as a open source plugin
+ return db.Migrator().AutoMigrate(
+ // TODO add your models in here
+ )
+}
+
+func (plugin {{ .PluginName }}) SubTaskMetas() []core.SubTaskMeta {
+ return []core.SubTaskMeta{
+ // TODO add your sub task here
+ }
+}
+
+func (plugin {{ .PluginName }}) PrepareTaskData(taskCtx core.TaskContext, options map[string]interface{}) (interface{}, error) {
+ var op tasks.{{ .PluginName }}Options
+ err := mapstructure.Decode(options, &op)
+ if err != nil {
+ return nil, err
+ }
+
+ // apiClient, err := tasks.New{{ .PluginName }}ApiClient(taskCtx)
+ // if err != nil {
+ // return nil, err
+ // }
+
+ return &tasks.{{ .PluginName }}TaskData{
+ Options: &op,
+ // TODO you can init and stash some handler to deal data at all subtasks, Such as apiClient as below.
+ // NOTES: In task_data.go/TaskData should declare `ApiClient`
+ // ApiClient: apiClient,
+ }, nil
+}
+
+// PkgPath information lost when compiled as plugin(.so)
+func (plugin {{ .PluginName }}) RootPkgPath() string {
+ return "github.com/apache/incubator-devlake/plugins/{{ .pluginName }}"
+}
+
+func (plugin {{ .PluginName }}) ApiResources() map[string]map[string]core.ApiResourceHandler {
+ return nil
+}
+
+// standalone mode for debugging
+func main() {
+ cmd := &cobra.Command{Use: "{{ .pluginName }}"}
+
+ // TODO add your cmd flag if necessary
+ // yourFlag := cmd.Flags().IntP("yourFlag", "y", 8, "TODO add description here")
+ // _ = cmd.MarkFlagRequired("yourFlag")
+
+ cmd.Run = func(cmd *cobra.Command, args []string) {
+ runner.DirectRun(cmd, args, PluginEntry, []string{}, map[string]interface{}{
+ // TODO add more custom params here
+ })
+ }
+ runner.RunCmd(cmd)
+}
diff --git a/generator/template/plugin/task_data.go-template b/generator/template/plugin/task_data.go-template
new file mode 100644
index 00000000..4498596e
--- /dev/null
+++ b/generator/template/plugin/task_data.go-template
@@ -0,0 +1,38 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package plugin
+
+import (
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+type {{ .PluginName }}ApiParams struct {
+}
+
+type {{ .PluginName }}Options struct {
+ // TODO add some custom options here if necessary
+ // options means some custom params required by plugin running.
+ // Such As How many rows do your want
+ // You can use it in sub tasks and you need pass it in main.go and pipelines.
+ Tasks []string `json:"tasks,omitempty"`
+}
+
+type {{ .PluginName }}TaskData struct {
+ Options *{{ .PluginName }}Options
+ // ApiClient *helper.ApiAsyncClient
+}
diff --git a/plugins/icla/models/committer.go b/plugins/icla/models/committer.go
new file mode 100644
index 00000000..b75b50aa
--- /dev/null
+++ b/plugins/icla/models/committer.go
@@ -0,0 +1,32 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "github.com/apache/incubator-devlake/models/common"
+)
+
+type IclaCommitter struct {
+ UserName string `gorm:"primaryKey;type:varchar(255)"`
+ Name string `gorm:"primaryKey;type:varchar(255)"`
+ common.NoPKModel
+}
+
+func (IclaCommitter) TableName() string {
+ return "_tool_icla_committer"
+}
diff --git a/plugins/icla/plugin_main.go b/plugins/icla/plugin_main.go
new file mode 100644
index 00000000..5bcf2345
--- /dev/null
+++ b/plugins/icla/plugin_main.go
@@ -0,0 +1,103 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/icla/models"
+ "github.com/apache/incubator-devlake/plugins/icla/tasks"
+ "github.com/apache/incubator-devlake/runner"
+ "github.com/mitchellh/mapstructure"
+ "github.com/spf13/cobra"
+ "github.com/spf13/viper"
+ "gorm.io/gorm"
+)
+
+// make sure interface is implemented
+var _ core.PluginMeta = (*Icla)(nil)
+var _ core.PluginInit = (*Icla)(nil)
+var _ core.PluginTask = (*Icla)(nil)
+var _ core.PluginApi = (*Icla)(nil)
+
+// Export a variable named PluginEntry for Framework to search and load
+var PluginEntry Icla //nolint
+
+type Icla struct{}
+
+func (plugin Icla) Description() string {
+ return "collect some Icla data"
+}
+
+func (plugin Icla) Init(config *viper.Viper, logger core.Logger, db *gorm.DB) error {
+ // AutoSchemas is a **develop** script to auto migrate models easily.
+ // FIXME Don't submit it as a open source plugin
+ return db.Migrator().AutoMigrate(
+ // TODO add your models in here
+ &models.IclaCommitter{},
+ )
+}
+
+func (plugin Icla) SubTaskMetas() []core.SubTaskMeta {
+ return []core.SubTaskMeta{
+ tasks.CollectCommitterMeta,
+ tasks.ExtractCommitterMeta,
+ }
+}
+
+func (plugin Icla) PrepareTaskData(taskCtx core.TaskContext, options map[string]interface{}) (interface{}, error) {
+ var op tasks.IclaOptions
+ err := mapstructure.Decode(options, &op)
+ if err != nil {
+ return nil, err
+ }
+
+ apiClient, err := tasks.NewIclaApiClient(taskCtx)
+ if err != nil {
+ return nil, err
+ }
+
+ return &tasks.IclaTaskData{
+ Options: &op,
+ ApiClient: apiClient,
+ }, nil
+}
+
+// PkgPath information lost when compiled as plugin(.so)
+func (plugin Icla) RootPkgPath() string {
+ return "github.com/apache/incubator-devlake/plugins/icla"
+}
+
+func (plugin Icla) ApiResources() map[string]map[string]core.ApiResourceHandler {
+ return nil
+}
+
+// standalone mode for debugging
+func main() {
+ cmd := &cobra.Command{Use: "icla"}
+
+ // TODO add your cmd flag if necessary
+ // yourFlag := cmd.Flags().IntP("yourFlag", "y", 8, "TODO add description here")
+ // _ = cmd.MarkFlagRequired("yourFlag")
+
+ cmd.Run = func(cmd *cobra.Command, args []string) {
+ runner.DirectRun(cmd, args, PluginEntry, []string{}, map[string]interface{}{
+ // TODO add more custom params here
+ })
+ }
+ runner.RunCmd(cmd)
+}
diff --git a/plugins/icla/tasks/api_client.go b/plugins/icla/tasks/api_client.go
new file mode 100644
index 00000000..6e4944ab
--- /dev/null
+++ b/plugins/icla/tasks/api_client.go
@@ -0,0 +1,69 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "fmt"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/apache/incubator-devlake/utils"
+ "net/http"
+)
+
+const ENDPOINT = "https://people.apache.org/"
+
+func NewIclaApiClient(taskCtx core.TaskContext) (*helper.ApiAsyncClient, error) {
+ // load and process configuration
+ token := taskCtx.GetConfig("ICLA_TOKEN")
+ if token == "" {
+ println("invalid ICLA_TOKEN, but ignore this error now")
+ }
+ userRateLimit, err := utils.StrToIntOr(taskCtx.GetConfig("ICLA_API_REQUESTS_PER_HOUR"), 18000)
+ if err != nil {
+ return nil, err
+ }
+ proxy := taskCtx.GetConfig("ICLA_PROXY")
+
+ // real request apiClient
+ apiClient, err := helper.NewApiClient(ENDPOINT, nil, 0, proxy, taskCtx.GetContext())
+ if err != nil {
+ return nil, err
+ }
+ // set token
+ apiClient.SetHeaders(map[string]string{
+ "Authorization": fmt.Sprintf("Bearer %v", token),
+ })
+
+ // TODO add some check after request if necessary
+ apiClient.SetAfterFunction(func(res *http.Response) error {
+ if res.StatusCode == http.StatusUnauthorized {
+ return fmt.Errorf("authentication failed, please check your Bearer Auth Token")
+ }
+ return nil
+ })
+
+ // create async api client
+ asyncApiClient, err := helper.CreateAsyncApiClient(taskCtx, apiClient, &helper.ApiRateLimitCalculator{
+ UserRateLimitPerHour: userRateLimit,
+ })
+ if err != nil {
+ return nil, err
+ }
+
+ return asyncApiClient, nil
+}
diff --git a/plugins/icla/tasks/committer_collector.go b/plugins/icla/tasks/committer_collector.go
new file mode 100644
index 00000000..484b2f1c
--- /dev/null
+++ b/plugins/icla/tasks/committer_collector.go
@@ -0,0 +1,73 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "net/http"
+ "net/url"
+)
+
+const RAW_COMMITTER_TABLE = "icla_committer"
+
+var _ core.SubTaskEntryPoint = CollectCommitter
+
+func CollectCommitter(taskCtx core.SubTaskContext) error {
+ data := taskCtx.GetData().(*IclaTaskData)
+
+ collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: IclaApiParams{},
+ Table: RAW_COMMITTER_TABLE,
+ },
+ ApiClient: data.ApiClient,
+ Incremental: false,
+ UrlTemplate: "public/icla-info.json",
+ Query: func(reqData *helper.RequestData) (url.Values, error) {
+ query := url.Values{}
+ return query, nil
+ },
+ ResponseParser: func(res *http.Response) ([]json.RawMessage, error) {
+ body := &struct {
+ LastUpdated string `json:"last_updated"`
+ Committers json.RawMessage `json:"committers"`
+ }{}
+ err := helper.UnmarshalResponse(res, body)
+ if err != nil {
+ return nil, err
+ }
+ println("receive data:", len(body.Committers))
+ return []json.RawMessage{body.Committers}, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return collector.Execute()
+}
+
+var CollectCommitterMeta = core.SubTaskMeta{
+ Name: "CollectCommitter",
+ EntryPoint: CollectCommitter,
+ EnabledByDefault: true,
+ Description: "Collect Committer data from Icla api",
+}
diff --git a/plugins/icla/tasks/committer_extractor.go b/plugins/icla/tasks/committer_extractor.go
new file mode 100644
index 00000000..74ca28ca
--- /dev/null
+++ b/plugins/icla/tasks/committer_extractor.go
@@ -0,0 +1,64 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/apache/incubator-devlake/plugins/icla/models"
+)
+
+var _ core.SubTaskEntryPoint = ExtractCommitter
+
+func ExtractCommitter(taskCtx core.SubTaskContext) error {
+ extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
+ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: IclaApiParams{},
+ Table: RAW_COMMITTER_TABLE,
+ },
+ Extract: func(resData *helper.RawData) ([]interface{}, error) {
+ names := &map[string]string{}
+ err := json.Unmarshal(resData.Data, names)
+ if err != nil {
+ return nil, err
+ }
+ extractedModels := make([]interface{}, 0)
+ for userName, name := range *names {
+ extractedModels = append(extractedModels, &models.IclaCommitter{
+ UserName: userName,
+ Name: name,
+ })
+ }
+ return extractedModels, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return extractor.Execute()
+}
+
+var ExtractCommitterMeta = core.SubTaskMeta{
+ Name: "ExtractCommitter",
+ EntryPoint: ExtractCommitter,
+ EnabledByDefault: true,
+ Description: "Extract raw data into tool layer table {{ .plugin_name }}_{{ .extractor_data_name }}",
+}
diff --git a/plugins/icla/tasks/task_data.go b/plugins/icla/tasks/task_data.go
new file mode 100644
index 00000000..f6b2de5f
--- /dev/null
+++ b/plugins/icla/tasks/task_data.go
@@ -0,0 +1,36 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import "github.com/apache/incubator-devlake/plugins/helper"
+
+type IclaApiParams struct {
+}
+
+type IclaOptions struct {
+ // TODO add some custom options here if necessary
+ // options means some custom params required by plugin running.
+ // Such As How many rows do your want
+ // You can use it in sub tasks and you need pass it in main.go and pipelines.
+ Tasks []string `json:"tasks,omitempty"`
+}
+
+type IclaTaskData struct {
+ Options *IclaOptions
+ ApiClient *helper.ApiAsyncClient
+}