[incubator-devlake] 01/01: fix(tapd): convert unicode to string

[wa...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

warren pushed a commit to branch fix-tapd-convert-unicode
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git

commit 322952225c6da9f09ecbc867e2d5cf4a4e2d9a74
Author: Warren Chen <yi...@merico.dev>
AuthorDate: Mon Feb 13 15:48:09 2023 +0800

    fix(tapd): convert unicode to string
---
 .../tapd/tasks/story_changelog_extractor.go        | 56 +++++++++++++++++-----
 1 file changed, 44 insertions(+), 12 deletions(-)

diff --git a/backend/plugins/tapd/tasks/story_changelog_extractor.go b/backend/plugins/tapd/tasks/story_changelog_extractor.go
index 67b9eda28..f9055b1c8 100644
--- a/backend/plugins/tapd/tasks/story_changelog_extractor.go
+++ b/backend/plugins/tapd/tasks/story_changelog_extractor.go
@@ -19,28 +19,30 @@ package tasks
 
 import (
 	"encoding/json"
-	"github.com/apache/incubator-devlake/core/errors"
-	"github.com/apache/incubator-devlake/core/plugin"
-	"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
-	"github.com/apache/incubator-devlake/plugins/tapd/models"
+	"github.com/apache/incubator-devlake/errors"
+	"strconv"
 	"strings"
+
+	"github.com/apache/incubator-devlake/plugins/core"
+	"github.com/apache/incubator-devlake/plugins/helper"
+	"github.com/apache/incubator-devlake/plugins/tapd/models"
 )
 
-var _ plugin.SubTaskEntryPoint = ExtractStoryChangelog
+var _ core.SubTaskEntryPoint = ExtractStoryChangelog
 
-var ExtractStoryChangelogMeta = plugin.SubTaskMeta{
+var ExtractStoryChangelogMeta = core.SubTaskMeta{
 	Name:             "extractStoryChangelog",
 	EntryPoint:       ExtractStoryChangelog,
 	EnabledByDefault: true,
 	Description:      "Extract raw workspace data into tool layer table _tool_tapd_iterations",
-	DomainTypes:      []string{plugin.DOMAIN_TYPE_TICKET},
+	DomainTypes:      []string{core.DOMAIN_TYPE_TICKET},
 }
 
-func ExtractStoryChangelog(taskCtx plugin.SubTaskContext) errors.Error {
+func ExtractStoryChangelog(taskCtx core.SubTaskContext) errors.Error {
 	rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_STORY_CHANGELOG_TABLE, false)
-	extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
+	extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
 		RawDataSubTaskArgs: *rawDataSubTaskArgs,
-		Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
+		Extract: func(row *helper.RawData) ([]interface{}, errors.Error) {
 			var storyChangelogBody struct {
 				WorkitemChange models.TapdStoryChangelog
 			}
@@ -88,14 +90,23 @@ func ExtractStoryChangelog(taskCtx plugin.SubTaskContext) errors.Error {
 						default:
 							item.ValueBeforeParsed = valueBeforeMap.(string)
 						}
+						err = convertUnicode(&item)
+						if err != nil {
+							return nil, err
+						}
 						results = append(results, &item)
 					}
 				default:
 					item.ConnectionId = data.Options.ConnectionId
 					item.ChangelogId = storyChangelog.Id
 					item.Field = fc.Field
-					item.ValueAfterParsed = strings.Trim(string(fc.ValueAfterParsed), `"`)
-					item.ValueBeforeParsed = strings.Trim(string(fc.ValueBeforeParsed), `"`)
+					item.ValueAfterParsed = valueAfterMap.(string)
+					// as ValueAfterParsed is string, valueBeforeMap is always string
+					item.ValueBeforeParsed = valueBeforeMap.(string)
+				}
+				err = convertUnicode(&item)
+				if err != nil {
+					return nil, err
 				}
 				if item.Field == "iteration_id" {
 					iterationFrom, iterationTo, err := parseIterationChangelog(taskCtx, item.ValueBeforeParsed, item.ValueAfterParsed)
@@ -118,3 +129,24 @@ func ExtractStoryChangelog(taskCtx plugin.SubTaskContext) errors.Error {
 
 	return extractor.Execute()
 }
+
+func unicodeToZh(s string) (string, error) {
+	str, err := strconv.Unquote(strings.Replace(strconv.Quote(s), `\\u`, `\u`, -1))
+	if err != nil {
+		return "", err
+	}
+	return str, nil
+}
+
+func convertUnicode(item *models.TapdStoryChangelogItem) errors.Error {
+	var err errors.Error
+	item.ValueAfterParsed, err = errors.Convert01(unicodeToZh(item.ValueAfterParsed))
+	if err != nil {
+		return err
+	}
+	item.ValueBeforeParsed, err = errors.Convert01(unicodeToZh(item.ValueBeforeParsed))
+	if err != nil {
+		return err
+	}
+	return nil
+}