You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by zh...@apache.org on 2023/05/19 13:02:34 UTC
[incubator-devlake] branch main updated: fix: invalid byte sequence for encoding UTF8: 0x00 (#5244)
This is an automated email from the ASF dual-hosted git repository.
zhangliang2022 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new b4894bf90 fix: invalid byte sequence for encoding UTF8: 0x00 (#5244)
b4894bf90 is described below
commit b4894bf90f7673424df5f0fc201a05e774ce95ec
Author: Liang Zhang <li...@merico.dev>
AuthorDate: Fri May 19 21:02:29 2023 +0800
fix: invalid byte sequence for encoding UTF8: 0x00 (#5244)
---
.../jira/e2e/raw_tables/_raw_jira_api_issues.csv | 2 +-
.../plugins/jira/tasks/apiv2models/changelog.go | 13 +++--
backend/plugins/jira/tasks/apiv2models/comment.go | 4 +-
backend/plugins/jira/tasks/apiv2models/issue.go | 1 +
backend/plugins/jira/tasks/apiv2models/util.go | 33 ++++++++++++
.../plugins/jira/tasks/apiv2models/util_test.go | 58 ++++++++++++++++++++++
6 files changed, 105 insertions(+), 6 deletions(-)
diff --git a/backend/plugins/jira/e2e/raw_tables/_raw_jira_api_issues.csv b/backend/plugins/jira/e2e/raw_tables/_raw_jira_api_issues.csv
index 347027c36..f4e09684b 100644
--- a/backend/plugins/jira/e2e/raw_tables/_raw_jira_api_issues.csv
+++ b/backend/plugins/jira/e2e/raw_tables/_raw_jira_api_issues.csv
@@ -2,7 +2,7 @@
12441,"{""ConnectionId"":2,""BoardId"":8}","{""id"": ""10063"", ""key"": ""EE-1"", ""self"": ""https://merico.atlassian.net/rest/agile/1.0/issue/10063"", ""expand"": ""operations,versionedRepresentations,editmeta,changelog,renderedFields"", ""fields"": {""epic"": null, ""votes"": {""self"": ""https://merico.atlassian.net/rest/api/2/issue/EE-1/votes"", ""votes"": 0, ""hasVoted"": false}, ""labels"": [""frontEnd"",""Saas""], ""sprint"": null, ""status"": {""id"": ""10068"", ""name"": ""已完成 [...]
12442,"{""ConnectionId"":2,""BoardId"":8}","{""id"": ""10064"", ""key"": ""EE-2"", ""self"": ""https://merico.atlassian.net/rest/agile/1.0/issue/10064"", ""expand"": ""operations,versionedRepresentations,editmeta,changelog,renderedFields"", ""fields"": {""epic"": null, ""votes"": {""self"": ""https://merico.atlassian.net/rest/api/2/issue/EE-2/votes"", ""votes"": 0, ""hasVoted"": false}, ""labels"": [""frontEnd"",""Saas""], ""sprint"": null, ""status"": {""id"": ""10068"", ""name"": ""已完成 [...]
12443,"{""ConnectionId"":2,""BoardId"":8}","{""id"": ""10065"", ""key"": ""EE-3"", ""self"": ""https://merico.atlassian.net/rest/agile/1.0/issue/10065"", ""expand"": ""operations,versionedRepresentations,editmeta,changelog,renderedFields"", ""fields"": {""epic"": null, ""votes"": {""self"": ""https://merico.atlassian.net/rest/api/2/issue/EE-3/votes"", ""votes"": 0, ""hasVoted"": false}, ""labels"": [""frontEnd"",""Saas""], ""sprint"": null, ""status"": {""id"": ""10068"", ""name"": ""已完成 [...]
-12444,"{""ConnectionId"":2,""BoardId"":8}","{""id"": ""10066"", ""key"": ""EE-4"", ""self"": ""https://merico.atlassian.net/rest/agile/1.0/issue/10066"", ""expand"": ""operations,versionedRepresentations,editmeta,changelog,renderedFields"", ""fields"": {""epic"": null, ""votes"": {""self"": ""https://merico.atlassian.net/rest/api/2/issue/EE-4/votes"", ""votes"": 0, ""hasVoted"": false}, ""labels"": [""frontEnd"",""Saas""], ""sprint"": null, ""status"": {""id"": ""10068"", ""name"": ""已完成 [...]
+12444,"{""ConnectionId"":2,""BoardId"":8}","{""id"": ""10066"", ""key"": ""EE-4"", ""self"": ""https://merico.atlassian.net/rest/agile/1.0/issue/10066"", ""expand"": ""operations,versionedRepresentations,editmeta,changelog,renderedFields"", ""fields"": {""epic"": null, ""votes"": {""self"": ""https://merico.atlassian.net/rest/api/2/issue/EE-4/votes"", ""votes"": 0, ""hasVoted"": false}, ""labels"": [""frontEnd"",""Saas""], ""sprint"": null, ""status"": {""id"": ""10068"", ""name"": ""已完成 [...]
12445,"{""ConnectionId"":2,""BoardId"":8}","{""id"": ""10067"", ""key"": ""EE-5"", ""self"": ""https://merico.atlassian.net/rest/agile/1.0/issue/10067"", ""expand"": ""operations,versionedRepresentations,editmeta,changelog,renderedFields"", ""fields"": {""epic"": null, ""votes"": {""self"": ""https://merico.atlassian.net/rest/api/2/issue/EE-5/votes"", ""votes"": 0, ""hasVoted"": false}, ""labels"": [""frontEnd"",""Saas""], ""sprint"": null, ""status"": {""id"": ""10068"", ""name"": ""已完成 [...]
12446,"{""ConnectionId"":2,""BoardId"":8}","{""id"": ""10068"", ""key"": ""EE-6"", ""self"": ""https://merico.atlassian.net/rest/agile/1.0/issue/10068"", ""expand"": ""operations,versionedRepresentations,editmeta,changelog,renderedFields"", ""fields"": {""epic"": null, ""votes"": {""self"": ""https://merico.atlassian.net/rest/api/2/issue/EE-6/votes"", ""votes"": 0, ""hasVoted"": false}, ""labels"": [""frontEnd"",""Saas""], ""sprint"": null, ""status"": {""id"": ""10068"", ""name"": ""已完成 [...]
12447,"{""ConnectionId"":2,""BoardId"":8}","{""id"": ""10070"", ""key"": ""EE-8"", ""self"": ""https://merico.atlassian.net/rest/agile/1.0/issue/10070"", ""expand"": ""operations,versionedRepresentations,editmeta,changelog,renderedFields"", ""fields"": {""epic"": null, ""votes"": {""self"": ""https://merico.atlassian.net/rest/api/2/issue/EE-8/votes"", ""votes"": 0, ""hasVoted"": false}, ""labels"": [""frontEnd"",""Saas""], ""sprint"": null, ""status"": {""id"": ""10068"", ""name"": ""已完成 [...]
diff --git a/backend/plugins/jira/tasks/apiv2models/changelog.go b/backend/plugins/jira/tasks/apiv2models/changelog.go
index 9e0ae4371..7c7cadf54 100644
--- a/backend/plugins/jira/tasks/apiv2models/changelog.go
+++ b/backend/plugins/jira/tasks/apiv2models/changelog.go
@@ -18,9 +18,10 @@ limitations under the License.
package apiv2models
import (
+ "time"
+
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/jira/models"
- "time"
)
type Changelog struct {
@@ -31,7 +32,7 @@ type Changelog struct {
}
func (c Changelog) ToToolLayer(connectionId, issueId uint64, issueUpdated *time.Time) (*models.JiraIssueChangelogs, *models.JiraAccount) {
- return &models.JiraIssueChangelogs{
+ changelog := &models.JiraIssueChangelogs{
ConnectionId: connectionId,
ChangelogId: c.ID,
IssueId: issueId,
@@ -40,7 +41,9 @@ func (c Changelog) ToToolLayer(connectionId, issueId uint64, issueUpdated *time.
AuthorActive: c.Author.Active,
Created: c.Created.ToTime(),
IssueUpdated: issueUpdated,
- }, c.Author.ToToolLayer(connectionId)
+ }
+ stripZeroByte(changelog)
+ return changelog, c.Author.ToToolLayer(connectionId)
}
type ChangelogItem struct {
@@ -53,7 +56,7 @@ type ChangelogItem struct {
}
func (c ChangelogItem) ToToolLayer(connectionId, changelogId uint64) *models.JiraIssueChangelogItems {
- return &models.JiraIssueChangelogItems{
+ item := &models.JiraIssueChangelogItems{
ConnectionId: connectionId,
ChangelogId: changelogId,
Field: c.Field,
@@ -63,6 +66,8 @@ func (c ChangelogItem) ToToolLayer(connectionId, changelogId uint64) *models.Jir
ToValue: c.ToValue,
ToString: c.ToString,
}
+ stripZeroByte(item)
+ return item
}
func (c ChangelogItem) ExtractUser(connectionId uint64) []*models.JiraAccount {
diff --git a/backend/plugins/jira/tasks/apiv2models/comment.go b/backend/plugins/jira/tasks/apiv2models/comment.go
index c02957a8b..96968f5ac 100644
--- a/backend/plugins/jira/tasks/apiv2models/comment.go
+++ b/backend/plugins/jira/tasks/apiv2models/comment.go
@@ -18,9 +18,10 @@ limitations under the License.
package apiv2models
import (
+ "time"
+
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/jira/models"
- "time"
)
type Comment struct {
@@ -49,5 +50,6 @@ func (c Comment) ToToolLayer(connectionId uint64, issueId uint64, issueUpdated *
result.CreatorAccountId = c.Author.getAccountId()
result.CreatorDisplayName = c.Author.DisplayName
}
+ stripZeroByte(result)
return result
}
diff --git a/backend/plugins/jira/tasks/apiv2models/issue.go b/backend/plugins/jira/tasks/apiv2models/issue.go
index 8a0651acd..803a5f926 100644
--- a/backend/plugins/jira/tasks/apiv2models/issue.go
+++ b/backend/plugins/jira/tasks/apiv2models/issue.go
@@ -216,6 +216,7 @@ func (i Issue) toToolLayer(connectionId uint64) *models.JiraIssue {
if i.Fields.Timespent != nil {
result.SpentMinutes = *i.Fields.Timespent / 60
}
+ stripZeroByte(result)
return result
}
diff --git a/backend/plugins/jira/tasks/apiv2models/util.go b/backend/plugins/jira/tasks/apiv2models/util.go
new file mode 100644
index 000000000..3e15cd319
--- /dev/null
+++ b/backend/plugins/jira/tasks/apiv2models/util.go
@@ -0,0 +1,33 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package apiv2models
+
+import (
+ "reflect"
+ "strings"
+)
+
+func stripZeroByte(ifc interface{}) {
+ v := reflect.ValueOf(ifc).Elem()
+ for i := 0; i < v.NumField(); i++ {
+ if v.Field(i).Type() == reflect.TypeOf("") {
+ stripped := strings.ReplaceAll(v.Field(i).String(), "\u0000", "")
+ v.Field(i).Set(reflect.ValueOf(stripped))
+ }
+ }
+}
diff --git a/backend/plugins/jira/tasks/apiv2models/util_test.go b/backend/plugins/jira/tasks/apiv2models/util_test.go
new file mode 100644
index 000000000..49f529b40
--- /dev/null
+++ b/backend/plugins/jira/tasks/apiv2models/util_test.go
@@ -0,0 +1,58 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package apiv2models
+
+import (
+ "github.com/apache/incubator-devlake/plugins/jira/models"
+ "testing"
+)
+
+func Test_stripZeroByte(t *testing.T) {
+ type args struct {
+ ifc interface{}
+ }
+ tests := []struct {
+ name string
+ args args
+ }{
+ {
+ name: "Test_stripZeroByte",
+ args: args{
+ ifc: &models.JiraIssueChangelogItems{
+ Field: "home\u0000",
+ FromString: "Earth",
+ ToString: "Mars\u0000",
+ },
+ },
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ stripZeroByte(tt.args.ifc)
+ if tt.args.ifc.(*models.JiraIssueChangelogItems).Field != "home" {
+ t.Errorf("stripZeroByte() = %v, want %v", tt.args.ifc.(*models.JiraIssueChangelogItems).Field, "home")
+ }
+ if tt.args.ifc.(*models.JiraIssueChangelogItems).FromString != "Earth" {
+ t.Errorf("stripZeroByte() = %v, want %v", tt.args.ifc.(*models.JiraIssueChangelogItems).FromString, "Earth")
+ }
+ if tt.args.ifc.(*models.JiraIssueChangelogItems).ToString != "Mars" {
+ t.Errorf("stripZeroByte() = %v, want %v", tt.args.ifc.(*models.JiraIssueChangelogItems).ToString, "Mars")
+ }
+ })
+ }
+}