You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by li...@apache.org on 2023/03/09 01:48:35 UTC
[incubator-devlake] branch main updated: feat: nullable DataFlowTester (#4618)
This is an automated email from the ASF dual-hosted git repository.
likyh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new c293c943a feat: nullable DataFlowTester (#4618)
c293c943a is described below
commit c293c943ae50e880340711b247334a67af9524ba
Author: mindlesscloud <li...@merico.dev>
AuthorDate: Thu Mar 9 09:48:29 2023 +0800
feat: nullable DataFlowTester (#4618)
---
backend/helpers/e2ehelper/data_flow_tester.go | 52 ++++++++++++---
backend/helpers/e2ehelper/nullable_test.go | 78 ++++++++++++++++++++++
.../e2ehelper/testdata/issue_changelogs.csv | 22 +++---
3 files changed, 131 insertions(+), 21 deletions(-)
diff --git a/backend/helpers/e2ehelper/data_flow_tester.go b/backend/helpers/e2ehelper/data_flow_tester.go
index e52a27726..d4b862383 100644
--- a/backend/helpers/e2ehelper/data_flow_tester.go
+++ b/backend/helpers/e2ehelper/data_flow_tester.go
@@ -89,6 +89,8 @@ type TableOptions struct {
// IgnoreTypes similar to IgnoreFields, this will ignore the fields contained in the type. Useful for ignoring embedded
// types and their fields in the target model
IgnoreTypes []interface{}
+ // if Nullable is set to be true, only the string `NULL` will be taken as NULL
+ Nullable bool
}
// NewDataFlowTester create a *DataFlowTester to help developer test their subtasks data flow
@@ -135,8 +137,7 @@ func (t *DataFlowTester) ImportCsvIntoRawTable(csvRelPath string, rawTableName s
}
}
-// ImportCsvIntoTabler imports records from specified csv file into target tabler, note that existing data would be deleted first.
-func (t *DataFlowTester) ImportCsvIntoTabler(csvRelPath string, dst schema.Tabler) {
+func (t *DataFlowTester) importCsv(csvRelPath string, dst schema.Tabler, nullable bool) {
csvIter, _ := pluginhelper.NewCsvFileIterator(csvRelPath)
defer csvIter.Close()
t.FlushTabler(dst)
@@ -144,8 +145,14 @@ func (t *DataFlowTester) ImportCsvIntoTabler(csvRelPath string, dst schema.Table
for csvIter.HasNext() {
toInsertValues := csvIter.Fetch()
for i := range toInsertValues {
- if toInsertValues[i].(string) == `` {
- toInsertValues[i] = nil
+ if nullable {
+ if toInsertValues[i].(string) == `NULL` {
+ toInsertValues[i] = nil
+ }
+ } else {
+ if toInsertValues[i].(string) == `` {
+ toInsertValues[i] = nil
+ }
}
}
result := t.Db.Model(dst).Create(toInsertValues)
@@ -156,6 +163,16 @@ func (t *DataFlowTester) ImportCsvIntoTabler(csvRelPath string, dst schema.Table
}
}
+// ImportCsvIntoTabler imports records from specified csv file into target tabler, the empty string will be taken as NULL. note that existing data would be deleted first.
+func (t *DataFlowTester) ImportCsvIntoTabler(csvRelPath string, dst schema.Tabler) {
+ t.importCsv(csvRelPath, dst, false)
+}
+
+// ImportNullableCsvIntoTabler imports records from specified csv file into target tabler, the `NULL` will be taken as NULL. note that existing data would be deleted first.
+func (t *DataFlowTester) ImportNullableCsvIntoTabler(csvRelPath string, dst schema.Tabler) {
+ t.importCsv(csvRelPath, dst, true)
+}
+
// FlushRawTable migrate table and deletes all records from specified table
func (t *DataFlowTester) FlushRawTable(rawTableName string) {
// flush target table
@@ -272,7 +289,11 @@ func (t *DataFlowTester) CreateSnapshot(dst schema.Tabler, opts TableOptions) {
if value.Valid {
values[i] = value.Time.In(location).Format("2006-01-02T15:04:05.000-07:00")
} else {
- values[i] = ``
+ if opts.Nullable {
+ values[i] = "NULL"
+ } else {
+ values[i] = ""
+ }
}
case *bool:
if *forScanValues[i].(*bool) {
@@ -285,14 +306,22 @@ func (t *DataFlowTester) CreateSnapshot(dst schema.Tabler, opts TableOptions) {
if value.Valid {
values[i] = value.String
} else {
- values[i] = ``
+ if opts.Nullable {
+ values[i] = "NULL"
+ } else {
+ values[i] = ""
+ }
}
case *sql.NullInt64:
value := *forScanValues[i].(*sql.NullInt64)
if value.Valid {
values[i] = strconv.FormatInt(value.Int64, 10)
} else {
- values[i] = ``
+ if opts.Nullable {
+ values[i] = "NULL"
+ } else {
+ values[i] = ""
+ }
}
case *string:
values[i] = fmt.Sprint(*forScanValues[i].(*string))
@@ -333,7 +362,10 @@ func (t *DataFlowTester) ExportRawTable(rawTableName string, csvRelPath string)
}
}
-func formatDbValue(value interface{}) string {
+func formatDbValue(value interface{}, nullable bool) string {
+ if nullable && value == nil {
+ return "NULL"
+ }
location, _ := time.LoadLocation(`UTC`)
switch value := value.(type) {
case time.Time:
@@ -457,7 +489,7 @@ func (t *DataFlowTester) VerifyTableWithOptions(dst schema.Tabler, opts TableOpt
actualTotal++
pkValues := make([]string, 0, len(pkColumns))
for _, pkc := range pkColumns {
- pkValues = append(pkValues, formatDbValue(actual[pkc.Name()]))
+ pkValues = append(pkValues, formatDbValue(actual[pkc.Name()], opts.Nullable))
}
expected, ok := csvMap[strings.Join(pkValues, `-`)]
assert.True(t.T, ok, fmt.Sprintf(`%s not found (with params from csv %s)`, dst.TableName(), pkValues))
@@ -465,7 +497,7 @@ func (t *DataFlowTester) VerifyTableWithOptions(dst schema.Tabler, opts TableOpt
continue
}
for _, field := range targetFields {
- assert.Equal(t.T, expected[field], formatDbValue(actual[field]), fmt.Sprintf(`%s.%s not match (with params from csv %s)`, dst.TableName(), field, pkValues))
+ assert.Equal(t.T, expected[field], formatDbValue(actual[field], opts.Nullable), fmt.Sprintf(`%s.%s not match (with params from csv %s)`, dst.TableName(), field, pkValues))
}
}
diff --git a/backend/helpers/e2ehelper/nullable_test.go b/backend/helpers/e2ehelper/nullable_test.go
new file mode 100644
index 000000000..84d6386df
--- /dev/null
+++ b/backend/helpers/e2ehelper/nullable_test.go
@@ -0,0 +1,78 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2ehelper
+
+import (
+ "testing"
+)
+
+func TestNullableFlow(t *testing.T) {
+ dataflowTester := NewDataFlowTester(t, "", nil)
+
+ // nullable as false
+ dataflowTester.ImportCsvIntoTabler("./testdata/issue_changelogs.csv", &nullStringTest{})
+ dataflowTester.VerifyTableWithOptions(
+ &nullStringTest{},
+ TableOptions{
+ CSVRelPath: "./testdata/issue_changelogs.csv",
+ TargetFields: ColumnWithRawData(
+ "id",
+ "issue_id",
+ "author_id",
+ "author_name",
+ "field_id",
+ "field_name",
+ "original_from_value",
+ "original_to_value",
+ "from_value",
+ "to_value",
+ "created_date",
+ "_raw_data_params",
+ "_raw_data_table",
+ "_raw_data_id",
+ "_raw_data_remark",
+ ),
+ },
+ )
+ // nullable as true
+ dataflowTester.ImportNullableCsvIntoTabler("./testdata/issue_changelogs.csv", &nullStringTest{})
+ dataflowTester.VerifyTableWithOptions(
+ &nullStringTest{},
+ TableOptions{
+ Nullable: true,
+ CSVRelPath: "./testdata/issue_changelogs.csv",
+ TargetFields: ColumnWithRawData(
+ "id",
+ "issue_id",
+ "author_id",
+ "author_name",
+ "field_id",
+ "field_name",
+ "original_from_value",
+ "original_to_value",
+ "from_value",
+ "to_value",
+ "created_date",
+ "_raw_data_params",
+ "_raw_data_table",
+ "_raw_data_id",
+ "_raw_data_remark",
+ ),
+ },
+ )
+}
diff --git a/backend/helpers/e2ehelper/testdata/issue_changelogs.csv b/backend/helpers/e2ehelper/testdata/issue_changelogs.csv
index cb8150b37..d9d66efd3 100644
--- a/backend/helpers/e2ehelper/testdata/issue_changelogs.csv
+++ b/backend/helpers/e2ehelper/testdata/issue_changelogs.csv
@@ -1,15 +1,15 @@
id,issue_id,author_id,author_name,field_id,field_name,original_from_value,original_to_value,from_value,to_value,created_date,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-jira:JiraIssueChangelogItems:2:10645:Rank,jira:JiraIssue:2:10067,jira:JiraAccount:2:5e9711ba34f7b90c0fbc37d3,Rankin Zheng,,Rank,,Ranked lower,,,2020-06-12T00:17:32.778+00:00,"{""ConnectionId"":2,""BoardId"":8}",_raw_jira_api_issues,12445,
-jira:JiraIssueChangelogItems:2:10646:Fix Version,jira:JiraIssue:2:10064,jira:JiraAccount:2:5e9711ba34f7b90c0fbc37d3,Rankin Zheng,,Fix Version,,v2.7.0,,,2020-06-12T00:17:56.609+00:00,"{""ConnectionId"":2,""BoardId"":8}",_raw_jira_api_issues,12442,
-jira:JiraIssueChangelogItems:2:10647:Fix Version,jira:JiraIssue:2:10065,jira:JiraAccount:2:5e9711ba34f7b90c0fbc37d3,Rankin Zheng,,Fix Version,,v2.7.0,,,2020-06-12T00:17:56.680+00:00,"{""ConnectionId"":2,""BoardId"":8}",_raw_jira_api_issues,12443,
-jira:JiraIssueChangelogItems:2:10648:Fix Version,jira:JiraIssue:2:10066,jira:JiraAccount:2:5e9711ba34f7b90c0fbc37d3,Rankin Zheng,,Fix Version,,v2.7.0,,,2020-06-12T00:17:56.735+00:00,"{""ConnectionId"":2,""BoardId"":8}",_raw_jira_api_issues,12444,
-jira:JiraIssueChangelogItems:2:10649:Fix Version,jira:JiraIssue:2:10068,jira:JiraAccount:2:5e9711ba34f7b90c0fbc37d3,Rankin Zheng,,Fix Version,,v2.7.0,,,2020-06-12T00:17:56.787+00:00,"{""ConnectionId"":2,""BoardId"":8}",_raw_jira_api_issues,12446,
-jira:JiraIssueChangelogItems:2:10650:Fix Version,jira:JiraIssue:2:10067,jira:JiraAccount:2:5e9711ba34f7b90c0fbc37d3,Rankin Zheng,,Fix Version,,v2.7.0,,,2020-06-12T00:18:00.255+00:00,"{""ConnectionId"":2,""BoardId"":8}",_raw_jira_api_issues,12445,
-jira:JiraIssueChangelogItems:2:10655:Fix Version,jira:JiraIssue:2:10072,jira:JiraAccount:2:5e9711ba34f7b90c0fbc37d3,Rankin Zheng,,Fix Version,,v2.7.0,,,2020-06-12T00:19:34.103+00:00,"{""ConnectionId"":2,""BoardId"":8}",_raw_jira_api_issues,12449,
-jira:JiraIssueChangelogItems:2:10656:Fix Version,jira:JiraIssue:2:10070,jira:JiraAccount:2:5e9711ba34f7b90c0fbc37d3,Rankin Zheng,,Fix Version,,v2.7.0,,,2020-06-12T00:19:34.130+00:00,"{""ConnectionId"":2,""BoardId"":8}",_raw_jira_api_issues,12447,
-jira:JiraIssueChangelogItems:2:10657:Fix Version,jira:JiraIssue:2:10071,jira:JiraAccount:2:5e9711ba34f7b90c0fbc37d3,Rankin Zheng,,Fix Version,,v2.7.0,,,2020-06-12T00:19:34.157+00:00,"{""ConnectionId"":2,""BoardId"":8}",_raw_jira_api_issues,12448,
-jira:JiraIssueChangelogItems:2:10659:Epic Link,jira:JiraIssue:2:10063,jira:JiraAccount:2:5e9711ba34f7b90c0fbc37d3,Rankin Zheng,,Epic Link,,EE-11,,,2020-06-12T00:21:20.929+00:00,"{""ConnectionId"":2,""BoardId"":8}",_raw_jira_api_issues,12441,
-jira:JiraIssueChangelogItems:2:10660:Epic Link,jira:JiraIssue:2:10064,jira:JiraAccount:2:5e9711ba34f7b90c0fbc37d3,Rankin Zheng,,Epic Link,,EE-11,,,2020-06-12T00:21:20.980+00:00,"{""ConnectionId"":2,""BoardId"":8}",_raw_jira_api_issues,12442,
+jira:JiraIssueChangelogItems:2:10645:Rank,jira:JiraIssue:2:10067,jira:JiraAccount:2:5e9711ba34f7b90c0fbc37d3,Rankin Zheng,,Rank,,Ranked lower,NULL,,2020-06-12T00:17:32.778+00:00,"{""ConnectionId"":2,""BoardId"":8}",_raw_jira_api_issues,12445,
+jira:JiraIssueChangelogItems:2:10646:Fix Version,jira:JiraIssue:2:10064,jira:JiraAccount:2:5e9711ba34f7b90c0fbc37d3,Rankin Zheng,,Fix Version,,v2.7.0,NULL,,2020-06-12T00:17:56.609+00:00,"{""ConnectionId"":2,""BoardId"":8}",_raw_jira_api_issues,12442,
+jira:JiraIssueChangelogItems:2:10647:Fix Version,jira:JiraIssue:2:10065,jira:JiraAccount:2:5e9711ba34f7b90c0fbc37d3,Rankin Zheng,,Fix Version,,v2.7.0,NULL,,2020-06-12T00:17:56.680+00:00,"{""ConnectionId"":2,""BoardId"":8}",_raw_jira_api_issues,12443,
+jira:JiraIssueChangelogItems:2:10648:Fix Version,jira:JiraIssue:2:10066,jira:JiraAccount:2:5e9711ba34f7b90c0fbc37d3,Rankin Zheng,,Fix Version,,v2.7.0,NULL,,2020-06-12T00:17:56.735+00:00,"{""ConnectionId"":2,""BoardId"":8}",_raw_jira_api_issues,12444,
+jira:JiraIssueChangelogItems:2:10649:Fix Version,jira:JiraIssue:2:10068,jira:JiraAccount:2:5e9711ba34f7b90c0fbc37d3,Rankin Zheng,,Fix Version,,v2.7.0,NULL,,2020-06-12T00:17:56.787+00:00,"{""ConnectionId"":2,""BoardId"":8}",_raw_jira_api_issues,12446,
+jira:JiraIssueChangelogItems:2:10650:Fix Version,jira:JiraIssue:2:10067,jira:JiraAccount:2:5e9711ba34f7b90c0fbc37d3,Rankin Zheng,,Fix Version,,v2.7.0,NULL,,2020-06-12T00:18:00.255+00:00,"{""ConnectionId"":2,""BoardId"":8}",_raw_jira_api_issues,12445,
+jira:JiraIssueChangelogItems:2:10655:Fix Version,jira:JiraIssue:2:10072,jira:JiraAccount:2:5e9711ba34f7b90c0fbc37d3,Rankin Zheng,,Fix Version,,v2.7.0,NULL,,2020-06-12T00:19:34.103+00:00,"{""ConnectionId"":2,""BoardId"":8}",_raw_jira_api_issues,12449,
+jira:JiraIssueChangelogItems:2:10656:Fix Version,jira:JiraIssue:2:10070,jira:JiraAccount:2:5e9711ba34f7b90c0fbc37d3,Rankin Zheng,,Fix Version,,v2.7.0,NULL,,2020-06-12T00:19:34.130+00:00,"{""ConnectionId"":2,""BoardId"":8}",_raw_jira_api_issues,12447,
+jira:JiraIssueChangelogItems:2:10657:Fix Version,jira:JiraIssue:2:10071,jira:JiraAccount:2:5e9711ba34f7b90c0fbc37d3,Rankin Zheng,,Fix Version,,v2.7.0,NULL,,2020-06-12T00:19:34.157+00:00,"{""ConnectionId"":2,""BoardId"":8}",_raw_jira_api_issues,12448,
+jira:JiraIssueChangelogItems:2:10659:Epic Link,jira:JiraIssue:2:10063,jira:JiraAccount:2:5e9711ba34f7b90c0fbc37d3,Rankin Zheng,,Epic Link,,EE-11,NULL,,2020-06-12T00:21:20.929+00:00,"{""ConnectionId"":2,""BoardId"":8}",_raw_jira_api_issues,12441,
+jira:JiraIssueChangelogItems:2:10660:Epic Link,jira:JiraIssue:2:10064,jira:JiraAccount:2:5e9711ba34f7b90c0fbc37d3,Rankin Zheng,,Epic Link,,EE-11,NULL,,2020-06-12T00:21:20.980+00:00,"{""ConnectionId"":2,""BoardId"":8}",_raw_jira_api_issues,12442,
jira:JiraIssueChangelogItems:2:10661:Epic Link,jira:JiraIssue:2:10065,jira:JiraAccount:2:5e9711ba34f7b90c0fbc37d3,Rankin Zheng,,Epic Link,,EE-11,,,2020-06-12T00:21:21.038+00:00,"{""ConnectionId"":2,""BoardId"":8}",_raw_jira_api_issues,12443,
jira:JiraIssueChangelogItems:2:10662:Epic Link,jira:JiraIssue:2:10066,jira:JiraAccount:2:5e9711ba34f7b90c0fbc37d3,Rankin Zheng,,Epic Link,,EE-11,,,2020-06-12T00:21:21.089+00:00,"{""ConnectionId"":2,""BoardId"":8}",_raw_jira_api_issues,12444,
jira:JiraIssueChangelogItems:2:10665:Epic Link,jira:JiraIssue:2:10068,jira:JiraAccount:2:5e9711ba34f7b90c0fbc37d3,Rankin Zheng,,Epic Link,,EE-12,,,2020-06-12T00:22:49.463+00:00,"{""ConnectionId"":2,""BoardId"":8}",_raw_jira_api_issues,12446,