You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by GitBox <gi...@apache.org> on 2022/06/15 07:22:21 UTC
[GitHub] [incubator-devlake] klesh commented on a diff in pull request #2190: update e2e test helper
klesh commented on code in PR #2190:
URL: https://github.com/apache/incubator-devlake/pull/2190#discussion_r897614754
##########
helpers/e2ehelper/data_flow_tester.go:
##########
@@ -71,45 +77,67 @@ func NewDataFlowTester(t *testing.T, pluginName string, pluginMeta core.PluginMe
panic(err)
}
cfg := config.GetConfig()
+ e2eDbUrl := cfg.GetString(`E2E_DB_URL`)
+ if e2eDbUrl == `` {
+ panic(fmt.Errorf(`e2e can only run with E2E_DB_URL, please set it in .env`))
+ }
+ cfg.Set(`DB_URL`, cfg.GetString(`E2E_DB_URL`))
db, err := runner.NewGormDb(cfg, logger.Global)
if err != nil {
panic(err)
}
return &DataFlowTester{
Cfg: cfg,
Db: db,
+ Dal: dalgorm.NewDalgorm(db),
T: t,
Name: pluginName,
Plugin: pluginMeta,
Log: logger.Global,
}
}
-// ImportCsv imports records from specified csv file into target table, note that existing data would be deleted first.
-func (t *DataFlowTester) ImportCsv(csvRelPath string, tableName string) {
+// ImportCsvIntoRawTable imports records from specified csv file into target table, note that existing data would be deleted first.
+func (t *DataFlowTester) ImportCsvIntoRawTable(csvRelPath string, tableName string) {
Review Comment:
Why? I thought we agree that we should be able to import table other than `RawTable`
##########
helpers/e2ehelper/data_flow_tester.go:
##########
@@ -124,11 +152,81 @@ func (t *DataFlowTester) Subtask(subtaskMeta core.SubTaskMeta, taskData interfac
}
}
+// CreateSnapshot reads rows from database and write them into .csv file.
+func (t *DataFlowTester) CreateSnapshot(dst schema.Tabler, csvRelPath string, pkfields []string, targetfields []string) {
+ location, _ := time.LoadLocation(`UTC`)
+ var allFields []string
Review Comment:
Mind the linting error
##########
helpers/e2ehelper/data_flow_tester.go:
##########
@@ -71,45 +77,67 @@ func NewDataFlowTester(t *testing.T, pluginName string, pluginMeta core.PluginMe
panic(err)
}
cfg := config.GetConfig()
+ e2eDbUrl := cfg.GetString(`E2E_DB_URL`)
+ if e2eDbUrl == `` {
+ panic(fmt.Errorf(`e2e can only run with E2E_DB_URL, please set it in .env`))
+ }
+ cfg.Set(`DB_URL`, cfg.GetString(`E2E_DB_URL`))
db, err := runner.NewGormDb(cfg, logger.Global)
if err != nil {
panic(err)
}
return &DataFlowTester{
Cfg: cfg,
Db: db,
+ Dal: dalgorm.NewDalgorm(db),
T: t,
Name: pluginName,
Plugin: pluginMeta,
Log: logger.Global,
}
}
-// ImportCsv imports records from specified csv file into target table, note that existing data would be deleted first.
-func (t *DataFlowTester) ImportCsv(csvRelPath string, tableName string) {
+// ImportCsvIntoRawTable imports records from specified csv file into target table, note that existing data would be deleted first.
+func (t *DataFlowTester) ImportCsvIntoRawTable(csvRelPath string, tableName string) {
csvIter := pluginhelper.NewCsvFileIterator(csvRelPath)
defer csvIter.Close()
- // create table if not exists
- err := t.Db.Table(tableName).AutoMigrate(&helper.RawData{})
- if err != nil {
- panic(err)
- }
- t.FlushTable(tableName)
+ t.FlushRawTable(tableName)
// load rows and insert into target table
for csvIter.HasNext() {
- // make sure
- result := t.Db.Table(tableName).Create(csvIter.Fetch())
+ toInsertValues := csvIter.Fetch()
+ // FIXME Hack code
+ if t.Db.Dialector.Name() == `postgres` {
+ toInsertValues[`data`] = strings.Replace(toInsertValues[`data`].(string), `\`, `\\`, -1)
+ }
+ result := t.Db.Table(tableName).Create(toInsertValues)
if result.Error != nil {
panic(result.Error)
}
assert.Equal(t.T, int64(1), result.RowsAffected)
}
}
-// FlushTable deletes all records from specified table
-func (t *DataFlowTester) FlushTable(tableName string) {
+// MigrateRawTableAndFlush migrate table and deletes all records from specified table
+func (t *DataFlowTester) FlushRawTable(rawTableName string) {
Review Comment:
Again, we should be able to flush other kinds of tables as well, shouldn't we?
##########
helpers/e2ehelper/data_flow_tester.go:
##########
@@ -71,45 +77,67 @@ func NewDataFlowTester(t *testing.T, pluginName string, pluginMeta core.PluginMe
panic(err)
}
cfg := config.GetConfig()
+ e2eDbUrl := cfg.GetString(`E2E_DB_URL`)
+ if e2eDbUrl == `` {
+ panic(fmt.Errorf(`e2e can only run with E2E_DB_URL, please set it in .env`))
+ }
+ cfg.Set(`DB_URL`, cfg.GetString(`E2E_DB_URL`))
db, err := runner.NewGormDb(cfg, logger.Global)
if err != nil {
panic(err)
}
return &DataFlowTester{
Cfg: cfg,
Db: db,
+ Dal: dalgorm.NewDalgorm(db),
T: t,
Name: pluginName,
Plugin: pluginMeta,
Log: logger.Global,
}
}
-// ImportCsv imports records from specified csv file into target table, note that existing data would be deleted first.
-func (t *DataFlowTester) ImportCsv(csvRelPath string, tableName string) {
+// ImportCsvIntoRawTable imports records from specified csv file into target table, note that existing data would be deleted first.
+func (t *DataFlowTester) ImportCsvIntoRawTable(csvRelPath string, tableName string) {
csvIter := pluginhelper.NewCsvFileIterator(csvRelPath)
defer csvIter.Close()
- // create table if not exists
- err := t.Db.Table(tableName).AutoMigrate(&helper.RawData{})
- if err != nil {
- panic(err)
- }
- t.FlushTable(tableName)
+ t.FlushRawTable(tableName)
// load rows and insert into target table
for csvIter.HasNext() {
- // make sure
- result := t.Db.Table(tableName).Create(csvIter.Fetch())
+ toInsertValues := csvIter.Fetch()
+ // FIXME Hack code
+ if t.Db.Dialector.Name() == `postgres` {
+ toInsertValues[`data`] = strings.Replace(toInsertValues[`data`].(string), `\`, `\\`, -1)
Review Comment:
Maybe we can check if the `field` is `json.RawMessage`, and convert it to `[]byte` so it could be inserted for all kind of database properly?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org