You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2022/05/17 23:43:56 UTC
[incubator-doris] branch master updated: [regression test] add some case for json load regression test (#9614)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new b6f5c89f6c [regression test] add some case for json load regression test (#9614)
b6f5c89f6c is described below
commit b6f5c89f6c5dff7102af47aee205fa75f4a3f175
Author: carlvinhust2012 <hu...@126.com>
AuthorDate: Wed May 18 07:43:51 2022 +0800
[regression test] add some case for json load regression test (#9614)
Co-authored-by: hucheng01 <hu...@baidu.com>
---
.../data/load/stream_load/simple_object_json.json | 10 ++
.../suites/load/stream_load/test_json_load.groovy | 131 ++++++++++++++++++++-
2 files changed, 135 insertions(+), 6 deletions(-)
diff --git a/regression-test/data/load/stream_load/simple_object_json.json b/regression-test/data/load/stream_load/simple_object_json.json
new file mode 100644
index 0000000000..5a128142ae
--- /dev/null
+++ b/regression-test/data/load/stream_load/simple_object_json.json
@@ -0,0 +1,10 @@
+{"id": 1, "city": "beijing", "code": 2345671}
+{"id": 2, "city": "shanghai", "code": 2345672}
+{"id": 3, "city": "guangzhou", "code": 2345673}
+{"id": 4, "city": "shenzhen", "code": 2345674}
+{"id": 5, "city": "hangzhou", "code": 2345675}
+{"id": 6, "city": "nanjing", "code": 2345676}
+{"id": 7, "city": "wuhan", "code": 2345677}
+{"id": 8, "city": "chengdu", "code": 2345678}
+{"id": 9, "city": "xian", "code": 2345679}
+{"id": 10, "city": "hefei", "code": 23456710}
\ No newline at end of file
diff --git a/regression-test/suites/load/stream_load/test_json_load.groovy b/regression-test/suites/load/stream_load/test_json_load.groovy
index cd5d17e712..305de78eac 100644
--- a/regression-test/suites/load/stream_load/test_json_load.groovy
+++ b/regression-test/suites/load/stream_load/test_json_load.groovy
@@ -36,7 +36,7 @@ suite("test_json_load", "load") {
assertTrue(result1[0][0] == 0, "Create table should update 0 rows")
// insert 1 row to check whether the table is ok
- def result2 = sql "INSERT INTO test_json_load (id, city, code) VALUES (200, 'shenzhen', 0755)"
+ def result2 = sql "INSERT INTO test_json_load (id, city, code) VALUES (200, 'changsha', 3456789)"
assertTrue(result2.size() == 1)
assertTrue(result2[0].size() == 1)
assertTrue(result2[0][0] == 1, "Insert should update 1 rows")
@@ -96,6 +96,39 @@ suite("test_json_load", "load") {
}
}
}
+
+ def load_from_hdfs1 = {testTablex, label, hdfsFilePath, format, brokerName, hdfsUser, hdfsPasswd ->
+ def result1= sql """
+ LOAD LABEL ${label} (
+ DATA INFILE("${hdfsFilePath}")
+ INTO TABLE ${testTablex}
+ FORMAT as "${format}"
+ PRECEDING FILTER id > 1 and id < 10)
+ with BROKER "${brokerName}"
+ ("username"="${hdfsUser}", "password"="${hdfsPasswd}");
+ """
+
+ assertTrue(result1.size() == 1)
+ assertTrue(result1[0].size() == 1)
+ assertTrue(result1[0][0] == 0, "Query OK, 0 rows affected")
+ }
+
+ def load_from_hdfs2 = {testTablex, label, hdfsFilePath, format, brokerName, hdfsUser, hdfsPasswd ->
+ def result1= sql """
+ LOAD LABEL ${label} (
+ DATA INFILE("${hdfsFilePath}")
+ INTO TABLE ${testTablex}
+ FORMAT as "${format}"
+ PRECEDING FILTER id < 10
+ where id > 1 and id < 5)
+ with BROKER "${brokerName}"
+ ("username"="${hdfsUser}", "password"="${hdfsPasswd}");
+ """
+
+ assertTrue(result1.size() == 1)
+ assertTrue(result1[0].size() == 1)
+ assertTrue(result1[0][0] == 0, "Query OK, 0 rows affected")
+ }
// case1: import simple json
try {
@@ -165,8 +198,30 @@ suite("test_json_load", "load") {
} finally {
try_sql("DROP TABLE IF EXISTS ${testTable}")
}
+
+ // case4: import json and apply jsonpaths & exprs
+ try {
+ sql "DROP TABLE IF EXISTS ${testTable}"
+
+ create_test_table2.call(testTable)
+
+ load_json_data.call('true', '', 'json', 'code = id * 10 + 200', '[\"$.id\"]',
+ '', '', '', 'simple_json.json')
+
+ def result3 = sql "select * from test_json_load order by id"
+ assertTrue(result3.size() == 11)
+ assertTrue(result3[0].size() == 2)
+ assertTrue(result3[0][0] == 1)
+ assertTrue(result3[0][1] == 210)
+ assertTrue(result3[9].size() == 2)
+ assertTrue(result3[9][0] == 10)
+ assertTrue(result3[9][1] == 300)
+
+ } finally {
+ try_sql("DROP TABLE IF EXISTS ${testTable}")
+ }
- // case4: import json with line reader
+ // case5: import json with line reader
try {
sql "DROP TABLE IF EXISTS ${testTable}"
@@ -188,7 +243,7 @@ suite("test_json_load", "load") {
try_sql("DROP TABLE IF EXISTS ${testTable}")
}
- // case5: import json use exprs and jsonpaths
+ // case6: import json use exprs and jsonpaths
try {
sql "DROP TABLE IF EXISTS ${testTable}"
@@ -210,7 +265,7 @@ suite("test_json_load", "load") {
try_sql("DROP TABLE IF EXISTS ${testTable}")
}
- // case6: import json use where
+ // case7: import json use where
try {
sql "DROP TABLE IF EXISTS ${testTable}"
@@ -234,7 +289,7 @@ suite("test_json_load", "load") {
try_sql("DROP TABLE IF EXISTS ${testTable}")
}
- // case7: import json use fuzzy_parse
+ // case8: import json use fuzzy_parse
try {
sql "DROP TABLE IF EXISTS ${testTable}"
@@ -258,7 +313,7 @@ suite("test_json_load", "load") {
try_sql("DROP TABLE IF EXISTS ${testTable}")
}
- // case8: import json use json_root
+ // case9: import json use json_root
try {
sql "DROP TABLE IF EXISTS ${testTable}"
@@ -280,4 +335,68 @@ suite("test_json_load", "load") {
} finally {
try_sql("DROP TABLE IF EXISTS ${testTable}")
}
+
+ // if 'enableHdfs' in regression-conf.groovy has been set to true,
+ // the test will run these case as below.
+ if (enableHdfs()) {
+ brokerName =getBrokerName()
+ hdfsUser = getHdfsUser()
+ hdfsPasswd = getHdfsPasswd()
+ def hdfs_file_path = uploadToHdfs "stream_load/simple_object_json.json"
+ def format = "json"
+
+ // case10: import json use pre-filter exprs
+ try {
+ sql "DROP TABLE IF EXISTS ${testTable}"
+
+ create_test_table1.call(testTable)
+
+ def test_load_label = UUID.randomUUID().toString().replaceAll("-", "")
+ load_from_hdfs1.call(testTable, test_load_label, hdfs_file_path, format,
+ brokerName, hdfsUser, hdfsPasswd)
+
+ // wait to load finished
+ sleep(5000)
+
+ def result3 = sql "select * from test_json_load order by id"
+ assertTrue(result3.size() == 9)
+ assertTrue(result3[0].size() == 3)
+ assertTrue(result3[0][0] == 2)
+ assertTrue(result3[0][1] == "shanghai")
+ assertTrue(result3[0][2] == 2345672)
+ assertTrue(result3[7].size() == 3)
+ assertTrue(result3[7][0] == 9)
+ assertTrue(result3[7][1] == "xian")
+ assertTrue(result3[7][2] == 2345679)
+ } finally {
+ try_sql("DROP TABLE IF EXISTS ${testTable}")
+ }
+
+ // case11: import json use pre-filter and where exprs
+ try {
+ sql "DROP TABLE IF EXISTS ${testTable}"
+
+ create_test_table1.call(testTable)
+
+ def test_load_label = UUID.randomUUID().toString().replaceAll("-", "")
+ load_from_hdfs2.call(testTable, test_load_label, hdfs_file_path, format,
+ brokerName, hdfsUser, hdfsPasswd)
+
+ // wait to load finished
+ sleep(5000)
+
+ def result3 = sql "select * from test_json_load order by id"
+ assertTrue(result3.size() == 4)
+ assertTrue(result3[0].size() == 3)
+ assertTrue(result3[0][0] == 2)
+ assertTrue(result3[0][1] == "shanghai")
+ assertTrue(result3[0][2] == 2345672)
+ assertTrue(result3[2].size() == 3)
+ assertTrue(result3[2][0] == 4)
+ assertTrue(result3[2][1] == "shenzhen")
+ assertTrue(result3[2][2] == 2345674)
+ } finally {
+ try_sql("DROP TABLE IF EXISTS ${testTable}")
+ }
+ }
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org