You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@carbondata.apache.org by GitBox <gi...@apache.org> on 2021/09/16 06:43:30 UTC
[GitHub] [carbondata] Indhumathi27 commented on a change in pull request #4218: [CARBONDATA-4285] Alter add complex columns with global sort compaction is failed
Indhumathi27 commented on a change in pull request #4218:
URL: https://github.com/apache/carbondata/pull/4218#discussion_r709818240
##########
File path: integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/alterTable/TestAlterTableAddColumns.scala
##########
@@ -583,4 +583,78 @@ class TestAlterTableAddColumns extends QueryTest with BeforeAndAfterAll {
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.ENABLE_VECTOR_READER, "true")
}
+
+ test("test the complex columns with global sort compaction") {
+ sql("DROP TABLE IF EXISTS alter_global1")
+ sql("CREATE TABLE alter_global1(intField INT) STORED AS carbondata " +
+ "TBLPROPERTIES('sort_columns'='intField','sort_scope'='global_sort')")
+ sql("insert into alter_global1 values(1)")
+ sql("insert into alter_global1 values(2)")
+ sql("insert into alter_global1 values(3)")
+ sql( "ALTER TABLE alter_global1 ADD COLUMNS(str1 array<int>)")
+ sql("insert into alter_global1 values(4, array(1))")
+ checkAnswer(sql("select * from alter_global1"),
+ Seq(Row(1, null), Row(2, null), Row(3, null), Row(4, make(Array(1)))))
+ val addedColumns = addedColumnsInSchemaEvolutionEntry("alter_global1")
+ assert(addedColumns.size == 1)
+ sql("alter table alter_global1 compact 'minor'")
+ checkAnswer(sql("select * from alter_global1"),
+ Seq(Row(1, null), Row(2, null), Row(3, null), Row(4, make(Array(1)))))
+ sql("DROP TABLE IF EXISTS alter_global1")
+ }
+
Review comment:
remove extra line here
##########
File path: integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/alterTable/TestAlterTableAddColumns.scala
##########
@@ -583,4 +583,78 @@ class TestAlterTableAddColumns extends QueryTest with BeforeAndAfterAll {
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.ENABLE_VECTOR_READER, "true")
}
+
+ test("test the complex columns with global sort compaction") {
+ sql("DROP TABLE IF EXISTS alter_global1")
+ sql("CREATE TABLE alter_global1(intField INT) STORED AS carbondata " +
+ "TBLPROPERTIES('sort_columns'='intField','sort_scope'='global_sort')")
+ sql("insert into alter_global1 values(1)")
+ sql("insert into alter_global1 values(2)")
+ sql("insert into alter_global1 values(3)")
+ sql( "ALTER TABLE alter_global1 ADD COLUMNS(str1 array<int>)")
+ sql("insert into alter_global1 values(4, array(1))")
+ checkAnswer(sql("select * from alter_global1"),
+ Seq(Row(1, null), Row(2, null), Row(3, null), Row(4, make(Array(1)))))
+ val addedColumns = addedColumnsInSchemaEvolutionEntry("alter_global1")
+ assert(addedColumns.size == 1)
+ sql("alter table alter_global1 compact 'minor'")
+ checkAnswer(sql("select * from alter_global1"),
+ Seq(Row(1, null), Row(2, null), Row(3, null), Row(4, make(Array(1)))))
+ sql("DROP TABLE IF EXISTS alter_global1")
+ }
+
+
+ test("test the multi-level complex columns with global sort compaction") {
+ sql("DROP TABLE IF EXISTS alter_global2")
+ sql("CREATE TABLE alter_global2(intField INT) STORED AS carbondata " +
+ "TBLPROPERTIES('sort_columns'='intField','sort_scope'='global_sort')")
+ sql("insert into alter_global2 values(1)")
+ // multi-level nested array
+ sql(
+ "ALTER TABLE alter_global2 ADD COLUMNS(arr1 array<array<int>>, arr2 array<struct<a1:string," +
+ "map1:Map<string, string>>>) ")
+ sql(
+ "insert into alter_global2 values(1, array(array(1,2)), array(named_struct('a1','st'," +
+ "'map1', map('a','b'))))")
+ // multi-level nested struct
+ sql("ALTER TABLE alter_global2 ADD COLUMNS(struct1 struct<s1:string, arr: array<int>>," +
+ " struct2 struct<num:double,contact:map<string,array<int>>>) ")
+ sql("insert into alter_global2 values(1, " +
+ "array(array(1,2)), array(named_struct('a1','st','map1', map('a','b'))), " +
+ "named_struct('s1','hi','arr',array(1,2)), named_struct('num',2.3,'contact',map('ph'," +
+ "array(1,2))))")
+ // multi-level nested map
+ sql(
+ "ALTER TABLE alter_global2 ADD COLUMNS(map1 map<string,array<string>>, map2 map<string," +
+ "struct<d:int, s:struct<im:string>>>)")
+ sql("insert into alter_global2 values(1, " +
+ "array(array(1,2)), array(named_struct('a1','st','map1', map('a','b'))), " +
+ "named_struct('s1','hi','arr',array(1,2)), named_struct('num',2.3,'contact',map('ph'," +
+ "array(1,2))),map('a',array('hi')), map('a',named_struct('d',23,'s',named_struct('im'," +
+ "'sh'))))")
+ checkAnswer(sql("select * from alter_global2"),
+ Seq(Row(1, null, null, null, null, null, null),
+ Row(1, make(Array(make(Array(1, 2)))), make(Array(Row("st", Map("a" -> "b")))),
+ null, null, null, null),
+ Row(1, make(Array(make(Array(1, 2)))), make(Array(Row("st", Map("a" -> "b")))),
+ Row("hi", make(Array(1, 2))), Row(2.3, Map("ph" -> make(Array(1, 2)))), null, null),
+ Row(1, make(Array(make(Array(1, 2)))), make(Array(Row("st", Map("a" -> "b")))),
+ Row("hi", make(Array(1, 2))), Row(2.3, Map("ph" -> make(Array(1, 2)))),
+ Map("a" -> make(Array("hi"))), Map("a" -> Row(23, Row("sh"))))
+ ))
+ val addedColumns = addedColumnsInSchemaEvolutionEntry("alter_global2")
+ assert(addedColumns.size == 6)
+ sql("alter table alter_global2 compact 'minor'")
+ checkAnswer(sql("select * from alter_global2"),
Review comment:
check answer can be reused here
##########
File path: integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/alterTable/TestAlterTableAddColumns.scala
##########
@@ -583,4 +583,78 @@ class TestAlterTableAddColumns extends QueryTest with BeforeAndAfterAll {
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.ENABLE_VECTOR_READER, "true")
}
+
+ test("test the complex columns with global sort compaction") {
+ sql("DROP TABLE IF EXISTS alter_global1")
+ sql("CREATE TABLE alter_global1(intField INT) STORED AS carbondata " +
+ "TBLPROPERTIES('sort_columns'='intField','sort_scope'='global_sort')")
+ sql("insert into alter_global1 values(1)")
+ sql("insert into alter_global1 values(2)")
+ sql("insert into alter_global1 values(3)")
+ sql( "ALTER TABLE alter_global1 ADD COLUMNS(str1 array<int>)")
+ sql("insert into alter_global1 values(4, array(1))")
+ checkAnswer(sql("select * from alter_global1"),
+ Seq(Row(1, null), Row(2, null), Row(3, null), Row(4, make(Array(1)))))
+ val addedColumns = addedColumnsInSchemaEvolutionEntry("alter_global1")
+ assert(addedColumns.size == 1)
+ sql("alter table alter_global1 compact 'minor'")
+ checkAnswer(sql("select * from alter_global1"),
+ Seq(Row(1, null), Row(2, null), Row(3, null), Row(4, make(Array(1)))))
+ sql("DROP TABLE IF EXISTS alter_global1")
+ }
+
+
+ test("test the multi-level complex columns with global sort compaction") {
+ sql("DROP TABLE IF EXISTS alter_global2")
+ sql("CREATE TABLE alter_global2(intField INT) STORED AS carbondata " +
+ "TBLPROPERTIES('sort_columns'='intField','sort_scope'='global_sort')")
+ sql("insert into alter_global2 values(1)")
+ // multi-level nested array
+ sql(
+ "ALTER TABLE alter_global2 ADD COLUMNS(arr1 array<array<int>>, arr2 array<struct<a1:string," +
+ "map1:Map<string, string>>>) ")
+ sql(
+ "insert into alter_global2 values(1, array(array(1,2)), array(named_struct('a1','st'," +
+ "'map1', map('a','b'))))")
+ // multi-level nested struct
+ sql("ALTER TABLE alter_global2 ADD COLUMNS(struct1 struct<s1:string, arr: array<int>>," +
+ " struct2 struct<num:double,contact:map<string,array<int>>>) ")
+ sql("insert into alter_global2 values(1, " +
+ "array(array(1,2)), array(named_struct('a1','st','map1', map('a','b'))), " +
+ "named_struct('s1','hi','arr',array(1,2)), named_struct('num',2.3,'contact',map('ph'," +
+ "array(1,2))))")
+ // multi-level nested map
+ sql(
+ "ALTER TABLE alter_global2 ADD COLUMNS(map1 map<string,array<string>>, map2 map<string," +
+ "struct<d:int, s:struct<im:string>>>)")
+ sql("insert into alter_global2 values(1, " +
Review comment:
please add a insert with null and empty data also
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: dev-unsubscribe@carbondata.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org