You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@carbondata.apache.org by GitBox <gi...@apache.org> on 2021/03/31 15:25:33 UTC
[GitHub] [carbondata] akkio-97 opened a new pull request #4115: [WIP]
akkio-97 opened a new pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115
### Why is this PR needed?
### What changes were proposed in this PR?
### Does this PR introduce any user interface change?
- No
- Yes. (please explain the change and update document)
### Is any new testcase added?
- No
- Yes
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] akkio-97 commented on a change in pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
akkio-97 commented on a change in pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#discussion_r610393520
##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
##########
@@ -275,6 +275,56 @@ class AlterTableColumnSchemaGenerator(
allColumns ++= Seq(columnSchema)
}
newCols ++= Seq(columnSchema)
+ if(field.dataType== Some("Array") ) {
+ columnSchema.setNumberOfChild(field.children.size)
+ val childField = field.children.get(0)
+ val childSchema: ColumnSchema = TableNewProcessor.createColumnSchema(
+ childField,
Review comment:
Right, thrift is updated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] akkio-97 commented on a change in pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
akkio-97 commented on a change in pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#discussion_r615904251
##########
File path: integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/alterTable/TestAlterTableAddColumns.scala
##########
@@ -136,14 +139,274 @@ class TestAlterTableAddColumns extends QueryTest with BeforeAndAfterAll {
sql(s"""DROP TABLE IF EXISTS ${ tableName }""")
}
+ def addedColumnsInSchemaEvolutionEntry(tableName: String): Int = {
+ val carbonTable = CarbonEnv.getCarbonTable(None, tableName)(sqlContext.sparkSession)
+ val schemaEvolutionList = carbonTable.getTableInfo
+ .getFactTable
+ .getSchemaEvolution()
+ .getSchemaEvolutionEntryList()
+ var addedColumns = 0
+ for (i <- 0 until schemaEvolutionList.size()) {
+ addedColumns += schemaEvolutionList.get(i).getAdded.size()
+ }
+ addedColumns
+ }
+
+ test("Test adding of array of all primitive datatypes") {
+ sql("DROP TABLE IF EXISTS alter_com")
+ sql("CREATE TABLE alter_com(intfield int) STORED AS carbondata")
+ sql(
+ "ALTER TABLE alter_com ADD COLUMNS(arr1 array<short>, arr2 array<int>, arr3 " +
+ "array<long>, arr4 array<double>, arr5 array<decimal(8,2)>, arr6 array<string>, arr7 " +
+ "array<char(5)>, arr8 array<varchar(50)>, arr9 array<boolean>, arr10 array<date>, arr11 " +
+ "array<timestamp> )")
+ val columns = sql("desc table alter_com").collect()
+ assert(columns.size.equals(12))
+ sql(
+ "insert into alter_com values(1,array(1,5),array(1,2),array(1,2,3),array(1.2d,2.3d),array" +
+ "(4.5,6.7),array('hello','world'),array('a','bcd'),array('abcd','efg'),array(true,false)," +
+ "array('2017-02-01','2018-09-11'),array('2017-02-01 00:01:00','2018-02-01 02:21:00') )")
+ checkAnswer(sql(
+ "select * from alter_com"),
+ Seq(Row(1,
+ make(Array(1, 5)),
+ make(Array(1, 2)),
+ make(Array(1, 2, 3)),
+ make(Array(1.2, 2.3)),
+ make(Array(java.math.BigDecimal.valueOf(4.5).setScale(2),
+ java.math.BigDecimal.valueOf(6.7).setScale(2))),
+ make(Array("hello", "world")),
+ make(Array("a", "bcd")),
+ make(Array("abcd", "efg")),
+ make(Array(true, false)),
+ make(Array(Date.valueOf("2017-02-01"),
+ Date.valueOf("2018-09-11"))),
+ make(Array(Timestamp.valueOf("2017-02-01 00:01:00"),
+ Timestamp.valueOf("2018-02-01 02:21:00")))
+ )))
+
+ val addedColumns = addedColumnsInSchemaEvolutionEntry("alter_com")
+ assert(addedColumns == 11)
Review comment:
Done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] akashrn5 commented on pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
akashrn5 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-822983721
LGTM
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] akkio-97 commented on a change in pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
akkio-97 commented on a change in pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#discussion_r613786506
##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
##########
@@ -275,6 +275,64 @@ class AlterTableColumnSchemaGenerator(
allColumns ++= Seq(columnSchema)
}
newCols ++= Seq(columnSchema)
+ if (DataTypes.isArrayType(columnSchema.getDataType)) {
+ columnSchema.setNumberOfChild(field.children.size)
+ val childField = field.children.get(0)
+ // TODO: should support adding multi-level complex columns: CARBONDATA-4164
+ if (childField.children != Some(null)) {
+ throw new UnsupportedOperationException("Only single-level complex types are allowed")
+ }
+ val childSchema: ColumnSchema = TableNewProcessor.createColumnSchema(
Review comment:
done
##########
File path: integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/alterTable/TestAlterTableAddColumns.scala
##########
@@ -136,6 +140,221 @@ class TestAlterTableAddColumns extends QueryTest with BeforeAndAfterAll {
sql(s"""DROP TABLE IF EXISTS ${ tableName }""")
}
+ test("Test adding of array of all primitive datatypes") {
+ sql("DROP TABLE IF EXISTS alter_com")
+ sql("CREATE TABLE alter_com(intfield int) STORED AS carbondata")
+ sql(
+ "ALTER TABLE alter_com ADD COLUMNS(arr1 array<short>, arr2 array<int>, arr3 " +
+ "array<long>, arr4 array<double>, arr5 array<decimal(8,2)>, arr6 array<string>, arr7 " +
+ "array<char(5)>, arr8 array<varchar(50)>, arr9 array<boolean>, arr10 array<date>, arr11 " +
+ "array<timestamp> )")
+ sql("desc table alter_com").show(false)
+ sql(
+ "insert into alter_com values(1,array(1,5),array(1,2),array(1,2,3),array(1.2d,2.3d),array" +
+ "(4.5,6.7),array('hello','world'),array('a','bcd'),array('abcd','efg'),array(true,false)," +
+ "array('2017-02-01','2018-09-11'),array('2017-02-01 00:01:00','2018-02-01 02:21:00') )")
+ checkAnswer(sql(
+ "select * from alter_com"),
+ Seq(Row(1, make(Array(1, 5)),
+ make(Array(1, 2)),
+ make(Array(1, 2, 3)),
+ make(Array(1.2, 2.3)),
+ make(Array(java.math.BigDecimal.valueOf(4.5).setScale(2),
+ java.math.BigDecimal.valueOf(6.7).setScale(2))),
+ make(Array("hello", "world")),
+ make(Array("a", "bcd")),
+ make(Array("abcd", "efg")),
+ make(Array(true, false)),
+ make(Array(Date.valueOf("2017-02-01"), Date.valueOf("2018-09-11"))),
+ make(Array(Timestamp.valueOf("2017-02-01 00:01:00"),
+ Timestamp.valueOf("2018-02-01 02:21:00")))
+ )))
+ sql("DROP TABLE IF EXISTS alter_com")
+ }
+
+ test("Test adding of struct of all primitive datatypes") {
+ sql("DROP TABLE IF EXISTS alter_com")
+ sql(
+ "CREATE TABLE alter_com(intField INT, structField struct<a:short,b:int,c:long,d:double," +
+ "e:decimal(8,2),f:string,g:char(5),h:varchar(50),i:boolean,j:date,k:timestamp>) STORED AS " +
+ "carbondata")
+ sql(
+ "insert into alter_com values(1, named_struct('a',1,'b',2,'c',3,'d',1.23,'e',2.34,'f'," +
+ "'hello','g','abc','h','def','i',true,'j','2017-02-01','k','2018-02-01 02:00:00.0') ) ")
+ sql("DROP TABLE IF EXISTS alter_com")
+ }
+
+ def insertIntoTableForArrayType(): Unit = {
+ sql("insert into alter_com values(4,array(2),array(1,2,3,4),array('abc','def'))")
+ sql("insert into alter_com values(5,array(1,2),array(1), array('Hulk','Thor'))")
+ sql(
+ "insert into alter_com values(6,array(1,2,3),array(1234,8,33333),array('Iron','Man'," +
+ "'Jarvis'))")
+ }
+
+ def checkRestulForArrayType(): Unit = {
+ val totalRows = sql("select * from alter_com").collect()
+ val a = sql("select * from alter_com where array_contains(arr2,2)").collect
+ val b = sql("select * from alter_com where array_contains(arr3,'Thor')").collect
+ val c = sql("select * from alter_com where intField = 1").collect
+ assert(totalRows.size == 6)
+ assert(a.size == 1)
+ assert(b.size == 1)
+ // check default value for newly added array column that is index - 3 and 4
+ assert(c(0)(2) == null && c(0)(3) == null)
+ }
+
+ test("Test alter add for arrays enabling local dictionary") {
+ sql("DROP TABLE IF EXISTS alter_com")
+ sql(
+ "CREATE TABLE alter_com(intField INT, arr1 array<int>) " +
+ "STORED AS carbondata TBLPROPERTIES('local_dictionary_enable'='true')")
+
+ sql("insert into alter_com values(1,array(1) )")
+ sql("insert into alter_com values(2,array(9,0) )")
+ sql("insert into alter_com values(3,array(11,12,13) )")
+
+ sql(
+ "ALTER TABLE alter_com ADD COLUMNS(arr2 array<int>, arr3 array<string>) TBLPROPERTIES" +
+ "('LOCAL_DICTIONARY_INCLUDE'='arr3')")
+ val schema = sql("describe alter_com").collect()
+ assert(schema.size == 4)
+
+ // For the previous segments the default value for newly added array column is null
+ insertIntoTableForArrayType
+ checkRestulForArrayType
+ sql("DROP TABLE IF EXISTS alter_com")
+
+ }
+
+ test("Test alter add for arrays disabling local dictionary") {
+ sql("DROP TABLE IF EXISTS alter_com")
+ sql(
+ "CREATE TABLE alter_com(intField INT, arr1 array<int>) " +
+ "STORED AS carbondata TBLPROPERTIES('local_dictionary_enable'='false')")
+
+ sql("insert into alter_com values(1,array(1) )")
+ sql("insert into alter_com values(2,array(9,0) )")
+ sql("insert into alter_com values(3,array(11,12,13) )")
+ sql(
+ "ALTER TABLE alter_com ADD COLUMNS(arr2 array<int>, arr3 array<string>) TBLPROPERTIES" +
+ "('LOCAL_DICTIONARY_EXCLUDE'='arr3')")
+ val schema = sql("describe alter_com").collect()
+ assert(schema.size == 4)
+
+ // For the previous segments the default value for newly added array column is null
+ insertIntoTableForArrayType
+ checkRestulForArrayType
+ sql("DROP TABLE IF EXISTS alter_com")
+
+ }
+
+ def insertIntoTableForStructType(): Unit = {
+ sql(
+ "insert into alter_struct values(2, named_struct('id1', 'id2','name1','name2'), " +
+ "named_struct('a','id2','b', 'abc2'), 'hello world', 5, named_struct('c','id3'," +
+ "'d', 'abc3','e', 22), array(1,2,3) )")
+ sql(
+ "insert into alter_struct values(3, named_struct('id1', 'id3','name1','name3'), " +
+ "named_struct('a','id2.1','b', 'abc2.1'), 'India', 5, named_struct('c','id3.1'," +
+ "'d', 'abc3.1','e', 25), array(4,5) )")
+
+ }
+
+ def checkResultForStructType(): Unit = {
+ val totalRows = sql("select * from alter_struct").collect()
+ val a = sql("select * from alter_struct where struct1.a = 'id2' ").collect()
+ val b = sql(
+ "select * from alter_struct where struct1.a = 'id2' or struct2.c = 'id3.1' or " +
+ "array_contains(arr,5) ").collect()
+ val c = sql("select * from alter_struct where roll = 1").collect()
+
+ assert(totalRows.size == 3)
+ assert(a.size == 1)
+ assert(b.size == 2)
+ // check default value for newly added struct column
+ assert(c(0)(2) == null)
+ }
+
+ test("Test alter add for structs enabling local dictionary") {
Review comment:
done
##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
##########
@@ -275,6 +275,64 @@ class AlterTableColumnSchemaGenerator(
allColumns ++= Seq(columnSchema)
}
newCols ++= Seq(columnSchema)
+ if (DataTypes.isArrayType(columnSchema.getDataType)) {
+ columnSchema.setNumberOfChild(field.children.size)
+ val childField = field.children.get(0)
+ // TODO: should support adding multi-level complex columns: CARBONDATA-4164
+ if (childField.children != Some(null)) {
+ throw new UnsupportedOperationException("Only single-level complex types are allowed")
+ }
+ val childSchema: ColumnSchema = TableNewProcessor.createColumnSchema(
+ childField,
+ new java.util.ArrayList[Encoding](),
+ isDimensionCol = true,
+ childField.precision,
+ childField.scale,
+ childField.schemaOrdinal + currentSchemaOrdinal,
+ alterTableModel.highCardinalityDims,
+ alterTableModel.databaseName.getOrElse(dbName),
+ isSortColumn(childField.name.getOrElse(childField.column)),
+ isVarcharColumn(childField.name.getOrElse(childField.column)))
+
+ if (childSchema.getDataType == DataTypes.VARCHAR) {
+ // put the new long string columns in 'longStringCols'
+ // and add them after old long string columns
+ longStringCols ++= Seq(childSchema)
+ } else {
+ allColumns ++= Seq(childSchema)
+ }
+ newCols ++= Seq(childSchema)
+ } else if (DataTypes.isStructType(columnSchema.getDataType)) {
+ val noOfChildren = field.children.get.size
+ columnSchema.setNumberOfChild(noOfChildren)
+ for (i <- 0 to noOfChildren - 1) {
+ val childField = field.children.get(i)
+ // TODO: should support adding multi-level complex columns: CARBONDATA-4164
+ if (childField.children != Some(null)) {
+ throw new UnsupportedOperationException("Only single-level complex types are allowed")
Review comment:
done
##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
##########
@@ -275,6 +275,64 @@ class AlterTableColumnSchemaGenerator(
allColumns ++= Seq(columnSchema)
}
newCols ++= Seq(columnSchema)
+ if (DataTypes.isArrayType(columnSchema.getDataType)) {
+ columnSchema.setNumberOfChild(field.children.size)
+ val childField = field.children.get(0)
+ // TODO: should support adding multi-level complex columns: CARBONDATA-4164
+ if (childField.children != Some(null)) {
+ throw new UnsupportedOperationException("Only single-level complex types are allowed")
Review comment:
done
##########
File path: index/secondary-index/src/test/scala/org/apache/carbondata/spark/testsuite/secondaryindex/TestSIWithComplexArrayType.scala
##########
@@ -43,6 +43,83 @@ class TestSIWithComplexArrayType extends QueryTest with BeforeAndAfterEach {
sql("drop table if exists complextable5")
}
+ test("Test alter add array column before creating SI") {
+ sql("drop table if exists complextable")
+ sql("create table complextable (id string, country array<string>, name string) stored as carbondata")
+ sql("insert into complextable select 1,array('china', 'us'), 'b'")
+ sql("insert into complextable select 2,array('pak'), 'v'")
+
+ sql("drop index if exists index_11 on complextable")
+ sql(
+ "ALTER TABLE complextable ADD COLUMNS(arr2 array<string>)")
+ sql("insert into complextable select 3,array('china'), 'f',array('hello','world')")
+ sql("insert into complextable select 4,array('india'),'g',array('iron','man','jarvis')")
+
+ val result1 = sql("select * from complextable where array_contains(arr2,'iron')")
+ val result2 = sql("select * from complextable where arr2[0]='iron'")
+ sql("create index index_11 on table complextable(arr2) as 'carbondata'")
+ val df1 = sql(" select * from complextable where array_contains(arr2,'iron')")
+ val df2 = sql(" select * from complextable where arr2[0]='iron'")
+ if (!isFilterPushedDownToSI(df1.queryExecution.sparkPlan)) {
+ assert(false)
+ } else {
+ assert(true)
+ }
+ if (!isFilterPushedDownToSI(df2.queryExecution.sparkPlan)) {
+ assert(false)
+ } else {
+ assert(true)
+ }
+ val doNotHitSIDf = sql(" select * from complextable where array_contains(arr2,'iron') and array_contains(arr2,'man')")
+ if (isFilterPushedDownToSI(doNotHitSIDf.queryExecution.sparkPlan)) {
+ assert(false)
+ } else {
+ assert(true)
+ }
+ checkAnswer(result1, df1)
+ checkAnswer(result2, df2)
+ }
+
+ test("Test alter add array column after creating SI") {
Review comment:
done
##########
File path: integration/spark/src/test/scala/org/apache/carbondata/view/rewrite/TestAllOperationsOnMV.scala
##########
@@ -79,6 +90,12 @@ class TestAllOperationsOnMV extends QueryTest with BeforeAndAfterEach {
}.getMessage.contains("Cannot add columns in a materialized view table default.dm1")
}
+ test("test alter add complex column on MV table") {
+ intercept[ProcessMetaDataException] {
Review comment:
done
##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
##########
@@ -473,6 +528,11 @@ class AlterTableColumnSchemaGenerator(
for (elem <- alterTableModel.tableProperties) {
if (elem._1.toLowerCase.startsWith(defaultValueString)) {
if (col.getColumnName.equalsIgnoreCase(elem._1.substring(defaultValueString.length))) {
+ if (DataTypes.isArrayType(col.getDataType) || DataTypes.isStructType(col.getDataType) ||
+ DataTypes.isStructType(col.getDataType)) {
Review comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-817557252
Build Failed with Spark 2.4.5, Please check CI http://121.244.95.60:12602/job/ApacheCarbon_PR_Builder_2.4.5/3397/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [CARBONDATA-4160] Alter carbon schema by adding single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-812529533
Build Failed with Spark 2.3.4, Please check CI http://121.244.95.60:12602/job/ApacheCarbonPRBuilder2.3/5119/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-822476499
Build Success with Spark 2.3.4, Please check CI http://121.244.95.60:12602/job/ApacheCarbonPRBuilder2.3/5208/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-822355050
Build Success with Spark 2.4.5, Please check CI http://121.244.95.60:12602/job/ApacheCarbon_PR_Builder_2.4.5/3457/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] akashrn5 commented on a change in pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
akashrn5 commented on a change in pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#discussion_r609832608
##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
##########
@@ -275,6 +275,56 @@ class AlterTableColumnSchemaGenerator(
allColumns ++= Seq(columnSchema)
}
newCols ++= Seq(columnSchema)
+ if(field.dataType== Some("Array") ) {
+ columnSchema.setNumberOfChild(field.children.size)
+ val childField = field.children.get(0)
+ val childSchema: ColumnSchema = TableNewProcessor.createColumnSchema(
+ childField,
Review comment:
@akkio-97 , my bad, i debugged a test case where there were no dimensions
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-819839099
Build Success with Spark 2.4.5, Please check CI http://121.244.95.60:12602/job/ApacheCarbon_PR_Builder_2.4.5/3431/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] Indhumathi27 commented on a change in pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
Indhumathi27 commented on a change in pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#discussion_r613876286
##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParserUtil.scala
##########
@@ -683,12 +683,6 @@ object CarbonSparkSqlParserUtil {
fields: List[Field],
tblProp: Option[List[(String, String)]]
): CarbonAlterTableAddColumnCommand = {
- fields.foreach { f =>
- if (CarbonParserUtil.isComplexType(f.dataType.get)) {
Review comment:
I think, still Alter add Map type is not supported. Please handle
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [WIP]
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-811664877
Build Failed with Spark 2.4.5, Please check CI http://121.244.95.60:12602/job/ApacheCarbon_PR_Builder_2.4.5/3356/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] akkio-97 edited a comment on pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
akkio-97 edited a comment on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-819673737
> @akkio-97 For alter add columns, currently we support a property to provide default value in alter add ddl, like "DEFAULT.VALUE.column_name=somevalue". Please check and handle this scenario also.
1) This is a part of the query flow. Because only when the select command is fired the data assigned will be inserted into the row.
2) The data which we assign as default is not parsed through queryUtil flow and is just a byte array. This data is not encoded so existing parsers and decoders cannot be used here to extract actual data and insert into the row.
3) We need to first write a new parser and then create ArrayQueryType and PrimitiveQueryType. After which insertion can be done.
4) This I think can be handled in a separate PR
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-820272574
Build Success with Spark 2.4.5, Please check CI http://121.244.95.60:12602/job/ApacheCarbon_PR_Builder_2.4.5/3438/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-822254971
Build Success with Spark 2.3.4, Please check CI http://121.244.95.60:12602/job/ApacheCarbonPRBuilder2.3/5203/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] akkio-97 commented on a change in pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
akkio-97 commented on a change in pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#discussion_r613397286
##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParserUtil.scala
##########
@@ -683,12 +683,6 @@ object CarbonSparkSqlParserUtil {
fields: List[Field],
tblProp: Option[List[(String, String)]]
): CarbonAlterTableAddColumnCommand = {
- fields.foreach { f =>
Review comment:
done
##########
File path: integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/alterTable/TestAlterTableAddColumns.scala
##########
@@ -136,6 +140,209 @@ class TestAlterTableAddColumns extends QueryTest with BeforeAndAfterAll {
sql(s"""DROP TABLE IF EXISTS ${ tableName }""")
}
+ test("Test adding of array of all primitive datatypes") {
+ sql("DROP TABLE IF EXISTS alter_com")
+ sql("CREATE TABLE alter_com(intfield int) STORED AS carbondata")
+ sql(
+ "ALTER TABLE alter_com ADD COLUMNS(arr1 array<short>, arr2 array<int>, arr3 " +
+ "array<long>, arr4 array<double>, arr5 array<decimal(8,2)>, arr6 array<string>, arr7 " +
+ "array<char(5)>, arr8 array<varchar(50)>, arr9 array<boolean>, arr10 array<date>, arr11 " +
+ "array<timestamp> )")
+ sql("desc table alter_com").show(false)
+ sql(
+ "insert into alter_com values(1,array(1,5),array(1,2),array(1,2,3),array(1.2d,2.3d),array" +
+ "(4.5,6.7),array('hello','world'),array('a','bcd'),array('abcd','efg'),array(true,false)," +
+ "array('2017-02-01','2018-09-11'),array('2017-02-01 00:01:00','2018-02-01 02:21:00') )")
+ checkAnswer(sql(
+ "select * from alter_com"),
+ Seq(Row(1, make(Array(1, 5)),
+ make(Array(1, 2)),
+ make(Array(1, 2, 3)),
+ make(Array(1.2, 2.3)),
+ make(Array(java.math.BigDecimal.valueOf(4.5).setScale(2),
+ java.math.BigDecimal.valueOf(6.7).setScale(2))),
+ make(Array("hello", "world")),
+ make(Array("a", "bcd")),
+ make(Array("abcd", "efg")),
+ make(Array(true, false)),
+ make(Array(Date.valueOf("2017-02-01"), Date.valueOf("2018-09-11"))),
+ make(Array(Timestamp.valueOf("2017-02-01 00:01:00"),
+ Timestamp.valueOf("2018-02-01 02:21:00")))
+ )))
+ sql("DROP TABLE IF EXISTS alter_com")
+ }
+
+ test("Test adding of struct of all primitive datatypes") {
+ sql("DROP TABLE IF EXISTS alter_com")
+ sql(
+ "CREATE TABLE alter_com(intField INT, structField struct<a:short,b:int,c:long,d:double," +
+ "e:decimal(8,2),f:string,g:char(5),h:varchar(50),i:boolean,j:date,k:timestamp>) STORED AS " +
+ "carbondata")
+ sql(
+ "insert into alter_com values(1, named_struct('a',1,'b',2,'c',3,'d',1.23,'e',2.34,'f'," +
+ "'hello','g','abc','h','def','i',true,'j','2017-02-01','k','2018-02-01 02:00:00.0') ) ")
+ sql("select * from alter_com").show(false)
+ sql("DROP TABLE IF EXISTS alter_com")
+ }
+
+ def insertIntoTableForArrayType(): Unit = {
+ sql("insert into alter_com values(4,array(2),array(1,2,3,4),array('abc','def'))")
+ sql("insert into alter_com values(5,array(1,2),array(1), array('Hulk','Thor'))")
+ sql(
+ "insert into alter_com values(6,array(1,2,3),array(1234,8,33333),array('Iron','Man'," +
+ "'Jarvis'))")
+ }
+
+ def checkRestulForArrayType(): Unit = {
+ val totalRows = sql("select * from alter_com").collect()
+ val a = sql("select * from alter_com where array_contains(arr2,2)").collect
+ val b = sql("select * from alter_com where array_contains(arr3,'Thor')").collect
+ val c = sql("select * from alter_com where intField = 1").collect
+ assert(totalRows.size == 6)
+ assert(a.size == 1)
+ assert(b.size == 1)
+ // check default value for newly added array column that is index - 3 and 4
+ assert(c(0)(2) == null && c(0)(3) == null)
+ }
+
+ test("Test alter add for arrays enabling local dictionary") {
+ sql("DROP TABLE IF EXISTS alter_com")
+ sql(
+ "CREATE TABLE alter_com(intField INT, arr1 array<int>) " +
+ "STORED AS carbondata TBLPROPERTIES('local_dictionary_enable'='true')")
+
+ sql("insert into alter_com values(1,array(1) )")
+ sql("insert into alter_com values(2,array(9,0) )")
+ sql("insert into alter_com values(3,array(11,12,13) )")
+
+ sql(
+ "ALTER TABLE alter_com ADD COLUMNS(arr2 array<int>, arr3 array<string>) TBLPROPERTIES" +
+ "('LOCAL_DICTIONARY_INCLUDE'='arr3')")
+ val schema = sql("describe alter_com").collect()
+ assert(schema.size == 4)
+
+ // For the previous segments the default value for newly added array column is null
+ sql("insert into alter_com values(4,array(2),array(1,2,3,4),array('abc','def'))")
+ sql("insert into alter_com values(5,array(1,2),array(1), array('Hulk','Thor'))")
+ sql(
+ "insert into alter_com values(6,array(1,2,3),array(1234,8,33333),array('Iron','Man'," +
+ "'Jarvis'))")
+
+ val totalRows = sql("select * from alter_com").collect()
+ val a = sql("select * from alter_com where array_contains(arr2,2)").collect
+ val b = sql("select * from alter_com where array_contains(arr3,'Thor')").collect
+ val c = sql("select * from alter_com where intField = 1").collect
+ assert(totalRows.size == 6)
+ assert(a.size == 1)
+ assert(b.size == 1)
+ // check default value for newly added array column that is index - 3 and 4
+ assert(c(0)(2) == null && c(0)(3) == null)
+ sql("DROP TABLE IF EXISTS alter_com")
+
+ }
+
+ test("Test alter add for arrays disabling local dictionary") {
+ sql("DROP TABLE IF EXISTS alter_com")
+ sql(
+ "CREATE TABLE alter_com(intField INT, arr1 array<int>) " +
+ "STORED AS carbondata TBLPROPERTIES('local_dictionary_enable'='false')")
+
+ sql("insert into alter_com values(1,array(1) )")
+ sql("insert into alter_com values(2,array(9,0) )")
+ sql("insert into alter_com values(3,array(11,12,13) )")
+ sql(
+ "ALTER TABLE alter_com ADD COLUMNS(arr2 array<int>, arr3 array<string>) TBLPROPERTIES" +
+ "('LOCAL_DICTIONARY_EXCLUDE'='arr3')")
+ val schema = sql("describe alter_com").collect()
+ assert(schema.size == 4)
+
+ // For the previous segments the default value for newly added array column is null
+ sql("insert into alter_com values(4,array(2),array(1,2,3,4),array('abc','def'))")
+ sql("insert into alter_com values(5,array(1,2),array(1), array('Hulk','Thor'))")
+ sql(
+ "insert into alter_com values(6,array(1,2,3),array(1234,80,3333),array('Iron','Man'," +
+ "'Jarvis'))")
+
+ val totalRows = sql("select * from alter_com").collect()
+ val a = sql("select * from alter_com where array_contains(arr2,2)").collect
+ val b = sql("select * from alter_com where array_contains(arr3,'Thor')").collect
+ val c = sql("select * from alter_com where intField = 1").collect
+ assert(totalRows.size == 6)
+ assert(a.size == 1)
+ assert(b.size == 1)
+ // check default value for newly added array column that is index - 3 and 4
+ assert(c(0)(2) == null && c(0)(3) == null)
+ sql("DROP TABLE IF EXISTS alter_com")
+
+ }
+
+ def insertIntoTableForStructType(): Unit = {
+ sql(
+ "insert into alter_struct values(2, named_struct('id1', 'id2','name1','name2'), " +
+ "named_struct('a','id2','b', 'abc2'), 'hello world', 5, named_struct('c','id3'," +
+ "'d', 'abc3','e', 22), array(1,2,3) )")
+ sql(
+ "insert into alter_struct values(3, named_struct('id1', 'id3','name1','name3'), " +
+ "named_struct('a','id2.1','b', 'abc2.1'), 'India', 5, named_struct('c','id3.1'," +
+ "'d', 'abc3.1','e', 25), array(4,5) )")
+
+ }
+
+ def checkResultForStructType(): Unit = {
+ val totalRows = sql("select * from alter_struct").collect()
+ val a = sql("select * from alter_struct where struct1.a = 'id2' ").collect()
+ val b = sql(
+ "select * from alter_struct where struct1.a = 'id2' or struct2.c = 'id3.1' or " +
+ "array_contains(arr,5) ").collect()
+ val c = sql("select * from alter_struct where roll = 1").collect()
+
+ assert(totalRows.size == 3)
+ assert(a.size == 1)
+ assert(b.size == 2)
+ // check default value for newly added struct column
+ assert(c(0)(2) == null)
+ }
+
+ test("Test alter add for structs enabling local dictionary") {
+ sql("DROP TABLE IF EXISTS alter_struct")
+ sql(
+ "create table alter_struct(roll int, department struct<id1:string,name1:string>) STORED AS " +
+ "carbondata TBLPROPERTIES('local_dictionary_enable'='true')")
+ sql(
+ "insert into alter_struct values(1, named_struct('id1', 'id1','name1','name1'))")
+ sql(
+ "ALTER TABLE alter_struct ADD COLUMNS(struct1 struct<a:string,b:string>, temp string," +
+ " intField int, struct2 struct<c:string,d:string,e:int>, arr array<int>) TBLPROPERTIES " +
+ "('LOCAL_DICTIONARY_INCLUDE'='struct1, struct2')")
+ val schema = sql("describe alter_struct").collect()
+ assert(schema.size == 7)
+ // For the previous segments the default value for newly added struct column is null
+ insertIntoTableForStructType
+ checkResultForStructType
+ sql("DROP TABLE IF EXISTS alter_struct")
+
+ }
+
+ test("Test alter add for structs, disabling local dictionary") {
+ sql("DROP TABLE IF EXISTS alter_struct")
+ sql(
+ "create table alter_struct(roll int, department struct<id1:string,name1:string>) STORED AS " +
+ "carbondata TBLPROPERTIES('local_dictionary_enable'='false')")
+ sql(
+ "insert into alter_struct values(1, named_struct('id1', 'id1','name1','name1'))")
+ sql(
+ "ALTER TABLE alter_struct ADD COLUMNS(struct1 struct<a:string,b:string>, temp string," +
+ " intField int, struct2 struct<c:string,d:string,e:int>, arr array<int>) TBLPROPERTIES " +
+ "('LOCAL_DICTIONARY_EXCLUDE'='struct1, struct2')")
+ val schema = sql("describe alter_struct").collect()
+ assert(schema.size == 7)
+ // For the previous segments the default value for newly added struct column is null
+ insertIntoTableForStructType
+ checkResultForStructType
+ sql("DROP TABLE IF EXISTS alter_struct")
+
+ }
+
Review comment:
done
##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
##########
@@ -275,6 +275,56 @@ class AlterTableColumnSchemaGenerator(
allColumns ++= Seq(columnSchema)
}
newCols ++= Seq(columnSchema)
+ if(field.dataType== Some("Array") ) {
Review comment:
done
##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
##########
@@ -534,10 +581,10 @@ object TableNewProcessor {
if (highCardinalityDims.contains(colName)) {
encoders.remove(Encoding.DICTIONARY)
}
- if (dataType == DataTypes.DATE) {
- encoders.add(Encoding.DICTIONARY)
- encoders.add(Encoding.DIRECT_DICTIONARY)
- } else if (dataType == DataTypes.TIMESTAMP && !highCardinalityDims.contains(colName)) {
+ // complex children does not require encoding
+ if (!colName.contains('.') && (dataType == DataTypes.DATE ||
Review comment:
done
##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
##########
@@ -275,6 +275,56 @@ class AlterTableColumnSchemaGenerator(
allColumns ++= Seq(columnSchema)
}
newCols ++= Seq(columnSchema)
+ if(field.dataType== Some("Array") ) {
+ columnSchema.setNumberOfChild(field.children.size)
+ val childField = field.children.get(0)
+ val childSchema: ColumnSchema = TableNewProcessor.createColumnSchema(
+ childField,
+ new java.util.ArrayList[Encoding](),
+ isDimensionCol = true,
+ childField.precision,
+ childField.scale,
+ childField.schemaOrdinal + currentSchemaOrdinal,
+ alterTableModel.highCardinalityDims,
+ alterTableModel.databaseName.getOrElse(dbName),
+ isSortColumn(childField.name.getOrElse(childField.column)),
+ isVarcharColumn(childField.name.getOrElse(childField.column)))
+
+ if (childSchema.getDataType == DataTypes.VARCHAR) {
+ // put the new long string columns in 'longStringCols'
+ // and add them after old long string columns
+ longStringCols ++= Seq(childSchema)
+ } else {
+ allColumns ++= Seq(childSchema)
+ }
+ newCols ++= Seq(childSchema)
+ } else if (field.dataType == Some("Struct")) {
Review comment:
done
##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
##########
@@ -275,6 +275,56 @@ class AlterTableColumnSchemaGenerator(
allColumns ++= Seq(columnSchema)
}
newCols ++= Seq(columnSchema)
+ if(field.dataType== Some("Array") ) {
Review comment:
done
##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
##########
@@ -275,6 +275,56 @@ class AlterTableColumnSchemaGenerator(
allColumns ++= Seq(columnSchema)
}
newCols ++= Seq(columnSchema)
+ if(field.dataType== Some("Array") ) {
+ columnSchema.setNumberOfChild(field.children.size)
+ val childField = field.children.get(0)
+ val childSchema: ColumnSchema = TableNewProcessor.createColumnSchema(
+ childField,
Review comment:
Sorting is not supported for complex columns
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-821022713
Build Success with Spark 2.3.4, Please check CI http://121.244.95.60:12602/job/ApacheCarbonPRBuilder2.3/5198/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] akkio-97 commented on pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
akkio-97 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-819783720
retest this please
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] akashrn5 commented on pull request #4115: [CARBONDATA-4160] Alter carbon schema by adding single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
akashrn5 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-813866247
@akkio-97 , please create a single parent jira for alter support for complex types, and then create sub tasks based on the different phase activities. After that you can give the sub jira name in title and change title as support Add column
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [WIP]
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-812424124
Build Failed with Spark 2.4.5, Please check CI http://121.244.95.60:12602/job/ApacheCarbon_PR_Builder_2.4.5/3365/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [WIP]
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-812049200
Build Failed with Spark 2.4.5, Please check CI http://121.244.95.60:12602/job/ApacheCarbon_PR_Builder_2.4.5/3362/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [CARBONDATA-4160] Alter carbon schema by adding single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-813232378
Build Success with Spark 2.4.5, Please check CI http://121.244.95.60:12602/job/ApacheCarbon_PR_Builder_2.4.5/3372/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-820277896
Build Success with Spark 2.3.4, Please check CI http://121.244.95.60:12602/job/ApacheCarbonPRBuilder2.3/5190/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-819837007
Build Failed with Spark 2.3.4, Please check CI http://121.244.95.60:12602/job/ApacheCarbonPRBuilder2.3/5183/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] akkio-97 commented on pull request #4115: [CARBONDATA-4160] Alter carbon schema by adding single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
akkio-97 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-812524349
retest this please
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [CARBONDATA-4160] Alter carbon schema by adding single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-812471222
Build Failed with Spark 2.4.5, Please check CI http://121.244.95.60:12602/job/ApacheCarbon_PR_Builder_2.4.5/3366/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [CARBONDATA-4160] Alter carbon schema by adding single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-813232084
Build Success with Spark 2.3.4, Please check CI http://121.244.95.60:12602/job/ApacheCarbonPRBuilder2.3/5123/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] akkio-97 commented on a change in pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
akkio-97 commented on a change in pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#discussion_r614576721
##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
##########
@@ -319,10 +373,6 @@ class AlterTableColumnSchemaGenerator(
sys.error(s"Duplicate column found with name: $name")
})
- if (newCols.exists(_.getDataType.isComplexType)) {
Review comment:
done
##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
##########
@@ -255,6 +276,10 @@ class AlterTableColumnSchemaGenerator(
// add new dimension columns
alterTableModel.dimCols.foreach(field => {
+ if (field.dataType.get.equals("Map")) {
Review comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] Indhumathi27 commented on a change in pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
Indhumathi27 commented on a change in pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#discussion_r613746573
##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
##########
@@ -473,6 +528,11 @@ class AlterTableColumnSchemaGenerator(
for (elem <- alterTableModel.tableProperties) {
if (elem._1.toLowerCase.startsWith(defaultValueString)) {
if (col.getColumnName.equalsIgnoreCase(elem._1.substring(defaultValueString.length))) {
+ if (DataTypes.isArrayType(col.getDataType) || DataTypes.isStructType(col.getDataType) ||
+ DataTypes.isStructType(col.getDataType)) {
Review comment:
Same check twice for Struct type. Please check
##########
File path: integration/spark/src/test/scala/org/apache/carbondata/view/rewrite/TestAllOperationsOnMV.scala
##########
@@ -79,6 +90,12 @@ class TestAllOperationsOnMV extends QueryTest with BeforeAndAfterEach {
}.getMessage.contains("Cannot add columns in a materialized view table default.dm1")
}
+ test("test alter add complex column on MV table") {
+ intercept[ProcessMetaDataException] {
Review comment:
This test case already exists. Please remove duplicate test cases
##########
File path: index/secondary-index/src/test/scala/org/apache/carbondata/spark/testsuite/secondaryindex/TestSIWithComplexArrayType.scala
##########
@@ -43,6 +43,83 @@ class TestSIWithComplexArrayType extends QueryTest with BeforeAndAfterEach {
sql("drop table if exists complextable5")
}
+ test("Test alter add array column before creating SI") {
+ sql("drop table if exists complextable")
+ sql("create table complextable (id string, country array<string>, name string) stored as carbondata")
+ sql("insert into complextable select 1,array('china', 'us'), 'b'")
+ sql("insert into complextable select 2,array('pak'), 'v'")
+
+ sql("drop index if exists index_11 on complextable")
+ sql(
+ "ALTER TABLE complextable ADD COLUMNS(arr2 array<string>)")
+ sql("insert into complextable select 3,array('china'), 'f',array('hello','world')")
+ sql("insert into complextable select 4,array('india'),'g',array('iron','man','jarvis')")
+
+ val result1 = sql("select * from complextable where array_contains(arr2,'iron')")
+ val result2 = sql("select * from complextable where arr2[0]='iron'")
+ sql("create index index_11 on table complextable(arr2) as 'carbondata'")
+ val df1 = sql(" select * from complextable where array_contains(arr2,'iron')")
+ val df2 = sql(" select * from complextable where arr2[0]='iron'")
+ if (!isFilterPushedDownToSI(df1.queryExecution.sparkPlan)) {
+ assert(false)
+ } else {
+ assert(true)
+ }
+ if (!isFilterPushedDownToSI(df2.queryExecution.sparkPlan)) {
+ assert(false)
+ } else {
+ assert(true)
+ }
+ val doNotHitSIDf = sql(" select * from complextable where array_contains(arr2,'iron') and array_contains(arr2,'man')")
+ if (isFilterPushedDownToSI(doNotHitSIDf.queryExecution.sparkPlan)) {
+ assert(false)
+ } else {
+ assert(true)
+ }
+ checkAnswer(result1, df1)
+ checkAnswer(result2, df2)
+ }
+
+ test("Test alter add array column after creating SI") {
Review comment:
I think this testcase validation is not required, as it is smiiliar to testcases that you have added in TestAlterTableAddColumns. Add column on maintable wont affect SI / MV, as they are created with other columns. you can remove those test cases.
##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
##########
@@ -275,6 +275,64 @@ class AlterTableColumnSchemaGenerator(
allColumns ++= Seq(columnSchema)
}
newCols ++= Seq(columnSchema)
+ if (DataTypes.isArrayType(columnSchema.getDataType)) {
+ columnSchema.setNumberOfChild(field.children.size)
+ val childField = field.children.get(0)
+ // TODO: should support adding multi-level complex columns: CARBONDATA-4164
+ if (childField.children != Some(null)) {
+ throw new UnsupportedOperationException("Only single-level complex types are allowed")
Review comment:
```suggestion
throw new UnsupportedOperationException("Alter add columns with single-level complex types are only allowed")
```
##########
File path: integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/alterTable/TestAlterTableAddColumns.scala
##########
@@ -136,6 +140,221 @@ class TestAlterTableAddColumns extends QueryTest with BeforeAndAfterAll {
sql(s"""DROP TABLE IF EXISTS ${ tableName }""")
}
+ test("Test adding of array of all primitive datatypes") {
+ sql("DROP TABLE IF EXISTS alter_com")
+ sql("CREATE TABLE alter_com(intfield int) STORED AS carbondata")
+ sql(
+ "ALTER TABLE alter_com ADD COLUMNS(arr1 array<short>, arr2 array<int>, arr3 " +
+ "array<long>, arr4 array<double>, arr5 array<decimal(8,2)>, arr6 array<string>, arr7 " +
+ "array<char(5)>, arr8 array<varchar(50)>, arr9 array<boolean>, arr10 array<date>, arr11 " +
+ "array<timestamp> )")
+ sql("desc table alter_com").show(false)
+ sql(
+ "insert into alter_com values(1,array(1,5),array(1,2),array(1,2,3),array(1.2d,2.3d),array" +
+ "(4.5,6.7),array('hello','world'),array('a','bcd'),array('abcd','efg'),array(true,false)," +
+ "array('2017-02-01','2018-09-11'),array('2017-02-01 00:01:00','2018-02-01 02:21:00') )")
+ checkAnswer(sql(
+ "select * from alter_com"),
+ Seq(Row(1, make(Array(1, 5)),
+ make(Array(1, 2)),
+ make(Array(1, 2, 3)),
+ make(Array(1.2, 2.3)),
+ make(Array(java.math.BigDecimal.valueOf(4.5).setScale(2),
+ java.math.BigDecimal.valueOf(6.7).setScale(2))),
+ make(Array("hello", "world")),
+ make(Array("a", "bcd")),
+ make(Array("abcd", "efg")),
+ make(Array(true, false)),
+ make(Array(Date.valueOf("2017-02-01"), Date.valueOf("2018-09-11"))),
+ make(Array(Timestamp.valueOf("2017-02-01 00:01:00"),
+ Timestamp.valueOf("2018-02-01 02:21:00")))
+ )))
+ sql("DROP TABLE IF EXISTS alter_com")
+ }
+
+ test("Test adding of struct of all primitive datatypes") {
+ sql("DROP TABLE IF EXISTS alter_com")
+ sql(
+ "CREATE TABLE alter_com(intField INT, structField struct<a:short,b:int,c:long,d:double," +
+ "e:decimal(8,2),f:string,g:char(5),h:varchar(50),i:boolean,j:date,k:timestamp>) STORED AS " +
+ "carbondata")
+ sql(
+ "insert into alter_com values(1, named_struct('a',1,'b',2,'c',3,'d',1.23,'e',2.34,'f'," +
+ "'hello','g','abc','h','def','i',true,'j','2017-02-01','k','2018-02-01 02:00:00.0') ) ")
+ sql("DROP TABLE IF EXISTS alter_com")
+ }
+
+ def insertIntoTableForArrayType(): Unit = {
+ sql("insert into alter_com values(4,array(2),array(1,2,3,4),array('abc','def'))")
+ sql("insert into alter_com values(5,array(1,2),array(1), array('Hulk','Thor'))")
+ sql(
+ "insert into alter_com values(6,array(1,2,3),array(1234,8,33333),array('Iron','Man'," +
+ "'Jarvis'))")
+ }
+
+ def checkRestulForArrayType(): Unit = {
+ val totalRows = sql("select * from alter_com").collect()
+ val a = sql("select * from alter_com where array_contains(arr2,2)").collect
+ val b = sql("select * from alter_com where array_contains(arr3,'Thor')").collect
+ val c = sql("select * from alter_com where intField = 1").collect
+ assert(totalRows.size == 6)
+ assert(a.size == 1)
+ assert(b.size == 1)
+ // check default value for newly added array column that is index - 3 and 4
+ assert(c(0)(2) == null && c(0)(3) == null)
+ }
+
+ test("Test alter add for arrays enabling local dictionary") {
+ sql("DROP TABLE IF EXISTS alter_com")
+ sql(
+ "CREATE TABLE alter_com(intField INT, arr1 array<int>) " +
+ "STORED AS carbondata TBLPROPERTIES('local_dictionary_enable'='true')")
+
+ sql("insert into alter_com values(1,array(1) )")
+ sql("insert into alter_com values(2,array(9,0) )")
+ sql("insert into alter_com values(3,array(11,12,13) )")
+
+ sql(
+ "ALTER TABLE alter_com ADD COLUMNS(arr2 array<int>, arr3 array<string>) TBLPROPERTIES" +
+ "('LOCAL_DICTIONARY_INCLUDE'='arr3')")
+ val schema = sql("describe alter_com").collect()
+ assert(schema.size == 4)
+
+ // For the previous segments the default value for newly added array column is null
+ insertIntoTableForArrayType
+ checkRestulForArrayType
+ sql("DROP TABLE IF EXISTS alter_com")
+
+ }
+
+ test("Test alter add for arrays disabling local dictionary") {
+ sql("DROP TABLE IF EXISTS alter_com")
+ sql(
+ "CREATE TABLE alter_com(intField INT, arr1 array<int>) " +
+ "STORED AS carbondata TBLPROPERTIES('local_dictionary_enable'='false')")
+
+ sql("insert into alter_com values(1,array(1) )")
+ sql("insert into alter_com values(2,array(9,0) )")
+ sql("insert into alter_com values(3,array(11,12,13) )")
+ sql(
+ "ALTER TABLE alter_com ADD COLUMNS(arr2 array<int>, arr3 array<string>) TBLPROPERTIES" +
+ "('LOCAL_DICTIONARY_EXCLUDE'='arr3')")
+ val schema = sql("describe alter_com").collect()
+ assert(schema.size == 4)
+
+ // For the previous segments the default value for newly added array column is null
+ insertIntoTableForArrayType
+ checkRestulForArrayType
+ sql("DROP TABLE IF EXISTS alter_com")
+
+ }
+
+ def insertIntoTableForStructType(): Unit = {
+ sql(
+ "insert into alter_struct values(2, named_struct('id1', 'id2','name1','name2'), " +
+ "named_struct('a','id2','b', 'abc2'), 'hello world', 5, named_struct('c','id3'," +
+ "'d', 'abc3','e', 22), array(1,2,3) )")
+ sql(
+ "insert into alter_struct values(3, named_struct('id1', 'id3','name1','name3'), " +
+ "named_struct('a','id2.1','b', 'abc2.1'), 'India', 5, named_struct('c','id3.1'," +
+ "'d', 'abc3.1','e', 25), array(4,5) )")
+
+ }
+
+ def checkResultForStructType(): Unit = {
+ val totalRows = sql("select * from alter_struct").collect()
+ val a = sql("select * from alter_struct where struct1.a = 'id2' ").collect()
+ val b = sql(
+ "select * from alter_struct where struct1.a = 'id2' or struct2.c = 'id3.1' or " +
+ "array_contains(arr,5) ").collect()
+ val c = sql("select * from alter_struct where roll = 1").collect()
+
+ assert(totalRows.size == 3)
+ assert(a.size == 1)
+ assert(b.size == 2)
+ // check default value for newly added struct column
+ assert(c(0)(2) == null)
+ }
+
+ test("Test alter add for structs enabling local dictionary") {
Review comment:
modify this testcase as below. in testcase, keep create table with/without local dictionary and move all other steps to method and reuse. you can pass local dict enable/disable as param, to make changes in alter add.
Please handle for Array testcase also.
##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
##########
@@ -275,6 +275,64 @@ class AlterTableColumnSchemaGenerator(
allColumns ++= Seq(columnSchema)
}
newCols ++= Seq(columnSchema)
+ if (DataTypes.isArrayType(columnSchema.getDataType)) {
+ columnSchema.setNumberOfChild(field.children.size)
+ val childField = field.children.get(0)
+ // TODO: should support adding multi-level complex columns: CARBONDATA-4164
+ if (childField.children != Some(null)) {
+ throw new UnsupportedOperationException("Only single-level complex types are allowed")
+ }
+ val childSchema: ColumnSchema = TableNewProcessor.createColumnSchema(
+ childField,
+ new java.util.ArrayList[Encoding](),
+ isDimensionCol = true,
+ childField.precision,
+ childField.scale,
+ childField.schemaOrdinal + currentSchemaOrdinal,
+ alterTableModel.highCardinalityDims,
+ alterTableModel.databaseName.getOrElse(dbName),
+ isSortColumn(childField.name.getOrElse(childField.column)),
+ isVarcharColumn(childField.name.getOrElse(childField.column)))
+
+ if (childSchema.getDataType == DataTypes.VARCHAR) {
+ // put the new long string columns in 'longStringCols'
+ // and add them after old long string columns
+ longStringCols ++= Seq(childSchema)
+ } else {
+ allColumns ++= Seq(childSchema)
+ }
+ newCols ++= Seq(childSchema)
+ } else if (DataTypes.isStructType(columnSchema.getDataType)) {
+ val noOfChildren = field.children.get.size
+ columnSchema.setNumberOfChild(noOfChildren)
+ for (i <- 0 to noOfChildren - 1) {
+ val childField = field.children.get(i)
+ // TODO: should support adding multi-level complex columns: CARBONDATA-4164
+ if (childField.children != Some(null)) {
+ throw new UnsupportedOperationException("Only single-level complex types are allowed")
Review comment:
Same comment as above
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] Indhumathi27 commented on a change in pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
Indhumathi27 commented on a change in pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#discussion_r614571362
##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
##########
@@ -255,6 +276,10 @@ class AlterTableColumnSchemaGenerator(
// add new dimension columns
alterTableModel.dimCols.foreach(field => {
+ if (field.dataType.get.equals("Map")) {
Review comment:
Can use CarbonCommonConstants.MAP constant
##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
##########
@@ -319,10 +373,6 @@ class AlterTableColumnSchemaGenerator(
sys.error(s"Duplicate column found with name: $name")
})
- if (newCols.exists(_.getDataType.isComplexType)) {
Review comment:
Please update document [Alter-add](https://github.com/apache/carbondata/blob/master/docs/ddl-of-carbondata.md#add-columns)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [WIP]
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-811702843
Build Failed with Spark 2.3.4, Please check CI http://121.244.95.60:12602/job/ApacheCarbonPRBuilder2.3/5108/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] akashrn5 commented on a change in pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
akashrn5 commented on a change in pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#discussion_r613021868
##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
##########
@@ -275,6 +275,56 @@ class AlterTableColumnSchemaGenerator(
allColumns ++= Seq(columnSchema)
}
newCols ++= Seq(columnSchema)
+ if(field.dataType== Some("Array") ) {
+ columnSchema.setNumberOfChild(field.children.size)
+ val childField = field.children.get(0)
+ val childSchema: ColumnSchema = TableNewProcessor.createColumnSchema(
+ childField,
+ new java.util.ArrayList[Encoding](),
+ isDimensionCol = true,
+ childField.precision,
+ childField.scale,
+ childField.schemaOrdinal + currentSchemaOrdinal,
+ alterTableModel.highCardinalityDims,
+ alterTableModel.databaseName.getOrElse(dbName),
+ isSortColumn(childField.name.getOrElse(childField.column)),
+ isVarcharColumn(childField.name.getOrElse(childField.column)))
+
+ if (childSchema.getDataType == DataTypes.VARCHAR) {
+ // put the new long string columns in 'longStringCols'
+ // and add them after old long string columns
+ longStringCols ++= Seq(childSchema)
+ } else {
+ allColumns ++= Seq(childSchema)
+ }
+ newCols ++= Seq(childSchema)
+ } else if (field.dataType == Some("Struct")) {
Review comment:
same as above
##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
##########
@@ -275,6 +275,56 @@ class AlterTableColumnSchemaGenerator(
allColumns ++= Seq(columnSchema)
}
newCols ++= Seq(columnSchema)
+ if(field.dataType== Some("Array") ) {
Review comment:
instead of hardcoding, please use existing APIs like `org.apache.carbondata.core.metadata.datatype.DataTypes#isArrayType`
##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableAddColumnCommand.scala
##########
@@ -85,6 +85,8 @@ private[sql] case class CarbonAlterTableAddColumnCommand(
timeStamp = System.currentTimeMillis
val schemaEvolutionEntry = new org.apache.carbondata.core.metadata.schema.SchemaEvolutionEntry
schemaEvolutionEntry.setTimeStamp(timeStamp)
+ // filter out complex children columns
+ newCols = newCols.filter(x => !x.isComplexColumn)
schemaEvolutionEntry.setAdded(newCols.toList.asJava)
Review comment:
can we add in Schema Evolution entry, when you add columns schema as added, if you are just adding parent make sure, that column schema as its all child columns, just confirm if its there already, else please handle it.
##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
##########
@@ -534,10 +581,10 @@ object TableNewProcessor {
if (highCardinalityDims.contains(colName)) {
encoders.remove(Encoding.DICTIONARY)
}
- if (dataType == DataTypes.DATE) {
- encoders.add(Encoding.DICTIONARY)
- encoders.add(Encoding.DIRECT_DICTIONARY)
- } else if (dataType == DataTypes.TIMESTAMP && !highCardinalityDims.contains(colName)) {
+ // complex children does not require encoding
+ if (!colName.contains('.') && (dataType == DataTypes.DATE ||
Review comment:
use static constant for point
##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParserUtil.scala
##########
@@ -683,12 +683,6 @@ object CarbonSparkSqlParserUtil {
fields: List[Field],
tblProp: Option[List[(String, String)]]
): CarbonAlterTableAddColumnCommand = {
- fields.foreach { f =>
Review comment:
i think since you are supporting only for single level complex column, if user tries to add multi level you should throw exception, and wherever you handle this, please add a TODO and mention jira ID which you have already created to support multilevel
##########
File path: integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/alterTable/TestAlterTableAddColumns.scala
##########
@@ -136,6 +140,209 @@ class TestAlterTableAddColumns extends QueryTest with BeforeAndAfterAll {
sql(s"""DROP TABLE IF EXISTS ${ tableName }""")
}
+ test("Test adding of array of all primitive datatypes") {
+ sql("DROP TABLE IF EXISTS alter_com")
+ sql("CREATE TABLE alter_com(intfield int) STORED AS carbondata")
+ sql(
+ "ALTER TABLE alter_com ADD COLUMNS(arr1 array<short>, arr2 array<int>, arr3 " +
+ "array<long>, arr4 array<double>, arr5 array<decimal(8,2)>, arr6 array<string>, arr7 " +
+ "array<char(5)>, arr8 array<varchar(50)>, arr9 array<boolean>, arr10 array<date>, arr11 " +
+ "array<timestamp> )")
+ sql("desc table alter_com").show(false)
+ sql(
+ "insert into alter_com values(1,array(1,5),array(1,2),array(1,2,3),array(1.2d,2.3d),array" +
+ "(4.5,6.7),array('hello','world'),array('a','bcd'),array('abcd','efg'),array(true,false)," +
+ "array('2017-02-01','2018-09-11'),array('2017-02-01 00:01:00','2018-02-01 02:21:00') )")
+ checkAnswer(sql(
+ "select * from alter_com"),
+ Seq(Row(1, make(Array(1, 5)),
+ make(Array(1, 2)),
+ make(Array(1, 2, 3)),
+ make(Array(1.2, 2.3)),
+ make(Array(java.math.BigDecimal.valueOf(4.5).setScale(2),
+ java.math.BigDecimal.valueOf(6.7).setScale(2))),
+ make(Array("hello", "world")),
+ make(Array("a", "bcd")),
+ make(Array("abcd", "efg")),
+ make(Array(true, false)),
+ make(Array(Date.valueOf("2017-02-01"), Date.valueOf("2018-09-11"))),
+ make(Array(Timestamp.valueOf("2017-02-01 00:01:00"),
+ Timestamp.valueOf("2018-02-01 02:21:00")))
+ )))
+ sql("DROP TABLE IF EXISTS alter_com")
+ }
+
+ test("Test adding of struct of all primitive datatypes") {
+ sql("DROP TABLE IF EXISTS alter_com")
+ sql(
+ "CREATE TABLE alter_com(intField INT, structField struct<a:short,b:int,c:long,d:double," +
+ "e:decimal(8,2),f:string,g:char(5),h:varchar(50),i:boolean,j:date,k:timestamp>) STORED AS " +
+ "carbondata")
+ sql(
+ "insert into alter_com values(1, named_struct('a',1,'b',2,'c',3,'d',1.23,'e',2.34,'f'," +
+ "'hello','g','abc','h','def','i',true,'j','2017-02-01','k','2018-02-01 02:00:00.0') ) ")
+ sql("select * from alter_com").show(false)
+ sql("DROP TABLE IF EXISTS alter_com")
+ }
+
+ def insertIntoTableForArrayType(): Unit = {
+ sql("insert into alter_com values(4,array(2),array(1,2,3,4),array('abc','def'))")
+ sql("insert into alter_com values(5,array(1,2),array(1), array('Hulk','Thor'))")
+ sql(
+ "insert into alter_com values(6,array(1,2,3),array(1234,8,33333),array('Iron','Man'," +
+ "'Jarvis'))")
+ }
+
+ def checkRestulForArrayType(): Unit = {
+ val totalRows = sql("select * from alter_com").collect()
+ val a = sql("select * from alter_com where array_contains(arr2,2)").collect
+ val b = sql("select * from alter_com where array_contains(arr3,'Thor')").collect
+ val c = sql("select * from alter_com where intField = 1").collect
+ assert(totalRows.size == 6)
+ assert(a.size == 1)
+ assert(b.size == 1)
+ // check default value for newly added array column that is index - 3 and 4
+ assert(c(0)(2) == null && c(0)(3) == null)
+ }
+
+ test("Test alter add for arrays enabling local dictionary") {
+ sql("DROP TABLE IF EXISTS alter_com")
+ sql(
+ "CREATE TABLE alter_com(intField INT, arr1 array<int>) " +
+ "STORED AS carbondata TBLPROPERTIES('local_dictionary_enable'='true')")
+
+ sql("insert into alter_com values(1,array(1) )")
+ sql("insert into alter_com values(2,array(9,0) )")
+ sql("insert into alter_com values(3,array(11,12,13) )")
+
+ sql(
+ "ALTER TABLE alter_com ADD COLUMNS(arr2 array<int>, arr3 array<string>) TBLPROPERTIES" +
+ "('LOCAL_DICTIONARY_INCLUDE'='arr3')")
+ val schema = sql("describe alter_com").collect()
+ assert(schema.size == 4)
+
+ // For the previous segments the default value for newly added array column is null
+ sql("insert into alter_com values(4,array(2),array(1,2,3,4),array('abc','def'))")
+ sql("insert into alter_com values(5,array(1,2),array(1), array('Hulk','Thor'))")
+ sql(
+ "insert into alter_com values(6,array(1,2,3),array(1234,8,33333),array('Iron','Man'," +
+ "'Jarvis'))")
+
+ val totalRows = sql("select * from alter_com").collect()
+ val a = sql("select * from alter_com where array_contains(arr2,2)").collect
+ val b = sql("select * from alter_com where array_contains(arr3,'Thor')").collect
+ val c = sql("select * from alter_com where intField = 1").collect
+ assert(totalRows.size == 6)
+ assert(a.size == 1)
+ assert(b.size == 1)
+ // check default value for newly added array column that is index - 3 and 4
+ assert(c(0)(2) == null && c(0)(3) == null)
+ sql("DROP TABLE IF EXISTS alter_com")
+
+ }
+
+ test("Test alter add for arrays disabling local dictionary") {
+ sql("DROP TABLE IF EXISTS alter_com")
+ sql(
+ "CREATE TABLE alter_com(intField INT, arr1 array<int>) " +
+ "STORED AS carbondata TBLPROPERTIES('local_dictionary_enable'='false')")
+
+ sql("insert into alter_com values(1,array(1) )")
+ sql("insert into alter_com values(2,array(9,0) )")
+ sql("insert into alter_com values(3,array(11,12,13) )")
+ sql(
+ "ALTER TABLE alter_com ADD COLUMNS(arr2 array<int>, arr3 array<string>) TBLPROPERTIES" +
+ "('LOCAL_DICTIONARY_EXCLUDE'='arr3')")
+ val schema = sql("describe alter_com").collect()
+ assert(schema.size == 4)
+
+ // For the previous segments the default value for newly added array column is null
+ sql("insert into alter_com values(4,array(2),array(1,2,3,4),array('abc','def'))")
+ sql("insert into alter_com values(5,array(1,2),array(1), array('Hulk','Thor'))")
+ sql(
+ "insert into alter_com values(6,array(1,2,3),array(1234,80,3333),array('Iron','Man'," +
+ "'Jarvis'))")
+
+ val totalRows = sql("select * from alter_com").collect()
+ val a = sql("select * from alter_com where array_contains(arr2,2)").collect
+ val b = sql("select * from alter_com where array_contains(arr3,'Thor')").collect
+ val c = sql("select * from alter_com where intField = 1").collect
+ assert(totalRows.size == 6)
+ assert(a.size == 1)
+ assert(b.size == 1)
+ // check default value for newly added array column that is index - 3 and 4
+ assert(c(0)(2) == null && c(0)(3) == null)
+ sql("DROP TABLE IF EXISTS alter_com")
+
+ }
+
+ def insertIntoTableForStructType(): Unit = {
+ sql(
+ "insert into alter_struct values(2, named_struct('id1', 'id2','name1','name2'), " +
+ "named_struct('a','id2','b', 'abc2'), 'hello world', 5, named_struct('c','id3'," +
+ "'d', 'abc3','e', 22), array(1,2,3) )")
+ sql(
+ "insert into alter_struct values(3, named_struct('id1', 'id3','name1','name3'), " +
+ "named_struct('a','id2.1','b', 'abc2.1'), 'India', 5, named_struct('c','id3.1'," +
+ "'d', 'abc3.1','e', 25), array(4,5) )")
+
+ }
+
+ def checkResultForStructType(): Unit = {
+ val totalRows = sql("select * from alter_struct").collect()
+ val a = sql("select * from alter_struct where struct1.a = 'id2' ").collect()
+ val b = sql(
+ "select * from alter_struct where struct1.a = 'id2' or struct2.c = 'id3.1' or " +
+ "array_contains(arr,5) ").collect()
+ val c = sql("select * from alter_struct where roll = 1").collect()
+
+ assert(totalRows.size == 3)
+ assert(a.size == 1)
+ assert(b.size == 2)
+ // check default value for newly added struct column
+ assert(c(0)(2) == null)
+ }
+
+ test("Test alter add for structs enabling local dictionary") {
+ sql("DROP TABLE IF EXISTS alter_struct")
+ sql(
+ "create table alter_struct(roll int, department struct<id1:string,name1:string>) STORED AS " +
+ "carbondata TBLPROPERTIES('local_dictionary_enable'='true')")
+ sql(
+ "insert into alter_struct values(1, named_struct('id1', 'id1','name1','name1'))")
+ sql(
+ "ALTER TABLE alter_struct ADD COLUMNS(struct1 struct<a:string,b:string>, temp string," +
+ " intField int, struct2 struct<c:string,d:string,e:int>, arr array<int>) TBLPROPERTIES " +
+ "('LOCAL_DICTIONARY_INCLUDE'='struct1, struct2')")
+ val schema = sql("describe alter_struct").collect()
+ assert(schema.size == 7)
+ // For the previous segments the default value for newly added struct column is null
+ insertIntoTableForStructType
+ checkResultForStructType
+ sql("DROP TABLE IF EXISTS alter_struct")
+
+ }
+
+ test("Test alter add for structs, disabling local dictionary") {
+ sql("DROP TABLE IF EXISTS alter_struct")
+ sql(
+ "create table alter_struct(roll int, department struct<id1:string,name1:string>) STORED AS " +
+ "carbondata TBLPROPERTIES('local_dictionary_enable'='false')")
+ sql(
+ "insert into alter_struct values(1, named_struct('id1', 'id1','name1','name1'))")
+ sql(
+ "ALTER TABLE alter_struct ADD COLUMNS(struct1 struct<a:string,b:string>, temp string," +
+ " intField int, struct2 struct<c:string,d:string,e:int>, arr array<int>) TBLPROPERTIES " +
+ "('LOCAL_DICTIONARY_EXCLUDE'='struct1, struct2')")
+ val schema = sql("describe alter_struct").collect()
+ assert(schema.size == 7)
+ // For the previous segments the default value for newly added struct column is null
+ insertIntoTableForStructType
+ checkResultForStructType
+ sql("DROP TABLE IF EXISTS alter_struct")
+
+ }
+
Review comment:
also add a test case to validate adding of multi level complex column
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-820983730
Build Failed with Spark 2.4.5, Please check CI http://121.244.95.60:12602/job/ApacheCarbon_PR_Builder_2.4.5/3446/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [WIP]
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-811646724
Build Failed with Spark 2.4.5, Please check CI http://121.244.95.60:12602/job/ApacheCarbon_PR_Builder_2.4.5/3355/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] akkio-97 commented on pull request #4115: [CARBONDATA-4160] Alter carbon schema by adding single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
akkio-97 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-813150060
retest this please
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [CARBONDATA-4160] Alter carbon schema by adding single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-812529197
Build Failed with Spark 2.4.5, Please check CI http://121.244.95.60:12602/job/ApacheCarbon_PR_Builder_2.4.5/3368/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] akkio-97 commented on a change in pull request #4115: [WIP]
Posted by GitBox <gi...@apache.org>.
akkio-97 commented on a change in pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#discussion_r606105876
##########
File path: integration/spark/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala
##########
@@ -248,16 +248,6 @@ class AlterTableValidationTestCase extends QueryTest with BeforeAndAfterAll {
}
}
- test("test adding complex datatype column") {
Review comment:
not required anymore
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] akkio-97 commented on pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
akkio-97 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-820958582
retest this please
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] Indhumathi27 commented on a change in pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
Indhumathi27 commented on a change in pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#discussion_r613755261
##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
##########
@@ -275,6 +275,64 @@ class AlterTableColumnSchemaGenerator(
allColumns ++= Seq(columnSchema)
}
newCols ++= Seq(columnSchema)
+ if (DataTypes.isArrayType(columnSchema.getDataType)) {
+ columnSchema.setNumberOfChild(field.children.size)
+ val childField = field.children.get(0)
+ // TODO: should support adding multi-level complex columns: CARBONDATA-4164
+ if (childField.children != Some(null)) {
Review comment:
```suggestion
if (!childField.children.contains(null)) {
```
Handle in all places
##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
##########
@@ -275,6 +275,64 @@ class AlterTableColumnSchemaGenerator(
allColumns ++= Seq(columnSchema)
}
newCols ++= Seq(columnSchema)
+ if (DataTypes.isArrayType(columnSchema.getDataType)) {
+ columnSchema.setNumberOfChild(field.children.size)
+ val childField = field.children.get(0)
+ // TODO: should support adding multi-level complex columns: CARBONDATA-4164
+ if (childField.children != Some(null)) {
+ throw new UnsupportedOperationException("Only single-level complex types are allowed")
+ }
+ val childSchema: ColumnSchema = TableNewProcessor.createColumnSchema(
Review comment:
Can move getting child schema to new method and reuse for array and struct type
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-820764494
Build Failed with Spark 2.4.5, Please check CI http://121.244.95.60:12602/job/ApacheCarbon_PR_Builder_2.4.5/3442/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] Indhumathi27 commented on pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
Indhumathi27 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-822311739
LGTM
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] akkio-97 commented on a change in pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
akkio-97 commented on a change in pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#discussion_r614672631
##########
File path: docs/ddl-of-carbondata.md
##########
@@ -694,7 +694,7 @@ CarbonData DDL statements are documented here,which includes:
```
ALTER TABLE carbon ADD COLUMNS (a1 INT, b1 STRING) TBLPROPERTIES('DEFAULT.VALUE.a1'='10')
```
- **NOTE:** Add Complex datatype columns is not supported.
+ **NOTE:** Adding of only single-level Complex datatype columns(only array and struct) is supported.
Review comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] akashrn5 commented on a change in pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
akashrn5 commented on a change in pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#discussion_r614776990
##########
File path: integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/alterTable/TestAlterTableAddColumns.scala
##########
@@ -136,14 +139,228 @@ class TestAlterTableAddColumns extends QueryTest with BeforeAndAfterAll {
sql(s"""DROP TABLE IF EXISTS ${ tableName }""")
}
+ test("Test adding of array of all primitive datatypes") {
Review comment:
@akkio-97 please add test case for add columns with comment for complex column and then also add a test case to verify the SchemaEvolutionEntry after add column
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-823039698
Build Success with Spark 2.3.4, Please check CI http://121.244.95.60:12602/job/ApacheCarbonPRBuilder2.3/5215/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-820361276
Build Success with Spark 2.4.5, Please check CI http://121.244.95.60:12602/job/ApacheCarbon_PR_Builder_2.4.5/3440/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-819740885
Build Success with Spark 2.4.5, Please check CI http://121.244.95.60:12602/job/ApacheCarbon_PR_Builder_2.4.5/3430/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-820435272
Build Success with Spark 2.4.5, Please check CI http://121.244.95.60:12602/job/ApacheCarbon_PR_Builder_2.4.5/3441/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [WIP]
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-811156835
Build Failed with Spark 2.4.5, Please check CI http://121.244.95.60:12602/job/ApacheCarbon_PR_Builder_2.4.5/3353/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [WIP]
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-812422053
Build Success with Spark 2.3.4, Please check CI http://121.244.95.60:12602/job/ApacheCarbonPRBuilder2.3/5116/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] akkio-97 commented on a change in pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
akkio-97 commented on a change in pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#discussion_r613828144
##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
##########
@@ -275,6 +275,64 @@ class AlterTableColumnSchemaGenerator(
allColumns ++= Seq(columnSchema)
}
newCols ++= Seq(columnSchema)
+ if (DataTypes.isArrayType(columnSchema.getDataType)) {
+ columnSchema.setNumberOfChild(field.children.size)
+ val childField = field.children.get(0)
+ // TODO: should support adding multi-level complex columns: CARBONDATA-4164
+ if (childField.children != Some(null)) {
Review comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-817701999
Build Success with Spark 2.4.5, Please check CI http://121.244.95.60:12602/job/ApacheCarbon_PR_Builder_2.4.5/3400/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-817705872
Build Success with Spark 2.3.4, Please check CI http://121.244.95.60:12602/job/ApacheCarbonPRBuilder2.3/5152/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] akkio-97 commented on a change in pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
akkio-97 commented on a change in pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#discussion_r615567566
##########
File path: integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/alterTable/TestAlterTableAddColumns.scala
##########
@@ -136,14 +139,228 @@ class TestAlterTableAddColumns extends QueryTest with BeforeAndAfterAll {
sql(s"""DROP TABLE IF EXISTS ${ tableName }""")
}
+ test("Test adding of array of all primitive datatypes") {
Review comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-821137636
Build Failed with Spark 2.4.5, Please check CI http://121.244.95.60:12602/job/ApacheCarbon_PR_Builder_2.4.5/3448/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-820429606
Build Success with Spark 2.3.4, Please check CI http://121.244.95.60:12602/job/ApacheCarbonPRBuilder2.3/5193/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] akkio-97 commented on a change in pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
akkio-97 commented on a change in pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#discussion_r614670056
##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
##########
@@ -235,6 +236,26 @@ class AlterTableColumnSchemaGenerator(
dataType.toString.equals("STRUCT")
}
+ private def createChildSchema(childField: Field, currentSchemaOrdinal: Int): ColumnSchema = {
+ // TODO: should support adding multi-level complex columns: CARBONDATA-4164
+ if (!childField.children.contains(null)) {
Review comment:
isDefined cannot be used because it contains Some(null) in case of single-level and a field in case of multi-level.
So isDefined is true in both cases
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-820942569
Build Failed with Spark 2.3.4, Please check CI http://121.244.95.60:12602/job/ApacheCarbonPRBuilder2.3/5197/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] akkio-97 commented on a change in pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
akkio-97 commented on a change in pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#discussion_r614670056
##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
##########
@@ -235,6 +236,26 @@ class AlterTableColumnSchemaGenerator(
dataType.toString.equals("STRUCT")
}
+ private def createChildSchema(childField: Field, currentSchemaOrdinal: Int): ColumnSchema = {
+ // TODO: should support adding multi-level complex columns: CARBONDATA-4164
+ if (!childField.children.contains(null)) {
Review comment:
isDefined cannot be used because it contains Some(null)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-817556023
Build Failed with Spark 2.3.4, Please check CI http://121.244.95.60:12602/job/ApacheCarbonPRBuilder2.3/5148/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [WIP]
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-811716619
Build Failed with Spark 2.3.4, Please check CI http://121.244.95.60:12602/job/ApacheCarbonPRBuilder2.3/5109/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] Indhumathi27 commented on a change in pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
Indhumathi27 commented on a change in pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#discussion_r613747475
##########
File path: integration/spark/src/test/scala/org/apache/carbondata/view/rewrite/TestAllOperationsOnMV.scala
##########
@@ -79,6 +90,12 @@ class TestAllOperationsOnMV extends QueryTest with BeforeAndAfterEach {
}.getMessage.contains("Cannot add columns in a materialized view table default.dm1")
}
+ test("test alter add complex column on MV table") {
+ intercept[ProcessMetaDataException] {
Review comment:
This test case already exists. Not required to check for complex type. Please remove duplicate test cases
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-820136561
Build Success with Spark 2.3.4, Please check CI http://121.244.95.60:12602/job/ApacheCarbonPRBuilder2.3/5185/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-823032415
Build Success with Spark 2.4.5, Please check CI http://121.244.95.60:12602/job/ApacheCarbon_PR_Builder_2.4.5/3467/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-821077689
Build Failed with Spark 2.3.4, Please check CI http://121.244.95.60:12602/job/ApacheCarbonPRBuilder2.3/5199/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] asfgit closed pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
asfgit closed pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] Indhumathi27 commented on pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
Indhumathi27 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-816453149
@akkio-97 For alter add columns, currently we support a property to provide default value in alter add ddl, like "DEFAULT.VALUE.column_name=somevalue". Please check and handle this scenario also.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] akkio-97 commented on a change in pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
akkio-97 commented on a change in pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#discussion_r613397041
##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableAddColumnCommand.scala
##########
@@ -85,6 +85,8 @@ private[sql] case class CarbonAlterTableAddColumnCommand(
timeStamp = System.currentTimeMillis
val schemaEvolutionEntry = new org.apache.carbondata.core.metadata.schema.SchemaEvolutionEntry
schemaEvolutionEntry.setTimeStamp(timeStamp)
+ // filter out complex children columns
+ newCols = newCols.filter(x => !x.isComplexColumn)
schemaEvolutionEntry.setAdded(newCols.toList.asJava)
Review comment:
We only maintain number of children in each parent schema. Each column schema is independent, hence it is a flat structure.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-821081138
Build Failed with Spark 2.4.5, Please check CI http://121.244.95.60:12602/job/ApacheCarbon_PR_Builder_2.4.5/3447/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] Indhumathi27 commented on pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
Indhumathi27 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-820083693
retest this please
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] akashrn5 commented on a change in pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
akashrn5 commented on a change in pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#discussion_r609828600
##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
##########
@@ -275,6 +275,56 @@ class AlterTableColumnSchemaGenerator(
allColumns ++= Seq(columnSchema)
}
newCols ++= Seq(columnSchema)
+ if(field.dataType== Some("Array") ) {
+ columnSchema.setNumberOfChild(field.children.size)
+ val childField = field.children.get(0)
+ val childSchema: ColumnSchema = TableNewProcessor.createColumnSchema(
+ childField,
Review comment:
then how come when i debug your test case, i get the wrong order?

--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-820360590
Build Success with Spark 2.3.4, Please check CI http://121.244.95.60:12602/job/ApacheCarbonPRBuilder2.3/5192/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] akkio-97 commented on pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
akkio-97 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-819673737
> @akkio-97 For alter add columns, currently we support a property to provide default value in alter add ddl, like "DEFAULT.VALUE.column_name=somevalue". Please check and handle this scenario also.
1) This is a part of the query flow. Because only when the select command is fired the data assigned will be inserted to the row.
2) The data which we assign as default is not parsed through queryUtil flow and is just a byte array. This data is not encoded so existing parsers and decoders cannot be used here to extract actual data and insert into the row.
3) We need to first write a new parser and then create ArrayQueryType and PrimitiveQueryType. After which insertion can be done.
4) This I think can be handled in a separate PR
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] akkio-97 commented on pull request #4115: [WIP]
Posted by GitBox <gi...@apache.org>.
akkio-97 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-811643311
retest this please
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] akkio-97 commented on a change in pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
akkio-97 commented on a change in pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#discussion_r614670056
##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
##########
@@ -235,6 +236,26 @@ class AlterTableColumnSchemaGenerator(
dataType.toString.equals("STRUCT")
}
+ private def createChildSchema(childField: Field, currentSchemaOrdinal: Int): ColumnSchema = {
+ // TODO: should support adding multi-level complex columns: CARBONDATA-4164
+ if (!childField.children.contains(null)) {
Review comment:
have put check on size
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-821138733
Build Failed with Spark 2.3.4, Please check CI http://121.244.95.60:12602/job/ApacheCarbonPRBuilder2.3/5200/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] akkio-97 commented on a change in pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
akkio-97 commented on a change in pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#discussion_r613397852
##########
File path: index/secondary-index/src/test/scala/org/apache/carbondata/spark/testsuite/secondaryindex/TestSIWithComplexArrayType.scala
##########
@@ -43,6 +43,83 @@ class TestSIWithComplexArrayType extends QueryTest with BeforeAndAfterEach {
sql("drop table if exists complextable5")
}
+ test("Test alter add array column before creating SI") {
+ sql("drop table if exists complextable")
+ sql("create table complextable (id string, country array<string>, name string) stored as carbondata")
+ sql("insert into complextable select 1,array('china', 'us'), 'b'")
+ sql("insert into complextable select 2,array('pak'), 'v'")
+
+ sql("drop index if exists index_11 on complextable")
+ sql(
+ "ALTER TABLE complextable ADD COLUMNS(arr2 array<string>)")
+ sql("insert into complextable select 3,array('china'), 'f',array('hello','world')")
+ sql("insert into complextable select 4,array('india'),'g',array('iron','man','jarvis')")
+
+ val result1 = sql("select * from complextable where array_contains(arr2,'iron')")
+ val result2 = sql("select * from complextable where arr2[0]='iron'")
+ sql("create index index_11 on table complextable(arr2) as 'carbondata'")
+ val df1 = sql(" select * from complextable where array_contains(arr2,'iron')")
+ val df2 = sql(" select * from complextable where arr2[0]='iron'")
+ if (!isFilterPushedDownToSI(df1.queryExecution.sparkPlan)) {
+ assert(false)
+ } else {
+ assert(true)
+ }
+ if (!isFilterPushedDownToSI(df2.queryExecution.sparkPlan)) {
+ assert(false)
+ } else {
+ assert(true)
+ }
+ val doNotHitSIDf = sql(" select * from complextable where array_contains(arr2,'iron') and array_contains(arr2,'man')")
+ if (isFilterPushedDownToSI(doNotHitSIDf.queryExecution.sparkPlan)) {
+ assert(false)
+ } else {
+ assert(true)
+ }
+ checkAnswer(result1, df1)
+ checkAnswer(result2, df2)
+ }
+
+ test("Test alter add array column after creating SI") {
+ sql("drop table if exists complextable")
+ sql("create table complextable (id string, country array<string>, name string) stored as carbondata")
+ sql("insert into complextable select 1,array('china', 'us'), 'b'")
+ sql("insert into complextable select 2,array('pak'), 'v'")
+
+ sql("drop index if exists index_11 on complextable")
+ sql("create index index_11 on table complextable(country) as 'carbondata'")
+
+ sql(
Review comment:
done
##########
File path: integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/alterTable/TestAlterTableAddColumns.scala
##########
@@ -136,6 +137,155 @@ class TestAlterTableAddColumns extends QueryTest with BeforeAndAfterAll {
sql(s"""DROP TABLE IF EXISTS ${ tableName }""")
}
+ test("Test alter add for arrays enabling local dictionary") {
+ sql("DROP TABLE IF EXISTS alter_com")
+ sql(
+ "CREATE TABLE alter_com(intField INT, arr1 array<int>) " +
+ "STORED AS carbondata TBLPROPERTIES('local_dictionary_enable'='true')")
+
+ sql("insert into alter_com values(1,array(1) )")
+ sql("insert into alter_com values(2,array(9,0) )")
+ sql("insert into alter_com values(3,array(11,12,13) )")
+
+ sql(
+ "ALTER TABLE alter_com ADD COLUMNS(arr2 array<int>, arr3 array<string>) TBLPROPERTIES" +
+ "('LOCAL_DICTIONARY_INCLUDE'='arr3')")
+ val schema = sql("describe alter_com").collect()
+ assert(schema.size == 4)
+
+ // For the previous segments the default value for newly added array column is null
+ sql("insert into alter_com values(4,array(2),array(1,2,3,4),array('abc','def'))")
+ sql("insert into alter_com values(5,array(1,2),array(1), array('Hulk','Thor'))")
+ sql(
+ "insert into alter_com values(6,array(1,2,3),array(1234,8,33333),array('Iron','Man'," +
+ "'Jarvis'))")
+
+ val totalRows = sql("select * from alter_com").collect()
+ val a = sql("select * from alter_com where array_contains(arr2,2)").collect
+ val b = sql("select * from alter_com where array_contains(arr3,'Thor')").collect
+ val c = sql("select * from alter_com where intField = 1").collect
+ assert(totalRows.size == 6)
+ assert(a.size == 1)
+ assert(b.size == 1)
+ // check default value for newly added array column that is index - 3 and 4
+ assert(c(0)(2) == null && c(0)(3) == null)
+ sql("DROP TABLE IF EXISTS alter_com")
+
+ }
+
+ test("Test alter add for arrays disabling local dictionary") {
+ sql("DROP TABLE IF EXISTS alter_com")
+ sql(
+ "CREATE TABLE alter_com(intField INT, arr1 array<int>) " +
+ "STORED AS carbondata TBLPROPERTIES('local_dictionary_enable'='false')")
+
+ sql("insert into alter_com values(1,array(1) )")
+ sql("insert into alter_com values(2,array(9,0) )")
+ sql("insert into alter_com values(3,array(11,12,13) )")
+ sql(
+ "ALTER TABLE alter_com ADD COLUMNS(arr2 array<double>, arr3 array<string>) TBLPROPERTIES" +
+ "('LOCAL_DICTIONARY_EXCLUDE'='arr3')")
+ val schema = sql("describe alter_com").collect()
+ assert(schema.size == 4)
+
+ // For the previous segments the default value for newly added array column is null
+ sql("insert into alter_com values(4,array(2),array(1,2,3,4),array('abc','def'))")
+ sql("insert into alter_com values(5,array(1,2),array(1), array('Hulk','Thor'))")
+ sql(
+ "insert into alter_com values(6,array(1,2,3),array(1234,80,3333),array('Iron','Man'," +
+ "'Jarvis'))")
+
+ val totalRows = sql("select * from alter_com").collect()
+ val a = sql("select * from alter_com where array_contains(arr2,2)").collect
+ val b = sql("select * from alter_com where array_contains(arr3,'Thor')").collect
+ val c = sql("select * from alter_com where intField = 1").collect
+ assert(totalRows.size == 6)
+ assert(a.size == 1)
+ assert(b.size == 1)
+ // check default value for newly added array column that is index - 3 and 4
+ assert(c(0)(2) == null && c(0)(3) == null)
+ sql("DROP TABLE IF EXISTS alter_com")
+
+ }
+
+ test("Test alter add for structs enabling local dictionary") {
+ sql("DROP TABLE IF EXISTS alter_struct")
+ sql(
+ "create table alter_struct(roll int, department struct<id1:string,name1:string>) STORED AS " +
+ "carbondata TBLPROPERTIES('local_dictionary_enable'='true')")
+ sql(
+ "insert into alter_struct values(1, named_struct('id1', 'id1','name1','name1'))")
+ sql(
+ "ALTER TABLE alter_struct ADD COLUMNS(struct1 struct<a:string,b:string>, temp string," +
+ " intField int, struct2 struct<c:string,d:string,e:int>, arr array<int>) TBLPROPERTIES " +
+ "('LOCAL_DICTIONARY_INCLUDE'='struct1, struct2')")
+ val schema = sql("describe alter_struct").collect()
+ assert(schema.size == 7)
+
+ sql(
+ "insert into alter_struct values(2, named_struct('id1', 'id1','name1','name1'), " +
+ "named_struct('a','id2','b', 'abc2'), 'hello world', 5, named_struct('c','id3'," +
+ "'d', 'abc3','e', 22), array(1,2,3) )")
+ sql(
+ "insert into alter_struct values(3, named_struct('id1', 'id1','name1','name1'), " +
+ "named_struct('a','id2.1','b', 'abc2.1'), 'India', 5, named_struct('c','id3.1'," +
+ "'d', 'abc3.1','e', 25), array(4,5) )")
+
+ val totalRows = sql("select * from alter_struct").collect()
+ val a = sql("select * from alter_struct where struct1.a = 'id2' ").collect()
+ val b = sql(
+ "select * from alter_struct where struct1.a = 'id2' or struct2.c = 'id3.1' or " +
+ "array_contains(arr,5) ").collect()
+ val c = sql("select * from alter_struct where roll = 1").collect()
+
+ assert(totalRows.size == 3)
+ assert(a.size == 1)
+ assert(b.size == 2)
+ // check default value for newly added struct column
+ assert(c(0)(2) == null)
+ sql("DROP TABLE IF EXISTS alter_struct")
+
+ }
+
+ test("Test alter add for structs, disabling local dictionary") {
+ sql("DROP TABLE IF EXISTS alter_struct")
+ sql(
+ "create table alter_struct(roll int, department struct<id1:string,name1:string>) STORED AS " +
+ "carbondata TBLPROPERTIES('local_dictionary_enable'='false')")
Review comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [WIP]
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-811888643
Build Failed with Spark 2.4.5, Please check CI http://121.244.95.60:12602/job/ApacheCarbon_PR_Builder_2.4.5/3360/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [WIP]
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-811716003
Build Failed with Spark 2.4.5, Please check CI http://121.244.95.60:12602/job/ApacheCarbon_PR_Builder_2.4.5/3358/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] akashrn5 commented on a change in pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
akashrn5 commented on a change in pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#discussion_r613037958
##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
##########
@@ -275,6 +275,56 @@ class AlterTableColumnSchemaGenerator(
allColumns ++= Seq(columnSchema)
}
newCols ++= Seq(columnSchema)
+ if(field.dataType== Some("Array") ) {
Review comment:
also please correct the code style, please use Ctrl + Alt + Shift + L in all the places where u have changed and use carbondata style config
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [CARBONDATA-4160] Alter carbon schema by adding single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-813170864
Build Failed with Spark 2.4.5, Please check CI http://121.244.95.60:12602/job/ApacheCarbon_PR_Builder_2.4.5/3370/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-820763395
Build Success with Spark 2.3.4, Please check CI http://121.244.95.60:12602/job/ApacheCarbonPRBuilder2.3/5194/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [CARBONDATA-4160] Alter carbon schema by adding single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-812471606
Build Failed with Spark 2.3.4, Please check CI http://121.244.95.60:12602/job/ApacheCarbonPRBuilder2.3/5117/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] ydvpankaj99 commented on pull request #4115: [WIP]
Posted by GitBox <gi...@apache.org>.
ydvpankaj99 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-811850075
retest this please
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] akkio-97 commented on a change in pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
akkio-97 commented on a change in pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#discussion_r609792211
##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
##########
@@ -275,6 +275,56 @@ class AlterTableColumnSchemaGenerator(
allColumns ++= Seq(columnSchema)
}
newCols ++= Seq(columnSchema)
+ if(field.dataType== Some("Array") ) {
+ columnSchema.setNumberOfChild(field.children.size)
+ val childField = field.children.get(0)
+ val childSchema: ColumnSchema = TableNewProcessor.createColumnSchema(
+ childField,
Review comment:
The order is maintained like above. The thrift column schema does not contain new columns.
The new dimensions(including complex) are added after the old dimensions only. After which all measure columns are added.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] akkio-97 commented on a change in pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
akkio-97 commented on a change in pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#discussion_r613940772
##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
##########
@@ -275,6 +295,39 @@ class AlterTableColumnSchemaGenerator(
allColumns ++= Seq(columnSchema)
}
newCols ++= Seq(columnSchema)
+ if (DataTypes.isArrayType(columnSchema.getDataType)) {
+ columnSchema.setNumberOfChild(field.children.size)
+ val childField = field.children.get(0)
+ val childSchema: ColumnSchema = createChildSchema(childField, currentSchemaOrdinal)
+ if (childSchema.getDataType == DataTypes.VARCHAR) {
+ // put the new long string columns in 'longStringCols'
+ // and add them after old long string columns
+ longStringCols ++= Seq(childSchema)
+ } else {
+ allColumns ++= Seq(childSchema)
+ }
+ newCols ++= Seq(childSchema)
+ } else if (DataTypes.isStructType(columnSchema.getDataType)) {
+ val noOfChildren = field.children.get.size
+ columnSchema.setNumberOfChild(noOfChildren)
+ for (i <- 0 to noOfChildren - 1) {
+ val childField = field.children.get(i)
+ // TODO: should support adding multi-level complex columns: CARBONDATA-4164
+ if (!childField.children.contains(null)) {
Review comment:
done
##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParserUtil.scala
##########
@@ -683,12 +683,6 @@ object CarbonSparkSqlParserUtil {
fields: List[Field],
tblProp: Option[List[(String, String)]]
): CarbonAlterTableAddColumnCommand = {
- fields.foreach { f =>
- if (CarbonParserUtil.isComplexType(f.dataType.get)) {
Review comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-819736041
Build Failed with Spark 2.3.4, Please check CI http://121.244.95.60:12602/job/ApacheCarbonPRBuilder2.3/5182/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] Indhumathi27 commented on pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
Indhumathi27 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-822986620
retest this please
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [WIP]
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-811665501
Build Failed with Spark 2.3.4, Please check CI http://121.244.95.60:12602/job/ApacheCarbonPRBuilder2.3/5107/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [WIP]
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-811889516
Build Failed with Spark 2.3.4, Please check CI http://121.244.95.60:12602/job/ApacheCarbonPRBuilder2.3/5111/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] akashrn5 commented on a change in pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
akashrn5 commented on a change in pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#discussion_r609832608
##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
##########
@@ -275,6 +275,56 @@ class AlterTableColumnSchemaGenerator(
allColumns ++= Seq(columnSchema)
}
newCols ++= Seq(columnSchema)
+ if(field.dataType== Some("Array") ) {
+ columnSchema.setNumberOfChild(field.children.size)
+ val childField = field.children.get(0)
+ val childSchema: ColumnSchema = TableNewProcessor.createColumnSchema(
+ childField,
Review comment:
@akkio-97 , my bad, i debugged a test case where there were no dimensions. And just to correct you, thrift will be changed if you can see the thriftTable object which you receive after process method call, so that it will be updated in hive metastore freshly and carbon metadata cache also by removing old table.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] akkio-97 commented on pull request #4115: [CARBONDATA-4160] Alter carbon schema by adding single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
akkio-97 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-813192053
retest this please
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-822476020
Build Success with Spark 2.4.5, Please check CI http://121.244.95.60:12602/job/ApacheCarbon_PR_Builder_2.4.5/3460/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] akashrn5 commented on a change in pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
akashrn5 commented on a change in pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#discussion_r615698795
##########
File path: integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/alterTable/TestAlterTableAddColumns.scala
##########
@@ -136,14 +139,274 @@ class TestAlterTableAddColumns extends QueryTest with BeforeAndAfterAll {
sql(s"""DROP TABLE IF EXISTS ${ tableName }""")
}
+ def addedColumnsInSchemaEvolutionEntry(tableName: String): Int = {
+ val carbonTable = CarbonEnv.getCarbonTable(None, tableName)(sqlContext.sparkSession)
+ val schemaEvolutionList = carbonTable.getTableInfo
+ .getFactTable
+ .getSchemaEvolution()
+ .getSchemaEvolutionEntryList()
+ var addedColumns = 0
+ for (i <- 0 until schemaEvolutionList.size()) {
+ addedColumns += schemaEvolutionList.get(i).getAdded.size()
+ }
+ addedColumns
+ }
+
+ test("Test adding of array of all primitive datatypes") {
+ sql("DROP TABLE IF EXISTS alter_com")
+ sql("CREATE TABLE alter_com(intfield int) STORED AS carbondata")
+ sql(
+ "ALTER TABLE alter_com ADD COLUMNS(arr1 array<short>, arr2 array<int>, arr3 " +
+ "array<long>, arr4 array<double>, arr5 array<decimal(8,2)>, arr6 array<string>, arr7 " +
+ "array<char(5)>, arr8 array<varchar(50)>, arr9 array<boolean>, arr10 array<date>, arr11 " +
+ "array<timestamp> )")
+ val columns = sql("desc table alter_com").collect()
+ assert(columns.size.equals(12))
+ sql(
+ "insert into alter_com values(1,array(1,5),array(1,2),array(1,2,3),array(1.2d,2.3d),array" +
+ "(4.5,6.7),array('hello','world'),array('a','bcd'),array('abcd','efg'),array(true,false)," +
+ "array('2017-02-01','2018-09-11'),array('2017-02-01 00:01:00','2018-02-01 02:21:00') )")
+ checkAnswer(sql(
+ "select * from alter_com"),
+ Seq(Row(1,
+ make(Array(1, 5)),
+ make(Array(1, 2)),
+ make(Array(1, 2, 3)),
+ make(Array(1.2, 2.3)),
+ make(Array(java.math.BigDecimal.valueOf(4.5).setScale(2),
+ java.math.BigDecimal.valueOf(6.7).setScale(2))),
+ make(Array("hello", "world")),
+ make(Array("a", "bcd")),
+ make(Array("abcd", "efg")),
+ make(Array(true, false)),
+ make(Array(Date.valueOf("2017-02-01"),
+ Date.valueOf("2018-09-11"))),
+ make(Array(Timestamp.valueOf("2017-02-01 00:01:00"),
+ Timestamp.valueOf("2018-02-01 02:21:00")))
+ )))
+
+ val addedColumns = addedColumnsInSchemaEvolutionEntry("alter_com")
+ assert(addedColumns == 11)
Review comment:
also add an assert of added column names and datatypes in a test case
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] akkio-97 commented on pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
akkio-97 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-820908025
retest this please
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] ydvpankaj99 commented on pull request #4115: [WIP]
Posted by GitBox <gi...@apache.org>.
ydvpankaj99 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-811697138
retest this please
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [WIP]
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-811648692
Build Failed with Spark 2.3.4, Please check CI http://121.244.95.60:12602/job/ApacheCarbonPRBuilder2.3/5106/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] Indhumathi27 commented on a change in pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
Indhumathi27 commented on a change in pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#discussion_r613866130
##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
##########
@@ -275,6 +295,39 @@ class AlterTableColumnSchemaGenerator(
allColumns ++= Seq(columnSchema)
}
newCols ++= Seq(columnSchema)
+ if (DataTypes.isArrayType(columnSchema.getDataType)) {
+ columnSchema.setNumberOfChild(field.children.size)
+ val childField = field.children.get(0)
+ val childSchema: ColumnSchema = createChildSchema(childField, currentSchemaOrdinal)
+ if (childSchema.getDataType == DataTypes.VARCHAR) {
+ // put the new long string columns in 'longStringCols'
+ // and add them after old long string columns
+ longStringCols ++= Seq(childSchema)
+ } else {
+ allColumns ++= Seq(childSchema)
+ }
+ newCols ++= Seq(childSchema)
+ } else if (DataTypes.isStructType(columnSchema.getDataType)) {
+ val noOfChildren = field.children.get.size
+ columnSchema.setNumberOfChild(noOfChildren)
+ for (i <- 0 to noOfChildren - 1) {
+ val childField = field.children.get(i)
+ // TODO: should support adding multi-level complex columns: CARBONDATA-4164
+ if (!childField.children.contains(null)) {
Review comment:
Please remove this check, since it is already handled in createChildschema method
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [WIP]
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-811153877
Build Failed with Spark 2.3.4, Please check CI http://121.244.95.60:12602/job/ApacheCarbonPRBuilder2.3/5104/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] Indhumathi27 commented on a change in pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
Indhumathi27 commented on a change in pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#discussion_r614614639
##########
File path: integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/alterTable/TestAlterTableAddColumns.scala
##########
@@ -136,6 +140,232 @@ class TestAlterTableAddColumns extends QueryTest with BeforeAndAfterAll {
sql(s"""DROP TABLE IF EXISTS ${ tableName }""")
}
+ test("Test adding of array of all primitive datatypes") {
+ sql("DROP TABLE IF EXISTS alter_com")
+ sql("CREATE TABLE alter_com(intfield int) STORED AS carbondata")
+ sql(
+ "ALTER TABLE alter_com ADD COLUMNS(arr1 array<short>, arr2 array<int>, arr3 " +
+ "array<long>, arr4 array<double>, arr5 array<decimal(8,2)>, arr6 array<string>, arr7 " +
+ "array<char(5)>, arr8 array<varchar(50)>, arr9 array<boolean>, arr10 array<date>, arr11 " +
+ "array<timestamp> )")
+ val columns = sql("desc table alter_com").collect()
+ assert(columns.size.equals(12))
+ sql(
+ "insert into alter_com values(1,array(1,5),array(1,2),array(1,2,3),array(1.2d,2.3d),array" +
+ "(4.5,6.7),array('hello','world'),array('a','bcd'),array('abcd','efg'),array(true,false)," +
+ "array('2017-02-01','2018-09-11'),array('2017-02-01 00:01:00','2018-02-01 02:21:00') )")
+ checkAnswer(sql(
+ "select * from alter_com"),
+ Seq(Row(1, make(Array(1, 5)),
+ make(Array(1, 2)),
+ make(Array(1, 2, 3)),
+ make(Array(1.2, 2.3)),
+ make(Array(java.math.BigDecimal.valueOf(4.5).setScale(2),
+ java.math.BigDecimal.valueOf(6.7).setScale(2))),
+ make(Array("hello", "world")),
+ make(Array("a", "bcd")),
+ make(Array("abcd", "efg")),
+ make(Array(true, false)),
+ make(Array(Date.valueOf("2017-02-01"), Date.valueOf("2018-09-11"))),
+ make(Array(Timestamp.valueOf("2017-02-01 00:01:00"),
+ Timestamp.valueOf("2018-02-01 02:21:00")))
+ )))
+ sql("DROP TABLE IF EXISTS alter_com")
+ }
+
+ test("Test adding of struct of all primitive datatypes") {
+ sql("DROP TABLE IF EXISTS alter_com")
+ sql(
+ "CREATE TABLE alter_com(intField INT) STORED AS " +
+ "carbondata")
+ sql(
+ "ALTER TABLE alter_com ADD COLUMNS(structField struct<a:short,b:int,c:long,d:double, " +
+ "e:decimal(8,2),f:string,g:char(5),h:varchar(50),i:boolean,j:date,k:timestamp>)")
+ sql(
+ "insert into alter_com values(1, named_struct('a',1,'b',2,'c',3,'d',1.23,'e',2.34,'f'," +
+ "'hello','g','abc','h','def','i',true,'j','2017-02-01','k','2018-02-01 02:00:00.0') ) ")
+ val rows = sql("select structField from alter_com").collect()
+ assert(rows(0)(0).asInstanceOf[GenericRowWithSchema].size == 11)
+ sql("DROP TABLE IF EXISTS alter_com")
+ }
+
+ def insertIntoTableForArrayType(): Unit = {
+ sql("insert into alter_com values(4,array(2),array(1,2,3,4),array('abc','def'))")
+ sql("insert into alter_com values(5,array(1,2),array(1), array('Hulk','Thor'))")
+ sql(
+ "insert into alter_com values(6,array(1,2,3),array(1234,8,33333),array('Iron','Man'," +
+ "'Jarvis'))")
+ }
+
+ def checkRestulForArrayType(): Unit = {
+ val totalRows = sql("select * from alter_com").collect()
+ val a = sql("select * from alter_com where array_contains(arr2,2)").collect
+ val b = sql("select * from alter_com where array_contains(arr3,'Thor')").collect
+ val c = sql("select * from alter_com where intField = 1").collect
+ assert(totalRows.size == 6)
+ assert(a.size == 1)
+ assert(b.size == 1)
+ // check default value for newly added array column that is index - 3 and 4
+ assert(c(0)(2) == null && c(0)(3) == null)
+ }
+
+ def createTableForComplexTypes(dictionary: String, complexType: String): Unit = {
+ if (complexType.equals("ARRAY")) {
+ sql("DROP TABLE IF EXISTS alter_com")
+ sql(
+ "CREATE TABLE alter_com(intField INT, arr1 array<int>) " +
+ "STORED AS carbondata")
+ sql("insert into alter_com values(1,array(1) )")
+ sql("insert into alter_com values(2,array(9,0) )")
+ sql("insert into alter_com values(3,array(11,12,13) )")
+ sql(
+ s"ALTER TABLE alter_com ADD COLUMNS(arr2 array<int>, arr3 array<string>) TBLPROPERTIES" +
+ s"('$dictionary'='arr3')")
+ val schema = sql("describe alter_com").collect()
+ assert(schema.size == 4)
+ } else if (complexType.equals("STRUCT")) {
+ sql("DROP TABLE IF EXISTS alter_struct")
+ sql(
+ "create table alter_struct(roll int, department struct<id1:string,name1:string>) STORED " +
+ "AS carbondata")
+ sql(
+ "insert into alter_struct values(1, named_struct('id1', 'id1','name1','name1'))")
+ sql(
+ "ALTER TABLE alter_struct ADD COLUMNS(struct1 struct<a:string,b:string>, temp string," +
+ " intField int, struct2 struct<c:string,d:string,e:int>, arr array<int>) TBLPROPERTIES " +
+ "('LOCAL_DICTIONARY_INCLUDE'='struct1, struct2')")
+ val schema = sql("describe alter_struct").collect()
+ assert(schema.size == 7)
+ }
+
+ }
+
+ test("Test alter add for arrays enabling local dictionary") {
+ createTableForComplexTypes("LOCAL_DICTIONARY_INCLUDE", "ARRAY")
+ // For the previous segments the default value for newly added array column is null
+ insertIntoTableForArrayType
+ checkRestulForArrayType
+ sql(s"ALTER TABLE alter_com ADD COLUMNS(arr4 array<int>) ")
+ sql(s"ALTER TABLE alter_com ADD COLUMNS(arr5 array<int>, str struct<a:int,b:string>) ")
+ sql(
+ "insert into alter_com values(2,array(9,0),array(1,2,3),array(4,5),array(6,7),array(8,9)," +
+ "named_struct('a',1,'b','abcde') )")
+ val rows = sql("select * from alter_com").collect()
+ assert(rows.size == 7)
+ val columns = sql("desc table alter_com").collect()
+ assert(columns.size == 7)
+ sql("DROP TABLE IF EXISTS alter_com")
+
+ }
+
+ test("Test alter add for arrays disabling local dictionary") {
+ createTableForComplexTypes("LOCAL_DICTIONARY_EXCLUDE", "ARRAY")
+ // For the previous segments the default value for newly added array column is null
+ insertIntoTableForArrayType
+ checkRestulForArrayType
+ sql("DROP TABLE IF EXISTS alter_com")
+
Review comment:
remove unwanted line in all testcases
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [WIP]
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-812048745
Build Failed with Spark 2.3.4, Please check CI http://121.244.95.60:12602/job/ApacheCarbonPRBuilder2.3/5113/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] ydvpankaj99 commented on pull request #4115: [WIP]
Posted by GitBox <gi...@apache.org>.
ydvpankaj99 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-811709153
retest this please
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-820136083
Build Failed with Spark 2.4.5, Please check CI http://121.244.95.60:12602/job/ApacheCarbon_PR_Builder_2.4.5/3433/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] akashrn5 commented on a change in pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
akashrn5 commented on a change in pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#discussion_r609663124
##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
##########
@@ -275,6 +275,56 @@ class AlterTableColumnSchemaGenerator(
allColumns ++= Seq(columnSchema)
}
newCols ++= Seq(columnSchema)
+ if(field.dataType== Some("Array") ) {
+ columnSchema.setNumberOfChild(field.children.size)
+ val childField = field.children.get(0)
+ val childSchema: ColumnSchema = TableNewProcessor.createColumnSchema(
+ childField,
Review comment:
@akkio-97 i think here once you added column, the column will be at last, i mean to say like below
existing dim + new added dim + existing measures + new measures, same has to be present in thrift column schema also, you can refer the existing test cases to know better.
Also, have a test cases, with sort column, measures in sort column, dim in sort column etc,
please recheck and correct this and confirm.
##########
File path: core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecutorImpl.java
##########
@@ -585,7 +585,9 @@ public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) th
}
} else {
GenericQueryType complexType = complexDimensionInfoMap.get(dimensionChunkIndex[i]);
- complexType.fillRequiredBlockData(rawBlockletColumnChunks);
+ if (complexType != null) {
Review comment:
before this PR, why null check was not there?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [CARBONDATA-4160] Alter carbon schema by adding single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-813171380
Build Success with Spark 2.3.4, Please check CI http://121.244.95.60:12602/job/ApacheCarbonPRBuilder2.3/5121/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] akkio-97 commented on a change in pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
akkio-97 commented on a change in pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#discussion_r613398079
##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
##########
@@ -275,6 +275,56 @@ class AlterTableColumnSchemaGenerator(
allColumns ++= Seq(columnSchema)
}
newCols ++= Seq(columnSchema)
+ if(field.dataType== Some("Array") ) {
+ columnSchema.setNumberOfChild(field.children.size)
+ val childField = field.children.get(0)
Review comment:
done
##########
File path: core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
##########
@@ -280,6 +280,9 @@ public static void fillQueryDimensionChunkIndexes(
private static void fillParentDetails(Map<Integer, Integer> dimensionToBlockIndexMap,
CarbonDimension dimension, Map<Integer, GenericQueryType> complexTypeMap) {
+ if (!dimensionToBlockIndexMap.containsKey(dimension.getOrdinal())) {
Review comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] Indhumathi27 commented on a change in pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
Indhumathi27 commented on a change in pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#discussion_r608358732
##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
##########
@@ -275,6 +275,56 @@ class AlterTableColumnSchemaGenerator(
allColumns ++= Seq(columnSchema)
}
newCols ++= Seq(columnSchema)
+ if(field.dataType== Some("Array") ) {
+ columnSchema.setNumberOfChild(field.children.size)
+ val childField = field.children.get(0)
Review comment:
Please handle if array<complex>/struct<complex> column is provided for add column
##########
File path: integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/alterTable/TestAlterTableAddColumns.scala
##########
@@ -136,6 +137,155 @@ class TestAlterTableAddColumns extends QueryTest with BeforeAndAfterAll {
sql(s"""DROP TABLE IF EXISTS ${ tableName }""")
}
+ test("Test alter add for arrays enabling local dictionary") {
+ sql("DROP TABLE IF EXISTS alter_com")
+ sql(
+ "CREATE TABLE alter_com(intField INT, arr1 array<int>) " +
+ "STORED AS carbondata TBLPROPERTIES('local_dictionary_enable'='true')")
+
+ sql("insert into alter_com values(1,array(1) )")
+ sql("insert into alter_com values(2,array(9,0) )")
+ sql("insert into alter_com values(3,array(11,12,13) )")
+
+ sql(
+ "ALTER TABLE alter_com ADD COLUMNS(arr2 array<int>, arr3 array<string>) TBLPROPERTIES" +
+ "('LOCAL_DICTIONARY_INCLUDE'='arr3')")
+ val schema = sql("describe alter_com").collect()
+ assert(schema.size == 4)
+
+ // For the previous segments the default value for newly added array column is null
+ sql("insert into alter_com values(4,array(2),array(1,2,3,4),array('abc','def'))")
+ sql("insert into alter_com values(5,array(1,2),array(1), array('Hulk','Thor'))")
+ sql(
+ "insert into alter_com values(6,array(1,2,3),array(1234,8,33333),array('Iron','Man'," +
+ "'Jarvis'))")
+
+ val totalRows = sql("select * from alter_com").collect()
+ val a = sql("select * from alter_com where array_contains(arr2,2)").collect
+ val b = sql("select * from alter_com where array_contains(arr3,'Thor')").collect
+ val c = sql("select * from alter_com where intField = 1").collect
+ assert(totalRows.size == 6)
+ assert(a.size == 1)
+ assert(b.size == 1)
+ // check default value for newly added array column that is index - 3 and 4
+ assert(c(0)(2) == null && c(0)(3) == null)
+ sql("DROP TABLE IF EXISTS alter_com")
+
+ }
+
+ test("Test alter add for arrays disabling local dictionary") {
+ sql("DROP TABLE IF EXISTS alter_com")
+ sql(
+ "CREATE TABLE alter_com(intField INT, arr1 array<int>) " +
+ "STORED AS carbondata TBLPROPERTIES('local_dictionary_enable'='false')")
+
+ sql("insert into alter_com values(1,array(1) )")
+ sql("insert into alter_com values(2,array(9,0) )")
+ sql("insert into alter_com values(3,array(11,12,13) )")
+ sql(
+ "ALTER TABLE alter_com ADD COLUMNS(arr2 array<double>, arr3 array<string>) TBLPROPERTIES" +
+ "('LOCAL_DICTIONARY_EXCLUDE'='arr3')")
+ val schema = sql("describe alter_com").collect()
+ assert(schema.size == 4)
+
+ // For the previous segments the default value for newly added array column is null
+ sql("insert into alter_com values(4,array(2),array(1,2,3,4),array('abc','def'))")
+ sql("insert into alter_com values(5,array(1,2),array(1), array('Hulk','Thor'))")
+ sql(
+ "insert into alter_com values(6,array(1,2,3),array(1234,80,3333),array('Iron','Man'," +
+ "'Jarvis'))")
+
+ val totalRows = sql("select * from alter_com").collect()
+ val a = sql("select * from alter_com where array_contains(arr2,2)").collect
+ val b = sql("select * from alter_com where array_contains(arr3,'Thor')").collect
+ val c = sql("select * from alter_com where intField = 1").collect
+ assert(totalRows.size == 6)
+ assert(a.size == 1)
+ assert(b.size == 1)
+ // check default value for newly added array column that is index - 3 and 4
+ assert(c(0)(2) == null && c(0)(3) == null)
+ sql("DROP TABLE IF EXISTS alter_com")
+
+ }
+
+ test("Test alter add for structs enabling local dictionary") {
+ sql("DROP TABLE IF EXISTS alter_struct")
+ sql(
+ "create table alter_struct(roll int, department struct<id1:string,name1:string>) STORED AS " +
+ "carbondata TBLPROPERTIES('local_dictionary_enable'='true')")
+ sql(
+ "insert into alter_struct values(1, named_struct('id1', 'id1','name1','name1'))")
+ sql(
+ "ALTER TABLE alter_struct ADD COLUMNS(struct1 struct<a:string,b:string>, temp string," +
+ " intField int, struct2 struct<c:string,d:string,e:int>, arr array<int>) TBLPROPERTIES " +
+ "('LOCAL_DICTIONARY_INCLUDE'='struct1, struct2')")
+ val schema = sql("describe alter_struct").collect()
+ assert(schema.size == 7)
+
+ sql(
+ "insert into alter_struct values(2, named_struct('id1', 'id1','name1','name1'), " +
+ "named_struct('a','id2','b', 'abc2'), 'hello world', 5, named_struct('c','id3'," +
+ "'d', 'abc3','e', 22), array(1,2,3) )")
+ sql(
+ "insert into alter_struct values(3, named_struct('id1', 'id1','name1','name1'), " +
+ "named_struct('a','id2.1','b', 'abc2.1'), 'India', 5, named_struct('c','id3.1'," +
+ "'d', 'abc3.1','e', 25), array(4,5) )")
+
+ val totalRows = sql("select * from alter_struct").collect()
+ val a = sql("select * from alter_struct where struct1.a = 'id2' ").collect()
+ val b = sql(
+ "select * from alter_struct where struct1.a = 'id2' or struct2.c = 'id3.1' or " +
+ "array_contains(arr,5) ").collect()
+ val c = sql("select * from alter_struct where roll = 1").collect()
+
+ assert(totalRows.size == 3)
+ assert(a.size == 1)
+ assert(b.size == 2)
+ // check default value for newly added struct column
+ assert(c(0)(2) == null)
+ sql("DROP TABLE IF EXISTS alter_struct")
+
+ }
+
+ test("Test alter add for structs, disabling local dictionary") {
+ sql("DROP TABLE IF EXISTS alter_struct")
+ sql(
+ "create table alter_struct(roll int, department struct<id1:string,name1:string>) STORED AS " +
+ "carbondata TBLPROPERTIES('local_dictionary_enable'='false')")
Review comment:
Please move common queries to a method, for Local Dict enable/disable, for both array and struct
##########
File path: core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
##########
@@ -280,6 +280,9 @@ public static void fillQueryDimensionChunkIndexes(
private static void fillParentDetails(Map<Integer, Integer> dimensionToBlockIndexMap,
CarbonDimension dimension, Map<Integer, GenericQueryType> complexTypeMap) {
+ if (!dimensionToBlockIndexMap.containsKey(dimension.getOrdinal())) {
Review comment:
Please add comments why this change is required
##########
File path: index/secondary-index/src/test/scala/org/apache/carbondata/spark/testsuite/secondaryindex/TestSIWithComplexArrayType.scala
##########
@@ -43,6 +43,83 @@ class TestSIWithComplexArrayType extends QueryTest with BeforeAndAfterEach {
sql("drop table if exists complextable5")
}
+ test("Test alter add array column before creating SI") {
+ sql("drop table if exists complextable")
+ sql("create table complextable (id string, country array<string>, name string) stored as carbondata")
+ sql("insert into complextable select 1,array('china', 'us'), 'b'")
+ sql("insert into complextable select 2,array('pak'), 'v'")
+
+ sql("drop index if exists index_11 on complextable")
+ sql(
+ "ALTER TABLE complextable ADD COLUMNS(arr2 array<string>)")
+ sql("insert into complextable select 3,array('china'), 'f',array('hello','world')")
+ sql("insert into complextable select 4,array('india'),'g',array('iron','man','jarvis')")
+
+ val result1 = sql("select * from complextable where array_contains(arr2,'iron')")
+ val result2 = sql("select * from complextable where arr2[0]='iron'")
+ sql("create index index_11 on table complextable(arr2) as 'carbondata'")
+ val df1 = sql(" select * from complextable where array_contains(arr2,'iron')")
+ val df2 = sql(" select * from complextable where arr2[0]='iron'")
+ if (!isFilterPushedDownToSI(df1.queryExecution.sparkPlan)) {
+ assert(false)
+ } else {
+ assert(true)
+ }
+ if (!isFilterPushedDownToSI(df2.queryExecution.sparkPlan)) {
+ assert(false)
+ } else {
+ assert(true)
+ }
+ val doNotHitSIDf = sql(" select * from complextable where array_contains(arr2,'iron') and array_contains(arr2,'man')")
+ if (isFilterPushedDownToSI(doNotHitSIDf.queryExecution.sparkPlan)) {
+ assert(false)
+ } else {
+ assert(true)
+ }
+ checkAnswer(result1, df1)
+ checkAnswer(result2, df2)
+ }
+
+ test("Test alter add array column after creating SI") {
+ sql("drop table if exists complextable")
+ sql("create table complextable (id string, country array<string>, name string) stored as carbondata")
+ sql("insert into complextable select 1,array('china', 'us'), 'b'")
+ sql("insert into complextable select 2,array('pak'), 'v'")
+
+ sql("drop index if exists index_11 on complextable")
+ sql("create index index_11 on table complextable(country) as 'carbondata'")
+
+ sql(
Review comment:
Please add testcases for Add column of array & struct of all primitive types
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-822256718
Build Success with Spark 2.4.5, Please check CI http://121.244.95.60:12602/job/ApacheCarbon_PR_Builder_2.4.5/3451/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] CarbonDataQA2 commented on pull request #4115: [WIP]
Posted by GitBox <gi...@apache.org>.
CarbonDataQA2 commented on pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#issuecomment-811704829
Build Failed with Spark 2.4.5, Please check CI http://121.244.95.60:12602/job/ApacheCarbon_PR_Builder_2.4.5/3357/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [carbondata] akashrn5 commented on a change in pull request #4115: [CARBONDATA-4163] Support adding of single-level complex columns(array/struct)
Posted by GitBox <gi...@apache.org>.
akashrn5 commented on a change in pull request #4115:
URL: https://github.com/apache/carbondata/pull/4115#discussion_r614629919
##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
##########
@@ -235,6 +236,26 @@ class AlterTableColumnSchemaGenerator(
dataType.toString.equals("STRUCT")
}
+ private def createChildSchema(childField: Field, currentSchemaOrdinal: Int): ColumnSchema = {
+ // TODO: should support adding multi-level complex columns: CARBONDATA-4164
+ if (!childField.children.contains(null)) {
Review comment:
.contains, can we replace with `isDefined`? thats more clear
##########
File path: docs/ddl-of-carbondata.md
##########
@@ -694,7 +694,7 @@ CarbonData DDL statements are documented here,which includes:
```
ALTER TABLE carbon ADD COLUMNS (a1 INT, b1 STRING) TBLPROPERTIES('DEFAULT.VALUE.a1'='10')
```
- **NOTE:** Add Complex datatype columns is not supported.
+ **NOTE:** Adding of only single-level Complex datatype columns(only array and struct) is supported.
Review comment:
please give example if adding complex column array and struct
##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
##########
@@ -235,6 +236,26 @@ class AlterTableColumnSchemaGenerator(
dataType.toString.equals("STRUCT")
}
+ private def createChildSchema(childField: Field, currentSchemaOrdinal: Int): ColumnSchema = {
+ // TODO: should support adding multi-level complex columns: CARBONDATA-4164
+ if (!childField.children.contains(null)) {
+ throw new UnsupportedOperationException(
+ "Alter add columns with single-level complex types are only allowed")
Review comment:
```suggestion
"Alter add columns with nested complex types is not allowed")
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org