[carbondata] branch master updated: [CARBONDATA-3723] Fix insert failure when partition column is a first column of projection in MV

[ku...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git


The following commit(s) were added to refs/heads/master by this push:
     new 85623be  [CARBONDATA-3723] Fix insert failure when partition column is a first column of projection in MV
85623be is described below

commit 85623bebb97a05774d69fe03c4ded8be8c5c6cd4
Author: ajantha-bhat <aj...@gmail.com>
AuthorDate: Wed Feb 26 12:25:43 2020 +0530

    [CARBONDATA-3723] Fix insert failure when partition column is a first column of projection in MV
    
    Why is this PR needed?
    For Alter table drop and add column scenarios, partition columns
    may not be in the end of create-order column. So, added a logic
    to keep the partition column in the end in #3634.
    
    But partition may not be the last projection column during mv creation.
    So now, create order partition is kept last, but the projection is not last.
    Hence the insert gives wrong data or fails.
    
    What changes were proposed in this PR?
    rearrage partition to last in create order columns, only for alter table scenarios
    
    This closes #3640
---
 .../command/management/CarbonInsertIntoCommand.scala   | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoCommand.scala
index 25887e7..0832d62 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoCommand.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoCommand.scala
@@ -37,7 +37,7 @@ import org.apache.carbondata.common.logging.LogServiceFactory
 import org.apache.carbondata.converter.SparkDataTypeConverterImpl
 import org.apache.carbondata.core.datastore.impl.FileFactory
 import org.apache.carbondata.core.indexstore.PartitionSpec
-import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, TableInfo}
+import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, TableInfo, TableSchema}
 import org.apache.carbondata.core.metadata.schema.table.column.{CarbonColumn, ColumnSchema}
 import org.apache.carbondata.core.statusmanager.{LoadMetadataDetails, SegmentStatus, SegmentStatusManager}
 import org.apache.carbondata.core.util.{CarbonProperties, DataTypeUtil, ThreadLocalSessionInfo}
@@ -274,6 +274,20 @@ case class CarbonInsertIntoCommand(databaseNameOp: Option[String],
     Seq.empty
   }
 
+  private def isAlteredSchema(tableSchema: TableSchema): Boolean = {
+    if (tableInfo.getFactTable.getSchemaEvolution != null) {
+      tableInfo
+        .getFactTable
+        .getSchemaEvolution
+        .getSchemaEvolutionEntryList.asScala.exists(entry =>
+        (entry.getAdded != null && entry.getAdded.size() > 0) ||
+        (entry.getRemoved != null && entry.getRemoved.size() > 0)
+      )
+    } else {
+      false
+    }
+  }
+
   def getReArrangedLogicalPlan(
       reArrangedIndex: Seq[Int],
       selectedColumnSchema: Seq[ColumnSchema],
@@ -468,7 +482,7 @@ case class CarbonInsertIntoCommand(databaseNameOp: Option[String],
     }
     var createOrderColumns = table.getCreateOrderColumn.asScala
     val createOrderMap = mutable.Map[String, Int]()
-    if (partitionColumnNames != null) {
+    if (partitionColumnNames != null && isAlteredSchema(tableInfo.getFactTable)) {
       // For alter table drop/add column scenarios, partition column may not be in the end.
       // Need to keep it in the end.
       createOrderColumns = createOrderColumns.filterNot(col =>