You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/11/08 04:15:44 UTC
[10/24] carbondata git commit: [CARBONDATA-1593] Add partition to
table cause NoSuchTableException
[CARBONDATA-1593] Add partition to table cause NoSuchTableException
AlterTableSplitCarbonPartition's processSchema method doesn't provide db info to sparkSession.catalog.refreshTable, this will cause NoSuchTableException when we add partitions to carbondata table.
This closes #1452
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/e9454499
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/e9454499
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/e9454499
Branch: refs/heads/pre-aggregate
Commit: e9454499dcb89ed69b1e18f79b3003ea8e5d8d25
Parents: 0586146
Author: wyp <wy...@163.com>
Authored: Mon Oct 30 12:49:53 2017 +0800
Committer: lionelcao <wh...@gmail.com>
Committed: Thu Nov 2 10:10:45 2017 +0800
----------------------------------------------------------------------
.../AlterTableSplitCarbonPartitionCommand.scala | 3 +-
.../partition/TestAlterPartitionTable.scala | 57 ++++++++++++++++++++
2 files changed, 59 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e9454499/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/AlterTableSplitCarbonPartitionCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/AlterTableSplitCarbonPartitionCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/AlterTableSplitCarbonPartitionCommand.scala
index e16dfc9..12bf31e 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/AlterTableSplitCarbonPartitionCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/AlterTableSplitCarbonPartitionCommand.scala
@@ -23,6 +23,7 @@ import java.util
import scala.collection.JavaConverters._
import org.apache.spark.sql.{CarbonEnv, Row, SparkSession}
+import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.execution.command.{AlterTableSplitPartitionModel, DataProcessCommand, RunnableCommand, SchemaProcessCommand}
import org.apache.spark.sql.hive.{CarbonMetaStore, CarbonRelation}
import org.apache.spark.util.{AlterTableUtil, PartitionUtils}
@@ -109,7 +110,7 @@ case class AlterTableSplitCarbonPartitionCommand(
CarbonUtil.writeThriftTableToSchemaFile(schemaFilePath, thriftTable)
// update the schema modified time
carbonMetaStore.updateAndTouchSchemasUpdatedTime(storePath)
- sparkSession.catalog.refreshTable(tableName)
+ sparkSession.sessionState.catalog.refreshTable(TableIdentifier(tableName, Option(dbName)))
Seq.empty
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e9454499/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
index 5e81044..04de9a3 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
@@ -779,6 +779,63 @@ class TestAlterPartitionTable extends QueryTest with BeforeAndAfterAll {
.contains("Data in range info must be the same type with the partition field's type"))
}
+ test("Add partition to table in or not in default database") {
+ sql("DROP TABLE IF EXISTS carbon_table_default_db")
+ sql(
+ """
+ | CREATE TABLE carbon_table_default_db(id INT, name STRING) PARTITIONED BY (dt STRING)
+ | STORED BY 'carbondata' TBLPROPERTIES('PARTITION_TYPE'='RANGE', 'RANGE_INFO'='2015,2016')
+ """.stripMargin)
+ sql("ALTER TABLE carbon_table_default_db ADD PARTITION ('2017')")
+
+ val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default_carbon_table_default_db")
+ val partitionInfo = carbonTable.getPartitionInfo(carbonTable.getFactTableName)
+ val partitionIds = partitionInfo.getPartitionIds
+ val range_info = partitionInfo.getRangeInfo
+ assert(partitionIds == List(0, 1, 2, 3).map(Integer.valueOf(_)).asJava)
+ assert(partitionInfo.getMaxPartitionId == 3)
+ assert(partitionInfo.getNumPartitions == 4)
+ assert(range_info.get(0) == "2015")
+ assert(range_info.get(1) == "2016")
+ assert(range_info.get(2) == "2017")
+
+ sql("CREATE DATABASE IF NOT EXISTS carbondb")
+ sql("DROP TABLE IF EXISTS carbondb.carbontable")
+ sql(
+ """
+ | CREATE TABLE carbondb.carbontable(id INT, name STRING) PARTITIONED BY (dt STRING)
+ | STORED BY 'carbondata' TBLPROPERTIES('PARTITION_TYPE'='RANGE', 'RANGE_INFO'='2015,2016')
+ """.stripMargin)
+ sql("ALTER TABLE carbondb.carbontable ADD PARTITION ('2017')")
+
+ val carbonTable1 = CarbonMetadata.getInstance().getCarbonTable("carbondb_carbontable")
+ val partitionInfo1 = carbonTable1.getPartitionInfo(carbonTable1.getFactTableName)
+ val partitionIds1 = partitionInfo1.getPartitionIds
+ val range_info1 = partitionInfo1.getRangeInfo
+ assert(partitionIds1 == List(0, 1, 2, 3).map(Integer.valueOf(_)).asJava)
+ assert(partitionInfo1.getMaxPartitionId == 3)
+ assert(partitionInfo1.getNumPartitions == 4)
+ assert(range_info1.get(0) == "2015")
+ assert(range_info1.get(1) == "2016")
+ assert(range_info1.get(2) == "2017")
+ }
+
+ test("test exception when alter partition's table doesn't exist in a perticular database") {
+ val exception_test_add_partition: Exception = intercept[Exception] {
+ sql("CREATE DATABASE IF NOT EXISTS carbondb")
+ sql("USE default")
+ sql(
+ """
+ | CREATE TABLE carbon_table_in_default_db(id INT, name STRING)
+ | PARTITIONED BY (dt STRING) STORED BY 'carbondata'
+ | TBLPROPERTIES('PARTITION_TYPE'='RANGE', 'RANGE_INFO'='2015,2016')
+ """.stripMargin)
+ sql("ALTER TABLE carbondb.carbon_table_in_default_db ADD PARTITION ('2017')")
+ }
+ assert(exception_test_add_partition.getMessage
+ .contains("Table or view 'carbon_table_in_default_db' not found in database 'carbondb'"))
+ }
+
def validateDataFiles(tableUniqueName: String, segmentId: String, partitions: Seq[Int]): Unit = {
val carbonTable = CarbonMetadata.getInstance().getCarbonTable(tableUniqueName)
val dataFiles = getDataFiles(carbonTable, segmentId)