You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2016/08/01 10:05:39 UTC
[41/47] incubator-carbondata git commit: Problem: Exception thrown
while executing drop table command in spark-sql cli (#888)
Problem: Exception thrown while executing drop table command in spark-sql cli (#888)
Analysis: Whenever drop table operation is performed, first the cube is deleted from carbon metastore and then command is executed to delete the same table from hive metastore. While deleting from hive metastore the call comes back to carbon metastore catalog where we again check for table existence but table is not found and exception is thrown. Call comes again to carbon because the command is executed as spark sql command and not as hive sql command and we catch the logical plan when drop command is executed as spark sql command.
Impact: drop table operation
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/b82a960b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/b82a960b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/b82a960b
Branch: refs/heads/master
Commit: b82a960bd58acd83d1f6ecf79340128170178414
Parents: 87f0dec
Author: manishgupta88 <to...@gmail.com>
Authored: Fri Jul 29 15:31:33 2016 +0530
Committer: sujith71955 <su...@gmail.com>
Committed: Fri Jul 29 18:01:33 2016 +0800
----------------------------------------------------------------------
.../spark/sql/hive/CarbonMetastoreCatalog.scala | 12 +---------
.../spark/rdd/CarbonGlobalDictionaryRDD.scala | 14 ++++++------
.../spark/util/GlobalDictionaryUtil.scala | 6 ++---
.../resources/complexTypeDecimalNestedHive.csv | 8 +++++++
.../test/resources/emptyDimensionDataHive.csv | 20 +++++++++++++++++
.../dataload/TestLoadDataWithNullMeasures.scala | 1 +
.../TestDataWithDicExcludeAndInclude.scala | 22 ++++++++++++++-----
.../dataload/TestLoadDataWithHiveSyntax.scala | 23 +++++++++++++++-----
.../NoDictionaryColumnTestCase.scala | 6 +++--
.../NullMeasureValueTestCaseFilter.scala | 7 +++---
10 files changed, 82 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b82a960b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastoreCatalog.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastoreCatalog.scala b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastoreCatalog.scala
index 7ec1b87..e18e1d3 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastoreCatalog.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastoreCatalog.scala
@@ -495,17 +495,7 @@ class CarbonMetastoreCatalog(hive: HiveContext, val storePath: String, client: C
org.carbondata.core.carbon.metadata.CarbonMetadata.getInstance
.removeTable(schemaName + "_" + cubeName)
- try {
- sqlContext.sql(s"DROP TABLE $schemaName.$cubeName").collect()
- } catch {
- case e: Exception =>
- LOGGER.audit(
- s"Error While deleting the table $schemaName.$cubeName during drop Table" + e.getMessage)
- }
-
- logInfo(s"Table $cubeName of $schemaName Database dropped syccessfully.")
- LOGGER.info("Table " + cubeName + " of " + schemaName + " Database dropped syccessfully.")
-
+ sqlContext.asInstanceOf[HiveContext].runSqlHive(s"DROP TABLE IF EXISTS $schemaName.$cubeName")
}
private def getTimestampFileAndType(schemaName: String, cubeName: String) = {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b82a960b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
index a1870e9..70d0cb1 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
@@ -156,7 +156,7 @@ case class DictionaryLoadModel(table: CarbonTableIdentifier,
rowCountPercentage: Double,
columnIdentifier: Array[ColumnIdentifier],
isFirstLoad: Boolean,
- hdfstemplocation: String,
+ hdfsTempLocation: String,
lockType: String,
zooKeeperUrl: String) extends Serializable
@@ -301,11 +301,11 @@ class CarbonGlobalDictionaryGenerateRDD(
val carbonTablePath = pathService.getCarbonTablePath(model.columnIdentifier(split.index),
model.hdfsLocation, model.table)
CarbonProperties.getInstance.addProperty(CarbonCommonConstants.HDFS_TEMP_LOCATION,
- model.hdfstemplocation)
+ model.hdfsTempLocation)
CarbonProperties.getInstance.addProperty(CarbonCommonConstants.LOCK_TYPE,
- model.lockType)
- CarbonProperties.getInstance.addProperty(CarbonCommonConstants.ZOOKEEPER_URL,
- model.zooKeeperUrl)
+ model.lockType)
+ CarbonProperties.getInstance.addProperty(CarbonCommonConstants.ZOOKEEPER_URL,
+ model.zooKeeperUrl)
val dictLock = CarbonLockFactory
.getCarbonLockObj(carbonTablePath.getRelativeDictionaryDirectory,
model.columnIdentifier(split.index).getColumnId + LockUsage.LOCK)
@@ -408,8 +408,8 @@ class CarbonGlobalDictionaryGenerateRDD(
.clearDictionaryCache(dictionaryForDistinctValueLookUp);
}
org.carbondata.core.util.CarbonUtil.clearDictionaryCache(dictionaryForSortIndexWriting);
- if (dictLock != null) {
- if (isDictionaryLocked && dictLock.unlock()) {
+ if (dictLock != null && isDictionaryLocked) {
+ if (dictLock.unlock()) {
logInfo(s"Dictionary ${
model.primDimensions(split.index).getColName
} Unlocked Successfully.")
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b82a960b/integration/spark/src/main/scala/org/carbondata/spark/util/GlobalDictionaryUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/util/GlobalDictionaryUtil.scala b/integration/spark/src/main/scala/org/carbondata/spark/util/GlobalDictionaryUtil.scala
index 0e15803..1c45230 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/util/GlobalDictionaryUtil.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/util/GlobalDictionaryUtil.scala
@@ -297,8 +297,8 @@ object GlobalDictionaryUtil extends Logging {
val dictFilePaths = dictDetail.dictFilePaths
val dictFileExists = dictDetail.dictFileExists
val columnIdentifier = dictDetail.columnIdentifiers
- val hdfstemplocation = CarbonProperties.getInstance.getProperty(
- CarbonCommonConstants.HDFS_TEMP_LOCATION, System.getProperty("java.io.tmpdir"))
+ val hdfsTempLocation = CarbonProperties.getInstance.
+ getProperty(CarbonCommonConstants.HDFS_TEMP_LOCATION, System.getProperty("java.io.tmpdir"))
val lockType = CarbonProperties.getInstance
.getProperty(CarbonCommonConstants.LOCK_TYPE, CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS)
val zookeeperUrl = CarbonProperties.getInstance.getProperty(CarbonCommonConstants.ZOOKEEPER_URL)
@@ -331,7 +331,7 @@ object GlobalDictionaryUtil extends Logging {
rowCountPercentage,
columnIdentifier,
carbonLoadModel.getLoadMetadataDetails.size() == 0,
- hdfstemplocation,
+ hdfsTempLocation,
lockType,
zookeeperUrl)
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b82a960b/integration/spark/src/test/resources/complexTypeDecimalNestedHive.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/complexTypeDecimalNestedHive.csv b/integration/spark/src/test/resources/complexTypeDecimalNestedHive.csv
new file mode 100644
index 0000000..ae67723
--- /dev/null
+++ b/integration/spark/src/test/resources/complexTypeDecimalNestedHive.csv
@@ -0,0 +1,8 @@
+1.2,2015/7/23,china,aaa1,phone197,ASD69643,15000,3.113:imei$3.33:imsi
+2,2015/7/24,china,aaa2,phone756,ASD42892,15001,3.123:imei$7.33:imsi
+4.3,2015/7/26,china,aaa4,phone2435,ASD66902,15003,3.123:imei$56.33:imsi
+5,2015/7/27,china,aaa5,phone2441,ASD90633,15004,3.133:imei$5.33:imsi
+6.5,2015/7/28,china,aaa6,phone294,ASD59961,15005,3.133:imei$54.33:imsi
+8,2015/7/30,china,aaa8,phone1848,ASD57308,15007,32.13:imei$56.33:imsi
+9.1,2015/7/18,china,aaa9,phone706,ASD86717,15008,3.213:imei$44.33:imsi
+10,2015/7/19,usa,aaa10,phone685,ASD30505,15009,32.13:imei$33.33:imsi
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b82a960b/integration/spark/src/test/resources/emptyDimensionDataHive.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/emptyDimensionDataHive.csv b/integration/spark/src/test/resources/emptyDimensionDataHive.csv
new file mode 100644
index 0000000..2e89887
--- /dev/null
+++ b/integration/spark/src/test/resources/emptyDimensionDataHive.csv
@@ -0,0 +1,20 @@
+1,2015/7/23,,aaa1,,,15000
+2,2015/7/24,,,,,15001
+3,2015/7/25,china,aaa3,phone1904,ASD37014,15002
+4,2015/7/26,china,aaa4,,ASD66902,15003
+5,2015/7/27,china,aaa5,phone2441,,15004
+6,2015/7/28,,aaa6,phone294,ASD59961,15005
+7,2015/7/29,china,aaa7,phone610,ASD14875,15006
+8,2015/7/30,china,aaa8,,ASD57308,15007
+9,2015/7/18,china,aaa9,phone706,ASD86717,15008
+10,2015/7/19,usa,aaa10,phone685,ASD30505,15009
+11,2015/7/18,china,aaa11,phone1554,ASD26101,15010
+12,2015/7/19,,aaa12,phone1781,ASD85711,15011
+13,2015/7/20,china,,phone943,,15012
+14,2015/7/21,china,aaa14,phone1954,ASD80468,15013
+15,2015/7/22,china,aaa15,phone451,ASD1954,15014
+16,2015/7/23,china,aaa16,phone390,ASD38513,15015
+17,2015/7/24,,aaa17,phone1929,ASD86213,15016
+18,2015/7/25,usa,aaa18,,ASD88812,15017
+19,2015/7/26,china,aaa19,phone2151,ASD9316,15018
+20,2015/7/27,china,aaa20,phone2625,ASD62597,15019
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b82a960b/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithNullMeasures.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithNullMeasures.scala b/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithNullMeasures.scala
index 09f44ad..3a920b8 100644
--- a/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithNullMeasures.scala
+++ b/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithNullMeasures.scala
@@ -29,6 +29,7 @@ import org.scalatest.BeforeAndAfterAll
*/
class TestLoadDataWithNullMeasures extends QueryTest with BeforeAndAfterAll {
override def beforeAll {
+ sql("DROP TABLE IF EXISTS carbontable")
sql(
"CREATE TABLE carbontable (empno int, empname String, designation String, doj String, " +
"workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, " +
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b82a960b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestDataWithDicExcludeAndInclude.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestDataWithDicExcludeAndInclude.scala b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestDataWithDicExcludeAndInclude.scala
index 7ab1a33..0e333dd 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestDataWithDicExcludeAndInclude.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestDataWithDicExcludeAndInclude.scala
@@ -40,14 +40,22 @@ class TestLoadDataWithDictionaryExcludeAndInclude extends QueryTest with BeforeA
}
def dropTable() = {
- sql("DROP TABLE IF EXISTS t3")
+ sql("DROP TABLE IF EXISTS exclude_include_t3")
+ sql("DROP TABLE IF EXISTS exclude_include_hive_t3")
}
def buildTable() = {
try {
sql(
"""
- CREATE TABLE IF NOT EXISTS t3
+ CREATE TABLE exclude_include_hive_t3
+ (ID Int, date Timestamp, country String,
+ name String, phonetype String, serialname String, salary Int)
+ row format delimited fields terminated by ','
+ """)
+ sql(
+ """
+ CREATE TABLE exclude_include_t3
(ID Int, date Timestamp, country String,
name String, phonetype String, serialname String, salary Int)
STORED BY 'org.apache.carbondata.format'
@@ -65,7 +73,11 @@ class TestLoadDataWithDictionaryExcludeAndInclude extends QueryTest with BeforeA
.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
sql(
s"""
- LOAD DATA LOCAL INPATH '$filePath' into table t3
+ LOAD DATA LOCAL INPATH '$filePath' into table exclude_include_t3
+ """)
+ sql(
+ s"""
+ LOAD DATA LOCAL INPATH './src/test/resources/emptyDimensionDataHive.csv' into table exclude_include_hive_t3
""")
} catch {
case ex: Throwable => logError(ex.getMessage + "\r\n" + ex.getStackTraceString)
@@ -81,9 +93,7 @@ class TestLoadDataWithDictionaryExcludeAndInclude extends QueryTest with BeforeA
test("test load data with dictionary exclude & include and with empty dimension") {
checkAnswer(
- sql("select ID from t3"), Seq(Row(1), Row(2), Row(3), Row(4), Row(5), Row(6), Row(7),
- Row(8), Row(9), Row(10), Row(11), Row(12), Row(13), Row(14), Row(15), Row(16), Row
- (17), Row(18), Row(19), Row(20))
+ sql("select ID from exclude_include_t3"), sql("select ID from exclude_include_hive_t3")
)
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b82a960b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala
index 173fe6b..731ed89 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala
@@ -529,25 +529,38 @@ class TestLoadDataWithHiveSyntax extends QueryTest with BeforeAndAfterAll {
}
test("test data which contain column with decimal data type in array of struct."){
- sql("DROP TABLE IF EXISTS t3")
+ sql("DROP TABLE IF EXISTS complex_t3")
+ sql("DROP TABLE IF EXISTS complex_hive_t3")
sql(
"""
- CREATE TABLE IF NOT EXISTS t3
+ CREATE TABLE complex_t3
(ID decimal, date Timestamp, country String,
name String, phonetype String, serialname String, salary Int, complex
array<struct<a:decimal(4,2),str:string>>)
STORED BY 'org.apache.carbondata.format'
"""
)
+ sql(
+ """
+ CREATE TABLE complex_hive_t3
+ (ID decimal, date Timestamp, country String,
+ name String, phonetype String, serialname String, salary Int, complex
+ array<struct<a:decimal(4,2),str:string>>)
+ row format delimited fields terminated by ','
+ """
+ )
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
sql(s"""
- LOAD DATA LOCAL INPATH './src/test/resources/complexTypeDecimalNested.csv' into table t3
+ LOAD DATA LOCAL INPATH './src/test/resources/complexTypeDecimalNested.csv' into table complex_t3
""")
- checkAnswer(sql("select count(*) from t3"),Seq(Row(8)))
- checkAnswer(sql("select id from t3 where salary = 15000"),Seq(Row(1)))
+ sql(s"""
+ LOAD DATA LOCAL INPATH './src/test/resources/complexTypeDecimalNestedHive.csv' into table complex_hive_t3
+ """)
+ checkAnswer(sql("select count(*) from complex_t3"),sql("select count(*) from complex_hive_t3"))
+ checkAnswer(sql("select id from complex_t3 where salary = 15000"),sql("select id from complex_hive_t3 where salary = 15000"))
}
test("test data loading when delimiter is '|' and data with header") {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b82a960b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/detailquery/NoDictionaryColumnTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/detailquery/NoDictionaryColumnTestCase.scala b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/detailquery/NoDictionaryColumnTestCase.scala
index 7ec5e9a..9e9fa90 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/detailquery/NoDictionaryColumnTestCase.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/detailquery/NoDictionaryColumnTestCase.scala
@@ -31,6 +31,8 @@ import org.scalatest.BeforeAndAfterAll
class NoDictionaryColumnTestCase extends QueryTest with BeforeAndAfterAll {
override def beforeAll {
+ sql("DROP TABLE IF EXISTS carbonTable")
+ sql("DROP TABLE IF EXISTS hiveTable")
sql("CREATE TABLE carbonTable (imei String, age Int, num BigInt) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('DICTIONARY_EXCLUDE'='age,num')")
sql("LOAD DATA LOCAL INPATH './src/test/resources/datawithNegtiveNumber.csv' INTO TABLE carbonTable")
sql("CREATE TABLE hiveTable (imei String, age Int, num BigInt) ROW FORMAT DELIMITED FIELDS TERMINATED BY ','")
@@ -52,7 +54,7 @@ class NoDictionaryColumnTestCase extends QueryTest with BeforeAndAfterAll {
}
override def afterAll {
- sql("DROP TABLE carbonTable")
- sql("DROP TABLE hiveTable")
+ sql("DROP TABLE IF EXISTS carbonTable")
+ sql("DROP TABLE IF EXISTS hiveTable")
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b82a960b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/NullMeasureValueTestCaseFilter.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/NullMeasureValueTestCaseFilter.scala b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/NullMeasureValueTestCaseFilter.scala
index 2449f64..d81974d 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/NullMeasureValueTestCaseFilter.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/NullMeasureValueTestCaseFilter.scala
@@ -10,8 +10,9 @@ import org.scalatest.BeforeAndAfterAll
class NullMeasureValueTestCaseFilter extends QueryTest with BeforeAndAfterAll {
override def beforeAll {
+ sql("drop table if exists t3")
sql(
- "CREATE TABLE IF NOT EXISTS t3 (ID Int, date Timestamp, country String, name String, " +
+ "CREATE TABLE t3 (ID bigInt, date Timestamp, country String, name String, " +
"phonetype String, serialname String, salary Int) STORED BY 'org.apache.carbondata.format'"
)
CarbonProperties.getInstance()
@@ -22,13 +23,13 @@ class NullMeasureValueTestCaseFilter extends QueryTest with BeforeAndAfterAll {
test("select ID from t3 where salary is not null") {
checkAnswer(
sql("select ID from t3 where salary is not null"),
- Seq(Row(1.0),Row(4.0)))
+ Seq(Row(1),Row(4)))
}
test("select ID from t3 where salary is null") {
checkAnswer(
sql("select ID from t3 where salary is null"),
- Seq(Row(2.0),Row(3.0)))
+ Seq(Row(2),Row(3)))
}
override def afterAll {