You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2017/06/15 11:50:37 UTC
[32/42] carbondata git commit: Fixed database cascade in spark 2.1
and alter table in vector mode.
Fixed database cascade in spark 2.1 and alter table in vector mode.
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/809d8806
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/809d8806
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/809d8806
Branch: refs/heads/branch-1.1
Commit: 809d880684f68a43bb83eccf23c5409fe02bf15f
Parents: 105b7c3
Author: ravipesala <ra...@gmail.com>
Authored: Sat Jun 3 13:11:57 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Jun 15 13:25:54 2017 +0530
----------------------------------------------------------------------
...tCreateTableWithDatabaseNameCaseChange.scala | 24 ++++++++++++++++++++
.../spark/sql/test/SparkTestQueryExecutor.scala | 1 +
.../vectorreader/ColumnarVectorWrapper.java | 14 ++++++------
.../execution/command/CarbonHiveCommands.scala | 7 +++---
4 files changed, 35 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/809d8806/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateTableWithDatabaseNameCaseChange.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateTableWithDatabaseNameCaseChange.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateTableWithDatabaseNameCaseChange.scala
index 87aac94..5bf55f9 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateTableWithDatabaseNameCaseChange.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateTableWithDatabaseNameCaseChange.scala
@@ -51,6 +51,30 @@ class TestCreateTableWithDatabaseNameCaseChange extends QueryTest with BeforeAnd
}
}
+ test("test drop database cascade with case sensitive") {
+ // this test case will test the creation of table for different case for database name.
+ // In hive dbName folder is always created with small case in HDFS. Carbon should behave
+ // the same way. If table creation fails during second time creation it means in HDFS
+ // separate folders are created for the matching case in commands executed.
+ sql("drop database if exists AbCdEf cascade")
+ sql("create database AbCdEf")
+ sql("use AbCdEf")
+ sql("create table carbonTable(a int, b string)stored by 'carbondata'")
+ sql("use default")
+ sql("drop database if exists AbCdEf cascade")
+ sql("create database AbCdEf")
+ sql("use AbCdEf")
+ try {
+ sql("create table carbonTable(a int, b string)stored by 'carbondata'")
+ assert(true)
+ } catch {
+ case ex: Exception =>
+ assert(false)
+ }
+ sql("use default")
+ sql("drop database if exists AbCdEf cascade")
+ }
+
override def afterAll {
sql("use default")
sql("drop database if exists dbCaseChange cascade")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/809d8806/integration/spark/src/main/scala/org/apache/spark/sql/test/SparkTestQueryExecutor.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/test/SparkTestQueryExecutor.scala b/integration/spark/src/main/scala/org/apache/spark/sql/test/SparkTestQueryExecutor.scala
index 591cdf4..27df623 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/test/SparkTestQueryExecutor.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/test/SparkTestQueryExecutor.scala
@@ -42,6 +42,7 @@ object SparkTestQueryExecutor {
.addProperty(CarbonCommonConstants.STORE_LOCATION_TEMP_PATH,
System.getProperty("java.io.tmpdir"))
.addProperty(CarbonCommonConstants.LOCK_TYPE, CarbonCommonConstants.CARBON_LOCK_TYPE_LOCAL)
+ .addProperty(CarbonCommonConstants.STORE_LOCATION, TestQueryExecutor.storeLocation)
val sc = new SparkContext(new SparkConf()
.setAppName("CarbonSpark")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/809d8806/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapper.java
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapper.java b/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapper.java
index c3d2a87..5ab741b 100644
--- a/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapper.java
+++ b/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapper.java
@@ -60,7 +60,7 @@ class ColumnarVectorWrapper implements CarbonColumnVector {
if (filteredRowsExist) {
for (int i = 0; i < count; i++) {
if (!filteredRows[rowId]) {
- putShort(counter++, value);
+ columnVector.putShort(counter++, value);
}
rowId++;
}
@@ -79,7 +79,7 @@ class ColumnarVectorWrapper implements CarbonColumnVector {
if (filteredRowsExist) {
for (int i = 0; i < count; i++) {
if (!filteredRows[rowId]) {
- putInt(counter++, value);
+ columnVector.putInt(counter++, value);
}
rowId++;
}
@@ -98,7 +98,7 @@ class ColumnarVectorWrapper implements CarbonColumnVector {
if (filteredRowsExist) {
for (int i = 0; i < count; i++) {
if (!filteredRows[rowId]) {
- putLong(counter++, value);
+ columnVector.putLong(counter++, value);
}
rowId++;
}
@@ -116,7 +116,7 @@ class ColumnarVectorWrapper implements CarbonColumnVector {
@Override public void putDecimals(int rowId, int count, Decimal value, int precision) {
for (int i = 0; i < count; i++) {
if (!filteredRows[rowId]) {
- putDecimal(counter++, value, precision);
+ columnVector.putDecimal(counter++, value, precision);
}
rowId++;
}
@@ -132,7 +132,7 @@ class ColumnarVectorWrapper implements CarbonColumnVector {
if (filteredRowsExist) {
for (int i = 0; i < count; i++) {
if (!filteredRows[rowId]) {
- putDouble(counter++, value);
+ columnVector.putDouble(counter++, value);
}
rowId++;
}
@@ -150,7 +150,7 @@ class ColumnarVectorWrapper implements CarbonColumnVector {
@Override public void putBytes(int rowId, int count, byte[] value) {
for (int i = 0; i < count; i++) {
if (!filteredRows[rowId]) {
- putBytes(counter++, value);
+ columnVector.putByteArray(counter++, value);
}
rowId++;
}
@@ -172,7 +172,7 @@ class ColumnarVectorWrapper implements CarbonColumnVector {
if (filteredRowsExist) {
for (int i = 0; i < count; i++) {
if (!filteredRows[rowId]) {
- putNull(counter++);
+ columnVector.putNull(counter++);
}
rowId++;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/809d8806/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/CarbonHiveCommands.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/CarbonHiveCommands.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/CarbonHiveCommands.scala
index 2786620..b72f077 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/CarbonHiveCommands.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/CarbonHiveCommands.scala
@@ -31,13 +31,12 @@ case class CarbonDropDatabaseCommand(command: DropDatabaseCommand)
val rows = command.run(sparkSession)
if (command.cascade) {
val tablesInDB = CarbonEnv.getInstance(sparkSession).carbonMetastore.getAllTables()
- .filterNot(_.database.exists(_.equalsIgnoreCase(dbName)))
+ .filter(_.database.exists(_.equalsIgnoreCase(dbName)))
tablesInDB.foreach { tableName =>
- CarbonDropTableCommand(true, Some(dbName), tableName.table).run(sparkSession)
+ CarbonDropTableCommand(true, tableName.database, tableName.table).run(sparkSession)
}
}
- CarbonEnv.getInstance(sparkSession).carbonMetastore.dropDatabaseDirectory(dbName)
+ CarbonEnv.getInstance(sparkSession).carbonMetastore.dropDatabaseDirectory(dbName.toLowerCase)
rows
}
}
-