You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2018/09/05 12:43:22 UTC
carbondata git commit: [CARBONDATA-2866] Block schema in external
table
Repository: carbondata
Updated Branches:
refs/heads/master b611a861a -> af2c469bb
[CARBONDATA-2866] Block schema in external table
This PR blocks schema in CREATE EXTERNAL TABLE
This closes #2645
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/af2c469b
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/af2c469b
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/af2c469b
Branch: refs/heads/master
Commit: af2c469bb8cd9e9fd87c7e6cc9f3f8fb66c578ed
Parents: b611a86
Author: Jacky Li <ja...@qq.com>
Authored: Fri Aug 17 22:04:27 2018 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Wed Sep 5 18:13:13 2018 +0530
----------------------------------------------------------------------
.../createTable/TestCreateExternalTable.scala | 19 +++++++++++
.../TestNonTransactionalCarbonTable.scala | 36 +++++++++++---------
...ansactionalCarbonTableWithAvroDataType.scala | 28 +++++++--------
.../org/apache/spark/sql/CarbonSource.scala | 9 ++---
.../sql/parser/CarbonSparkSqlParserUtil.scala | 18 +++++++---
5 files changed, 68 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/af2c469b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateExternalTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateExternalTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateExternalTable.scala
index 6fb24c7..56f038e 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateExternalTable.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateExternalTable.scala
@@ -90,6 +90,25 @@ class TestCreateExternalTable extends QueryTest with BeforeAndAfterAll {
}
}
+ test("create external table with specified schema") {
+ assert(new File(originDataPath).exists())
+ sql("DROP TABLE IF EXISTS source")
+ val ex = intercept[AnalysisException] {
+ sql(
+ s"""
+ |CREATE EXTERNAL TABLE source (key INT)
+ |STORED BY 'carbondata'
+ |LOCATION '$storeLocation/origin'
+ """.stripMargin)
+ }
+ assert(ex.message.contains("Schema may not be specified for external table"))
+
+ sql("DROP TABLE IF EXISTS source")
+
+ // DROP TABLE should not delete data
+ assert(new File(originDataPath).exists())
+ }
+
test("create external table with empty folder") {
val exception = intercept[AnalysisException] {
sql(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/af2c469b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
index 6406a66..87d5622 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
@@ -278,16 +278,18 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
// with partition
sql(
- s"""CREATE EXTERNAL TABLE sdkOutputTable(name string) PARTITIONED BY (age int) STORED BY
- |'carbondata' LOCATION
- |'$writerPath' """.stripMargin)
+ s"""
+ | CREATE EXTERNAL TABLE sdkOutputTable
+ | STORED BY 'carbondata'
+ | LOCATION '$writerPath'
+ """.stripMargin)
sql("drop table if exists t1")
sql("create table if not exists t1 (name string, age int, height double) STORED BY 'org.apache.carbondata.format'")
- var i =0;
+ var i =0
while (i<50){
sql (s"""insert into t1 values ("aaaaa", 12, 20)""").show(200,false)
- i = i+1;
+ i = i+1
}
checkAnswer(sql("select count(*) from t1"),Seq(Row(50)))
val one = Future {
@@ -400,7 +402,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
// with partition
sql(
- s"""CREATE EXTERNAL TABLE sdkOutputTable(name string) PARTITIONED BY (age int) STORED BY
+ s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
@@ -413,7 +415,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
sql("DROP TABLE IF EXISTS sdkOutputTable")
// with partition
sql(
- s"""CREATE EXTERNAL TABLE sdkOutputTable(name string) PARTITIONED BY (age int) STORED BY
+ s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
@@ -425,7 +427,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
sql("DROP TABLE IF EXISTS sdkOutputTable")
// with partition
sql(
- s"""CREATE EXTERNAL TABLE sdkOutputTable(name string) PARTITIONED BY (age int) STORED BY
+ s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
@@ -437,7 +439,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
// with partition
sql(
- s"""CREATE EXTERNAL TABLE sdkOutputTable(name string) PARTITIONED BY (age int) STORED BY
+ s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
@@ -464,7 +466,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
// with partition
sql(
- s"""CREATE EXTERNAL TABLE sdkOutputTable(name string) PARTITIONED BY (age int) STORED BY
+ s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
@@ -487,7 +489,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
// with partition
sql(
- s"""CREATE EXTERNAL TABLE sdkOutputTable(name string) PARTITIONED BY (age int) STORED BY
+ s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
@@ -520,7 +522,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
// with partition
sql(
- s"""CREATE EXTERNAL TABLE sdkOutputTable(name string) PARTITIONED BY (age int) STORED BY
+ s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
@@ -558,7 +560,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
// with partition
sql(
- s"""CREATE EXTERNAL TABLE sdkOutputTable(name string) PARTITIONED BY (age int) STORED BY
+ s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
@@ -737,7 +739,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
// with schema
sql(
- s"""CREATE EXTERNAL TABLE sdkOutputTable(age int) STORED BY
+ s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
@@ -2260,7 +2262,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
writer.write(record)
writer.close()
sql(
- s"""CREATE EXTERNAL TABLE sdkOutputTable(dateType date, course_details struct<course_struct_course_time: date>) STORED BY
+ s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
checkAnswer(sql("select * from sdkOutputTable"), Seq(Row(java.sql.Date.valueOf("1970-04-12"), Row(java.sql.Date.valueOf("1970-01-11")))))
@@ -2306,7 +2308,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
writer.write(record)
writer.close()
sql(
- s"""CREATE EXTERNAL TABLE sdkOutputTable(dateType timestamp, course_details struct<course_struct_course_time: timestamp>) STORED BY
+ s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
checkAnswer(sql("select * from sdkOutputTable"), Seq(Row(Timestamp.valueOf("1970-01-02 16:00:00"), Row(Timestamp.valueOf("1970-01-02 16:00:00")))))
@@ -2353,7 +2355,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
writer.write(record)
writer.close()
sql(
- s"""CREATE EXTERNAL TABLE sdkOutputTable(dateType timestamp, course_details struct<course_struct_course_time: timestamp>) STORED BY
+ s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
checkAnswer(sql("select * from sdkOutputTable"), Seq(Row(Timestamp.valueOf("1970-01-02 16:00:00"), Row(Timestamp.valueOf("1970-01-02 16:00:00")))))
http://git-wip-us.apache.org/repos/asf/carbondata/blob/af2c469b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTableWithAvroDataType.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTableWithAvroDataType.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTableWithAvroDataType.scala
index b50407c..7616ea3 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTableWithAvroDataType.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTableWithAvroDataType.scala
@@ -180,7 +180,7 @@ class TestNonTransactionalCarbonTableWithAvroDataType extends QueryTest with Bef
writer.write(record)
writer.close()
sql(
- s"""CREATE EXTERNAL TABLE sdkOutputTable(id decimal(4,3)) STORED BY
+ s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
checkAnswer(sql("select * from sdkOutputTable"),
@@ -241,7 +241,7 @@ class TestNonTransactionalCarbonTableWithAvroDataType extends QueryTest with Bef
writer.write(record)
writer.close()
sql(
- s"""CREATE EXTERNAL TABLE sdkOutputTable(id decimal(4,3)) STORED BY
+ s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
checkAnswer(sql("select * from sdkOutputTable"), Seq(Row(Row(true, null, null))))
@@ -271,7 +271,7 @@ class TestNonTransactionalCarbonTableWithAvroDataType extends QueryTest with Bef
writer.write(record)
writer.close()
sql(
- s"""CREATE EXTERNAL TABLE sdkOutputTable(id decimal(4,3)) STORED BY
+ s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
checkAnswer(sql("select * from sdkOutputTable"), Seq(Row(Row("abc", null, null))))
@@ -301,7 +301,7 @@ class TestNonTransactionalCarbonTableWithAvroDataType extends QueryTest with Bef
writer.write(record)
writer.close()
sql(
- s"""CREATE EXTERNAL TABLE sdkOutputTable(id decimal(4,3)) STORED BY
+ s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
checkAnswer(sql("select * from sdkOutputTable"), Seq(Row(Row(null, 10, null))))
@@ -331,7 +331,7 @@ class TestNonTransactionalCarbonTableWithAvroDataType extends QueryTest with Bef
writer.write(record)
writer.close()
sql(
- s"""CREATE EXTERNAL TABLE sdkOutputTable(id decimal(4,3)) STORED BY
+ s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
checkAnswer(sql("select * from sdkOutputTable"), Seq(Row(Row(null))))
@@ -398,7 +398,7 @@ class TestNonTransactionalCarbonTableWithAvroDataType extends QueryTest with Bef
writer.write(record)
writer.close()
sql(
- s"""CREATE EXTERNAL TABLE sdkOutputTable(id decimal(4,3)) STORED BY
+ s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
checkAnswer(sql("select * from sdkOutputTable"), Seq(Row(Row("VAL_0"))))
@@ -434,7 +434,7 @@ class TestNonTransactionalCarbonTableWithAvroDataType extends QueryTest with Bef
writer.write(record)
writer.close()
sql(
- s"""CREATE EXTERNAL TABLE sdkOutputTable(id decimal(4,3)) STORED BY
+ s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
sql("select * from sdkOutputTable").show(false)
@@ -472,7 +472,7 @@ class TestNonTransactionalCarbonTableWithAvroDataType extends QueryTest with Bef
writer.write(record)
writer.close()
sql(
- s"""CREATE EXTERNAL TABLE sdkOutputTable(id decimal(4,3)) STORED BY
+ s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
checkAnswer(sql("select * from sdkOutputTable"),
@@ -505,7 +505,7 @@ class TestNonTransactionalCarbonTableWithAvroDataType extends QueryTest with Bef
writer.write(record)
writer.close()
sql(
- s"""CREATE EXTERNAL TABLE sdkOutputTable(id decimal(4,3)) STORED BY
+ s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
sql("describe formatted sdkOutputTable").show(false)
@@ -564,7 +564,7 @@ class TestNonTransactionalCarbonTableWithAvroDataType extends QueryTest with Bef
writer.write(record)
writer.close()
sql(
- s"""CREATE EXTERNAL TABLE sdkOutputTable(id decimal(4,3)) STORED BY
+ s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
checkAnswer(sql("select * from sdkOutputTable"),
@@ -659,7 +659,7 @@ class TestNonTransactionalCarbonTableWithAvroDataType extends QueryTest with Bef
writer.write(record)
writer.close()
sql(
- s"""CREATE EXTERNAL TABLE sdkOutputTable(id decimal(4,3)) STORED BY
+ s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
sql("describe formatted sdkOutputTable").show(false)
@@ -700,7 +700,7 @@ class TestNonTransactionalCarbonTableWithAvroDataType extends QueryTest with Bef
writer.write(record)
writer.close()
sql(
- s"""CREATE EXTERNAL TABLE sdkOutputTable(id decimal(4,3)) STORED BY
+ s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
checkExistence(sql("select * from sdkOutputTable"), true, "1010.00")
@@ -738,7 +738,7 @@ class TestNonTransactionalCarbonTableWithAvroDataType extends QueryTest with Bef
writer.write(record)
writer.close()
sql(
- s"""CREATE EXTERNAL TABLE sdkOutputTable(id struct<b:decimal(4,3)>) STORED BY
+ s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
checkExistence(sql("select * from sdkOutputTable"), true, "32.00")
@@ -784,7 +784,7 @@ class TestNonTransactionalCarbonTableWithAvroDataType extends QueryTest with Bef
writer.write(record)
writer.close()
sql(
- s"""CREATE EXTERNAL TABLE sdkOutputTable(id struct<b:decimal(4,3)>) STORED BY
+ s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
checkExistence(sql("select * from sdkOutputTable"), true, "32.0")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/af2c469b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
index 693a8c4..16cee96 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
@@ -324,9 +324,7 @@ object CarbonSource {
tableDesc.copy(storage = updatedFormat)
} else {
val tableInfo = CarbonUtil.convertGsonToTableInfo(properties.asJava)
- val isExternal = properties.getOrElse("isExternal", "false")
- val isTransactionalTable = properties.getOrElse("isTransactional", "true")
- .contains("true")
+ val isTransactionalTable = properties.getOrElse("isTransactional", "true").contains("true")
tableInfo.setTransactionalTable(isTransactionalTable)
if (isTransactionalTable && !metaStore.isReadFromHiveMetaStore) {
// save to disk
@@ -350,10 +348,7 @@ object CarbonSource {
query: Option[LogicalPlan]): Map[String, String] = {
val model = createTableInfoFromParams(properties, dataSchema, identifier, query, sparkSession)
val tableInfo: TableInfo = TableNewProcessor(model)
- val isExternal = properties.getOrElse("isExternal", "false")
- val isTransactionalTable = properties.getOrElse("isTransactional", "true")
- .contains("true")
- val tablePath = properties.getOrElse("path", "")
+ val isTransactionalTable = properties.getOrElse("isTransactional", "true").contains("true")
tableInfo.setTablePath(identifier.getTablePath)
tableInfo.setTransactionalTable(isTransactionalTable)
tableInfo.setDatabaseName(identifier.getDatabaseName)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/af2c469b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParserUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParserUtil.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParserUtil.scala
index 9752535..1594c18 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParserUtil.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParserUtil.scala
@@ -17,6 +17,7 @@
package org.apache.spark.sql.parser
+import scala.collection.JavaConverters._
import scala.collection.mutable
import org.antlr.v4.runtime.tree.TerminalNode
@@ -127,6 +128,16 @@ object CarbonSparkSqlParserUtil {
var isTransactionalTable: Boolean = true
val tableInfo = if (external) {
+ if (fields.nonEmpty) {
+ // user provided schema for this external table, this is not allow currently
+ // see CARBONDATA-2866
+ operationNotAllowed(
+ "Schema may not be specified for external table", columns)
+ }
+ if (partitionByStructFields.nonEmpty) {
+ operationNotAllowed(
+ "Partition is not supported for external table", partitionColumns)
+ }
// read table info from schema file in the provided table path
// external table also must convert table name to lower case
val identifier = AbsoluteTableIdentifier.from(
@@ -142,15 +153,14 @@ object CarbonSparkSqlParserUtil {
isTransactionalTable = false
SchemaReader.inferSchema(identifier, false)
}
- }
- else {
+ } else {
SchemaReader.getTableInfo(identifier)
}
- }
- catch {
+ } catch {
case e: Throwable =>
operationNotAllowed(s"Invalid table path provided: ${ tablePath.get } ", tableHeader)
}
+
// set "_external" property, so that DROP TABLE will not delete the data
if (provider.equalsIgnoreCase("'carbonfile'")) {
table.getFactTable.getTableProperties.put("_filelevelformat", "true")