You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2018/03/26 15:28:46 UTC
carbondata git commit: [CARBONDATA-2262] Support the syntax of
'STORED AS CARBONDATA' to create table
Repository: carbondata
Updated Branches:
refs/heads/master 2eb8f047c -> ee9df2e2c
[CARBONDATA-2262] Support the syntax of 'STORED AS CARBONDATA' to create table
This closes #2078
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/ee9df2e2
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/ee9df2e2
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/ee9df2e2
Branch: refs/heads/master
Commit: ee9df2e2cab3d78e791cbcc8980211418ff00161
Parents: 2eb8f04
Author: root <60...@qq.com>
Authored: Mon Mar 19 19:59:26 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Mon Mar 26 23:28:26 2018 +0800
----------------------------------------------------------------------
.../sql/commands/StoredAsCarbondataSuite.scala | 102 +++++++++++++++++++
.../spark/sql/parser/CarbonSparkSqlParser.scala | 5 +-
.../spark/sql/hive/CarbonSessionState.scala | 1 +
.../spark/sql/hive/CarbonSessionState.scala | 1 +
.../org/apache/spark/SparkCommandSuite.scala | 2 +-
5 files changed, 109 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ee9df2e2/integration/spark-common-test/src/test/scala/org/apache/carbondata/sql/commands/StoredAsCarbondataSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/sql/commands/StoredAsCarbondataSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/sql/commands/StoredAsCarbondataSuite.scala
new file mode 100644
index 0000000..2029e93
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/sql/commands/StoredAsCarbondataSuite.scala
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.sql.commands
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterEach
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+
+class StoredAsCarbondataSuite extends QueryTest with BeforeAndAfterEach {
+ override def beforeEach(): Unit = {
+ sql("DROP TABLE IF EXISTS carbon_table")
+ sql("DROP TABLE IF EXISTS tableSize3")
+ }
+
+ override def afterEach(): Unit = {
+ sql("DROP TABLE IF EXISTS carbon_table")
+ sql("DROP TABLE IF EXISTS tableSize3")
+ }
+
+ test("CARBONDATA-2262: Support the syntax of 'STORED AS CARBONDATA', upper case") {
+ sql("CREATE TABLE carbon_table(key INT, value STRING) STORED AS CARBONDATA")
+ sql("INSERT INTO carbon_table VALUES (28,'Bob')")
+ checkAnswer(sql("SELECT * FROM carbon_table"), Seq(Row(28, "Bob")))
+ }
+
+ test("CARBONDATA-2262: Support the syntax of 'STORED AS carbondata', low case") {
+ sql("CREATE TABLE carbon_table(key INT, value STRING) STORED AS carbondata")
+ sql("INSERT INTO carbon_table VALUES (28,'Bob')")
+ checkAnswer(sql("SELECT * FROM carbon_table"), Seq(Row(28, "Bob")))
+ }
+
+ test("CARBONDATA-2262: Support the syntax of 'STORED AS carbondata, get data size and index size after minor compaction") {
+ sql("CREATE TABLE tableSize3 (empno INT, workgroupcategory STRING, deptno INT, projectcode INT, attendance INT) STORED AS carbondata")
+ sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/data.csv' INTO TABLE tableSize3 OPTIONS ('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'FILEHEADER'='')""")
+ sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/data.csv' INTO TABLE tableSize3 OPTIONS ('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'FILEHEADER'='')""")
+ sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/data.csv' INTO TABLE tableSize3 OPTIONS ('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'FILEHEADER'='')""")
+ sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/data.csv' INTO TABLE tableSize3 OPTIONS ('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'FILEHEADER'='')""")
+ sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/data.csv' INTO TABLE tableSize3 OPTIONS ('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'FILEHEADER'='')""")
+ sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/data.csv' INTO TABLE tableSize3 OPTIONS ('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'FILEHEADER'='')""")
+ sql("ALTER TABLE tableSize3 COMPACT 'minor'")
+ checkExistence(sql("DESCRIBE FORMATTED tableSize3"), true, CarbonCommonConstants.TABLE_DATA_SIZE)
+ checkExistence(sql("DESCRIBE FORMATTED tableSize3"), true, CarbonCommonConstants.TABLE_INDEX_SIZE)
+ val res3 = sql("DESCRIBE FORMATTED tableSize3").collect()
+ .filter(row => row.getString(0).contains(CarbonCommonConstants.TABLE_DATA_SIZE) ||
+ row.getString(0).contains(CarbonCommonConstants.TABLE_INDEX_SIZE))
+ assert(res3.length == 2)
+ res3.foreach(row => assert(row.getString(1).trim.toLong > 0))
+ }
+
+ test("CARBONDATA-2262: Don't Support the syntax of 'STORED AS 'carbondata''") {
+ try {
+ sql("CREATE TABLE carbon_table(key INT, value STRING) STORED AS 'carbondata'")
+ } catch {
+ case e: Exception =>
+ assert(e.getMessage.contains("mismatched input"))
+ }
+ }
+
+ test("CARBONDATA-2262: Don't Support the syntax of 'stored by carbondata'") {
+ try {
+ sql("CREATE TABLE carbon_table(key INT, value STRING) STORED BY carbondata")
+ } catch {
+ case e: Exception =>
+ assert(e.getMessage.contains("mismatched input"))
+ }
+ }
+
+ test("CARBONDATA-2262: Don't Support the syntax of 'STORED AS ', null format") {
+ try {
+ sql("CREATE TABLE carbon_table(key INT, value STRING) STORED AS ")
+ } catch {
+ case e: Exception =>
+ assert(e.getMessage.contains("no viable alternative at input"))
+ }
+ }
+
+ test("CARBONDATA-2262: Don't Support the syntax of 'STORED AS carbon'") {
+ try {
+ sql("CREATE TABLE carbon_table(key INT, value STRING) STORED AS carbon")
+ } catch {
+ case e: Exception =>
+ assert(e.getMessage.contains("Operation not allowed: STORED AS with file format 'carbon'"))
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ee9df2e2/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
index 610f9bf..659f069 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
@@ -91,8 +91,11 @@ class CarbonHelperSqlAstBuilder(conf: SQLConf,
def getFileStorage(createFileFormat: CreateFileFormatContext): String = {
Option(createFileFormat) match {
case Some(value) =>
- if (value.children.get(1).getText.equalsIgnoreCase("by")) {
+ val result = value.children.get(1).getText
+ if (result.equalsIgnoreCase("by")) {
value.storageHandler().STRING().getSymbol.getText
+ } else if (result.equalsIgnoreCase("as") && value.children.size() > 1) {
+ value.children.get(2).getText
} else {
// The case of "STORED AS PARQUET/ORC"
""
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ee9df2e2/integration/spark2/src/main/spark2.1/org/apache/spark/sql/hive/CarbonSessionState.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/spark2.1/org/apache/spark/sql/hive/CarbonSessionState.scala b/integration/spark2/src/main/spark2.1/org/apache/spark/sql/hive/CarbonSessionState.scala
index e8ae67b..4c2304d 100644
--- a/integration/spark2/src/main/spark2.1/org/apache/spark/sql/hive/CarbonSessionState.scala
+++ b/integration/spark2/src/main/spark2.1/org/apache/spark/sql/hive/CarbonSessionState.scala
@@ -326,6 +326,7 @@ class CarbonSqlAstBuilder(conf: SQLConf, parser: CarbonSpark2SqlParser, sparkSes
val fileStorage = helper.getFileStorage(ctx.createFileFormat)
if (fileStorage.equalsIgnoreCase("'carbondata'") ||
+ fileStorage.equalsIgnoreCase("carbondata") ||
fileStorage.equalsIgnoreCase("'carbonfile'") ||
fileStorage.equalsIgnoreCase("'org.apache.carbondata.format'")) {
val createTableTuple = (ctx.createTableHeader, ctx.skewSpec, ctx.bucketSpec,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ee9df2e2/integration/spark2/src/main/spark2.2/org/apache/spark/sql/hive/CarbonSessionState.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/spark2.2/org/apache/spark/sql/hive/CarbonSessionState.scala b/integration/spark2/src/main/spark2.2/org/apache/spark/sql/hive/CarbonSessionState.scala
index 3cb9bd6..2ffc7db 100644
--- a/integration/spark2/src/main/spark2.2/org/apache/spark/sql/hive/CarbonSessionState.scala
+++ b/integration/spark2/src/main/spark2.2/org/apache/spark/sql/hive/CarbonSessionState.scala
@@ -325,6 +325,7 @@ class CarbonSqlAstBuilder(conf: SQLConf, parser: CarbonSpark2SqlParser, sparkSes
val fileStorage = helper.getFileStorage(ctx.createFileFormat)
if (fileStorage.equalsIgnoreCase("'carbondata'") ||
+ fileStorage.equalsIgnoreCase("carbondata") ||
fileStorage.equalsIgnoreCase("'carbonfile'") ||
fileStorage.equalsIgnoreCase("'org.apache.carbondata.format'")) {
val createTableTuple = (ctx.createTableHeader, ctx.skewSpec,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ee9df2e2/integration/spark2/src/test/scala/org/apache/spark/SparkCommandSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/SparkCommandSuite.scala b/integration/spark2/src/test/scala/org/apache/spark/SparkCommandSuite.scala
index b7cb63d..a9db10a 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/SparkCommandSuite.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/SparkCommandSuite.scala
@@ -31,7 +31,7 @@ class SparkCommandSuite extends Spark2QueryTest with BeforeAndAfterAll {
sql("DROP TABLE IF EXISTS src_orc")
}
- test("CARBONDATA-734: Support the syntax of 'STORED BY PARQUET/ORC'") {
+ test("CARBONDATA-734: Support the syntax of 'STORED AS PARQUET/ORC'") {
sql("CREATE TABLE src_pqt(key INT, value STRING) STORED AS PARQUET")
sql("CREATE TABLE src_orc(key INT, value STRING) STORED AS ORC")
}