You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2017/10/24 09:41:14 UTC
[01/16] carbondata git commit: [CARBONDATA-1597] Remove spark1
integration
Repository: carbondata
Updated Branches:
refs/heads/pre-aggregate 311a5b7e3 -> 5a67c98f1
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark2/pom.xml
----------------------------------------------------------------------
diff --git a/integration/spark2/pom.xml b/integration/spark2/pom.xml
index a09bd62..ed01728 100644
--- a/integration/spark2/pom.xml
+++ b/integration/spark2/pom.xml
@@ -38,16 +38,6 @@
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-spark-common</artifactId>
<version>${project.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-hive-thriftserver_2.10</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_2.10</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index e7dd0e5..39dc565 100644
--- a/pom.xml
+++ b/pom.xml
@@ -416,7 +416,7 @@
<profiles>
<profile>
- <!--This profile does not build spark module, so user should explicitly give spark profile also like spark-1.6 -->
+ <!--This profile does not build spark module, so user should explicitly give spark profile also like spark-2.1 -->
<id>build-with-format</id>
<modules>
<module>format</module>
@@ -426,15 +426,13 @@
<!-- This profile only should be used for release prepare to cover all the modules -->
<id>build-all</id>
<properties>
- <spark.version>1.6.2</spark.version>
- <scala.binary.version>2.10</scala.binary.version>
- <scala.version>2.10.4</scala.version>
+ <spark.version>2.1.0</spark.version>
+ <scala.binary.version>2.11</scala.binary.version>
+ <scala.version>2.11.8</scala.version>
<maven.test.skip>true</maven.test.skip>
</properties>
<modules>
<module>format</module>
- <module>integration/spark</module>
- <module>examples/spark</module>
<module>integration/spark2</module>
<module>examples/spark2</module>
<module>integration/hive</module>
@@ -467,65 +465,6 @@
</properties>
</profile>
<profile>
- <id>spark-1.5</id>
- <properties>
- <spark.version>1.5.2</spark.version>
- <scala.binary.version>2.10</scala.binary.version>
- <scala.version>2.10.4</scala.version>
- </properties>
- <modules>
- <module>integration/spark</module>
- <module>examples/spark</module>
- <module>examples/flink</module>
- </modules>
- </profile>
- <profile>
- <id>spark-1.6</id>
- <properties>
- <spark.version>1.6.2</spark.version>
- <scala.binary.version>2.10</scala.binary.version>
- <scala.version>2.10.4</scala.version>
- </properties>
- <modules>
- <module>integration/spark</module>
- <module>examples/spark</module>
- <module>examples/flink</module>
- </modules>
- <build>
- <plugins>
- <plugin>
- <groupId>org.eluder.coveralls</groupId>
- <artifactId>coveralls-maven-plugin</artifactId>
- <version>4.3.0</version>
- <configuration>
- <repoToken>Q8mcBW8dw34wjNiPAMx0SFQidspIwHw8g</repoToken>
- <sourceEncoding>UTF-8</sourceEncoding>
- <jacocoReports>
- <jacocoReport>${basedir}/target/carbondata-coverage-report/carbondata-coverage-report.xml
- </jacocoReport>
- </jacocoReports>
- <sourceDirectories>
- <sourceDirectory>${basedir}/common/src/main/java</sourceDirectory>
- <sourceDirectory>${basedir}/core/src/main/java</sourceDirectory>
- <sourceDirectory>${basedir}/processing/src/main/java</sourceDirectory>
- <sourceDirectory>${basedir}/hadoop/src/main/java</sourceDirectory>
- <sourceDirectory>${basedir}/integration/spark-common/src/main/scala</sourceDirectory>
- <sourceDirectory>${basedir}/integration/spark-common/src/main/java</sourceDirectory>
- <sourceDirectory>${basedir}/integration/spark-common-test/src/main/scala</sourceDirectory>
- <sourceDirectory>${basedir}/integration/spark-common-test/src/main/java</sourceDirectory>
- <sourceDirectory>${basedir}/integration/hive/src/main/scala</sourceDirectory>
- <sourceDirectory>${basedir}/integration/hive/src/main/java</sourceDirectory>
- <sourceDirectory>${basedir}/integration/presto/src/main/scala</sourceDirectory>
- <sourceDirectory>${basedir}/integration/presto/src/main/java</sourceDirectory>
- <sourceDirectory>${basedir}/integration/spark/src/main/scala</sourceDirectory>
- <sourceDirectory>${basedir}/integration/spark/src/main/java</sourceDirectory>
- </sourceDirectories>
- </configuration>
- </plugin>
- </plugins>
- </build>
- </profile>
- <profile>
<id>spark-2.1</id>
<activation>
<activeByDefault>true</activeByDefault>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/processing/pom.xml
----------------------------------------------------------------------
diff --git a/processing/pom.xml b/processing/pom.xml
index 03c8b57..7b53803 100644
--- a/processing/pom.xml
+++ b/processing/pom.xml
@@ -38,12 +38,6 @@
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-core</artifactId>
<version>${project.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_2.10</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
[11/16] carbondata git commit: [CARBONDATA-1444] Support Boolean data
type
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesLoadTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesLoadTest.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesLoadTest.scala
new file mode 100644
index 0000000..c19b0cd
--- /dev/null
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesLoadTest.scala
@@ -0,0 +1,744 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.spark.testsuite.booleantype
+
+import java.io.File
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
+
+class BooleanDataTypesLoadTest extends QueryTest with BeforeAndAfterEach with BeforeAndAfterAll {
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+
+ override def beforeEach(): Unit = {
+ sql("drop table if exists carbon_table")
+ sql("drop table if exists boolean_table")
+ sql("drop table if exists boolean_table2")
+ sql("drop table if exists boolean_table3")
+ sql("drop table if exists boolean_table4")
+ sql("drop table if exists badRecords")
+ sql("CREATE TABLE if not exists carbon_table(booleanField BOOLEAN) STORED BY 'carbondata'")
+ sql(
+ s"""
+ | CREATE TABLE badRecords(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | decimalField DECIMAL(18,2),
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ """.stripMargin)
+
+ sql(
+ s"""
+ | CREATE TABLE boolean_table(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+ }
+
+ override def afterAll(): Unit = {
+ sql("drop table if exists carbon_table")
+ sql("drop table if exists boolean_table")
+ sql("drop table if exists boolean_table2")
+ sql("drop table if exists boolean_table3")
+ sql("drop table if exists boolean_table4")
+ sql("drop table if exists badRecords")
+ }
+
+ test("Loading table: support boolean data type format") {
+ val fileLocation = s"$rootPath/integration/spark2/src/test/resources/bool/supportBooleanOnlyBoolean.csv"
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '$fileLocation'
+ | INTO TABLE carbon_table
+ | OPTIONS('FILEHEADER' = 'booleanField')
+ """.stripMargin)
+
+ checkAnswer(sql("select * from carbon_table where booleanField = true"),
+ Seq(Row(true), Row(true), Row(true), Row(true)))
+
+ checkAnswer(sql("select * from carbon_table"),
+ Seq(Row(true), Row(true), Row(true), Row(true), Row(false), Row(false), Row(false), Row(false), Row(null), Row(null), Row(null)))
+ }
+
+ test("Loading table: support boolean data type format, different format") {
+ val fileLocation = s"$rootPath/integration/spark2/src/test/resources/bool/supportBooleanDifferentFormat.csv"
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '$fileLocation'
+ | INTO TABLE carbon_table
+ | OPTIONS('FILEHEADER' = 'booleanField')
+ """.stripMargin)
+
+ checkAnswer(sql("select * from carbon_table where booleanField = true"),
+ Seq(Row(true), Row(true), Row(true), Row(true)))
+
+ checkAnswer(sql("select * from carbon_table"),
+ Seq(Row(true), Row(true), Row(true), Row(true), Row(false), Row(false), Row(false), Row(false)
+ , Row(null), Row(null), Row(null), Row(null), Row(null), Row(null), Row(null), Row(null), Row(null), Row(null), Row(null), Row(null)))
+ }
+
+ test("Loading table: support boolean and other data type") {
+ val storeLocation = s"$rootPath/integration/spark2/src/test/resources/bool/supportBoolean.csv"
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${storeLocation}'
+ | INTO TABLE boolean_table
+ | options('FILEHEADER'='shortField,booleanField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData,booleanField2')
+ """.stripMargin)
+
+ checkAnswer(
+ sql("select booleanField,intField from boolean_table"),
+ Seq(Row(true, 10), Row(false, 17), Row(false, 11),
+ Row(true, 10), Row(true, 10), Row(true, 14),
+ Row(false, 10), Row(false, 10), Row(false, 16), Row(false, 10))
+ )
+ }
+
+ test("Loading table: data columns is less than table defined columns") {
+ val storeLocation = s"$rootPath/integration/spark2/src/test/resources/bool/supportBoolean.csv"
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${storeLocation}'
+ | INTO TABLE boolean_table
+ | options('FILEHEADER'='shortField,booleanField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData,booleanField2')
+ """.stripMargin)
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from boolean_table"),
+ Seq(Row(true, 10, null), Row(false, 17, null), Row(false, 11, null),
+ Row(true, 10, null), Row(true, 10, null), Row(true, 14, null),
+ Row(false, 10, null), Row(false, 10, null), Row(false, 16, null), Row(false, 10, null))
+ )
+ }
+
+ test("Loading table: support boolean and other data type, data columns bigger than table defined columns") {
+ sql("drop table if exists boolean_table")
+ sql(
+ s"""
+ | CREATE TABLE boolean_table(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+
+ val storeLocation = s"$rootPath/integration/spark2/src/test/resources/bool/supportBooleanTwoBooleanColumns.csv"
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${storeLocation}'
+ | INTO TABLE boolean_table
+ | options('FILEHEADER'='shortField,booleanField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData,booleanField2')
+ """.stripMargin)
+
+ checkAnswer(
+ sql("select booleanField,intField from boolean_table"),
+ Seq(Row(true, 10), Row(false, 17), Row(false, 11),
+ Row(true, 10), Row(true, 10), Row(true, 14),
+ Row(false, 10), Row(false, 10), Row(false, 16), Row(false, 10))
+ )
+ }
+
+ test("Loading table: support boolean and other data type, with file header") {
+ sql("drop table if exists boolean_table")
+ sql(
+ s"""
+ | CREATE TABLE boolean_table(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+
+ val storeLocation = s"$rootPath/integration/spark2/src/test/resources/bool/supportBooleanWithFileHeader.csv"
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${storeLocation}'
+ | INTO TABLE boolean_table
+ """.stripMargin)
+
+ checkAnswer(
+ sql("select booleanField,intField from boolean_table"),
+ Seq(Row(true, 10), Row(false, 17), Row(false, 11),
+ Row(true, 10), Row(true, 10), Row(true, 14),
+ Row(false, 10), Row(false, 10), Row(false, 16), Row(false, 10))
+ )
+ }
+
+ test("Loading table: create with DICTIONARY_EXCLUDE, TABLE_BLOCKSIZE, NO_INVERTED_INDEX, SORT_SCOPE") {
+ sql("drop table if exists boolean_table")
+ sql(
+ s"""
+ | CREATE TABLE boolean_table(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_EXCLUDE'='charField','TABLE_BLOCKSIZE'='512','NO_INVERTED_INDEX'='charField', 'SORT_SCOPE'='GLOBAL_SORT')
+ """.stripMargin)
+
+ val storeLocation = s"$rootPath/integration/spark2/src/test/resources/bool/supportBoolean.csv"
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${storeLocation}'
+ | INTO TABLE boolean_table
+ | options('FILEHEADER'='shortField,booleanField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData,booleanField2')
+ """.stripMargin)
+
+ checkAnswer(
+ sql("select booleanField,intField from boolean_table"),
+ Seq(Row(true, 10), Row(false, 17), Row(false, 11),
+ Row(true, 10), Row(true, 10), Row(true, 14),
+ Row(false, 10), Row(false, 10), Row(false, 16), Row(false, 10))
+ )
+
+ checkAnswer(sql("select booleanField from boolean_table where booleanField = true"),
+ Seq(Row(true), Row(true), Row(true), Row(true)))
+
+ checkAnswer(sql("select count(*) from boolean_table where booleanField = true"),
+ Row(4))
+
+ checkAnswer(sql("select count(*) from boolean_table where booleanField = 'true'"),
+ Row(0))
+
+ checkAnswer(sql(
+ s"""
+ |select count(*)
+ |from boolean_table where booleanField = \"true\"
+ |""".stripMargin),
+ Row(0))
+
+ checkAnswer(sql("select booleanField from boolean_table where booleanField = false"),
+ Seq(Row(false), Row(false), Row(false), Row(false), Row(false), Row(false)))
+
+ checkAnswer(sql("select count(*) from boolean_table where booleanField = false"),
+ Row(6))
+
+ checkAnswer(sql("select count(*) from boolean_table where booleanField = 'false'"),
+ Row(0))
+
+ checkAnswer(sql("select count(*) from boolean_table where booleanField = null"),
+ Row(0))
+
+ checkAnswer(sql("select count(*) from boolean_table where booleanField = false or booleanField = true"),
+ Row(10))
+ }
+
+ test("Loading table: load with DELIMITER, QUOTECHAR, COMMENTCHAR, MULTILINE, ESCAPECHAR, COMPLEX_DELIMITER_LEVEL_1, SINGLE_PASS") {
+ sql("drop table if exists boolean_table")
+ sql(
+ s"""
+ | CREATE TABLE boolean_table(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_EXCLUDE'='charField','TABLE_BLOCKSIZE'='512','NO_INVERTED_INDEX'='charField', 'SORT_SCOPE'='GLOBAL_SORT')
+ """.stripMargin)
+
+ val storeLocation = s"$rootPath/integration/spark2/src/test/resources/bool/supportBooleanWithFileHeader.csv"
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${storeLocation}'
+ | INTO TABLE boolean_table
+ | options('DELIMITER'=',','QUOTECHAR'='"','COMMENTCHAR'='#','MULTILINE'='true','ESCAPECHAR'='\','COMPLEX_DELIMITER_LEVEL_1'='#','COMPLEX_DELIMITER_LEVEL_2'=':','SINGLE_PASS'='TRUE')
+ """.stripMargin)
+
+ checkAnswer(
+ sql("select booleanField,intField from boolean_table"),
+ Seq(Row(true, 10), Row(false, 17), Row(false, 11),
+ Row(true, 10), Row(true, 10), Row(true, 14),
+ Row(false, 10), Row(false, 10), Row(false, 16), Row(false, 10))
+ )
+
+ checkAnswer(sql("select booleanField from boolean_table where booleanField = true"),
+ Seq(Row(true), Row(true), Row(true), Row(true)))
+
+ checkAnswer(sql("select count(*) from boolean_table where booleanField = true"),
+ Row(4))
+
+ checkAnswer(sql("select count(*) from boolean_table where booleanField = 'true'"),
+ Row(0))
+
+ checkAnswer(sql(
+ s"""
+ |select count(*)
+ |from boolean_table where booleanField = \"true\"
+ |""".stripMargin),
+ Row(0))
+
+ checkAnswer(sql("select booleanField from boolean_table where booleanField = false"),
+ Seq(Row(false), Row(false), Row(false), Row(false), Row(false), Row(false)))
+
+ checkAnswer(sql("select count(*) from boolean_table where booleanField = false"),
+ Row(6))
+
+ checkAnswer(sql("select count(*) from boolean_table where booleanField = 'false'"),
+ Row(0))
+
+ checkAnswer(sql("select count(*) from boolean_table where booleanField = null"),
+ Row(0))
+
+ checkAnswer(sql("select count(*) from boolean_table where booleanField = false or booleanField = true"),
+ Row(10))
+ }
+
+ test("Loading table: bad_records_action is FORCE") {
+ val fileLocation = s"$rootPath/integration/spark2/src/test/resources/bool/supportBooleanDifferentFormat.csv"
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '$fileLocation'
+ | INTO TABLE carbon_table
+ | OPTIONS('FILEHEADER' = 'booleanField','bad_records_logger_enable'='true','bad_records_action'='FORCE')
+ """.stripMargin)
+
+ checkAnswer(sql("select * from carbon_table where booleanField = true"),
+ Seq(Row(true), Row(true), Row(true), Row(true)))
+
+ checkAnswer(sql("select * from carbon_table"),
+ Seq(Row(true), Row(true), Row(true), Row(true),
+ Row(false), Row(false), Row(false), Row(false),
+ Row(null), Row(null), Row(null), Row(null), Row(null), Row(null),
+ Row(null), Row(null), Row(null), Row(null), Row(null), Row(null)))
+ }
+
+ test("Loading table: bad_records_action is FORCE, support boolean and other data type") {
+ val storeLocation = s"$rootPath/integration/spark2/src/test/resources/bool/supportBooleanBadRecords.csv"
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${storeLocation}'
+ | INTO TABLE badRecords
+ | options('bad_records_logger_enable'='true','bad_records_action'='FORCE','FILEHEADER'='shortField,booleanField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData,booleanField2')
+ """.stripMargin)
+
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from badRecords"),
+ Seq(Row(true, 10, true), Row(null, 17, true), Row(null, 11, true),
+ Row(null, 10, true), Row(null, 10, true), Row(true, 14, null),
+ Row(false, 10, null), Row(false, 10, null), Row(false, 16, null), Row(false, 10, false))
+ )
+ }
+
+ test("Loading table: bad_records_action is IGNORE, support boolean and other data type") {
+ val storeLocation = s"$rootPath/integration/spark2/src/test/resources/bool/supportBooleanBadRecords.csv"
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${storeLocation}'
+ | INTO TABLE badRecords
+ | options('bad_records_logger_enable'='true','bad_records_action'='IGNORE','FILEHEADER'='shortField,booleanField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData,booleanField2')
+ """.stripMargin)
+
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from badRecords"),
+ Seq(Row(true, 10, true), Row(false, 10, false))
+ )
+ }
+
+ test("Loading table: bad_records_action is REDIRECT, support boolean and other data type") {
+ val storeLocation = s"$rootPath/integration/spark2/src/test/resources/bool/supportBooleanBadRecords.csv"
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${storeLocation}'
+ | INTO TABLE badRecords
+ | options('bad_records_logger_enable'='true','bad_records_action'='REDIRECT','FILEHEADER'='shortField,booleanField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData,booleanField2')
+ """.stripMargin)
+
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from badRecords"),
+ Seq(Row(true, 10, true), Row(false, 10, false))
+ )
+ }
+
+ test("Loading table: bad_records_action is FAIL") {
+ val storeLocation = s"$rootPath/integration/spark2/src/test/resources/bool/supportBooleanBadRecords.csv"
+ val exception_insert: Exception = intercept[Exception] {
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${storeLocation}'
+ | INTO TABLE badRecords
+ | options('bad_records_logger_enable'='true','bad_records_action'='FAIL','FILEHEADER'='shortField,booleanField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData,booleanField2')
+ """.stripMargin)
+ }
+ assert(exception_insert.getMessage.contains("The value with column name booleanfield and column data type BOOLEAN is not a valid BOOLEAN type"))
+ }
+
+ test("Loading overwrite: into and then overwrite table with another table: support boolean data type and other format") {
+ sql(
+ s"""
+ | CREATE TABLE boolean_table2(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+
+ sql(
+ s"""
+ | CREATE TABLE boolean_table3(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | stringField STRING,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='')
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE TABLE boolean_table4(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | stringField STRING,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='')
+ """.stripMargin)
+
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+ val storeLocation = s"$rootPath/integration/spark2/src/test/resources/bool/supportBooleanTwoBooleanColumns.csv"
+
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${storeLocation}'
+ | INTO TABLE boolean_table
+ | options('FILEHEADER'='shortField,booleanField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData,booleanField2')
+ """.stripMargin)
+
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${storeLocation}'
+ | overwrite INTO TABLE boolean_table
+ | options('FILEHEADER'='shortField,booleanField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData,booleanField2')
+ """.stripMargin)
+
+ sql("insert overwrite table boolean_table2 select * from boolean_table")
+ sql("insert overwrite table boolean_table3 select shortField,booleanField,intField,stringField,booleanField2 from boolean_table")
+ sql("insert overwrite table boolean_table4 select shortField,booleanField,intField,stringField,booleanField2 from boolean_table where shortField > 3")
+
+ checkAnswer(
+ sql("select booleanField,intField from boolean_table2"),
+ Seq(Row(true, 10), Row(false, 17), Row(false, 11),
+ Row(true, 10), Row(true, 10), Row(true, 14),
+ Row(false, 10), Row(false, 10), Row(false, 16), Row(false, 10))
+ )
+
+ checkAnswer(
+ sql("select booleanField,intField from boolean_table3"),
+ Seq(Row(true, 10), Row(false, 17), Row(false, 11),
+ Row(true, 10), Row(true, 10), Row(true, 14),
+ Row(false, 10), Row(false, 10), Row(false, 16), Row(false, 10))
+ )
+
+ checkAnswer(
+ sql("select booleanField,intField from boolean_table4"),
+ Seq(Row(false, 17), Row(false, 16))
+ )
+
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from boolean_table2"),
+ Seq(Row(true, 10, true), Row(false, 17, true), Row(false, 11, true),
+ Row(true, 10, true), Row(true, 10, true), Row(true, 14, false),
+ Row(false, 10, false), Row(false, 10, false), Row(false, 16, false), Row(false, 10, false))
+ )
+
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from boolean_table3"),
+ Seq(Row(true, 10, true), Row(false, 17, true), Row(false, 11, true),
+ Row(true, 10, true), Row(true, 10, true), Row(true, 14, false),
+ Row(false, 10, false), Row(false, 10, false), Row(false, 16, false), Row(false, 10, false))
+ )
+ }
+
+ test("Loading overwrite: support boolean data type format, different format") {
+ val fileLocation = s"$rootPath/integration/spark2/src/test/resources/bool/supportBooleanDifferentFormat.csv"
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '$fileLocation'
+ | INTO TABLE carbon_table
+ | OPTIONS('FILEHEADER' = 'booleanField')
+ """.stripMargin)
+
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '$fileLocation'
+ | OVERWRITE INTO TABLE carbon_table
+ | OPTIONS('FILEHEADER' = 'booleanField')
+ """.stripMargin)
+
+
+ checkAnswer(sql("select * from carbon_table where booleanField = true"),
+ Seq(Row(true), Row(true), Row(true), Row(true)))
+
+ checkAnswer(sql("select * from carbon_table"),
+ Seq(Row(true), Row(true), Row(true), Row(true), Row(false), Row(false), Row(false), Row(false)
+ , Row(null), Row(null), Row(null), Row(null), Row(null), Row(null), Row(null), Row(null), Row(null), Row(null), Row(null), Row(null)))
+ }
+
+ test("Loading overwrite: support boolean and other data type") {
+ val storeLocation = s"$rootPath/integration/spark2/src/test/resources/bool/supportBoolean.csv"
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${storeLocation}'
+ | INTO TABLE boolean_table
+ | options('FILEHEADER'='shortField,booleanField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData,booleanField2')
+ """.stripMargin)
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${storeLocation}'
+ | OVERWRITE INTO TABLE boolean_table
+ | options('FILEHEADER'='shortField,booleanField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData,booleanField2')
+ """.stripMargin)
+ checkAnswer(
+ sql("select booleanField,intField from boolean_table"),
+ Seq(Row(true, 10), Row(false, 17), Row(false, 11),
+ Row(true, 10), Row(true, 10), Row(true, 14),
+ Row(false, 10), Row(false, 10), Row(false, 16), Row(false, 10))
+ )
+ }
+
+ test("Comparing table: support boolean and other data type") {
+ sql(
+ s"""
+ | CREATE TABLE boolean_table2(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+
+ val storeLocation = s"$rootPath/integration/spark2/src/test/resources/bool/supportBooleanTwoBooleanColumns.csv"
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${storeLocation}'
+ | INTO TABLE boolean_table
+ | options('FILEHEADER'='shortField,booleanField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData,booleanField2')
+ """.stripMargin)
+
+ sql("insert into boolean_table2 select * from boolean_table where shortField = 1 and booleanField = true")
+
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from boolean_table"),
+ Seq(Row(true, 10, true), Row(false, 17, true), Row(false, 11, true),
+ Row(true, 10, true), Row(true, 10, true), Row(true, 14, false),
+ Row(false, 10, false), Row(false, 10, false), Row(false, 16, false), Row(false, 10, false))
+ )
+
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from boolean_table"),
+ Seq(Row(true, 10, true), Row(false, 17, true), Row(false, 11, true),
+ Row(true, 10, true), Row(true, 10, true), Row(true, 14, false),
+ Row(false, 10, false), Row(false, 10, false), Row(false, 16, false), Row(false, 10, false))
+ )
+
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from boolean_table2"),
+ Seq(Row(true, 10, true), Row(true, 10, true), Row(true, 10, true))
+ )
+
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from boolean_table where exists (select booleanField,intField,booleanField2 " +
+ "from boolean_table2 where boolean_table.intField=boolean_table2.intField)"),
+ Seq(Row(true, 10, true), Row(true, 10, true), Row(true, 10, true), Row(false, 10, false), Row(false, 10, false), Row(false, 10, false))
+ )
+ }
+
+ test("Loading table: unsafe, support boolean and other data type") {
+ initConf
+ sql("drop table if exists boolean_table")
+ sql(
+ s"""
+ | CREATE TABLE boolean_table(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+
+ val storeLocation = s"$rootPath/integration/spark2/src/test/resources/bool/supportBoolean.csv"
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${storeLocation}'
+ | INTO TABLE boolean_table
+ | options('FILEHEADER'='shortField,booleanField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData,booleanField2')
+ """.stripMargin)
+
+ checkAnswer(
+ sql("select booleanField,intField from boolean_table"),
+ Seq(Row(true, 10), Row(false, 17), Row(false, 11),
+ Row(true, 10), Row(true, 10), Row(true, 14),
+ Row(false, 10), Row(false, 10), Row(false, 16), Row(false, 10))
+ )
+ sql("drop table if exists boolean_table")
+ defaultConf
+ }
+
+ test("Loading table: unsafe, bad_records_action is IGNORE, support boolean and other data type") {
+ initConf
+ sql("drop table if exists badRecords")
+ sql(
+ s"""
+ | CREATE TABLE badRecords(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | decimalField DECIMAL(18,2),
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ """.stripMargin)
+ val storeLocation = s"$rootPath/integration/spark2/src/test/resources/bool/supportBooleanBadRecords.csv"
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${storeLocation}'
+ | INTO TABLE badRecords
+ | options('bad_records_logger_enable'='true','bad_records_action'='IGNORE','FILEHEADER'='shortField,booleanField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData,booleanField2')
+ """.stripMargin)
+
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from badRecords"),
+ Seq(Row(true, 10, true), Row(false, 10, false))
+ )
+ sql("drop table if exists badRecords")
+ defaultConf()
+ }
+
+ def initConf(): Unit ={
+ CarbonProperties.getInstance().
+ addProperty(CarbonCommonConstants.ENABLE_UNSAFE_COLUMN_PAGE_LOADING,
+ "true")
+ }
+
+ def defaultConf(): Unit ={
+ CarbonProperties.getInstance().
+ addProperty(CarbonCommonConstants.ENABLE_UNSAFE_COLUMN_PAGE_LOADING,
+ CarbonCommonConstants.ENABLE_DATA_LOADING_STATISTICS_DEFAULT)
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesParameterTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesParameterTest.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesParameterTest.scala
new file mode 100644
index 0000000..07c37b6
--- /dev/null
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesParameterTest.scala
@@ -0,0 +1,288 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.spark.testsuite.booleantype
+
+import java.io.File
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
+
+class BooleanDataTypesParameterTest extends QueryTest with BeforeAndAfterEach with BeforeAndAfterAll {
+ val filePath: String = s"$resourcesPath/globalsort"
+ val file1: String = resourcesPath + "/globalsort/sample1.csv"
+ val file2: String = resourcesPath + "/globalsort/sample2.csv"
+ val file3: String = resourcesPath + "/globalsort/sample3.csv"
+
+ override def beforeEach(): Unit = {
+ sql("drop table if exists boolean_one_column")
+ sql("drop table if exists boolean_table")
+ sql(
+ s"""CREATE TABLE if not exists boolean_one_column(
+ |booleanField BOOLEAN)
+ |STORED BY 'carbondata'
+ |""".stripMargin)
+ }
+
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+
+ override def beforeAll(): Unit = {
+ CarbonProperties.getInstance().
+ addProperty(CarbonCommonConstants.COMPACTION_SEGMENT_LEVEL_THRESHOLD,
+ CarbonCommonConstants.DEFAULT_SEGMENT_LEVEL_THRESHOLD)
+ }
+
+ override def afterAll(): Unit = {
+ sql("drop table if exists boolean_one_column")
+ sql("drop table if exists boolean_table")
+ }
+
+ test("ENABLE_AUTO_LOAD_MERGE: false, and Inserting and selecting table: one column boolean and many rows, should support") {
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE, "false")
+ sql("insert into boolean_one_column values(true)")
+ sql("insert into boolean_one_column values(True)")
+ sql("insert into boolean_one_column values(TRUE)")
+ sql("insert into boolean_one_column values('true')")
+ sql("insert into boolean_one_column values(False)")
+ sql("insert into boolean_one_column values(false)")
+ sql("insert into boolean_one_column values(FALSE)")
+ sql("insert into boolean_one_column values('false')")
+ sql("insert into boolean_one_column values('tr')")
+ sql("insert into boolean_one_column values(null)")
+ sql("insert into boolean_one_column values('truEe')")
+ sql("insert into boolean_one_column values('falsEe')")
+ sql("insert into boolean_one_column values('t')")
+ sql("insert into boolean_one_column values('f')")
+
+ checkAnswer(
+ sql("select * from boolean_one_column"),
+ Seq(Row(true), Row(true), Row(true), Row(true),
+ Row(false), Row(false), Row(false), Row(false),
+ Row(null), Row(null), Row(null), Row(null), Row(null), Row(null))
+ )
+
+ val segments = sql("SHOW SEGMENTS FOR TABLE boolean_one_column")
+ val SegmentSequenceIds = segments.collect().map { each => (each.toSeq) (0) }
+ assert(!SegmentSequenceIds.contains("0.1"))
+ assert(SegmentSequenceIds.length == 14)
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE,
+ CarbonCommonConstants.DEFAULT_ENABLE_AUTO_LOAD_MERGE)
+ }
+
+ test("ENABLE_AUTO_LOAD_MERGE: true, and Inserting and selecting table: one column boolean and many rows, should support") {
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE, "true")
+ sql("insert into boolean_one_column values(true)")
+ sql("insert into boolean_one_column values(True)")
+ sql("insert into boolean_one_column values(TRUE)")
+ sql("insert into boolean_one_column values('true')")
+ sql("insert into boolean_one_column values(False)")
+ sql("insert into boolean_one_column values(false)")
+ sql("insert into boolean_one_column values(FALSE)")
+ sql("insert into boolean_one_column values('false')")
+ sql("insert into boolean_one_column values('tr')")
+ sql("insert into boolean_one_column values(null)")
+ sql("insert into boolean_one_column values('truEe')")
+ sql("insert into boolean_one_column values('falsEe')")
+ sql("insert into boolean_one_column values('t')")
+ sql("insert into boolean_one_column values('f')")
+
+ checkAnswer(
+ sql("select * from boolean_one_column"),
+ Seq(Row(true), Row(true), Row(true), Row(true),
+ Row(false), Row(false), Row(false), Row(false),
+ Row(null), Row(null), Row(null), Row(null), Row(null), Row(null))
+ )
+
+ val segments = sql("SHOW SEGMENTS FOR TABLE boolean_one_column")
+ val SegmentSequenceIds = segments.collect().map { each => (each.toSeq) (0) }
+ assert(SegmentSequenceIds.contains("0.1"))
+ assert(SegmentSequenceIds.length == 18)
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE,
+ CarbonCommonConstants.DEFAULT_ENABLE_AUTO_LOAD_MERGE)
+ }
+
+ test("ENABLE_AUTO_LOAD_MERGE: false, and Loading table: support boolean and other data type") {
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE, "false")
+ sql(
+ s"""
+ | CREATE TABLE boolean_table(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+
+ val storeLocation = s"$rootPath/integration/spark2/src/test/resources/bool/supportBoolean.csv"
+ for (i <- 0 until 4) {
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${storeLocation}'
+ | INTO TABLE boolean_table
+ | options('FILEHEADER'='shortField,booleanField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData,booleanField2')
+ """.stripMargin)
+ }
+
+ checkAnswer(
+ sql("select count(*) from boolean_table"),
+ Seq(Row(40))
+ )
+ val segments = sql("SHOW SEGMENTS FOR TABLE boolean_table")
+ val SegmentSequenceIds = segments.collect().map { each => (each.toSeq) (0) }
+ assert(!SegmentSequenceIds.contains("0.1"))
+ assert(SegmentSequenceIds.length == 4)
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE,
+ CarbonCommonConstants.DEFAULT_ENABLE_AUTO_LOAD_MERGE)
+ }
+
+ test("ENABLE_AUTO_LOAD_MERGE: true, and Loading table: support boolean and other data type") {
+ //unfinish
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE, "true")
+ sql(
+ s"""
+ | CREATE TABLE boolean_table(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+
+ val storeLocation = s"$rootPath/integration/spark2/src/test/resources/bool/supportBooleanTwoBooleanColumns.csv"
+ for (i <- 0 until 4) {
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${storeLocation}'
+ | INTO TABLE boolean_table
+ | options('FILEHEADER'='shortField,booleanField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData,booleanField2')
+ """.stripMargin)
+ }
+
+ checkAnswer(
+ sql("select count(*) from boolean_table"),
+ Seq(Row(40))
+ )
+ val segments = sql("SHOW SEGMENTS FOR TABLE boolean_table")
+ val SegmentSequenceIds = segments.collect().map { each => (each.toSeq) (0) }
+ assert(SegmentSequenceIds.contains("0.1"))
+ assert(SegmentSequenceIds.length == 5)
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE,
+ CarbonCommonConstants.DEFAULT_ENABLE_AUTO_LOAD_MERGE)
+ }
+
+ test("ENABLE_AUTO_LOAD_MERGE: false, and sort_columns is boolean") {
+ sql("drop table if exists boolean_one_column")
+ sql(
+ s"""CREATE TABLE if not exists boolean_one_column(
+ |booleanField BOOLEAN)
+ |STORED BY 'carbondata'
+ |TBLPROPERTIES('sort_columns'='booleanField','SORT_SCOPE'='GLOBAL_SORT')
+ |""".stripMargin)
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE, "false")
+ sql("insert into boolean_one_column values(true)")
+ sql("insert into boolean_one_column values(True)")
+ sql("insert into boolean_one_column values(TRUE)")
+ sql("insert into boolean_one_column values('true')")
+ sql("insert into boolean_one_column values(False)")
+ sql("insert into boolean_one_column values(false)")
+ sql("insert into boolean_one_column values(FALSE)")
+ sql("insert into boolean_one_column values('false')")
+ sql("insert into boolean_one_column values('tr')")
+ sql("insert into boolean_one_column values(null)")
+ sql("insert into boolean_one_column values('truEe')")
+ sql("insert into boolean_one_column values('falsEe')")
+ sql("insert into boolean_one_column values('t')")
+ sql("insert into boolean_one_column values('f')")
+
+ checkAnswer(
+ sql("select * from boolean_one_column"),
+ Seq(Row(true), Row(true), Row(true), Row(true),
+ Row(false), Row(false), Row(false), Row(false),
+ Row(null), Row(null), Row(null), Row(null), Row(null), Row(null))
+ )
+
+ val segments = sql("SHOW SEGMENTS FOR TABLE boolean_one_column")
+ val SegmentSequenceIds = segments.collect().map { each => (each.toSeq) (0) }
+ assert(!SegmentSequenceIds.contains("0.1"))
+ assert(SegmentSequenceIds.length == 14)
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE,
+ CarbonCommonConstants.DEFAULT_ENABLE_AUTO_LOAD_MERGE)
+ }
+
+ test("ENABLE_AUTO_LOAD_MERGE: true, and sort_columns is boolean") {
+ sql("drop table if exists boolean_one_column")
+ sql(
+ s"""CREATE TABLE if not exists boolean_one_column(
+ |booleanField BOOLEAN)
+ |STORED BY 'carbondata'
+ |TBLPROPERTIES('sort_columns'='booleanField','SORT_SCOPE'='GLOBAL_SORT')
+ |""".stripMargin)
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE, "true")
+ sql("insert into boolean_one_column values(true)")
+ sql("insert into boolean_one_column values(True)")
+ sql("insert into boolean_one_column values(TRUE)")
+ sql("insert into boolean_one_column values('true')")
+ sql("insert into boolean_one_column values(False)")
+ sql("insert into boolean_one_column values(false)")
+ sql("insert into boolean_one_column values(FALSE)")
+ sql("insert into boolean_one_column values('false')")
+ sql("insert into boolean_one_column values('tr')")
+ sql("insert into boolean_one_column values(null)")
+ sql("insert into boolean_one_column values('truEe')")
+ sql("insert into boolean_one_column values('falsEe')")
+ sql("insert into boolean_one_column values('t')")
+ sql("insert into boolean_one_column values('f')")
+
+ checkAnswer(
+ sql("select * from boolean_one_column"),
+ Seq(Row(true), Row(true), Row(true), Row(true),
+ Row(false), Row(false), Row(false), Row(false),
+ Row(null), Row(null), Row(null), Row(null), Row(null), Row(null))
+ )
+
+ val segments = sql("SHOW SEGMENTS FOR TABLE boolean_one_column")
+ val SegmentSequenceIds = segments.collect().map { each => (each.toSeq) (0) }
+ assert(SegmentSequenceIds.contains("0.1"))
+ assert(SegmentSequenceIds.length == 18)
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE,
+ CarbonCommonConstants.DEFAULT_ENABLE_AUTO_LOAD_MERGE)
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesSortTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesSortTest.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesSortTest.scala
new file mode 100644
index 0000000..71c6c18
--- /dev/null
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesSortTest.scala
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.spark.testsuite.booleantype
+
+import java.io.File
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
+
+class BooleanDataTypesSortTest extends QueryTest with BeforeAndAfterEach with BeforeAndAfterAll {
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+
+ override def beforeEach(): Unit = {
+ sql("drop table if exists boolean_table")
+ sql("drop table if exists boolean_table2")
+ }
+
+ override def afterAll(): Unit = {
+ sql("drop table if exists boolean_table")
+ sql("drop table if exists boolean_table2")
+ }
+
+ test("Sort_columns: sort one boolean column") {
+ sql(
+ s"""
+ | CREATE TABLE boolean_table(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='booleanField','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+
+ sql(
+ s"""
+ | CREATE TABLE boolean_table2(
+ | shortField SHORT,
+ | booleanField BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='booleanField')
+ """.stripMargin)
+
+ val storeLocation = s"$rootPath/integration/spark2/src/test/resources/bool/supportBooleanTwoBooleanColumns.csv"
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${storeLocation}'
+ | INTO TABLE boolean_table
+ | options('FILEHEADER'='shortField,booleanField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData')
+ """.stripMargin)
+
+ sql("insert into boolean_table2 select shortField,booleanField from boolean_table where shortField = 1 and booleanField = true")
+
+ checkAnswer(
+ sql("select shortField,booleanField from boolean_table"),
+ Seq(Row(5, false), Row(1, false), Row(2, false), Row(1, false), Row(4, false), Row(1, false), Row(1, true), Row(1, true), Row(1, true), Row(3, true))
+ )
+
+ checkAnswer(
+ sql("select shortField,booleanField from boolean_table2"),
+ Seq(Row(1, true), Row(1, true), Row(1, true))
+ )
+ }
+
+ test("Sort_columns: sort two boolean columns and other data type column") {
+ sql(
+ s"""
+ | CREATE TABLE boolean_table(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='shortField,booleanField,booleanField2','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+
+ sql(
+ s"""
+ | CREATE TABLE boolean_table2(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='shortField,booleanField,booleanField2')
+ """.stripMargin)
+
+ val storeLocation = s"$rootPath/integration/spark2/src/test/resources/bool/supportBooleanTwoBooleanColumns.csv"
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${storeLocation}'
+ | INTO TABLE boolean_table
+ | options('FILEHEADER'='shortField,booleanField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData,booleanField2')
+ """.stripMargin)
+
+ sql("insert into boolean_table2 select shortField,booleanField,booleanField2 from boolean_table where shortField = 1 and booleanField = true")
+
+ checkAnswer(
+ sql("select shortField,booleanField,booleanField2 from boolean_table"),
+ Seq(Row(5, false, true), Row(1, false, false), Row(2, false, false), Row(1, false, false), Row(4, false, false),
+ Row(1, false, true), Row(1, true, true), Row(1, true, true), Row(1, true, true), Row(3, true, false))
+ )
+
+ checkAnswer(
+ sql("select shortField,booleanField,booleanField2 from boolean_table2"),
+ Seq(Row(1, true, true), Row(1, true, true), Row(1, true, true))
+ )
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeCarbonRowPage.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeCarbonRowPage.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeCarbonRowPage.java
index ff42e2d..1d8f941 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeCarbonRowPage.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeCarbonRowPage.java
@@ -127,7 +127,11 @@ public class UnsafeCarbonRowPage {
Object value = row[mesCount + dimensionSize];
if (null != value) {
DataType dataType = measureDataType[mesCount];
- if (dataType == DataTypes.SHORT) {
+ if (dataType == DataTypes.BOOLEAN) {
+ Boolean bval = (Boolean) value;
+ CarbonUnsafe.getUnsafe().putBoolean(baseObject, address + size, bval);
+ size += 1;
+ } else if (dataType == DataTypes.SHORT) {
Short sval = (Short) value;
CarbonUnsafe.getUnsafe().putShort(baseObject, address + size, sval);
size += 2;
@@ -209,7 +213,11 @@ public class UnsafeCarbonRowPage {
for (int mesCount = 0; mesCount < measureSize; mesCount++) {
if (isSet(nullSetWords, mesCount)) {
DataType dataType = measureDataType[mesCount];
- if (dataType == DataTypes.SHORT) {
+ if (dataType == DataTypes.BOOLEAN) {
+ Boolean bval = CarbonUnsafe.getUnsafe().getBoolean(baseObject, address + size);
+ size += 1;
+ rowToFill[dimensionSize + mesCount] = bval;
+ } else if (dataType == DataTypes.SHORT) {
Short sval = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size);
size += 2;
rowToFill[dimensionSize + mesCount] = sval;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateFileMerger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateFileMerger.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateFileMerger.java
index 3671316..fa50f1c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateFileMerger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateFileMerger.java
@@ -347,7 +347,9 @@ public class IntermediateFileMerger implements Callable<Void> {
if (null != NonDictionaryUtil.getMeasure(fieldIndex, row)) {
stream.write((byte) 1);
DataType dataType = aggType[counter];
- if (dataType == DataTypes.SHORT) {
+ if (dataType == DataTypes.BOOLEAN) {
+ stream.writeBoolean((boolean)NonDictionaryUtil.getMeasure(fieldIndex, row));
+ } else if (dataType == DataTypes.SHORT) {
stream.writeShort((short) NonDictionaryUtil.getMeasure(fieldIndex, row));
} else if (dataType == DataTypes.INT) {
stream.writeInt((int) NonDictionaryUtil.getMeasure(fieldIndex, row));
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortDataRows.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortDataRows.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortDataRows.java
index 5cc96c5..8a60657 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortDataRows.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortDataRows.java
@@ -295,7 +295,9 @@ public class SortDataRows {
if (null != value) {
stream.write((byte) 1);
DataType dataType = type[mesCount];
- if (dataType == DataTypes.SHORT) {
+ if (dataType == DataTypes.BOOLEAN) {
+ stream.writeBoolean((boolean) value);
+ } else if (dataType == DataTypes.SHORT) {
stream.writeShort((Short) value);
} else if (dataType == DataTypes.INT) {
stream.writeInt((Integer) value);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortTempFileChunkHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortTempFileChunkHolder.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortTempFileChunkHolder.java
index 5d339c7..afa4cd7 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortTempFileChunkHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortTempFileChunkHolder.java
@@ -345,7 +345,9 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
for (int i = 0; i < this.measureCount; i++) {
if (stream.readByte() == 1) {
DataType dataType = aggType[i];
- if (dataType == DataTypes.SHORT) {
+ if (dataType == DataTypes.BOOLEAN) {
+ measures[index++] = stream.readBoolean();
+ } else if (dataType == DataTypes.SHORT) {
measures[index++] = stream.readShort();
} else if (dataType == DataTypes.INT) {
measures[index++] = stream.readInt();
[07/16] carbondata git commit: [CARBONDATA-1597] Remove spark1
integration
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
deleted file mode 100644
index 6eeeaf9..0000000
--- a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ /dev/null
@@ -1,1088 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.spark.rdd
-
-import java.text.SimpleDateFormat
-import java.util
-import java.util.concurrent._
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable.ListBuffer
-import scala.util.Random
-import scala.util.control.Breaks._
-
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.Path
-import org.apache.hadoop.io.NullWritable
-import org.apache.hadoop.mapreduce.Job
-import org.apache.hadoop.mapreduce.lib.input.{FileInputFormat, FileSplit}
-import org.apache.spark.{SparkEnv, SparkException, TaskContext}
-import org.apache.spark.rdd.{DataLoadCoalescedRDD, DataLoadPartitionCoalescer, NewHadoopRDD, RDD}
-import org.apache.spark.sql.{CarbonEnv, DataFrame, Row, SQLContext}
-import org.apache.spark.sql.execution.command.{AlterTableModel, CompactionModel, ExecutionErrors, UpdateTableModel}
-import org.apache.spark.sql.hive.DistributionUtil
-import org.apache.spark.util.SparkUtil
-
-import org.apache.carbondata.common.constants.LoggerAction
-import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datastore.block.{Distributable, TableBlockInfo}
-import org.apache.carbondata.core.dictionary.server.DictionaryServer
-import org.apache.carbondata.core.locks.{CarbonLockFactory, ICarbonLock, LockUsage}
-import org.apache.carbondata.core.metadata.{CarbonTableIdentifier, ColumnarFormatVersion}
-import org.apache.carbondata.core.metadata.datatype.{DataType, DataTypes}
-import org.apache.carbondata.core.metadata.schema.partition.PartitionType
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.mutate.CarbonUpdateUtil
-import org.apache.carbondata.core.scan.partition.PartitionUtil
-import org.apache.carbondata.core.statusmanager.{LoadMetadataDetails, SegmentStatusManager}
-import org.apache.carbondata.core.util.{ByteUtil, CarbonProperties}
-import org.apache.carbondata.core.util.path.CarbonStorePath
-import org.apache.carbondata.processing.exception.DataLoadingException
-import org.apache.carbondata.processing.loading.{DataLoadProcessBuilder, FailureCauses}
-import org.apache.carbondata.processing.loading.csvinput.{BlockDetails, CSVInputFormat, StringArrayWritable}
-import org.apache.carbondata.processing.loading.exception.{CarbonDataLoadingException, NoRetryException}
-import org.apache.carbondata.processing.loading.model.CarbonLoadModel
-import org.apache.carbondata.processing.loading.sort.SortScopeOptions
-import org.apache.carbondata.processing.merger.{CarbonCompactionUtil, CarbonDataMergerUtil, CompactionType}
-import org.apache.carbondata.processing.splits.TableSplit
-import org.apache.carbondata.processing.util.{CarbonDataProcessorUtil, CarbonLoaderUtil, CarbonQueryUtil}
-import org.apache.carbondata.spark._
-import org.apache.carbondata.spark.load.DataLoadProcessBuilderOnSpark
-import org.apache.carbondata.spark.util.{CarbonScalaUtil, CommonUtil, Util}
-
-/**
- * This is the factory class which can create different RDD depends on user needs.
- *
- */
-object CarbonDataRDDFactory {
-
- private val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
-
- def alterTableForCompaction(sqlContext: SQLContext,
- alterTableModel: AlterTableModel,
- carbonLoadModel: CarbonLoadModel,
- storePath: String,
- storeLocation: String): Unit = {
- var compactionSize: Long = 0
- var compactionType: CompactionType = CompactionType.MINOR_COMPACTION
- if (alterTableModel.compactionType.equalsIgnoreCase("major")) {
- compactionSize = CarbonDataMergerUtil.getCompactionSize(CompactionType.MAJOR_COMPACTION)
- compactionType = CompactionType.MAJOR_COMPACTION
- } else if (alterTableModel.compactionType
- .equalsIgnoreCase(CompactionType.IUD_UPDDEL_DELTA_COMPACTION.toString)) {
- compactionType = CompactionType.IUD_UPDDEL_DELTA_COMPACTION
- if (alterTableModel.segmentUpdateStatusManager.get != None) {
- carbonLoadModel
- .setSegmentUpdateStatusManager((alterTableModel.segmentUpdateStatusManager.get))
- carbonLoadModel
- .setLoadMetadataDetails(alterTableModel.segmentUpdateStatusManager.get
- .getLoadMetadataDetails.toList.asJava)
- }
- }
- else {
- compactionType = CompactionType.MINOR_COMPACTION
- }
-
- LOGGER.audit(s"Compaction request received for table " +
- s"${ carbonLoadModel.getDatabaseName }.${ carbonLoadModel.getTableName }")
- val carbonTable = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
-
- if (null == carbonLoadModel.getLoadMetadataDetails) {
- CommonUtil.readLoadMetadataDetails(carbonLoadModel)
- }
- // reading the start time of data load.
- val loadStartTime = CarbonUpdateUtil.readCurrentTime();
- carbonLoadModel.setFactTimeStamp(loadStartTime)
-
- val isCompactionTriggerByDDl = true
- val compactionModel = CompactionModel(compactionSize,
- compactionType,
- carbonTable,
- isCompactionTriggerByDDl
- )
-
- val isConcurrentCompactionAllowed = CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.ENABLE_CONCURRENT_COMPACTION,
- CarbonCommonConstants.DEFAULT_ENABLE_CONCURRENT_COMPACTION
- )
- .equalsIgnoreCase("true")
-
- // if system level compaction is enabled then only one compaction can run in the system
- // if any other request comes at this time then it will create a compaction request file.
- // so that this will be taken up by the compaction process which is executing.
- if (!isConcurrentCompactionAllowed) {
- LOGGER.info("System level compaction lock is enabled.")
- handleCompactionForSystemLocking(sqlContext,
- carbonLoadModel,
- storePath,
- storeLocation,
- compactionType,
- carbonTable,
- compactionModel
- )
- } else {
- // normal flow of compaction
- val lock = CarbonLockFactory
- .getCarbonLockObj(carbonTable.getAbsoluteTableIdentifier.getCarbonTableIdentifier,
- LockUsage.COMPACTION_LOCK
- )
-
- if (lock.lockWithRetries()) {
- LOGGER.info("Acquired the compaction lock for table" +
- s" ${ carbonLoadModel.getDatabaseName }.${ carbonLoadModel.getTableName }")
- try {
- startCompactionThreads(sqlContext,
- carbonLoadModel,
- storePath,
- storeLocation,
- compactionModel,
- lock
- )
- } catch {
- case e: Exception =>
- LOGGER.error(s"Exception in start compaction thread. ${ e.getMessage }")
- lock.unlock()
- }
- } else {
- LOGGER.audit("Not able to acquire the compaction lock for table " +
- s"${ carbonLoadModel.getDatabaseName }.${ carbonLoadModel.getTableName }")
- LOGGER.error(s"Not able to acquire the compaction lock for table" +
- s" ${ carbonLoadModel.getDatabaseName }.${ carbonLoadModel.getTableName }")
- sys.error("Table is already locked for compaction. Please try after some time.")
- }
- }
- }
-
- def handleCompactionForSystemLocking(sqlContext: SQLContext,
- carbonLoadModel: CarbonLoadModel,
- storePath: String,
- storeLocation: String,
- compactionType: CompactionType,
- carbonTable: CarbonTable,
- compactionModel: CompactionModel): Unit = {
- val lock = CarbonLockFactory
- .getCarbonLockObj(CarbonCommonConstants.SYSTEM_LEVEL_COMPACTION_LOCK_FOLDER,
- LockUsage.SYSTEMLEVEL_COMPACTION_LOCK
- )
- if (lock.lockWithRetries()) {
- LOGGER.info(s"Acquired the compaction lock for table ${ carbonLoadModel.getDatabaseName }" +
- s".${ carbonLoadModel.getTableName }")
- try {
- startCompactionThreads(sqlContext,
- carbonLoadModel,
- storePath,
- storeLocation,
- compactionModel,
- lock
- )
- } catch {
- case e: Exception =>
- LOGGER.error(s"Exception in start compaction thread. ${ e.getMessage }")
- lock.unlock()
- // if the compaction is a blocking call then only need to throw the exception.
- if (compactionModel.isDDLTrigger) {
- throw e
- }
- }
- } else {
- LOGGER.audit("Not able to acquire the system level compaction lock for table " +
- s"${ carbonLoadModel.getDatabaseName }.${ carbonLoadModel.getTableName }")
- LOGGER.error("Not able to acquire the compaction lock for table " +
- s"${ carbonLoadModel.getDatabaseName }.${ carbonLoadModel.getTableName }")
- CarbonCompactionUtil
- .createCompactionRequiredFile(carbonTable.getMetaDataFilepath, compactionType)
- // do sys error only in case of DDL trigger.
- if (compactionModel.isDDLTrigger) {
- sys.error("Compaction is in progress, compaction request for table " +
- s"${ carbonLoadModel.getDatabaseName }.${ carbonLoadModel.getTableName }" +
- " is in queue.")
- } else {
- LOGGER.error("Compaction is in progress, compaction request for table " +
- s"${ carbonLoadModel.getDatabaseName }.${ carbonLoadModel.getTableName }" +
- " is in queue.")
- }
- }
- }
-
- def startCompactionThreads(sqlContext: SQLContext,
- carbonLoadModel: CarbonLoadModel,
- storePath: String,
- storeLocation: String,
- compactionModel: CompactionModel,
- compactionLock: ICarbonLock): Unit = {
- val executor: ExecutorService = Executors.newFixedThreadPool(1)
- // update the updated table status.
- CommonUtil.readLoadMetadataDetails(carbonLoadModel)
- val compactionThread = new Thread {
- override def run(): Unit = {
-
- try {
- // compaction status of the table which is triggered by the user.
- var triggeredCompactionStatus = false
- var exception: Exception = null
- try {
- DataManagementFunc.executeCompaction(carbonLoadModel: CarbonLoadModel,
- compactionModel: CompactionModel,
- executor, sqlContext, storeLocation
- )
- triggeredCompactionStatus = true
- } catch {
- case e: Exception =>
- LOGGER.error(s"Exception in compaction thread ${ e.getMessage }")
- exception = e
- }
- // continue in case of exception also, check for all the tables.
- val isConcurrentCompactionAllowed = CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.ENABLE_CONCURRENT_COMPACTION,
- CarbonCommonConstants.DEFAULT_ENABLE_CONCURRENT_COMPACTION
- ).equalsIgnoreCase("true")
-
- if (!isConcurrentCompactionAllowed) {
- LOGGER.info("System level compaction lock is enabled.")
- val skipCompactionTables = ListBuffer[CarbonTableIdentifier]()
- var table: CarbonTable = CarbonCompactionUtil
- .getNextTableToCompact(CarbonEnv.get.carbonMetastore.metadata.
- tablesMeta.map(_.carbonTable).toArray,
- skipCompactionTables.toList.asJava)
- while (null != table) {
- LOGGER.info("Compaction request has been identified for table " +
- s"${ table.getDatabaseName }." +
- s"${ table.getFactTableName }")
- val metadataPath = table.getMetaDataFilepath
- val compactionType = CarbonCompactionUtil.determineCompactionType(metadataPath)
-
- val newCarbonLoadModel = new CarbonLoadModel()
- DataManagementFunc.prepareCarbonLoadModel(table, newCarbonLoadModel)
-
- val compactionSize = CarbonDataMergerUtil
- .getCompactionSize(CompactionType.MAJOR_COMPACTION)
-
- val newcompactionModel = CompactionModel(compactionSize,
- compactionType,
- table,
- compactionModel.isDDLTrigger
- )
- // proceed for compaction
- try {
- DataManagementFunc.executeCompaction(newCarbonLoadModel,
- newcompactionModel,
- executor, sqlContext, storeLocation
- )
- } catch {
- case e: Exception =>
- LOGGER.error("Exception in compaction thread for table " +
- s"${ table.getDatabaseName }." +
- s"${ table.getFactTableName }")
- // not handling the exception. only logging as this is not the table triggered
- // by user.
- } finally {
- // delete the compaction required file in case of failure or success also.
- if (!CarbonCompactionUtil
- .deleteCompactionRequiredFile(metadataPath, compactionType)) {
- // if the compaction request file is not been able to delete then
- // add those tables details to the skip list so that it wont be considered next.
- skipCompactionTables.+=:(table.getCarbonTableIdentifier)
- LOGGER.error("Compaction request file can not be deleted for table " +
- s"${ table.getDatabaseName }." +
- s"${ table.getFactTableName }")
- }
- }
- // ********* check again for all the tables.
- table = CarbonCompactionUtil
- .getNextTableToCompact(CarbonEnv.get.carbonMetastore.metadata
- .tablesMeta.map(_.carbonTable).toArray, skipCompactionTables.asJava
- )
- }
- // giving the user his error for telling in the beeline if his triggered table
- // compaction is failed.
- if (!triggeredCompactionStatus) {
- throw new Exception("Exception in compaction " + exception.getMessage)
- }
- }
- } finally {
- executor.shutdownNow()
- DataManagementFunc.deletePartialLoadsInCompaction(carbonLoadModel)
- compactionLock.unlock()
- }
- }
- }
- // calling the run method of a thread to make the call as blocking call.
- // in the future we may make this as concurrent.
- compactionThread.run()
- }
-
- def loadCarbonData(sqlContext: SQLContext,
- carbonLoadModel: CarbonLoadModel,
- storePath: String,
- columnar: Boolean,
- partitionStatus: String = CarbonCommonConstants.STORE_LOADSTATUS_SUCCESS,
- result: Option[DictionaryServer],
- overwriteTable: Boolean,
- dataFrame: Option[DataFrame] = None,
- updateModel: Option[UpdateTableModel] = None): Unit = {
- val carbonTable = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
- val isAgg = false
- // for handling of the segment Merging.
- def handleSegmentMerging(): Unit = {
- LOGGER.info(s"compaction need status is" +
- s" ${ CarbonDataMergerUtil.checkIfAutoLoadMergingRequired() }")
- if (CarbonDataMergerUtil.checkIfAutoLoadMergingRequired()) {
- LOGGER.audit(s"Compaction request received for table " +
- s"${ carbonLoadModel.getDatabaseName }.${ carbonLoadModel.getTableName }")
- val compactionSize = 0
- val isCompactionTriggerByDDl = false
- val compactionModel = CompactionModel(compactionSize,
- CompactionType.MINOR_COMPACTION,
- carbonTable,
- isCompactionTriggerByDDl
- )
- var storeLocation = ""
- val configuredStore = Util.getConfiguredLocalDirs(SparkEnv.get.conf)
- if (null != configuredStore && configuredStore.nonEmpty) {
- storeLocation = configuredStore(Random.nextInt(configuredStore.length))
- }
- if (storeLocation == null) {
- storeLocation = System.getProperty("java.io.tmpdir")
- }
- storeLocation = storeLocation + "/carbonstore/" + System.nanoTime()
-
- val isConcurrentCompactionAllowed = CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.ENABLE_CONCURRENT_COMPACTION,
- CarbonCommonConstants.DEFAULT_ENABLE_CONCURRENT_COMPACTION
- )
- .equalsIgnoreCase("true")
-
- if (!isConcurrentCompactionAllowed) {
-
- handleCompactionForSystemLocking(sqlContext,
- carbonLoadModel,
- storePath,
- storeLocation,
- CompactionType.MINOR_COMPACTION,
- carbonTable,
- compactionModel
- )
- } else {
- val lock = CarbonLockFactory
- .getCarbonLockObj(carbonTable.getAbsoluteTableIdentifier.getCarbonTableIdentifier,
- LockUsage.COMPACTION_LOCK
- )
-
- if (lock.lockWithRetries()) {
- LOGGER.info("Acquired the compaction lock.")
- try {
- startCompactionThreads(sqlContext,
- carbonLoadModel,
- storePath,
- storeLocation,
- compactionModel,
- lock
- )
- } catch {
- case e: Exception =>
- LOGGER.error(s"Exception in start compaction thread. ${ e.getMessage }")
- lock.unlock()
- throw e
- }
- } else {
- LOGGER.audit("Not able to acquire the compaction lock for table " +
- s"${ carbonLoadModel.getDatabaseName }.${
- carbonLoadModel
- .getTableName
- }")
- LOGGER.error("Not able to acquire the compaction lock for table " +
- s"${ carbonLoadModel.getDatabaseName }.${
- carbonLoadModel
- .getTableName
- }")
- }
- }
- }
- }
-
- def updateStatus(loadStatus: String,
- stat: Array[(String, (LoadMetadataDetails, ExecutionErrors))]) = {
- val metadataDetails = if (stat != null && stat(0) != null) {
- stat(0)._2._1
- } else {
- new LoadMetadataDetails
- }
- CarbonLoaderUtil
- .populateNewLoadMetaEntry(metadataDetails,
- loadStatus,
- carbonLoadModel.getFactTimeStamp,
- true)
- val status = CarbonLoaderUtil.recordLoadMetadata(metadataDetails,
- carbonLoadModel, false, overwriteTable)
- if (!status) {
- val errorMessage = "Dataload failed due to failure in table status updation."
- LOGGER.audit("Data load is failed for " +
- s"${ carbonLoadModel.getDatabaseName }.${
- carbonLoadModel
- .getTableName
- }")
- LOGGER.error("Dataload failed due to failure in table status updation.")
- throw new Exception(errorMessage)
- }
- }
-
- try {
- LOGGER.audit(s"Data load request has been received for table" +
- s" ${ carbonLoadModel.getDatabaseName }.${ carbonLoadModel.getTableName }")
- // Check if any load need to be deleted before loading new data
- DataManagementFunc.deleteLoadsAndUpdateMetadata(carbonLoadModel.getDatabaseName,
- carbonLoadModel.getTableName, storePath, false, carbonTable)
- // get partition way from configuration
- // val isTableSplitPartition = CarbonProperties.getInstance().getProperty(
- // CarbonCommonConstants.TABLE_SPLIT_PARTITION,
- // CarbonCommonConstants.TABLE_SPLIT_PARTITION_DEFAULT_VALUE).toBoolean
- val isTableSplitPartition = false
- var blocksGroupBy: Array[(String, Array[BlockDetails])] = null
- var status: Array[(String, (LoadMetadataDetails, ExecutionErrors))] = null
- var res: Array[List[(String, (LoadMetadataDetails, ExecutionErrors))]] = null
-
- def loadDataFile(): Unit = {
- if (isTableSplitPartition) {
- /*
- * when data handle by table split partition
- * 1) get partition files, direct load or not will get the different files path
- * 2) get files blocks by using SplitUtils
- * 3) output Array[(partitionID,Array[BlockDetails])] to blocksGroupBy
- */
- var splits = Array[TableSplit]()
- if (carbonLoadModel.isDirectLoad) {
- // get all table Splits, this part means files were divide to different partitions
- splits = CarbonQueryUtil.getTableSplitsForDirectLoad(carbonLoadModel.getFactFilePath)
- // get all partition blocks from file list
- blocksGroupBy = splits.map {
- split =>
- val pathBuilder = new StringBuilder()
- for (path <- split.getPartition.getFilesPath.asScala) {
- pathBuilder.append(path).append(",")
- }
- if (pathBuilder.nonEmpty) {
- pathBuilder.substring(0, pathBuilder.size - 1)
- }
- (split.getPartition.getUniqueID, SparkUtil.getSplits(pathBuilder.toString(),
- sqlContext.sparkContext
- ))
- }
- } else {
- // get all table Splits,when come to this, means data have been partition
- splits = CarbonQueryUtil.getTableSplits(carbonLoadModel.getDatabaseName,
- carbonLoadModel.getTableName, null)
- // get all partition blocks from factFilePath/uniqueID/
- blocksGroupBy = splits.map {
- split =>
- val pathBuilder = new StringBuilder()
- pathBuilder.append(carbonLoadModel.getFactFilePath)
- if (!carbonLoadModel.getFactFilePath.endsWith("/")
- && !carbonLoadModel.getFactFilePath.endsWith("\\")) {
- pathBuilder.append("/")
- }
- pathBuilder.append(split.getPartition.getUniqueID).append("/")
- (split.getPartition.getUniqueID,
- SparkUtil.getSplits(pathBuilder.toString, sqlContext.sparkContext))
- }
- }
- } else {
- /*
- * when data load handle by node partition
- * 1)clone the hadoop configuration,and set the file path to the configuration
- * 2)use org.apache.hadoop.mapreduce.lib.input.TextInputFormat to get splits,size info
- * 3)use CarbonLoaderUtil.nodeBlockMapping to get mapping info of node and block,
- * for locally writing carbondata files(one file one block) in nodes
- * use NewCarbonDataLoadRDD to load data and write to carbondata files
- */
- val hadoopConfiguration = new Configuration(sqlContext.sparkContext.hadoopConfiguration)
- // FileUtils will skip file which is no csv, and return all file path which split by ','
- val filePaths = carbonLoadModel.getFactFilePath
- hadoopConfiguration.set(FileInputFormat.INPUT_DIR, filePaths)
- hadoopConfiguration.set(FileInputFormat.INPUT_DIR_RECURSIVE, "true")
- hadoopConfiguration.set("io.compression.codecs",
- """org.apache.hadoop.io.compress.GzipCodec,
- org.apache.hadoop.io.compress.DefaultCodec,
- org.apache.hadoop.io.compress.BZip2Codec""".stripMargin)
-
- CommonUtil.configSplitMaxSize(sqlContext.sparkContext, filePaths, hadoopConfiguration)
-
- val inputFormat = new org.apache.hadoop.mapreduce.lib.input.TextInputFormat
- val jobContext = new Job(hadoopConfiguration)
- val rawSplits = inputFormat.getSplits(jobContext).toArray
- val blockList = rawSplits.map { inputSplit =>
- val fileSplit = inputSplit.asInstanceOf[FileSplit]
- new TableBlockInfo(fileSplit.getPath.toString,
- fileSplit.getStart, "1",
- fileSplit.getLocations, fileSplit.getLength, ColumnarFormatVersion.V1, null
- ).asInstanceOf[Distributable]
- }
- // group blocks to nodes, tasks
- val startTime = System.currentTimeMillis
- val activeNodes = DistributionUtil
- .ensureExecutorsAndGetNodeList(blockList, sqlContext.sparkContext)
- val nodeBlockMapping =
- CarbonLoaderUtil
- .nodeBlockMapping(blockList.toSeq.asJava, -1, activeNodes.toList.asJava).asScala
- .toSeq
- val timeElapsed: Long = System.currentTimeMillis - startTime
- LOGGER.info("Total Time taken in block allocation: " + timeElapsed)
- LOGGER.info(s"Total no of blocks: ${ blockList.length }, " +
- s"No.of Nodes: ${nodeBlockMapping.size}")
- var str = ""
- nodeBlockMapping.foreach(entry => {
- val tableBlock = entry._2
- str = str + "#Node: " + entry._1 + " no.of.blocks: " + tableBlock.size()
- tableBlock.asScala.foreach(tableBlockInfo =>
- if (!tableBlockInfo.getLocations.exists(hostentry =>
- hostentry.equalsIgnoreCase(entry._1)
- )) {
- str = str + " , mismatch locations: " + tableBlockInfo.getLocations
- .foldLeft("")((a, b) => a + "," + b)
- }
- )
- str = str + "\n"
- }
- )
- LOGGER.info(str)
- blocksGroupBy = nodeBlockMapping.map(entry => {
- val blockDetailsList =
- entry._2.asScala.map(distributable => {
- val tableBlock = distributable.asInstanceOf[TableBlockInfo]
- new BlockDetails(new Path(tableBlock.getFilePath),
- tableBlock.getBlockOffset, tableBlock.getBlockLength, tableBlock.getLocations
- )
- }).toArray
- (entry._1, blockDetailsList)
- }
- ).toArray
- }
-
- status = new NewCarbonDataLoadRDD(sqlContext.sparkContext,
- new DataLoadResultImpl(),
- carbonLoadModel,
- blocksGroupBy,
- isTableSplitPartition).collect()
- }
-
- def loadDataFrame(): Unit = {
- try {
- val rdd = dataFrame.get.rdd
- val nodeNumOfData = rdd.partitions.flatMap[String, Array[String]] { p =>
- DataLoadPartitionCoalescer.getPreferredLocs(rdd, p).map(_.host)
- }.distinct.size
- val nodes = DistributionUtil.ensureExecutorsByNumberAndGetNodeList(nodeNumOfData,
- sqlContext.sparkContext)
- val newRdd = new DataLoadCoalescedRDD[Row](rdd, nodes.toArray.distinct)
- var numPartitions = DistributionUtil.getNodeList(sqlContext.sparkContext).length
- numPartitions = Math.max(1, Math.min(numPartitions, rdd.partitions.length))
- val coalesceRdd = rdd.coalesce(numPartitions, shuffle = false)
-
- status = new NewDataFrameLoaderRDD(sqlContext.sparkContext,
- new DataLoadResultImpl(),
- carbonLoadModel,
- newRdd).collect()
- } catch {
- case ex: Exception =>
- LOGGER.error(ex, "load data frame failed")
- throw ex
- }
- }
-
- def loadDataFrameForUpdate(): Unit = {
- val segmentUpdateParallelism = CarbonProperties.getInstance().getParallelismForSegmentUpdate
-
- def triggerDataLoadForSegment(key: String, taskNo: Int,
- iter: Iterator[Row]): Iterator[(String, (LoadMetadataDetails, ExecutionErrors))] = {
- val rddResult = new updateResultImpl()
- val LOGGER = LogServiceFactory.getLogService(this.getClass.getName)
- val resultIter = new Iterator[(String, (LoadMetadataDetails, ExecutionErrors))] {
- var partitionID = "0"
- val loadMetadataDetails = new LoadMetadataDetails
- val executionErrors = new ExecutionErrors(FailureCauses.NONE, "")
- var uniqueLoadStatusId = ""
- try {
- val segId = key
- val index = taskNo
- uniqueLoadStatusId = carbonLoadModel.getTableName +
- CarbonCommonConstants.UNDERSCORE +
- (index + "_0")
-
- // convert timestamp
- val timeStampInLong = updateModel.get.updatedTimeStamp + ""
- loadMetadataDetails.setPartitionCount(partitionID)
- loadMetadataDetails.setLoadName(segId)
- loadMetadataDetails.setLoadStatus(CarbonCommonConstants.STORE_LOADSTATUS_FAILURE)
- carbonLoadModel.setPartitionId(partitionID)
- carbonLoadModel.setSegmentId(segId)
- carbonLoadModel.setTaskNo(String.valueOf(index))
- carbonLoadModel.setFactTimeStamp(updateModel.get.updatedTimeStamp)
-
- // During Block Spill case Increment of File Count and proper adjustment of Block
- // naming is only done when AbstractFactDataWriter.java : initializeWriter get
- // CarbondataFileName as null. For handling Block Spill not setting the
- // CarbondataFileName in case of Update.
- // carbonLoadModel.setCarbondataFileName(newBlockName)
-
- // storeLocation = CarbonDataLoadRDD.initialize(carbonLoadModel, index)
- loadMetadataDetails.setLoadStatus(CarbonCommonConstants.STORE_LOADSTATUS_SUCCESS)
- UpdateDataLoad.DataLoadForUpdate(segId,
- index,
- iter,
- carbonLoadModel,
- loadMetadataDetails)
- } catch {
- case e: NoRetryException =>
- loadMetadataDetails
- .setLoadStatus(CarbonCommonConstants.STORE_LOADSTATUS_PARTIAL_SUCCESS)
- executionErrors.failureCauses = FailureCauses.BAD_RECORDS
- executionErrors.errorMsg = e.getMessage
- LOGGER.info("Bad Record Found")
- case e: Exception =>
- LOGGER.info("DataLoad failure")
- LOGGER.error(e)
- throw e
- }
-
- var finished = false
-
- override def hasNext: Boolean = !finished
-
- override def next(): (String, (LoadMetadataDetails, ExecutionErrors)) = {
- finished = true
- rddResult
- .getKey(uniqueLoadStatusId,
- (loadMetadataDetails, executionErrors))
- }
- }
- resultIter
- }
-
- val updateRdd = dataFrame.get.rdd
-
- // return directly if no rows to update
- val noRowsToUpdate = updateRdd.isEmpty()
- if (noRowsToUpdate) {
- res = Array[List[(String, (LoadMetadataDetails, ExecutionErrors))]]()
- return
- }
-
- // splitting as (key, value) i.e., (segment, updatedRows)
- val keyRDD = updateRdd.map(row =>
- (row.get(row.size - 1).toString, Row(row.toSeq.slice(0, row.size - 1): _*)))
-
- val loadMetadataDetails = SegmentStatusManager.readLoadMetadata(
- carbonTable.getMetaDataFilepath)
- val segmentIds = loadMetadataDetails.map(_.getLoadName)
- val segmentIdIndex = segmentIds.zipWithIndex.toMap
- val carbonTablePath = CarbonStorePath.getCarbonTablePath(carbonLoadModel.getStorePath,
- carbonTable.getCarbonTableIdentifier)
- val segmentId2maxTaskNo = segmentIds.map { segId =>
- (segId, CarbonUpdateUtil.getLatestTaskIdForSegment(segId, carbonTablePath))
- }.toMap
-
- class SegmentPartitioner(segIdIndex: Map[String, Int], parallelism: Int)
- extends org.apache.spark.Partitioner {
- override def numPartitions: Int = segmentIdIndex.size * parallelism
-
- override def getPartition(key: Any): Int = {
- val segId = key.asInstanceOf[String]
- // partitionId
- segmentIdIndex(segId) * parallelism + Random.nextInt(parallelism)
- }
- }
-
- val partitionByRdd = keyRDD.partitionBy(new SegmentPartitioner(segmentIdIndex,
- segmentUpdateParallelism))
-
- // because partitionId=segmentIdIndex*parallelism+RandomPart and RandomPart<parallelism,
- // so segmentIdIndex=partitionId/parallelism, this has been verified.
- res = partitionByRdd.map(_._2).mapPartitions { partition =>
- val partitionId = TaskContext.getPartitionId()
- val segIdIndex = partitionId / segmentUpdateParallelism
- val randomPart = partitionId - segIdIndex * segmentUpdateParallelism
- val segId = segmentIds(segIdIndex)
- val newTaskNo = segmentId2maxTaskNo(segId) + randomPart + 1
-
- List(triggerDataLoadForSegment(segId, newTaskNo, partition).toList).toIterator
- }.collect()
- }
-
- def loadDataForPartitionTable(): Unit = {
- try {
- val rdd = repartitionInputData(sqlContext, dataFrame, carbonLoadModel)
- status = new PartitionTableDataLoaderRDD(sqlContext.sparkContext,
- new DataLoadResultImpl(),
- carbonLoadModel,
- rdd).collect()
- } catch {
- case ex: Exception =>
- LOGGER.error(ex, "load data failed for partition table")
- throw ex
- }
- }
-
- // create new segment folder in carbon store
- if (!updateModel.isDefined) {
- CarbonLoaderUtil.checkAndCreateCarbonDataLocation(storePath,
- carbonLoadModel.getSegmentId, carbonTable)
- }
- var loadStatus = CarbonCommonConstants.STORE_LOADSTATUS_SUCCESS
- var errorMessage: String = "DataLoad failure"
- var executorMessage: String = ""
- val configuration = DataLoadProcessBuilder.createConfiguration(carbonLoadModel)
- val sortScope = CarbonDataProcessorUtil.getSortScope(configuration)
- try {
- if (updateModel.isDefined) {
- loadDataFrameForUpdate()
- } else if (carbonTable.getPartitionInfo(carbonTable.getFactTableName) != null) {
- loadDataForPartitionTable()
- } else if (configuration.isSortTable &&
- sortScope.equals(SortScopeOptions.SortScope.GLOBAL_SORT)) {
- LOGGER.audit("Using global sort for loading.")
- status = DataLoadProcessBuilderOnSpark.loadDataUsingGlobalSort(sqlContext.sparkContext,
- dataFrame, carbonLoadModel)
- } else if (dataFrame.isDefined) {
- loadDataFrame()
- } else {
- loadDataFile()
- }
- if (updateModel.isDefined) {
-
- res.foreach(resultOfSeg => resultOfSeg.foreach(
- resultOfBlock => {
- if (resultOfBlock._2._1.getLoadStatus
- .equalsIgnoreCase(CarbonCommonConstants.STORE_LOADSTATUS_FAILURE)) {
- loadStatus = CarbonCommonConstants.STORE_LOADSTATUS_FAILURE
- if (resultOfBlock._2._2.failureCauses == FailureCauses.NONE) {
- updateModel.get.executorErrors.failureCauses = FailureCauses.EXECUTOR_FAILURE
- updateModel.get.executorErrors.errorMsg = "Failure in the Executor."
- }
- else {
- updateModel.get.executorErrors = resultOfBlock._2._2
- }
- } else if (resultOfBlock._2._1.getLoadStatus
- .equalsIgnoreCase(CarbonCommonConstants.STORE_LOADSTATUS_PARTIAL_SUCCESS)) {
- loadStatus = CarbonCommonConstants.STORE_LOADSTATUS_PARTIAL_SUCCESS
- updateModel.get.executorErrors.failureCauses = resultOfBlock._2._2.failureCauses
- updateModel.get.executorErrors.errorMsg = resultOfBlock._2._2.errorMsg
- }
- }
- ))
-
- }
- else {
- val newStatusMap = scala.collection.mutable.Map.empty[String, String]
- if (status.nonEmpty) {
- status.foreach { eachLoadStatus =>
- val state = newStatusMap.get(eachLoadStatus._1)
- state match {
- case Some(CarbonCommonConstants.STORE_LOADSTATUS_FAILURE) =>
- newStatusMap.put(eachLoadStatus._1, eachLoadStatus._2._1.getLoadStatus)
- case Some(CarbonCommonConstants.STORE_LOADSTATUS_PARTIAL_SUCCESS)
- if eachLoadStatus._2._1.getLoadStatus ==
- CarbonCommonConstants.STORE_LOADSTATUS_SUCCESS =>
- newStatusMap.put(eachLoadStatus._1, eachLoadStatus._2._1.getLoadStatus)
- case _ =>
- newStatusMap.put(eachLoadStatus._1, eachLoadStatus._2._1.getLoadStatus)
- }
- }
-
- newStatusMap.foreach {
- case (key, value) =>
- if (value == CarbonCommonConstants.STORE_LOADSTATUS_FAILURE) {
- loadStatus = CarbonCommonConstants.STORE_LOADSTATUS_FAILURE
- } else if (value == CarbonCommonConstants.STORE_LOADSTATUS_PARTIAL_SUCCESS &&
- !loadStatus.equals(CarbonCommonConstants.STORE_LOADSTATUS_FAILURE)) {
- loadStatus = CarbonCommonConstants.STORE_LOADSTATUS_PARTIAL_SUCCESS
- }
- }
- } else {
- loadStatus = CarbonCommonConstants.STORE_LOADSTATUS_FAILURE
- }
-
- if (loadStatus != CarbonCommonConstants.STORE_LOADSTATUS_FAILURE &&
- partitionStatus == CarbonCommonConstants.STORE_LOADSTATUS_PARTIAL_SUCCESS) {
- loadStatus = partitionStatus
- }
- }
- } catch {
- case ex: Throwable =>
- loadStatus = CarbonCommonConstants.STORE_LOADSTATUS_FAILURE
- ex match {
- case sparkException: SparkException =>
- if (sparkException.getCause.isInstanceOf[DataLoadingException] ||
- sparkException.getCause.isInstanceOf[CarbonDataLoadingException]) {
- executorMessage = sparkException.getCause.getMessage
- errorMessage = errorMessage + ": " + executorMessage
- }
- case _ =>
- executorMessage = ex.getCause.getMessage
- errorMessage = errorMessage + ": " + executorMessage
- }
- LOGGER.info(errorMessage)
- LOGGER.error(ex)
- }
- // handle the status file updation for the update cmd.
- if (updateModel.isDefined) {
-
- if (loadStatus == CarbonCommonConstants.STORE_LOADSTATUS_FAILURE) {
- // updateModel.get.executorErrors.errorMsg = errorMessage
- if (updateModel.get.executorErrors.failureCauses == FailureCauses.NONE) {
- updateModel.get.executorErrors.failureCauses = FailureCauses.EXECUTOR_FAILURE
- if (null != executorMessage && !executorMessage.isEmpty) {
- updateModel.get.executorErrors.errorMsg = executorMessage
- } else {
- updateModel.get.executorErrors.errorMsg = "Update failed as the data load has failed."
- }
- }
- return
- } else if (loadStatus == CarbonCommonConstants.STORE_LOADSTATUS_PARTIAL_SUCCESS &&
- updateModel.get.executorErrors.failureCauses == FailureCauses.BAD_RECORDS &&
- carbonLoadModel.getBadRecordsAction.split(",")(1) == LoggerAction.FAIL.name) {
- return
- } else {
- // in success case handle updation of the table status file.
- // success case.
- val segmentDetails = new util.HashSet[String]()
-
- var resultSize = 0
-
- res.foreach(resultOfSeg => {
- resultSize = resultSize + resultOfSeg.size
- resultOfSeg.foreach(
- resultOfBlock => {
- segmentDetails.add(resultOfBlock._2._1.getLoadName)
- }
- )}
- )
-
- // this means that the update doesnt have any records to update so no need to do table
- // status file updation.
- if (resultSize == 0) {
- LOGGER.audit("Data update is successful with 0 rows updation for " +
- s"${carbonLoadModel.getDatabaseName}.${carbonLoadModel.getTableName}")
- return
- }
-
- if (
- CarbonUpdateUtil
- .updateTableMetadataStatus(segmentDetails,
- carbonTable,
- updateModel.get.updatedTimeStamp + "",
- true,
- new util.ArrayList[String](0))) {
- LOGGER.audit("Data update is successful for " +
- s"${ carbonLoadModel.getDatabaseName }.${ carbonLoadModel.getTableName }")
- }
- else {
- val errorMessage = "Data update failed due to failure in table status updation."
- LOGGER.audit("Data update is failed for " +
- s"${carbonLoadModel.getDatabaseName}.${carbonLoadModel.getTableName}")
- LOGGER.error("Data update failed due to failure in table status updation.")
- updateModel.get.executorErrors.errorMsg = errorMessage
- updateModel.get.executorErrors.failureCauses = FailureCauses
- .STATUS_FILE_UPDATION_FAILURE
- return
- }
-
- }
-
- return
- }
- if (loadStatus == CarbonCommonConstants.STORE_LOADSTATUS_FAILURE) {
- LOGGER.info("********starting clean up**********")
- CarbonLoaderUtil.deleteSegment(carbonLoadModel, carbonLoadModel.getSegmentId.toInt)
- LOGGER.info("********clean up done**********")
- LOGGER.audit(s"Data load is failed for " +
- s"${ carbonLoadModel.getDatabaseName }.${ carbonLoadModel.getTableName }")
- LOGGER.warn("Cannot write load metadata file as data load failed")
- updateStatus(loadStatus, status)
- throw new Exception(errorMessage)
- } else {
- // check if data load fails due to bad record and throw data load failure due to
- // bad record exception
- if (loadStatus == CarbonCommonConstants.STORE_LOADSTATUS_PARTIAL_SUCCESS &&
- status(0)._2._2.failureCauses == FailureCauses.BAD_RECORDS &&
- carbonLoadModel.getBadRecordsAction.split(",")(1) == LoggerAction.FAIL.name) {
- LOGGER.info("********starting clean up**********")
- CarbonLoaderUtil.deleteSegment(carbonLoadModel, carbonLoadModel.getSegmentId.toInt)
- LOGGER.info("********clean up done**********")
- LOGGER.audit(s"Data load is failed for " +
- s"${ carbonLoadModel.getDatabaseName }.${ carbonLoadModel.getTableName }")
- updateStatus(CarbonCommonConstants.STORE_LOADSTATUS_FAILURE, status)
- throw new Exception(status(0)._2._2.errorMsg)
- }
- if (!isAgg) {
- writeDictionary(carbonLoadModel, result)
- updateStatus(loadStatus, status)
- } else if (!carbonLoadModel.isRetentionRequest) {
- // TODO : Handle it
- LOGGER.info("********Database updated**********")
- }
-
- if (CarbonCommonConstants.STORE_LOADSTATUS_PARTIAL_SUCCESS.equals(loadStatus)) {
- LOGGER.audit("Data load is partially successful for " +
- s"${ carbonLoadModel.getDatabaseName }.${ carbonLoadModel.getTableName }")
- } else {
- LOGGER.audit("Data load is successful for " +
- s"${ carbonLoadModel.getDatabaseName }.${ carbonLoadModel.getTableName }")
- }
- try {
- // compaction handling
- handleSegmentMerging()
- } catch {
- case e: Exception =>
- throw new Exception(
- "Dataload is success. Auto-Compaction has failed. Please check logs.")
- }
- }
- }
-
- }
-
- /**
- * repartition the input data for partiton table.
- * @param sqlContext
- * @param dataFrame
- * @param carbonLoadModel
- * @return
- */
- private def repartitionInputData(sqlContext: SQLContext,
- dataFrame: Option[DataFrame],
- carbonLoadModel: CarbonLoadModel): RDD[Row] = {
- val carbonTable = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
- val partitionInfo = carbonTable.getPartitionInfo(carbonTable.getFactTableName)
- val partitionColumn = partitionInfo.getColumnSchemaList.get(0).getColumnName
- val partitionColumnDataType = partitionInfo.getColumnSchemaList.get(0).getDataType
- val columns = carbonLoadModel.getCsvHeaderColumns
- var partitionColumnIndex = -1
- breakable {
- for (i <- 0 until columns.length) {
- if (partitionColumn.equalsIgnoreCase(columns(i))) {
- partitionColumnIndex = i
- break
- }
- }
- }
- if (partitionColumnIndex == -1) {
- throw new DataLoadingException("Partition column not found.")
- }
-
- val dateFormatMap = CarbonDataProcessorUtil.getDateFormatMap(carbonLoadModel.getDateFormat())
- val specificFormat = Option(dateFormatMap.get(partitionColumn.toLowerCase))
- val timeStampFormat = if (specificFormat.isDefined) {
- new SimpleDateFormat(specificFormat.get)
- } else {
- val timestampFormatString = CarbonProperties.getInstance().getProperty(CarbonCommonConstants
- .CARBON_TIMESTAMP_FORMAT, CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
- new SimpleDateFormat(timestampFormatString)
- }
-
- val dateFormat = if (specificFormat.isDefined) {
- new SimpleDateFormat(specificFormat.get)
- } else {
- val dateFormatString = CarbonProperties.getInstance().getProperty(CarbonCommonConstants
- .CARBON_DATE_FORMAT, CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT)
- new SimpleDateFormat(dateFormatString)
- }
-
- // generate RDD[(K, V)] to use the partitionBy method of PairRDDFunctions
- val inputRDD: RDD[(String, Row)] = if (dataFrame.isDefined) {
- // input data from DataFrame
- val delimiterLevel1 = carbonLoadModel.getComplexDelimiterLevel1
- val delimiterLevel2 = carbonLoadModel.getComplexDelimiterLevel2
- val serializationNullFormat =
- carbonLoadModel.getSerializationNullFormat.split(CarbonCommonConstants.COMMA, 2)(1)
- dataFrame.get.rdd.map { row =>
- if (null != row && row.length > partitionColumnIndex &&
- null != row.get(partitionColumnIndex)) {
- (CarbonScalaUtil.getString(row.get(partitionColumnIndex), serializationNullFormat,
- delimiterLevel1, delimiterLevel2, timeStampFormat, dateFormat), row)
- } else {
- (null, row)
- }
- }
- } else {
- // input data from csv files
- val hadoopConfiguration = new Configuration()
- CommonUtil.configureCSVInputFormat(hadoopConfiguration, carbonLoadModel)
- hadoopConfiguration.set(FileInputFormat.INPUT_DIR, carbonLoadModel.getFactFilePath)
- val columnCount = columns.length
- new NewHadoopRDD[NullWritable, StringArrayWritable](
- sqlContext.sparkContext,
- classOf[CSVInputFormat],
- classOf[NullWritable],
- classOf[StringArrayWritable],
- hadoopConfiguration
- ).map { currentRow =>
- if (null == currentRow || null == currentRow._2) {
- val row = new StringArrayRow(new Array[String](columnCount))
- (null, row)
- } else {
- val row = new StringArrayRow(new Array[String](columnCount))
- val values = currentRow._2.get()
- if (values != null && values.length > partitionColumnIndex) {
- (currentRow._2.get()(partitionColumnIndex), row.setValues(currentRow._2.get()))
- } else {
- (null, row.setValues(currentRow._2.get()))
- }
- }
- }
- }
-
- val partitioner = PartitionFactory.getPartitioner(partitionInfo)
- if (partitionColumnDataType == DataTypes.STRING) {
- if (partitionInfo.getPartitionType == PartitionType.RANGE) {
- inputRDD.map { row => (ByteUtil.toBytes(row._1), row._2) }
- .partitionBy(partitioner)
- .map(_._2)
- } else {
- inputRDD.partitionBy(partitioner)
- .map(_._2)
- }
- } else {
- inputRDD.map { row =>
- (PartitionUtil.getDataBasedOnDataType(row._1, partitionColumnDataType, timeStampFormat,
- dateFormat), row._2)
- }
- .partitionBy(partitioner)
- .map(_._2)
- }
- }
-
- private def writeDictionary(carbonLoadModel: CarbonLoadModel,
- result: Option[DictionaryServer]) = {
- // write dictionary file and shutdown dictionary server
- val uniqueTableName: String = s"${ carbonLoadModel.getDatabaseName }_${
- carbonLoadModel.getTableName }"
- result match {
- case Some(server) =>
- try {
- server.writeTableDictionary(carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
- .getCarbonTableIdentifier.getTableId)
- } catch {
- case ex: Exception =>
- LOGGER.error(s"Error while writing dictionary file for $uniqueTableName")
- throw new Exception("Dataload failed due to error while writing dictionary file!")
- }
- case _ =>
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/main/scala/org/apache/carbondata/spark/thriftserver/CarbonThriftServer.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/thriftserver/CarbonThriftServer.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/thriftserver/CarbonThriftServer.scala
deleted file mode 100644
index f8275d1..0000000
--- a/integration/spark/src/main/scala/org/apache/carbondata/spark/thriftserver/CarbonThriftServer.scala
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.spark.thriftserver
-
-import java.io.File
-
-import org.apache.spark.{SparkConf, SparkContext}
-import org.apache.spark.sql.CarbonContext
-import org.apache.spark.sql.hive.thriftserver.HiveThriftServer2
-
-import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.util.CarbonProperties
-
-object CarbonThriftServer {
-
- def main(args: Array[String]): Unit = {
- val conf = new SparkConf()
- .setAppName("Carbon Thrift Server")
- if (!conf.contains("carbon.properties.filepath")) {
- val sparkHome = System.getenv.get("SPARK_HOME")
- if (sparkHome != null) {
- val file = new File(sparkHome + '/' + "conf" + '/' + "carbon.properties")
- if (file.exists()) {
- conf.set("carbon.properties.filepath", file.getCanonicalPath)
- System.setProperty("carbon.properties.filepath", file.getCanonicalPath)
- }
- }
- } else {
- System.setProperty("carbon.properties.filepath", conf.get("carbon.properties.filepath"))
- }
- if (org.apache.spark.SPARK_VERSION.startsWith("1.6")) {
- conf.set("spark.sql.hive.thriftServer.singleSession", "true")
- }
- val sc = new SparkContext(conf)
- val warmUpTime = CarbonProperties.getInstance().getProperty("carbon.spark.warmUpTime", "5000")
- try {
- Thread.sleep(Integer.parseInt(warmUpTime))
- } catch {
- case e: Exception =>
- val LOG = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
- LOG.error(s"Wrong value for carbon.spark.warmUpTime $warmUpTime " +
- "Using default Value and proceeding")
- Thread.sleep(30000)
- }
- val storePath = if (args.length > 0) args.head else null
- val cc = new CarbonContext(sc, storePath)
-
- HiveThriftServer2.startWithContext(cc)
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/main/scala/org/apache/carbondata/spark/util/CarbonSparkUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/util/CarbonSparkUtil.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/util/CarbonSparkUtil.scala
deleted file mode 100644
index 118249a..0000000
--- a/integration/spark/src/main/scala/org/apache/carbondata/spark/util/CarbonSparkUtil.scala
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.spark.util
-
-import scala.collection.JavaConverters._
-
-import org.apache.spark.sql.hive.{CarbonMetaData, DictionaryMap}
-
-import org.apache.carbondata.core.metadata.encoder.Encoding
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.util.CarbonUtil
-
-case class TransformHolder(rdd: Any, mataData: CarbonMetaData)
-
-object CarbonSparkUtil {
-
- def createSparkMeta(carbonTable: CarbonTable): CarbonMetaData = {
- val dimensionsAttr = carbonTable.getDimensionByTableName(carbonTable.getFactTableName)
- .asScala.map(x => x.getColName) // wf : may be problem
- val measureAttr = carbonTable.getMeasureByTableName(carbonTable.getFactTableName)
- .asScala.map(x => x.getColName)
- val dictionary =
- carbonTable.getDimensionByTableName(carbonTable.getFactTableName).asScala.map { f =>
- (f.getColName.toLowerCase,
- f.hasEncoding(Encoding.DICTIONARY) && !f.hasEncoding(Encoding.DIRECT_DICTIONARY) &&
- !f.getDataType.isComplexType)
- }
- CarbonMetaData(dimensionsAttr, measureAttr, carbonTable, DictionaryMap(dictionary.toMap))
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/main/scala/org/apache/carbondata/spark/util/QueryPlanUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/util/QueryPlanUtil.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/util/QueryPlanUtil.scala
deleted file mode 100644
index 4950227..0000000
--- a/integration/spark/src/main/scala/org/apache/carbondata/spark/util/QueryPlanUtil.scala
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.spark.util
-
-import scala.reflect.ClassTag
-
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.Path
-import org.apache.hadoop.mapred.JobConf
-import org.apache.hadoop.mapreduce.Job
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat
-
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
-import org.apache.carbondata.hadoop.api.CarbonTableInputFormat
-
-
-/**
- * All the utility functions for carbon plan creation
- */
-object QueryPlanUtil {
-
- /**
- * createCarbonInputFormat from query model
- */
- def createCarbonInputFormat(absoluteTableIdentifier: AbsoluteTableIdentifier) :
- (CarbonTableInputFormat[Array[Object]], Job) = {
- val carbonInputFormat = new CarbonTableInputFormat[Array[Object]]()
- val jobConf: JobConf = new JobConf(new Configuration)
- val job: Job = new Job(jobConf)
- FileInputFormat.addInputPath(job, new Path(absoluteTableIdentifier.getTablePath))
- (carbonInputFormat, job)
- }
-
- def createCarbonInputFormat[V: ClassTag](absoluteTableIdentifier: AbsoluteTableIdentifier,
- conf: Configuration) : CarbonTableInputFormat[V] = {
- val carbonInputFormat = new CarbonTableInputFormat[V]()
- val job: Job = new Job(conf)
- FileInputFormat.addInputPath(job, new Path(absoluteTableIdentifier.getTablePath))
- carbonInputFormat
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/main/scala/org/apache/spark/CarbonInputMetrics.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/CarbonInputMetrics.scala b/integration/spark/src/main/scala/org/apache/spark/CarbonInputMetrics.scala
deleted file mode 100644
index ea75ccb..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/CarbonInputMetrics.scala
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark
-
-import java.lang.Long
-
-import org.apache.spark.deploy.SparkHadoopUtil
-import org.apache.spark.executor.{DataReadMethod, InputMetrics}
-
-import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.hadoop.{CarbonMultiBlockSplit, InputMetricsStats}
-import org.apache.carbondata.spark.InitInputMetrics
-
-/**
- * It gives statistics of number of bytes and record read
- */
-class CarbonInputMetrics extends InitInputMetrics {
- @transient val LOGGER = LogServiceFactory.getLogService(this.getClass.getName)
- var inputMetrics: InputMetrics = _
- var bytesReadCallback: Option[() => scala.Long] = _
- var carbonMultiBlockSplit: CarbonMultiBlockSplit = _
-
- def initBytesReadCallback(context: TaskContext,
- carbonMultiBlockSplit: CarbonMultiBlockSplit) {
- inputMetrics = context.taskMetrics().getInputMetricsForReadMethod(DataReadMethod.Hadoop)
- this.carbonMultiBlockSplit = carbonMultiBlockSplit;
- bytesReadCallback = carbonMultiBlockSplit match {
- case _: CarbonMultiBlockSplit =>
- SparkHadoopUtil.get.getFSBytesReadOnThreadCallback()
- case _ => None
- }
- }
-
- def incrementRecordRead(recordRead: Long) {
- inputMetrics.incRecordsRead(recordRead)
- }
-
- def updateAndClose() {
- if (bytesReadCallback.isDefined) {
- inputMetrics.updateBytesRead()
- } else if (carbonMultiBlockSplit.isInstanceOf[CarbonMultiBlockSplit]) {
- // If we can't get the bytes read from the FS stats, fall back to the split size,
- // which may be inaccurate.
- try {
- inputMetrics.incBytesRead(carbonMultiBlockSplit.getLength)
- } catch {
- case e: java.io.IOException =>
- LOGGER.warn("Unable to get input size to set InputMetrics for task:" + e.getMessage)
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/main/scala/org/apache/spark/sql/CarbonBoundReference.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonBoundReference.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonBoundReference.scala
deleted file mode 100644
index a6c28a9..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonBoundReference.scala
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression, ExprId, LeafExpression, NamedExpression}
-import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
-import org.apache.spark.sql.sources.Filter
-import org.apache.spark.sql.types.DataType
-
-import org.apache.carbondata.core.scan.expression.ColumnExpression
-
-case class CastExpr(expr: Expression) extends Filter
-
-case class CarbonBoundReference(colExp: ColumnExpression, dataType: DataType, nullable: Boolean)
- extends LeafExpression with NamedExpression with CodegenFallback {
-
- type EvaluatedType = Any
-
- override def toString: String = s"input[" + colExp.getColIndex + "]"
-
- override def eval(input: InternalRow): Any = input.get(colExp.getColIndex, dataType)
-
- override def name: String = colExp.getColumnName
-
- override def toAttribute: Attribute = throw new UnsupportedOperationException
-
- override def exprId: ExprId = throw new UnsupportedOperationException
-
- override def qualifiers: Seq[String] = throw new UnsupportedOperationException
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/main/scala/org/apache/spark/sql/CarbonCatalystOperators.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonCatalystOperators.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonCatalystOperators.scala
deleted file mode 100644
index 024c54b..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonCatalystOperators.scala
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation
-import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.catalyst.plans.logical.{ UnaryNode, _ }
-import org.apache.spark.sql.hive.HiveContext
-import org.apache.spark.sql.optimizer.CarbonDecoderRelation
-import org.apache.spark.sql.types._
-
-import org.apache.carbondata.spark.CarbonAliasDecoderRelation
-import org.apache.carbondata.spark.util.CommonUtil
-
-/**
- * Top command
- */
-case class Top(count: Int, topOrBottom: Int, dim: NamedExpression, msr: NamedExpression,
- child: LogicalPlan) extends UnaryNode {
- def output: Seq[Attribute] = child.output
-
- override def references: AttributeSet = {
- val list = List(dim, msr)
- AttributeSet(list.flatMap(_.references))
- }
-}
-
-object getDB {
-
- def getDatabaseName(dbName: Option[String], sqlContext: SQLContext): String = {
- dbName.getOrElse(sqlContext.asInstanceOf[HiveContext].catalog.client.currentDatabase)
- }
-
-}
-
-/**
- * Shows Loads in a table
- */
-case class ShowLoadsCommand(databaseNameOp: Option[String], table: String, limit: Option[String])
- extends LogicalPlan with Command {
-
- override def children: Seq[LogicalPlan] = Seq.empty
-
- override def output: Seq[Attribute] = {
- Seq(AttributeReference("SegmentSequenceId", StringType, nullable = false)(),
- AttributeReference("Status", StringType, nullable = false)(),
- AttributeReference("Load Start Time", TimestampType, nullable = false)(),
- AttributeReference("Load End Time", TimestampType, nullable = false)())
- }
-}
-
-/**
- * Describe formatted for hive table
- */
-case class DescribeFormattedCommand(sql: String, tblIdentifier: TableIdentifier)
- extends LogicalPlan with Command {
- override def children: Seq[LogicalPlan] = Seq.empty
-
- override def output: Seq[AttributeReference] =
- Seq(AttributeReference("result", StringType, nullable = false)())
-}
-
-case class CarbonDictionaryCatalystDecoder(
- relations: Seq[CarbonDecoderRelation],
- profile: CarbonProfile,
- aliasMap: CarbonAliasDecoderRelation,
- isOuter: Boolean,
- child: LogicalPlan) extends UnaryNode {
- override def output: Seq[Attribute] = child.output
-}
-
-abstract class CarbonProfile(attributes: Seq[Attribute]) extends Serializable {
- def isEmpty: Boolean = attributes.isEmpty
-}
-
-case class IncludeProfile(attributes: Seq[Attribute]) extends CarbonProfile(attributes)
-
-case class ExcludeProfile(attributes: Seq[Attribute]) extends CarbonProfile(attributes)
-
-case class CreateDatabase(dbName: String, sql: String) extends LogicalPlan with Command {
- override def children: Seq[LogicalPlan] = Seq.empty
- override def output: Seq[AttributeReference] = {
- Seq()
- }
-}
-
-case class DropDatabase(dbName: String, isCascade: Boolean, sql: String)
- extends LogicalPlan with Command {
- override def children: Seq[LogicalPlan] = Seq.empty
- override def output: Seq[AttributeReference] = {
- Seq()
- }
-}
-
-case class UseDatabase(sql: String) extends LogicalPlan with Command {
- override def children: Seq[LogicalPlan] = Seq.empty
- override def output: Seq[AttributeReference] = {
- Seq()
- }
-}
-
-case class ProjectForUpdate(
- table: UnresolvedRelation,
- columns: List[String],
- children: Seq[LogicalPlan] ) extends LogicalPlan with Command {
- override def output: Seq[AttributeReference] = Seq.empty
-}
-
-case class UpdateTable(
- table: UnresolvedRelation,
- columns: List[String],
- selectStmt: String,
- filer: String) extends LogicalPlan {
- override def children: Seq[LogicalPlan] = Seq.empty
- override def output: Seq[AttributeReference] = Seq.empty
-}
-
-case class DeleteRecords(
- statement: String,
- table: UnresolvedRelation) extends LogicalPlan {
- override def children: Seq[LogicalPlan] = Seq.empty
- override def output: Seq[AttributeReference] = Seq.empty
-}
-
-case class ShowPartitions(
- table: TableIdentifier) extends LogicalPlan {
- override def children: Seq[LogicalPlan] = Seq.empty
- override def output: Seq[Attribute] = CommonUtil.partitionInfoOutput
-}
-
-/**
- * A logical plan representing insertion into Hive table.
- * This plan ignores nullability of ArrayType, MapType, StructType unlike InsertIntoTable
- * because Hive table doesn't have nullability for ARRAY, MAP, STRUCT types.
- */
-case class InsertIntoCarbonTable(
- table: CarbonDatasourceRelation,
- partition: Map[String, Option[String]],
- child: LogicalPlan,
- overwrite: Boolean,
- ifNotExists: Boolean)
- extends LogicalPlan with Command {
-
- override def children: Seq[LogicalPlan] = child :: Nil
- override def output: Seq[Attribute] = Seq.empty
-
- // This is the expected schema of the table prepared to be inserted into,
- // including dynamic partition columns.
- val tableOutput = table.carbonRelation.output
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/main/scala/org/apache/spark/sql/CarbonContext.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonContext.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonContext.scala
deleted file mode 100644
index da4b210..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonContext.scala
+++ /dev/null
@@ -1,197 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import java.io.File
-
-import scala.language.implicitConversions
-
-import org.apache.spark.SparkContext
-import org.apache.spark.sql.catalyst.ParserDialect
-import org.apache.spark.sql.catalyst.analysis.{Analyzer, OverrideCatalog}
-import org.apache.spark.sql.catalyst.optimizer.Optimizer
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.execution.ExtractPythonUDFs
-import org.apache.spark.sql.execution.datasources.{PreInsertCastAndRename, PreWriteCheck}
-import org.apache.spark.sql.hive._
-import org.apache.spark.sql.optimizer.CarbonOptimizer
-
-import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.stats.{QueryStatistic, QueryStatisticsConstants}
-import org.apache.carbondata.core.util.{CarbonProperties, CarbonTimeStatisticsFactory}
-
-class CarbonContext(
- val sc: SparkContext,
- val storePath: String,
- metaStorePath: String) extends HiveContext(sc) {
- self =>
-
- def this(sc: SparkContext) = {
- this(sc,
- null,
- new File(CarbonCommonConstants.METASTORE_LOCATION_DEFAULT_VAL).getCanonicalPath)
- }
-
- def this(sc: SparkContext, storePath: String) = {
- this(sc,
- storePath,
- new File(CarbonCommonConstants.METASTORE_LOCATION_DEFAULT_VAL).getCanonicalPath)
- }
-
- CarbonContext.addInstance(sc, this)
- CodeGenerateFactory.init(sc.version)
- udf.register("getTupleId", () => "")
- CarbonEnv.init(this)
-
- var lastSchemaUpdatedTime = System.currentTimeMillis()
- val hiveClientInterface = metadataHive
-
- protected[sql] override lazy val conf: SQLConf = new CarbonSQLConf
-
- @transient
- override lazy val catalog = {
- val carbonProperties = CarbonProperties.getInstance()
- if (storePath != null) {
- carbonProperties.addProperty(CarbonCommonConstants.STORE_LOCATION, storePath)
- // In case if it is in carbon.properties for backward compatible
- } else if (carbonProperties.getProperty(CarbonCommonConstants.STORE_LOCATION) == null) {
- carbonProperties.addProperty(CarbonCommonConstants.STORE_LOCATION,
- conf.getConfString("spark.sql.warehouse.dir"))
- }
- new CarbonMetastore(this, storePath, metadataHive, queryId) with OverrideCatalog
- }
-
- @transient
- override protected[sql] lazy val analyzer =
- new Analyzer(catalog, functionRegistry, conf) {
-
- override val extendedResolutionRules =
- catalog.ParquetConversions ::
- catalog.CreateTables ::
- CarbonIUDAnalysisRule ::
- CarbonPreInsertionCasts ::
- ExtractPythonUDFs ::
- ResolveHiveWindowFunction ::
- PreInsertCastAndRename ::
- Nil
-
- override val extendedCheckRules = Seq(
- PreWriteCheck(catalog)
- )
- }
-
- @transient
- override protected[sql] lazy val optimizer: Optimizer =
- CarbonOptimizer.optimizer(
- CodeGenerateFactory.createDefaultOptimizer(conf, sc),
- conf.asInstanceOf[CarbonSQLConf],
- sc.version)
-
- protected[sql] override def getSQLDialect(): ParserDialect = new CarbonSQLDialect(this)
-
- experimental.extraStrategies = {
- val carbonStrategy = new CarbonStrategies(self)
- Seq(carbonStrategy.CarbonTableScan, carbonStrategy.DDLStrategies)
- }
-
- override protected def configure(): Map[String, String] = {
- sc.hadoopConfiguration.addResource("hive-site.xml")
- if (sc.hadoopConfiguration.get(CarbonCommonConstants.HIVE_CONNECTION_URL) == null) {
- val metaStorePathAbsolute = new File(metaStorePath).getCanonicalPath
- val hiveMetaStoreDB = metaStorePathAbsolute + "/metastore_db"
- logDebug(s"metastore db is going to be created in location: $hiveMetaStoreDB")
- super.configure() ++ Map[String, String]((CarbonCommonConstants.HIVE_CONNECTION_URL,
- s"jdbc:derby:;databaseName=$hiveMetaStoreDB;create=true"),
- ("hive.metastore.warehouse.dir", metaStorePathAbsolute + "/hivemetadata"))
- } else {
- super.configure()
- }
- }
-
- @transient
- private val LOGGER = LogServiceFactory.getLogService(CarbonContext.getClass.getName)
-
- var queryId: String = ""
-
- override def sql(sql: String): DataFrame = {
- // queryId will be unique for each query, creting query detail holder
- queryId = System.nanoTime() + ""
- this.setConf("queryId", queryId)
-
- CarbonContext.updateCarbonPorpertiesPath(this)
- val sqlString = sql.toUpperCase
- LOGGER.info(s"Query [$sqlString]")
- val recorder = CarbonTimeStatisticsFactory.createDriverRecorder()
- val statistic = new QueryStatistic()
- val logicPlan: LogicalPlan = parseSql(sql)
- statistic.addStatistics(QueryStatisticsConstants.SQL_PARSE, System.currentTimeMillis())
- recorder.recordStatisticsForDriver(statistic, queryId)
- val result = new DataFrame(this, logicPlan)
-
- // We force query optimization to happen right away instead of letting it happen lazily like
- // when using the query DSL. This is so DDL commands behave as expected. This is only
- // generates the RDD lineage for DML queries, but do not perform any execution.
- result
- }
-
-}
-
-object CarbonContext {
-
- val datasourceName: String = "org.apache.carbondata.format"
-
- val datasourceShortName: String = "carbondata"
-
- @transient
- private val LOGGER = LogServiceFactory.getLogService(CarbonContext.getClass.getName)
-
- final def updateCarbonPorpertiesPath(hiveContext: HiveContext) {
- val carbonPropertiesFilePath = hiveContext.getConf("carbon.properties.filepath", null)
- val systemcarbonPropertiesFilePath = System.getProperty("carbon.properties.filepath", null)
- if (null != carbonPropertiesFilePath && null == systemcarbonPropertiesFilePath) {
- System.setProperty("carbon.properties.filepath",
- carbonPropertiesFilePath + "/" + "carbon.properties")
- }
- // configuring the zookeeper URl .
- val zooKeeperUrl = hiveContext.getConf("spark.deploy.zookeeper.url", "127.0.0.1:2181")
-
- CarbonProperties.getInstance().addProperty("spark.deploy.zookeeper.url", zooKeeperUrl)
-
- }
-
- // this cache is used to avoid creating multiple CarbonContext from same SparkContext,
- // to avoid the derby problem for metastore
- private val cache = collection.mutable.Map[SparkContext, CarbonContext]()
-
- def getInstance(sc: SparkContext): CarbonContext = {
- cache(sc)
- }
-
- def addInstance(sc: SparkContext, cc: CarbonContext): Unit = {
- if (cache.contains(sc)) {
- sys.error("creating multiple instances of CarbonContext is not " +
- "allowed using the same SparkContext instance")
- }
- cache(sc) = cc
- }
-}
-
-object SQLParser {
- def parse(sql: String, sqlContext: SQLContext): LogicalPlan = sqlContext.parseSql(sql)
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
deleted file mode 100644
index 2c2e954..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
+++ /dev/null
@@ -1,189 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import java.text.SimpleDateFormat
-import java.util.Date
-
-import scala.reflect.ClassTag
-
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.{FileStatus, Path}
-import org.apache.hadoop.mapred.JobConf
-import org.apache.hadoop.mapreduce.{Job, JobID}
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat
-import org.apache.spark._
-import org.apache.spark.annotation.DeveloperApi
-import org.apache.spark.mapreduce.SparkHadoopMapReduceUtil
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.sources.{Filter, HadoopFsRelation, OutputWriterFactory}
-import org.apache.spark.sql.types.StructType
-import org.apache.spark.util.SerializableConfiguration
-
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
-import org.apache.carbondata.core.scan.expression.logical.AndExpression
-import org.apache.carbondata.core.util.DataTypeUtil
-import org.apache.carbondata.core.util.path.CarbonTablePath
-import org.apache.carbondata.hadoop.{CarbonInputFormat, CarbonInputSplit, CarbonProjection}
-import org.apache.carbondata.hadoop.api.CarbonTableInputFormat
-import org.apache.carbondata.hadoop.util.{CarbonInputFormatUtil, SchemaReader}
-import org.apache.carbondata.processing.merger.TableMeta
-import org.apache.carbondata.spark.{CarbonFilters, CarbonOption}
-import org.apache.carbondata.spark.rdd.CarbonRDD
-import org.apache.carbondata.spark.readsupport.SparkRowReadSupportImpl
-import org.apache.carbondata.spark.util.SparkDataTypeConverterImpl
-
-private[sql] case class CarbonDatasourceHadoopRelation(
- sqlContext: SQLContext,
- paths: Array[String],
- parameters: Map[String, String],
- tableSchema: Option[StructType])
- extends HadoopFsRelation {
-
- lazy val schemaPath = new Path(CarbonTablePath.getSchemaFilePath(paths.head))
- if (!schemaPath.getFileSystem(new Configuration).exists(schemaPath)) {
- throw new IllegalArgumentException("invalid CarbonData file path: " + paths.head)
- }
-
- lazy val job = new Job(new JobConf())
- lazy val options = new CarbonOption(parameters)
- lazy val absIdentifier = AbsoluteTableIdentifier.fromTablePath(paths.head)
- lazy val relationRaw: CarbonRelation = {
- val carbonTable = SchemaReader.readCarbonTableFromStore(absIdentifier)
- if (carbonTable == null) {
- sys.error(s"CarbonData file path ${paths.head} is not valid")
- }
- CarbonRelation(
- carbonTable.getDatabaseName,
- carbonTable.getFactTableName,
- CarbonSparkUtil.createSparkMeta(carbonTable),
- new TableMeta(carbonTable.getCarbonTableIdentifier,
- paths.head, absIdentifier.getTablePath, carbonTable),
- None
- )(sqlContext)
- }
-
- override def dataSchema: StructType = tableSchema.getOrElse(relationRaw.schema)
-
- override def prepareJobForWrite(job: Job): OutputWriterFactory = {
- // TODO
- throw new UnsupportedOperationException
- }
-
- override def buildScan(
- requiredColumns: Array[String],
- filters: Array[Filter],
- inputFiles: Array[FileStatus]): RDD[Row] = {
- val conf = new Configuration(job.getConfiguration)
- filters.flatMap { filter =>
- CarbonFilters.createCarbonFilter(dataSchema, filter)
- }.reduceOption(new AndExpression(_, _))
- .foreach(CarbonTableInputFormat.setFilterPredicates(conf, _))
-
- val projection = new CarbonProjection
- requiredColumns.foreach(projection.addColumn)
- CarbonTableInputFormat.setColumnProjection(conf, projection)
- CarbonTableInputFormat.setCarbonReadSupport(conf, classOf[SparkRowReadSupportImpl])
-
- new CarbonHadoopFSRDD[Row](sqlContext.sparkContext,
- new SerializableConfiguration(conf),
- absIdentifier,
- classOf[CarbonTableInputFormat[Row]],
- classOf[Row]
- )
- }
-
-}
-
-class CarbonHadoopFSPartition(rddId: Int, val idx: Int,
- val carbonSplit: SerializableWritable[CarbonInputSplit])
- extends Partition {
-
- override val index: Int = idx
-
- override def hashCode(): Int = 41 * (41 + rddId) + idx
-}
-
-class CarbonHadoopFSRDD[V: ClassTag](
- @transient sc: SparkContext,
- conf: SerializableConfiguration,
- identifier: AbsoluteTableIdentifier,
- inputFormatClass: Class[_ <: CarbonTableInputFormat[V]],
- valueClass: Class[V])
- extends CarbonRDD[V](sc, Nil) with SparkHadoopMapReduceUtil {
-
- private val jobTrackerId: String = {
- val formatter = new SimpleDateFormat("yyyyMMddHHmm")
- formatter.format(new Date())
- }
- @transient protected val jobId = new JobID(jobTrackerId, id)
-
- override def internalCompute(split: Partition,
- context: TaskContext): Iterator[V] = {
- val attemptId = newTaskAttemptID(jobTrackerId, id, isMap = true, split.index, 0)
- val hadoopAttemptContext = newTaskAttemptContext(conf.value, attemptId)
- val job: Job = new Job(hadoopAttemptContext.getConfiguration)
- val format = CarbonInputFormatUtil.createCarbonInputFormat(identifier, job)
- CarbonInputFormat.setDataTypeConverter(hadoopAttemptContext.getConfiguration,
- new SparkDataTypeConverterImpl)
- hadoopAttemptContext.getConfiguration.set(FileInputFormat.INPUT_DIR, identifier.getTablePath)
- val reader =
- format.createRecordReader(split.asInstanceOf[CarbonHadoopFSPartition].carbonSplit.value,
- hadoopAttemptContext
- )
- reader.initialize(split.asInstanceOf[CarbonHadoopFSPartition].carbonSplit.value,
- hadoopAttemptContext
- )
- new Iterator[V] {
- private[this] var havePair = false
- private[this] var finished = false
-
- override def hasNext: Boolean = {
- if (context.isInterrupted) {
- throw new TaskKilledException
- }
- if (!finished && !havePair) {
- finished = !reader.nextKeyValue
- if (finished) {
- reader.close()
- }
- havePair = !finished
- }
- !finished
- }
-
- override def next(): V = {
- if (!hasNext) {
- throw new java.util.NoSuchElementException("End of stream")
- }
- havePair = false
- reader.getCurrentValue
- }
- }
- }
-
- override protected def getPartitions: Array[Partition] = {
- val jobContext = newJobContext(conf.value, jobId)
- val format = CarbonInputFormatUtil.createCarbonInputFormat(identifier, new Job(conf.value))
- jobContext.getConfiguration.set(FileInputFormat.INPUT_DIR, identifier.getTablePath)
- val splits = format.getSplits(jobContext).toArray
- val carbonInputSplits = splits
- .map(f => new SerializableWritable(f.asInstanceOf[CarbonInputSplit]))
- carbonInputSplits.zipWithIndex.map(f => new CarbonHadoopFSPartition(id, f._2, f._1))
- }
-}
[06/16] carbondata git commit: [CARBONDATA-1597] Remove spark1
integration
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
deleted file mode 100644
index 2e93a6c..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
+++ /dev/null
@@ -1,321 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import scala.collection.JavaConverters._
-import scala.language.implicitConversions
-
-import org.apache.hadoop.fs.Path
-import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.analysis.MultiInstanceRelation
-import org.apache.spark.sql.catalyst.expressions.AttributeReference
-import org.apache.spark.sql.catalyst.plans.logical._
-import org.apache.spark.sql.hive.{CarbonMetaData, CarbonMetastoreTypes}
-import org.apache.spark.sql.sources._
-import org.apache.spark.sql.types.{DataType, StructType}
-
-import org.apache.carbondata.core.datastore.impl.FileFactory
-import org.apache.carbondata.core.metadata.schema.table.column.{CarbonColumn, CarbonDimension}
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonImplicitDimension
-import org.apache.carbondata.core.statusmanager.SegmentStatusManager
-import org.apache.carbondata.core.util.path.CarbonStorePath
-import org.apache.carbondata.processing.merger.TableMeta
-import org.apache.carbondata.spark.{CarbonOption, _}
-
-/**
- * Carbon relation provider compliant to data source api.
- * Creates carbon relations
- */
-class CarbonSource extends RelationProvider
- with CreatableRelationProvider with HadoopFsRelationProvider with DataSourceRegister {
-
- override def shortName(): String = "carbondata"
-
- /**
- * Returns a new base relation with the given parameters.
- * Note: the parameters' keywords are case insensitive and this insensitivity is enforced
- * by the Map that is passed to the function.
- */
- override def createRelation(
- sqlContext: SQLContext,
- parameters: Map[String, String]): BaseRelation = {
- // if path is provided we can directly create Hadoop relation. \
- // Otherwise create datasource relation
- parameters.get("path") match {
- case Some(path) => CarbonDatasourceHadoopRelation(sqlContext, Array(path), parameters, None)
- case _ =>
- val options = new CarbonOption(parameters)
- val tableIdentifier = options.tableIdentifier.split("""\.""").toSeq
- val identifier = tableIdentifier match {
- case Seq(name) => TableIdentifier(name, None)
- case Seq(db, name) => TableIdentifier(name, Some(db))
- }
- CarbonDatasourceRelation(identifier, None)(sqlContext)
- }
- }
-
- override def createRelation(
- sqlContext: SQLContext,
- mode: SaveMode,
- parameters: Map[String, String],
- data: SchemaRDD): BaseRelation = {
-
- // To avoid derby problem, dataframe need to be writen and read using CarbonContext
- require(sqlContext.isInstanceOf[CarbonContext], "Error in saving dataframe to carbon file, " +
- "must use CarbonContext to save dataframe")
-
- // User should not specify path since only one store is supported in carbon currently,
- // after we support multi-store, we can remove this limitation
- require(!parameters.contains("path"), "'path' should not be specified, " +
- "the path to store carbon file is the 'storePath' specified when creating CarbonContext")
-
- val options = new CarbonOption(parameters)
- val storePath = CarbonContext.getInstance(sqlContext.sparkContext).storePath
- val tablePath = new Path(storePath + "/" + options.dbName + "/" + options.tableName)
- val isExists = tablePath.getFileSystem(sqlContext.sparkContext.hadoopConfiguration)
- .exists(tablePath)
- val (doSave, doAppend) = (mode, isExists) match {
- case (SaveMode.ErrorIfExists, true) =>
- sys.error(s"ErrorIfExists mode, path $storePath already exists.")
- case (SaveMode.Overwrite, true) =>
- val cc = CarbonContext.getInstance(sqlContext.sparkContext)
- cc.sql(s"DROP TABLE IF EXISTS ${ options.dbName }.${ options.tableName }")
- (true, false)
- case (SaveMode.Overwrite, false) | (SaveMode.ErrorIfExists, false) =>
- (true, false)
- case (SaveMode.Append, _) =>
- (false, true)
- case (SaveMode.Ignore, exists) =>
- (!exists, false)
- }
-
- if (doSave) {
- // save data when the save mode is Overwrite.
- new CarbonDataFrameWriter(data).saveAsCarbonFile(parameters)
- } else if (doAppend) {
- new CarbonDataFrameWriter(data).appendToCarbonFile(parameters)
- }
-
- createRelation(sqlContext, parameters)
- }
-
- override def createRelation(sqlContext: SQLContext,
- paths: Array[String],
- dataSchema: Option[StructType],
- partitionColumns: Option[StructType],
- parameters: Map[String, String]): HadoopFsRelation = {
- CarbonDatasourceHadoopRelation(sqlContext, paths, parameters, dataSchema)
- }
-}
-
-/**
- * Creates carbon relation compliant to data source api.
- * This relation is stored to hive metastore
- */
-private[sql] case class CarbonDatasourceRelation(
- tableIdentifier: TableIdentifier,
- alias: Option[String])
- (@transient context: SQLContext)
- extends BaseRelation with Serializable {
-
- lazy val carbonRelation: CarbonRelation = {
- CarbonEnv.get
- .carbonMetastore.lookupRelation1(tableIdentifier, None)(sqlContext)
- .asInstanceOf[CarbonRelation]
- }
-
- def getDatabaseName(): String = tableIdentifier.database.getOrElse("default")
-
- def getTable(): String = tableIdentifier.table
-
- def schema: StructType = carbonRelation.schema
-
- def sqlContext: SQLContext = context
-
- override def sizeInBytes: Long = carbonRelation.sizeInBytes
-}
-
-/**
- * Represents logical plan for one carbon table
- */
-case class CarbonRelation(
- databaseName: String,
- tableName: String,
- var metaData: CarbonMetaData,
- tableMeta: TableMeta,
- alias: Option[String])(@transient sqlContext: SQLContext)
- extends LeafNode with MultiInstanceRelation {
-
- def recursiveMethod(dimName: String, childDim: CarbonDimension): String = {
- childDim.getDataType.getName.toLowerCase match {
- case "array" => s"${
- childDim.getColName.substring(dimName.length + 1)
- }:array<${ getArrayChildren(childDim.getColName) }>"
- case "struct" => s"${
- childDim.getColName.substring(dimName.length + 1)
- }:struct<${ getStructChildren(childDim.getColName) }>"
- case dType => s"${ childDim.getColName.substring(dimName.length + 1) }:${ dType }"
- }
- }
-
- def getArrayChildren(dimName: String): String = {
- metaData.carbonTable.getChildren(dimName).asScala.map(childDim => {
- childDim.getDataType.getName.toLowerCase match {
- case "array" => s"array<${ getArrayChildren(childDim.getColName) }>"
- case "struct" => s"struct<${ getStructChildren(childDim.getColName) }>"
- case dType => addDecimalScaleAndPrecision(childDim, dType)
- }
- }).mkString(",")
- }
-
- def getStructChildren(dimName: String): String = {
- metaData.carbonTable.getChildren(dimName).asScala.map(childDim => {
- childDim.getDataType.getName.toLowerCase match {
- case "array" => s"${
- childDim.getColName.substring(dimName.length + 1)
- }:array<${ getArrayChildren(childDim.getColName) }>"
- case "struct" => s"${
- childDim.getColName.substring(dimName.length + 1)
- }:struct<${ metaData.carbonTable.getChildren(childDim.getColName)
- .asScala.map(f => s"${ recursiveMethod(childDim.getColName, f) }").mkString(",")
- }>"
- case dType => s"${ childDim.getColName
- .substring(dimName.length() + 1) }:${ addDecimalScaleAndPrecision(childDim, dType) }"
- }
- }).mkString(",")
- }
-
- override def newInstance(): LogicalPlan = {
- CarbonRelation(databaseName, tableName, metaData, tableMeta, alias)(sqlContext)
- .asInstanceOf[this.type]
- }
-
- val dimensionsAttr = {
- val sett = new java.util.LinkedHashSet(tableMeta.carbonTable
- .getDimensionByTableName(tableMeta.carbonTableIdentifier.getTableName).asScala.asJava)
- sett.asScala.toSeq.filter(dim => !dim.isInvisible ||
- (dim.isInvisible && dim.isInstanceOf[CarbonImplicitDimension]))
- .map(dim => {
- val dimval = metaData.carbonTable
- .getDimensionByName(metaData.carbonTable.getFactTableName, dim.getColName)
- val output: DataType = dimval.getDataType.getName.toLowerCase match {
- case "array" =>
- CarbonMetastoreTypes.toDataType(s"array<${ getArrayChildren(dim.getColName) }>")
- case "struct" =>
- CarbonMetastoreTypes.toDataType(s"struct<${ getStructChildren(dim.getColName) }>")
- case dType =>
- val dataType = addDecimalScaleAndPrecision(dimval, dType)
- CarbonMetastoreTypes.toDataType(dataType)
- }
-
- AttributeReference(
- dim.getColName,
- output,
- nullable = true)(qualifiers = tableName +: alias.toSeq)
- })
- }
-
- val measureAttr = {
- val factTable = tableMeta.carbonTable.getFactTableName
- new java.util.LinkedHashSet(
- tableMeta.carbonTable.
- getMeasureByTableName(tableMeta.carbonTable.getFactTableName).
- asScala.asJava).asScala.toSeq.filter(!_.getColumnSchema.isInvisible)
- .map(x => AttributeReference(x.getColName, CarbonMetastoreTypes.toDataType(
- metaData.carbonTable.getMeasureByName(factTable, x.getColName).getDataType.getName
- .toLowerCase match {
- case "float" => "double"
- case "decimal" => "decimal(" + x.getPrecision + "," + x.getScale + ")"
- case others => others
- }),
- nullable = true)(qualifiers = tableName +: alias.toSeq))
- }
-
- override val output = {
- val columns = tableMeta.carbonTable.getCreateOrderColumn(tableMeta.carbonTable.getFactTableName)
- .asScala
- columns.filter(!_.isInvisible).map { column =>
- if (column.isDimension()) {
- val output: DataType = column.getDataType.getName.toLowerCase match {
- case "array" =>
- CarbonMetastoreTypes.toDataType(s"array<${getArrayChildren(column.getColName)}>")
- case "struct" =>
- CarbonMetastoreTypes.toDataType(s"struct<${getStructChildren(column.getColName)}>")
- case dType =>
- val dataType = addDecimalScaleAndPrecision(column, dType)
- CarbonMetastoreTypes.toDataType(dataType)
- }
- AttributeReference(column.getColName, output,
- nullable = true
- )(qualifiers = tableName +: alias.toSeq)
- } else {
- AttributeReference(column.getColName, CarbonMetastoreTypes.toDataType(
- column.getDataType.getName.toLowerCase match {
- case "float" => "double"
- case "decimal" => "decimal(" + column.getColumnSchema.getPrecision + "," + column
- .getColumnSchema.getScale + ")"
- case others => others
- }
- ),
- nullable = true
- )(qualifiers = tableName +: alias.toSeq)
- }
- }
- }
- // TODO: Use data from the footers.
- override lazy val statistics = Statistics(sizeInBytes = this.sizeInBytes)
-
- override def equals(other: Any): Boolean = {
- other match {
- case p: CarbonRelation =>
- p.databaseName == databaseName && p.output == output && p.tableName == tableName
- case _ => false
- }
- }
-
- def addDecimalScaleAndPrecision(dimval: CarbonColumn, dataType: String): String = {
- var dType = dataType
- if (dimval.getDataType == org.apache.carbondata.core.metadata.datatype.DataTypes.DECIMAL) {
- dType +=
- "(" + dimval.getColumnSchema.getPrecision + "," + dimval.getColumnSchema.getScale + ")"
- }
- dType
- }
-
- private var tableStatusLastUpdateTime = 0L
-
- private var sizeInBytesLocalValue = 0L
-
- def sizeInBytes: Long = {
- val tableStatusNewLastUpdatedTime = SegmentStatusManager.getTableStatusLastModifiedTime(
- tableMeta.carbonTable.getAbsoluteTableIdentifier)
- if (tableStatusLastUpdateTime != tableStatusNewLastUpdatedTime) {
- val tablePath = CarbonStorePath.getCarbonTablePath(
- tableMeta.storePath,
- tableMeta.carbonTableIdentifier).getPath
- val fileType = FileFactory.getFileType(tablePath)
- if(FileFactory.isFileExist(tablePath, fileType)) {
- tableStatusLastUpdateTime = tableStatusNewLastUpdatedTime
- sizeInBytesLocalValue = FileFactory.getDirectorySize(tablePath)
- }
- }
- sizeInBytesLocalValue
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
deleted file mode 100644
index c14a61a..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
+++ /dev/null
@@ -1,259 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import scala.collection.JavaConverters._
-
-import org.apache.spark.TaskContext
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.errors.attachTree
-import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.execution.{SparkPlan, UnaryNode}
-import org.apache.spark.sql.hive.{CarbonMetastore, CarbonMetastoreTypes}
-import org.apache.spark.sql.optimizer.CarbonDecoderRelation
-import org.apache.spark.sql.types._
-
-import org.apache.carbondata.core.cache.{Cache, CacheProvider, CacheType}
-import org.apache.carbondata.core.cache.dictionary.{Dictionary, DictionaryColumnUniqueIdentifier}
-import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, ColumnIdentifier}
-import org.apache.carbondata.core.metadata.datatype.{DataTypes => CarbonDataTypes}
-import org.apache.carbondata.core.metadata.datatype.DataType
-import org.apache.carbondata.core.metadata.encoder.Encoding
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
-import org.apache.carbondata.core.util.{CarbonTimeStatisticsFactory, DataTypeUtil}
-import org.apache.carbondata.core.util.path.CarbonStorePath
-import org.apache.carbondata.spark.CarbonAliasDecoderRelation
-
-/**
- * It decodes the dictionary key to value
- */
-case class CarbonDictionaryDecoder(
- relations: Seq[CarbonDecoderRelation],
- profile: CarbonProfile,
- aliasMap: CarbonAliasDecoderRelation,
- child: SparkPlan)
- (@transient sqlContext: SQLContext)
- extends UnaryNode {
-
- override def otherCopyArgs: Seq[AnyRef] = sqlContext :: Nil
-
- override val output: Seq[Attribute] = {
- child.output.map { a =>
- val attr = aliasMap.getOrElse(a, a)
- val relation = relations.find(p => p.contains(attr))
- if (relation.isDefined && canBeDecoded(attr)) {
- val carbonTable = relation.get.carbonRelation.carbonRelation.metaData.carbonTable
- val carbonDimension = carbonTable
- .getDimensionByName(carbonTable.getFactTableName, attr.name)
- if (carbonDimension != null &&
- carbonDimension.hasEncoding(Encoding.DICTIONARY) &&
- !carbonDimension.hasEncoding(Encoding.DIRECT_DICTIONARY) &&
- !carbonDimension.isComplex()) {
- val newAttr = AttributeReference(a.name,
- convertCarbonToSparkDataType(carbonDimension,
- relation.get.carbonRelation.carbonRelation),
- a.nullable,
- a.metadata)(a.exprId,
- a.qualifiers).asInstanceOf[Attribute]
- newAttr
- } else {
- a
- }
- } else {
- a
- }
- }
- }
-
-
- def canBeDecoded(attr: Attribute): Boolean = {
- profile match {
- case ip: IncludeProfile if ip.attributes.nonEmpty =>
- ip.attributes
- .exists(a => a.name.equalsIgnoreCase(attr.name) && a.exprId == attr.exprId)
- case ep: ExcludeProfile =>
- !ep.attributes
- .exists(a => a.name.equalsIgnoreCase(attr.name) && a.exprId == attr.exprId)
- case _ => true
- }
- }
-
- def convertCarbonToSparkDataType(carbonDimension: CarbonDimension,
- relation: CarbonRelation): types.DataType = {
- carbonDimension.getDataType match {
- case CarbonDataTypes.STRING => StringType
- case CarbonDataTypes.SHORT => ShortType
- case CarbonDataTypes.INT => IntegerType
- case CarbonDataTypes.LONG => LongType
- case CarbonDataTypes.DOUBLE => DoubleType
- case CarbonDataTypes.BOOLEAN => BooleanType
- case CarbonDataTypes.DECIMAL =>
- val scale: Int = carbonDimension.getColumnSchema.getScale
- val precision: Int = carbonDimension.getColumnSchema.getPrecision
- if (scale == 0 && precision == 0) {
- DecimalType(18, 2)
- } else {
- DecimalType(precision, scale)
- }
- case CarbonDataTypes.TIMESTAMP => TimestampType
- case CarbonDataTypes.DATE => DateType
- case CarbonDataTypes.STRUCT =>
- CarbonMetastoreTypes
- .toDataType(s"struct<${ relation.getStructChildren(carbonDimension.getColName) }>")
- case CarbonDataTypes.ARRAY =>
- CarbonMetastoreTypes
- .toDataType(s"array<${ relation.getArrayChildren(carbonDimension.getColName) }>")
- }
- }
-
- val getDictionaryColumnIds = {
- val attributes = child.output
- val dictIds: Array[(String, ColumnIdentifier, DataType, CarbonDimension)] =
- attributes.map { a =>
- val attr = aliasMap.getOrElse(a, a)
- val relation = relations.find(p => p.contains(attr))
- if (relation.isDefined && canBeDecoded(attr)) {
- val carbonTable = relation.get.carbonRelation.carbonRelation.metaData.carbonTable
- val carbonDimension =
- carbonTable.getDimensionByName(carbonTable.getFactTableName, attr.name)
- if (carbonDimension != null &&
- carbonDimension.hasEncoding(Encoding.DICTIONARY) &&
- !carbonDimension.hasEncoding(Encoding.DIRECT_DICTIONARY) &&
- !carbonDimension.isComplex()) {
- (carbonTable.getFactTableName, carbonDimension.getColumnIdentifier,
- carbonDimension.getDataType, carbonDimension)
- } else {
- (null, null, null, null)
- }
- } else {
- (null, null, null, null)
- }
-
- }.toArray
- dictIds
- }
-
- override def outputsUnsafeRows: Boolean = true
-
- override def canProcessUnsafeRows: Boolean = true
-
- override def canProcessSafeRows: Boolean = true
-
- override def doExecute(): RDD[InternalRow] = {
- attachTree(this, "execute") {
- val storePath = sqlContext.catalog.asInstanceOf[CarbonMetastore].storePath
- val queryId = sqlContext.getConf("queryId", System.nanoTime() + "")
- val absoluteTableIdentifiers = relations.map { relation =>
- val carbonTable = relation.carbonRelation.carbonRelation.metaData.carbonTable
- (carbonTable.getFactTableName, carbonTable.getAbsoluteTableIdentifier)
- }.toMap
-
- val recorder = CarbonTimeStatisticsFactory.createExecutorRecorder(queryId)
- if (isRequiredToDecode) {
- val dataTypes = child.output.map { attr => attr.dataType }
- child.execute().mapPartitions { iter =>
- val cacheProvider: CacheProvider = CacheProvider.getInstance
- val forwardDictionaryCache: Cache[DictionaryColumnUniqueIdentifier, Dictionary] =
- cacheProvider.createCache(CacheType.FORWARD_DICTIONARY, storePath)
- val dicts: Seq[Dictionary] = getDictionary(absoluteTableIdentifiers,
- forwardDictionaryCache)
- val dictIndex = dicts.zipWithIndex.filter(x => x._1 != null).map(x => x._2)
- // add a task completion listener to clear dictionary that is a decisive factor for
- // LRU eviction policy
- val dictionaryTaskCleaner = TaskContext.get
- dictionaryTaskCleaner.addTaskCompletionListener(context =>
- dicts.foreach { dictionary =>
- if (null != dictionary) {
- dictionary.clear()
- }
- }
- )
- new Iterator[InternalRow] {
- val unsafeProjection = UnsafeProjection.create(output.map(_.dataType).toArray)
-
- override final def hasNext: Boolean = {
- iter.hasNext
- }
-
- override final def next(): InternalRow = {
- val row: InternalRow = iter.next()
- val data = row.toSeq(dataTypes).toArray
- dictIndex.foreach { index =>
- if (data(index) != null) {
- data(index) = DataTypeUtil.getDataBasedOnDataType(dicts(index)
- .getDictionaryValueForKeyInBytes(data(index).asInstanceOf[Int]),
- getDictionaryColumnIds(index)._4)
- }
- }
- val result = unsafeProjection(new GenericMutableRow(data))
- result
- }
- }
- }
- } else {
- child.execute()
- }
- }
- }
-
- private def isRequiredToDecode = {
- getDictionaryColumnIds.find(p => p._1 != null) match {
- case Some(value) => true
- case _ => false
- }
- }
-
- private def getDictionary(atiMap: Map[String, AbsoluteTableIdentifier],
- cache: Cache[DictionaryColumnUniqueIdentifier, Dictionary]) = {
- val dictionaryColumnIds = getDictionaryColumnIds.map { dictionaryId =>
- if (dictionaryId._2 != null) {
- new DictionaryColumnUniqueIdentifier(
- atiMap(dictionaryId._1).getCarbonTableIdentifier,
- dictionaryId._2, dictionaryId._3,
- CarbonStorePath.getCarbonTablePath(atiMap(dictionaryId._1)))
- } else {
- null
- }
- }
- try {
- val noDictionaryIndexes = new java.util.ArrayList[Int]()
- dictionaryColumnIds.zipWithIndex.foreach { columnIndex =>
- if (columnIndex._1 == null) {
- noDictionaryIndexes.add(columnIndex._2)
- }
- }
- val dict = cache.getAll(dictionaryColumnIds.filter(_ != null).toSeq.asJava);
- val finalDict = new java.util.ArrayList[Dictionary]()
- var dictIndex: Int = 0
- dictionaryColumnIds.zipWithIndex.foreach { columnIndex =>
- if (!noDictionaryIndexes.contains(columnIndex._2)) {
- finalDict.add(dict.get(dictIndex))
- dictIndex += 1
- } else {
- finalDict.add(null)
- }
- }
- finalDict.asScala
- } catch {
- case t: Throwable => Seq.empty
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonEnv.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
deleted file mode 100644
index 36cd6f2..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import org.apache.spark.scheduler.cluster.CoarseGrainedSchedulerBackend
-import org.apache.spark.sql.hive.{CarbonIUDAnalysisRule, CarbonMetastore}
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.hadoop.readsupport.impl.RawDataReadSupport
-import org.apache.carbondata.spark.rdd.SparkReadSupport
-
-case class CarbonEnv(carbonMetastore: CarbonMetastore)
-
-object CarbonEnv {
-
- @volatile private var carbonEnv: CarbonEnv = _
-
- // set readsupport class global so that the executor can get it.
- SparkReadSupport.readSupportClass = classOf[RawDataReadSupport]
-
- var initialized = false
-
- def init(sqlContext: SQLContext): Unit = {
- if (!initialized) {
- val cc = sqlContext.asInstanceOf[CarbonContext]
- val catalog = new CarbonMetastore(cc, cc.storePath, cc.hiveClientInterface, "")
- carbonEnv = CarbonEnv(catalog)
- CarbonIUDAnalysisRule.init(sqlContext)
- initialized = true
- CarbonProperties.getInstance.addProperty(CarbonCommonConstants.IS_DRIVER_INSTANCE, "true")
- }
- }
-
- def get: CarbonEnv = {
- if (initialized) carbonEnv
- else throw new RuntimeException("CarbonEnv not initialized")
- }
-}
-
-
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSQLConf.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSQLConf.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSQLConf.scala
deleted file mode 100644
index 6ed8c0d..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSQLConf.scala
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import org.apache.spark.sql.SQLConf.SQLConfEntry
-import org.apache.spark.sql.hive.CarbonSQLDialect
-
- /**
- * A trait that enables the setting and getting of mutable config parameters/hints.
- *
- */
-class CarbonSQLConf extends SQLConf {
-
- override def dialect: String = {
- getConf(SQLConf.DIALECT,
- classOf[CarbonSQLDialect].getCanonicalName)
- }
-
- override def caseSensitiveAnalysis: Boolean = getConf(SQLConf.CASE_SENSITIVE, false)
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/main/scala/org/apache/spark/sql/CarbonScan.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonScan.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonScan.scala
deleted file mode 100644
index a3c6343..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonScan.scala
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable.ArrayBuffer
-
-import org.apache.spark.CarbonInputMetrics
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.execution.LeafNode
-import org.apache.spark.sql.hive.CarbonMetastore
-
-import org.apache.carbondata.core.scan.model._
-import org.apache.carbondata.hadoop.{CarbonProjection, InputMetricsStats}
-import org.apache.carbondata.spark.CarbonFilters
-import org.apache.carbondata.spark.rdd.CarbonScanRDD
-
-case class CarbonScan(
- var columnProjection: Seq[Attribute],
- relationRaw: CarbonRelation,
- dimensionPredicatesRaw: Seq[Expression],
- useUnsafeCoversion: Boolean = true)(@transient val ocRaw: SQLContext) extends LeafNode {
- val carbonTable = relationRaw.metaData.carbonTable
- val selectedDims = scala.collection.mutable.MutableList[QueryDimension]()
- val selectedMsrs = scala.collection.mutable.MutableList[QueryMeasure]()
- @transient val carbonCatalog = ocRaw.catalog.asInstanceOf[CarbonMetastore]
-
- val attributesNeedToDecode = new java.util.LinkedHashSet[AttributeReference]()
- val unprocessedExprs = new ArrayBuffer[Expression]()
-
- val buildCarbonPlan: CarbonQueryPlan = {
- val plan: CarbonQueryPlan = new CarbonQueryPlan(relationRaw.databaseName, relationRaw.tableName)
- plan.setQueryId(ocRaw.getConf("queryId", System.nanoTime() + ""))
- processFilterExpressions(plan)
- plan
- }
-
- def processFilterExpressions(plan: CarbonQueryPlan) {
- if (dimensionPredicatesRaw.nonEmpty) {
- val expressionVal = CarbonFilters.processExpression(
- dimensionPredicatesRaw,
- attributesNeedToDecode,
- unprocessedExprs,
- carbonTable)
- expressionVal match {
- case Some(ce) =>
- // adding dimension used in expression in querystats
- plan.setFilterExpression(ce)
- case _ =>
- }
- }
- processExtraAttributes(plan)
- }
-
- private def processExtraAttributes(plan: CarbonQueryPlan) {
- if (attributesNeedToDecode.size() > 0) {
- val attributeOut = new ArrayBuffer[Attribute]() ++ columnProjection
-
- attributesNeedToDecode.asScala.foreach { attr =>
- if (!columnProjection.exists(_.name.equalsIgnoreCase(attr.name))) {
- attributeOut += attr
- }
- }
- columnProjection = attributeOut
- }
-
- val columns = carbonTable.getCreateOrderColumn(carbonTable.getFactTableName)
- columns.addAll(carbonTable.getImplicitDimensionByTableName(carbonTable.getFactTableName))
- val colAttr = new Array[Attribute](columns.size())
- columnProjection.foreach { attr =>
- val column =
- carbonTable.getColumnByName(carbonTable.getFactTableName, attr.name)
- if(column != null) {
- colAttr(columns.indexOf(column)) = attr
- }
- }
-
- columnProjection = colAttr.filter(f => f != null)
-
- var queryOrder: Integer = 0
- columnProjection.foreach { attr =>
- val carbonColumn = carbonTable.getColumnByName(carbonTable.getFactTableName, attr.name)
- if (carbonColumn != null) {
- if (carbonColumn.isDimension()) {
- val dim = new QueryDimension(attr.name)
- dim.setQueryOrder(queryOrder)
- queryOrder = queryOrder + 1
- selectedDims += dim
- } else {
- val m1 = new QueryMeasure(attr.name)
- m1.setQueryOrder(queryOrder)
- queryOrder = queryOrder + 1
- selectedMsrs += m1
- }
- }
- }
-
- // Fill the selected dimensions & measures obtained from
- // attributes to query plan for detailed query
- selectedDims.foreach(plan.addDimension)
- selectedMsrs.foreach(plan.addMeasure)
- }
-
- def inputRdd: CarbonScanRDD = {
- val projection = new CarbonProjection
- columnProjection.foreach { attr =>
- projection.addColumn(attr.name)
- }
- val inputMetricsStats: CarbonInputMetrics = new CarbonInputMetrics
- new CarbonScanRDD(
- ocRaw.sparkContext,
- projection,
- buildCarbonPlan.getFilterExpression,
- carbonTable.getAbsoluteTableIdentifier,
- carbonTable.getTableInfo.serialize(),
- carbonTable.getTableInfo, inputMetricsStats
- )
- }
-
- override def outputsUnsafeRows: Boolean =
- (attributesNeedToDecode.size() == 0) && useUnsafeCoversion
-
- override def doExecute(): RDD[InternalRow] = {
- val outUnsafeRows: Boolean = (attributesNeedToDecode.size() == 0) && useUnsafeCoversion
- inputRdd.mapPartitions { iter =>
- val unsafeProjection = UnsafeProjection.create(output.map(_.dataType).toArray)
- new Iterator[InternalRow] {
- override def hasNext: Boolean = iter.hasNext
-
- override def next(): InternalRow = {
- val value = iter.next
- if (outUnsafeRows) {
- unsafeProjection(value)
- } else {
- value
- }
- }
- }
- }
- }
-
- def output: Seq[Attribute] = {
- columnProjection
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSparkUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSparkUtil.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSparkUtil.scala
deleted file mode 100644
index bc62a55..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSparkUtil.scala
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import scala.collection.JavaConverters._
-
-import org.apache.spark.sql.hive.{CarbonMetaData, DictionaryMap}
-
-import org.apache.carbondata.core.metadata.encoder.Encoding
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.util.CarbonUtil
-
-case class TransformHolder(rdd: Any, mataData: CarbonMetaData)
-
-object CarbonSparkUtil {
-
- def createSparkMeta(carbonTable: CarbonTable): CarbonMetaData = {
- val dimensionsAttr = carbonTable.getDimensionByTableName(carbonTable.getFactTableName)
- .asScala.map(x => x.getColName)
- // wf : may be problem
- val measureAttr = carbonTable.getMeasureByTableName(carbonTable.getFactTableName)
- .asScala.map(x => x.getColName)
- val dictionary =
- carbonTable.getDimensionByTableName(carbonTable.getFactTableName).asScala.map { f =>
- (f.getColName.toLowerCase,
- f.hasEncoding(Encoding.DICTIONARY) && !f.hasEncoding(Encoding.DIRECT_DICTIONARY) &&
- !f.getDataType.isComplexType)
- }
- CarbonMetaData(dimensionsAttr, measureAttr, carbonTable, DictionaryMap(dictionary.toMap))
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
deleted file mode 100644
index 9dc9ee2..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
+++ /dev/null
@@ -1,589 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable.Map
-import scala.language.implicitConversions
-
-import org.apache.hadoop.hive.ql.lib.Node
-import org.apache.hadoop.hive.ql.parse._
-import org.apache.spark.sql.catalyst._
-import org.apache.spark.sql.catalyst.CarbonTableIdentifierImplicit._
-import org.apache.spark.sql.catalyst.analysis._
-import org.apache.spark.sql.catalyst.plans.logical._
-import org.apache.spark.sql.execution.ExplainCommand
-import org.apache.spark.sql.execution.command._
-import org.apache.spark.sql.execution.datasources.DescribeCommand
-import org.apache.spark.sql.hive.HiveQlWrapper
-import org.apache.spark.sql.types.StructField
-
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
-import org.apache.carbondata.spark.util.CommonUtil
-
-/**
- * Parser for All Carbon DDL, DML cases in Unified context
- */
-class CarbonSqlParser() extends CarbonDDLSqlParser {
-
- override def parse(input: String): LogicalPlan = {
- synchronized {
- // Initialize the Keywords.
- initLexical
- phrase(start)(new lexical.Scanner(input)) match {
- case Success(plan, _) => plan match {
- case x: LoadTable =>
- x.inputSqlString = input
- x
- case logicalPlan => logicalPlan
- }
- case failureOrError => sys.error(failureOrError.toString)
- }
- }
- }
-
- override protected lazy val start: Parser[LogicalPlan] = explainPlan | startCommand
-
- protected lazy val startCommand: Parser[LogicalPlan] =
- createDatabase | dropDatabase | loadManagement | describeTable |
- showPartitions | showLoads | alterTable | updateTable | deleteRecords | useDatabase |
- createTable
-
- protected lazy val loadManagement: Parser[LogicalPlan] =
- deleteLoadsByID | deleteLoadsByLoadDate | cleanFiles | loadDataNew
-
- protected lazy val createDatabase: Parser[LogicalPlan] =
- CREATE ~> (DATABASE | SCHEMA) ~> restInput ^^ {
- case statement =>
- val createDbSql = "CREATE DATABASE " + statement
- var dbName = ""
- // Get Ast node for create db command
- val node = HiveQlWrapper.getAst(createDbSql)
- node match {
- // get dbname
- case Token("TOK_CREATEDATABASE", children) =>
- dbName = BaseSemanticAnalyzer.unescapeIdentifier(children(0).getText)
- }
- CreateDatabase(convertDbNameToLowerCase(dbName), createDbSql)
- }
-
- protected lazy val dropDatabase: Parser[LogicalPlan] =
- DROP ~> (DATABASE | SCHEMA) ~> restInput ^^ {
- case statement =>
- val dropDbSql = "DROP DATABASE " + statement
- var dbName = ""
- var isCascade = false
- // Get Ast node for drop db command
- val node = HiveQlWrapper.getAst(dropDbSql)
- node match {
- case Token("TOK_DROPDATABASE", children) =>
- dbName = BaseSemanticAnalyzer.unescapeIdentifier(children(0).getText)
- // check whether cascade drop db
- children.collect {
- case t@Token("TOK_CASCADE", _) =>
- isCascade = true
- case _ => // Unsupport features
- }
- }
- DropDatabase(convertDbNameToLowerCase(dbName), isCascade, dropDbSql)
- }
-
- protected lazy val alterTable: Parser[LogicalPlan] =
- ALTER ~> TABLE ~> restInput ^^ {
- case statement =>
- try {
- val alterSql = "alter table " + statement
- // DDl will be parsed and we get the AST tree from the HiveQl
- val node = HiveQlWrapper.getAst(alterSql)
- // processing the AST tree
- nodeToPlanForAlterTable(node, alterSql)
- } catch {
- // MalformedCarbonCommandException need to be throw directly, parser will catch it
- case ce: MalformedCarbonCommandException =>
- throw ce
- }
- }
-
- /**
- * For handling the create table DDl systax compatible to Hive syntax
- */
- protected lazy val createTable: Parser[LogicalPlan] =
- restInput ^^ {
-
- case statement =>
- try {
- // DDl will be parsed and we get the AST tree from the HiveQl
- val node = HiveQlWrapper.getAst(statement)
- // processing the AST tree
- nodeToPlan(node)
- } catch {
- // MalformedCarbonCommandException need to be throw directly, parser will catch it
- case ce: MalformedCarbonCommandException =>
- throw ce
- case e: Exception =>
- sys.error("Parsing error") // no need to do anything.
- }
- }
-
- /**
- * This function will traverse the tree and logical plan will be formed using that.
- *
- * @param node
- * @return LogicalPlan
- */
- protected def nodeToPlan(node: Node): LogicalPlan = {
- node match {
- // if create table taken is found then only we will handle.
- case Token("TOK_CREATETABLE", children) =>
-
-
- var fields: Seq[Field] = Seq[Field]()
- var tableComment: String = ""
- var tableProperties = Map[String, String]()
- var partitionByFields: Seq[Field] = Seq[Field]()
- var partitionCols: Seq[PartitionerField] = Seq[PartitionerField]()
- var likeTableName: String = ""
- var storedBy: String = ""
- var ifNotExistPresent: Boolean = false
- var dbName: Option[String] = None
- var tableName: String = ""
- var bucketFields: Option[BucketFields] = None
-
- try {
-
- // Checking whether create table request is carbon table
- children.collect {
- case Token("TOK_STORAGEHANDLER", child :: Nil) =>
- storedBy = BaseSemanticAnalyzer.unescapeSQLString(child.getText).trim.toLowerCase
- case _ =>
- }
- if (!(storedBy.equals(CarbonContext.datasourceName) ||
- storedBy.equals(CarbonContext.datasourceShortName))) {
- sys.error("Not a carbon format request")
- }
-
- children.collect {
- // collecting all the field list
- case list@Token("TOK_TABCOLLIST", _) =>
- val cols = BaseSemanticAnalyzer.getColumns(list, true)
- if (cols != null) {
- val dupColsGrp = cols.asScala.groupBy(x => x.getName) filter {
- case (_, colList) => colList.size > 1
- }
- if (dupColsGrp.nonEmpty) {
- var columnName: String = ""
- dupColsGrp.toSeq.foreach(columnName += _._1 + ", ")
- columnName = columnName.substring(0, columnName.lastIndexOf(", "))
- val errorMessage = "Duplicate column name: " + columnName + " found in table " +
- ".Please check create table statement."
- throw new MalformedCarbonCommandException(errorMessage)
- }
- cols.asScala.map { col =>
- val columnName = col.getName()
- val dataType = Option(col.getType)
- val name = Option(col.getName())
- // This is to parse complex data types
- val x = '`' + col.getName + '`' + ' ' + col.getType
- val f: Field = anyFieldDef(new lexical.Scanner(x))
- match {
- case Success(field, _) => field
- case failureOrError => throw new MalformedCarbonCommandException(
- s"Unsupported data type: $col.getType")
- }
- // the data type of the decimal type will be like decimal(10,0)
- // so checking the start of the string and taking the precision and scale.
- // resetting the data type with decimal
- if (f.dataType.getOrElse("").startsWith("decimal")) {
- val (precision, scale) = getScaleAndPrecision(col.getType)
- f.precision = precision
- f.scale = scale
- f.dataType = Some("decimal")
- }
- if (f.dataType.getOrElse("").startsWith("char")) {
- f.dataType = Some("char")
- } else if (f.dataType.getOrElse("").startsWith("float")) {
- f.dataType = Some("float")
- }
- f.rawSchema = x
- fields ++= Seq(f)
- }
- }
-
- case Token("TOK_IFNOTEXISTS", _) =>
- ifNotExistPresent = true
-
- case t@Token("TOK_TABNAME", _) =>
- val (db, tblName) = extractDbNameTableName(t)
- dbName = db
- tableName = tblName.toLowerCase()
-
- case Token("TOK_TABLECOMMENT", child :: Nil) =>
- tableComment = BaseSemanticAnalyzer.unescapeSQLString(child.getText)
-
- case Token("TOK_TABLEPARTCOLS", list@Token("TOK_TABCOLLIST", _) :: Nil) =>
- val cols = BaseSemanticAnalyzer.getColumns(list(0), false)
- if (cols != null) {
- cols.asScala.map { col =>
- val columnName = col.getName()
- val dataType = Option(col.getType)
- val comment = col.getComment
- val rawSchema = '`' + col.getName + '`' + ' ' + col.getType
- val field = Field(columnName, dataType, Some(columnName), None)
-
- // the data type of the decimal type will be like decimal(10,0)
- // so checking the start of the string and taking the precision and scale.
- // resetting the data type with decimal
- if (field.dataType.getOrElse("").startsWith("decimal")) {
- val (precision, scale) = getScaleAndPrecision(col.getType)
- field.precision = precision
- field.scale = scale
- field.dataType = Some("decimal")
- }
- if (field.dataType.getOrElse("").startsWith("char")) {
- field.dataType = Some("char")
- } else if (field.dataType.getOrElse("").startsWith("float")) {
- field.dataType = Some("float")
- }
- field.rawSchema = rawSchema
- val partitionCol = new PartitionerField(columnName, dataType, comment)
- partitionCols ++= Seq(partitionCol)
- partitionByFields ++= Seq(field)
- }
- }
- case Token("TOK_TABLEPROPERTIES", list :: Nil) =>
- val propertySeq: Seq[(String, String)] = getProperties(list)
- val repeatedProperties = propertySeq.groupBy(_._1).filter(_._2.size > 1).keySet
- if (repeatedProperties.nonEmpty) {
- val repeatedPropStr: String = repeatedProperties.mkString(",")
- throw new MalformedCarbonCommandException("Table properties is repeated: " +
- repeatedPropStr)
- }
- tableProperties ++= propertySeq
-
- case Token("TOK_LIKETABLE", child :: Nil) =>
- likeTableName = child.getChild(0).getText()
- case Token("TOK_ALTERTABLE_BUCKETS",
- Token("TOK_TABCOLNAME", list) :: numberOfBuckets) =>
- val cols = list.map(_.getText)
- if (cols != null) {
- bucketFields = Some(BucketFields(cols,
- numberOfBuckets.head.getText.toInt))
- }
-
- case _ => // Unsupport features
- }
-
- // validate tblProperties
- if (!CommonUtil.validateTblProperties(tableProperties, fields)) {
- throw new MalformedCarbonCommandException("Invalid table properties")
- }
-
- if (partitionCols.nonEmpty) {
- if (!CommonUtil.validatePartitionColumns(tableProperties, partitionCols)) {
- throw new MalformedCarbonCommandException("Invalid partition definition")
- }
- // partition columns should not be part of the schema
- val colNames = fields.map(_.column)
- val badPartCols = partitionCols.map(_.partitionColumn).toSet.intersect(colNames.toSet)
- if (badPartCols.nonEmpty) {
- throw new MalformedCarbonCommandException(
- "Partition columns should not be specified in the schema: " +
- badPartCols.map("\"" + _ + "\"").mkString("[", ",", "]"))
- }
- fields ++= partitionByFields
- }
-
- // prepare table model of the collected tokens
- val tableModel: TableModel = prepareTableModel(ifNotExistPresent,
- dbName,
- tableName,
- fields,
- partitionCols,
- tableProperties,
- bucketFields)
-
- // get logical plan.
- CreateTable(tableModel)
- } catch {
- case ce: MalformedCarbonCommandException =>
- val message = if (tableName.isEmpty) {
- "Create table command failed. "
- }
- else if (dbName.isEmpty) {
- s"Create table command failed for $tableName. "
- }
- else {
- s"Create table command failed for ${ dbName.get }.$tableName. "
- }
- LOGGER.audit(message + ce.getMessage)
- throw ce
- }
-
- }
- }
-
- /**
- * This function will traverse the tree and logical plan will be formed using that.
- *
- * @param node
- * @return LogicalPlan
- */
- protected def nodeToPlanForAlterTable(node: Node, alterSql: String): LogicalPlan = {
- node match {
- // if create table taken is found then only we will handle.
- case Token("TOK_ALTERTABLE", children) =>
-
- var dbName: Option[String] = None
- var tableName: String = ""
- var compactionType: String = ""
-
- children.collect {
-
- case t@Token("TOK_TABNAME", _) =>
- val (db, tblName) = extractDbNameTableName(t)
- dbName = db
- tableName = tblName
-
- case Token("TOK_ALTERTABLE_COMPACT", child :: Nil) =>
- compactionType = BaseSemanticAnalyzer.unescapeSQLString(child.getText)
-
- case _ => // Unsupport features
- }
-
- val altertablemodel = AlterTableModel(dbName,
- tableName,
- None,
- compactionType,
- Some(System.currentTimeMillis()),
- alterSql)
- AlterTableCompaction(altertablemodel)
- }
- }
-
- protected lazy val loadDataNew: Parser[LogicalPlan] =
- LOAD ~> DATA ~> opt(LOCAL) ~> INPATH ~> stringLit ~ opt(OVERWRITE) ~
- (INTO ~> TABLE ~> (ident <~ ".").? ~ ident) ~
- (OPTIONS ~> "(" ~> repsep(loadOptions, ",") <~ ")").? <~ opt(";") ^^ {
- case filePath ~ isOverwrite ~ table ~ optionsList =>
- val (databaseNameOp, tableName) = table match {
- case databaseName ~ tableName => (databaseName, tableName.toLowerCase())
- }
- if (optionsList.isDefined) {
- validateOptions(optionsList)
- }
- val optionsMap = optionsList.getOrElse(List.empty[(String, String)]).toMap
- LoadTable(convertDbNameToLowerCase(databaseNameOp), tableName, filePath, Seq(), optionsMap,
- isOverwrite.isDefined)
- }
-
- protected lazy val describeTable: Parser[LogicalPlan] =
- ((DESCRIBE | DESC) ~> opt(EXTENDED | FORMATTED)) ~ (ident <~ ".").? ~ ident ^^ {
- case ef ~ db ~ tbl =>
- val tblIdentifier = db match {
- case Some(dbName) =>
- TableIdentifier(tbl.toLowerCase, Some(convertDbNameToLowerCase(dbName)))
- case None =>
- TableIdentifier(tbl.toLowerCase, None)
- }
- if (ef.isDefined && "FORMATTED".equalsIgnoreCase(ef.get)) {
- new DescribeFormattedCommand("describe formatted " + tblIdentifier,
- tblIdentifier)
- } else {
- new DescribeCommand(UnresolvedRelation(tblIdentifier, None), ef.isDefined)
- }
- }
-
- protected lazy val showLoads: Parser[LogicalPlan] =
- SHOW ~> SEGMENTS ~> FOR ~> TABLE ~> (ident <~ ".").? ~ ident ~
- (LIMIT ~> numericLit).? <~
- opt(";") ^^ {
- case databaseName ~ tableName ~ limit =>
- ShowLoadsCommand(convertDbNameToLowerCase(databaseName), tableName.toLowerCase(), limit)
- }
-
- protected lazy val deleteLoadsByID: Parser[LogicalPlan] =
- DELETE ~> FROM ~ TABLE ~> (ident <~ ".").? ~ ident ~
- (WHERE ~> (SEGMENT ~ "." ~ ID) ~> IN ~> "(" ~> repsep(segmentId, ",")) <~ ")" ~ opt(";") ^^ {
- case dbName ~ tableName ~ loadids =>
- DeleteLoadsById(loadids, convertDbNameToLowerCase(dbName), tableName.toLowerCase())
- }
-
- protected lazy val deleteLoadsByLoadDate: Parser[LogicalPlan] =
- DELETE ~> FROM ~> TABLE ~> (ident <~ ".").? ~ ident ~
- (WHERE ~> (SEGMENT ~ "." ~ STARTTIME ~> BEFORE) ~ stringLit) <~
- opt(";") ^^ {
- case database ~ table ~ condition =>
- condition match {
- case dateField ~ dateValue =>
- DeleteLoadsByLoadDate(convertDbNameToLowerCase(database),
- table.toLowerCase(),
- dateField,
- dateValue)
- }
- }
-
- protected lazy val cleanFiles: Parser[LogicalPlan] =
- CLEAN ~> FILES ~> FOR ~> TABLE ~> (ident <~ ".").? ~ ident <~ opt(";") ^^ {
- case databaseName ~ tableName =>
- CleanFiles(convertDbNameToLowerCase(databaseName), tableName.toLowerCase())
- }
-
- protected lazy val explainPlan: Parser[LogicalPlan] =
- (EXPLAIN ~> opt(EXTENDED)) ~ startCommand ^^ {
- case isExtended ~ logicalPlan =>
- logicalPlan match {
- case plan: CreateTable => ExplainCommand(logicalPlan, extended = isExtended.isDefined)
- case _ => ExplainCommand(OneRowRelation)
- }
- }
-
- protected lazy val deleteRecords: Parser[LogicalPlan] =
- (DELETE ~> FROM ~> table) ~ restInput.? <~ opt(";") ^^ {
- case table ~ rest =>
- val tableName = getTableName(table.tableIdentifier)
- val alias = table.alias.getOrElse("")
- DeleteRecords("select tupleId from " + tableName + " " + alias + rest.getOrElse(""), table)
- }
-
- protected lazy val updateTable: Parser[LogicalPlan] =
- UPDATE ~> table ~
- (SET ~> "(" ~> repsep(element, ",") <~ ")") ~
- ("=" ~> restInput) <~ opt(";") ^^ {
- case tab ~ columns ~ rest =>
- val (sel, where) = splitQuery(rest)
- val (selectStmt, relation) =
- if (!sel.toLowerCase.startsWith("select ")) {
- if (sel.trim.isEmpty) {
- sys.error("At least one source column has to be specified ")
- }
- // only list of expression are given, need to convert that list of expressions into
- // select statement on destination table
- val relation = tab match {
- case r@UnresolvedRelation(tableIdentifier, alias) =>
- updateRelation(r, tableIdentifier, alias)
- case _ => tab
- }
- ("select " + sel + " from " + getTableName(relation.tableIdentifier) + " " +
- relation.alias.get, relation)
- } else {
- (sel, updateRelation(tab, tab.tableIdentifier, tab.alias))
- }
- UpdateTable(relation, columns, selectStmt, where)
- }
- protected lazy val showPartitions: Parser[LogicalPlan] =
- (SHOW ~> PARTITIONS ~> table) <~ opt(";") ^^ {
- case table =>
- val tableName = getTableName(table.tableIdentifier)
- val alias = table.alias.getOrElse("")
- ShowPartitions(table.tableIdentifier)
- }
-
- private def splitQuery(query: String): (String, String) = {
- val stack = scala.collection.mutable.Stack[Char]()
- var foundSingleQuotes = false
- var foundDoubleQuotes = false
- var foundEscapeChar = false
- var ignoreChar = false
- var stop = false
- var bracketCount = 0
- val (selectStatement, where) = query.span {
- ch => {
- if (stop) {
- false
- } else {
- ignoreChar = false
- if (foundEscapeChar && (ch == '\'' || ch == '\"' || ch == '\\')) {
- foundEscapeChar = false
- ignoreChar = true
- }
- // If escaped single or double quotes found, no need to consider
- if (!ignoreChar) {
- if (ch == '\\') {
- foundEscapeChar = true
- } else if (ch == '\'') {
- foundSingleQuotes = !foundSingleQuotes
- } else if (ch == '\"') {
- foundDoubleQuotes = !foundDoubleQuotes
- }
- else if (ch == '(' && !foundSingleQuotes && !foundDoubleQuotes) {
- bracketCount = bracketCount + 1
- stack.push(ch)
- } else if (ch == ')' && !foundSingleQuotes && !foundDoubleQuotes) {
- bracketCount = bracketCount + 1
- stack.pop()
- if (0 == stack.size) {
- stop = true
- }
- }
- }
- true
- }
- }
- }
- if (bracketCount == 0 || bracketCount % 2 != 0) {
- sys.error("Parsing error, missing bracket ")
- }
- val select = selectStatement.trim
- (select.substring(1, select.length - 1).trim -> where.trim)
- }
-
-
- protected lazy val table: Parser[UnresolvedRelation] = {
- rep1sep(attributeName, ".") ~ opt(ident) ^^ {
- case tableIdent ~ alias => UnresolvedRelation(tableIdent, alias)
- }
- }
-
- protected lazy val attributeName: Parser[String] = acceptMatch("attribute name", {
- case lexical.Identifier(str) => str.toLowerCase
- case lexical.Keyword(str) if !lexical.delimiters.contains(str) => str.toLowerCase
- })
-
- private def updateRelation(
- r: UnresolvedRelation,
- tableIdentifier: Seq[String],
- alias: Option[String]): UnresolvedRelation = {
- alias match {
- case Some(_) => r
- case _ =>
- val tableAlias = tableIdentifier match {
- case Seq(dbName, tableName) => Some(tableName)
- case Seq(tableName) => Some(tableName)
- }
- UnresolvedRelation(tableIdentifier, tableAlias)
- }
- }
-
- private def getTableName(tableIdentifier: Seq[String]): String = {
- if (tableIdentifier.size > 1) {
- tableIdentifier(0) + "." + tableIdentifier(1)
- } else {
- tableIdentifier(0)
- }
- }
-
- protected lazy val element: Parser[String] =
- (ident <~ ".").? ~ ident ^^ {
- case table ~ column => column.toLowerCase
- }
-
- protected lazy val useDatabase: Parser[LogicalPlan] =
- USE ~> ident <~ opt(";") ^^ {
- case databaseName => UseDatabase(s"use ${ databaseName.toLowerCase }")
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/main/scala/org/apache/spark/sql/CodeGenerateFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CodeGenerateFactory.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CodeGenerateFactory.scala
deleted file mode 100644
index 065dfed..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CodeGenerateFactory.scala
+++ /dev/null
@@ -1,155 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import org.apache.spark.SparkContext
-import org.apache.spark.sql.catalyst.CatalystConf
-import org.apache.spark.sql.catalyst.optimizer.Optimizer
-import org.apache.spark.sql.catalyst.plans.logical.{Expand, LogicalPlan}
-import org.apache.spark.util.{ScalaCompilerUtil, Utils}
-
-private[sql] class CodeGenerateFactory(version: String) {
-
- val optimizerFactory = if (version.equals("1.6.2") || version.equals("1.6.3")) {
- ScalaCompilerUtil.compiledCode(CodeTemplates.spark1_6_OptimizerString)
- .asInstanceOf[AbstractCarbonOptimizerFactory]
- } else if (version.startsWith("1.6") || version.startsWith("1.5")) {
- ScalaCompilerUtil.compiledCode(CodeTemplates.defaultOptimizerString)
- .asInstanceOf[AbstractCarbonOptimizerFactory]
- } else {
- throw new UnsupportedOperationException(s"Spark version $version is not supported")
- }
-
- val expandFactory = if (version.startsWith("1.5")) {
- ScalaCompilerUtil.compiledCode(CodeTemplates.spark1_5ExpandString)
- .asInstanceOf[AbstractCarbonExpandFactory]
- } else if (version.startsWith("1.6")) {
- new AbstractCarbonExpandFactory {
- override def createExpand(expand: Expand, child: LogicalPlan): Expand = {
- val loader = Utils.getContextOrSparkClassLoader
- try {
- val cons = loader.loadClass("org.apache.spark.sql.catalyst.plans.logical.Expand")
- .getDeclaredConstructors
- cons.head.setAccessible(true)
- cons.head.newInstance(expand.projections, expand.output, child).asInstanceOf[Expand]
- } catch {
- case e: Exception => null
- }
- }
- }
- } else {
- throw new UnsupportedOperationException(s"Spark version $version is not supported")
- }
-
-}
-
-object CodeGenerateFactory {
-
- private var codeGenerateFactory: CodeGenerateFactory = _
-
- def init(version: String): Unit = {
- if (codeGenerateFactory == null) {
- codeGenerateFactory = new CodeGenerateFactory(version)
- }
- }
-
- def getInstance(): CodeGenerateFactory = {
- codeGenerateFactory
- }
-
- def createDefaultOptimizer(conf: CatalystConf, sc: SparkContext): Optimizer = {
- val name = "org.apache.spark.sql.catalyst.optimizer.DefaultOptimizer"
- val loader = Utils.getContextOrSparkClassLoader
- try {
- val cons = loader.loadClass(name + "$").getDeclaredConstructors
- cons.head.setAccessible(true)
- cons.head.newInstance().asInstanceOf[Optimizer]
- } catch {
- case e: Exception =>
- loader.loadClass(name).getConstructor(classOf[CatalystConf])
- .newInstance(conf).asInstanceOf[Optimizer]
- }
- }
-
-}
-
-object CodeTemplates {
-
- val spark1_6_OptimizerString =
- s"""
- import org.apache.spark.sql._;
- import org.apache.spark.sql.optimizer._;
- import org.apache.spark.sql.catalyst.plans.logical._;
- import org.apache.spark.sql.catalyst._;
- import org.apache.spark.sql.catalyst.optimizer.Optimizer;
-
- new AbstractCarbonOptimizerFactory {
- override def createOptimizer(optimizer: Optimizer, conf: CarbonSQLConf): Optimizer = {
- class CarbonOptimizer1(optimizer: Optimizer, conf: CarbonSQLConf)
- extends Optimizer(conf) {
- override val batches = Nil;
- override def execute(plan: LogicalPlan): LogicalPlan = {
- CarbonOptimizer.execute(plan, optimizer);
- }
- }
- new CarbonOptimizer1(optimizer, conf);
- }
- }
- """
-
- val defaultOptimizerString =
- s"""
- import org.apache.spark.sql._;
- import org.apache.spark.sql.optimizer._;
- import org.apache.spark.sql.catalyst.plans.logical._;
- import org.apache.spark.sql.catalyst._;
- import org.apache.spark.sql.catalyst.optimizer.Optimizer;
-
- new AbstractCarbonOptimizerFactory {
- override def createOptimizer(optimizer: Optimizer, conf: CarbonSQLConf): Optimizer = {
- class CarbonOptimizer2(optimizer: Optimizer, conf: CarbonSQLConf) extends Optimizer {
- val batches = Nil;
- override def execute(plan: LogicalPlan): LogicalPlan = {
- CarbonOptimizer.execute(plan, optimizer);
- }
- }
- new CarbonOptimizer2(optimizer, conf);
- }
- }
- """
-
- val spark1_5ExpandString =
- s"""
- import org.apache.spark.sql._
- import org.apache.spark.sql.catalyst.plans.logical.{Expand, LogicalPlan}
- new AbstractCarbonExpandFactory {
- override def createExpand(expand: Expand, child: LogicalPlan): Expand = {
- Expand(expand.bitmasks, expand.groupByExprs, expand.gid, child)
- }
- }
- """
-}
-
-abstract class AbstractCarbonOptimizerFactory {
- def createOptimizer(optimizer: Optimizer, conf: CarbonSQLConf) : Optimizer
-}
-
-abstract class AbstractCarbonExpandFactory {
- def createExpand(expand: Expand, child: LogicalPlan) : Expand
-}
-
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/main/scala/org/apache/spark/sql/CustomDeterministicExpression.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CustomDeterministicExpression.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CustomDeterministicExpression.scala
deleted file mode 100644
index d745be2..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CustomDeterministicExpression.scala
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.Expression
-import org.apache.spark.sql.catalyst.expressions.codegen.{CodeGenContext, GeneratedExpressionCode}
-import org.apache.spark.sql.types.{DataType, StringType}
-
-/**
- * Custom expression to override the deterministic property
- *
- */
-case class CustomDeterministicExpression(nonDt: Expression ) extends Expression with Serializable{
- override def nullable: Boolean = true
-
- override def eval(input: InternalRow): Any = null
-
- override protected def genCode(ctx: CodeGenContext,
- ev: GeneratedExpressionCode): String = ev.code
- override def deterministic: Boolean = true
-
- override def dataType: DataType = StringType
-
- override def children: Seq[Expression] = Seq()
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala b/integration/spark/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala
deleted file mode 100644
index dc2dd7b..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import scala.collection.JavaConverters._
-
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.{Expression => SparkExpression, GenericMutableRow}
-
-import org.apache.carbondata.core.scan.expression.{ColumnExpression, Expression, ExpressionResult, UnknownExpression}
-import org.apache.carbondata.core.scan.expression.conditional.ConditionalExpression
-import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException
-import org.apache.carbondata.core.scan.filter.intf.{ExpressionType, RowIntf}
-import org.apache.carbondata.spark.util.CarbonScalaUtil
-
-class SparkUnknownExpression(var sparkExp: SparkExpression)
- extends UnknownExpression with ConditionalExpression {
-
- private var evaluateExpression: (InternalRow) => Any = sparkExp.eval
- private var isExecutor: Boolean = false
- children.addAll(getColumnList())
-
- override def evaluate(carbonRowInstance: RowIntf): ExpressionResult = {
-
- val values = carbonRowInstance.getValues.toSeq.map {
- case s: String => org.apache.spark.unsafe.types.UTF8String.fromString(s)
- case d: java.math.BigDecimal => org.apache.spark.sql.types.Decimal.apply(d)
- case value => value
- }
- try {
- val result = evaluateExpression(
- new GenericMutableRow(values.map(a => a.asInstanceOf[Any]).toArray))
- val sparkRes = if (isExecutor) {
- result.asInstanceOf[InternalRow].get(0, sparkExp.dataType)
- } else {
- result
- }
- new ExpressionResult(CarbonScalaUtil.convertSparkToCarbonDataType(sparkExp.dataType),
- sparkRes
- )
- } catch {
- case e: Exception => throw new FilterUnsupportedException(e.getMessage)
- }
- }
-
- override def getFilterExpressionType: ExpressionType = {
- ExpressionType.UNKNOWN
- }
-
- override def getString: String = {
- sparkExp.toString()
- }
-
- def setEvaluateExpression(evaluateExpression: (InternalRow) => Any): Unit = {
- this.evaluateExpression = evaluateExpression
- isExecutor = true
- }
-
- override def findAndSetChild(oldExpr: Expression, newExpr: Expression): Unit = {}
-
- def getColumnList: java.util.List[ColumnExpression] = {
-
- val lst = new java.util.ArrayList[ColumnExpression]()
- getColumnListFromExpressionTree(sparkExp, lst)
- lst
- }
- def getLiterals: java.util.List[ExpressionResult] = {
-
- val lst = new java.util.ArrayList[ExpressionResult]()
- lst
- }
-
- def getAllColumnList: java.util.List[ColumnExpression] = {
- val lst = new java.util.ArrayList[ColumnExpression]()
- getAllColumnListFromExpressionTree(sparkExp, lst)
- lst
- }
-
- def isSingleColumn: Boolean = {
- val lst = new java.util.ArrayList[ColumnExpression]()
- getAllColumnListFromExpressionTree(sparkExp, lst)
- if (lst.size == 1 && lst.get(0).isDimension) {
- true
- } else {
- false
- }
- }
-
- def getColumnListFromExpressionTree(sparkCurrentExp: SparkExpression,
- list: java.util.List[ColumnExpression]): Unit = {
- sparkCurrentExp match {
- case carbonBoundRef: CarbonBoundReference =>
- val foundExp = list.asScala
- .find(p => p.getColumnName == carbonBoundRef.colExp.getColumnName)
- if (foundExp.isEmpty) {
- carbonBoundRef.colExp.setColIndex(list.size)
- list.add(carbonBoundRef.colExp)
- } else {
- carbonBoundRef.colExp.setColIndex(foundExp.get.getColIndex)
- }
- case _ => sparkCurrentExp.children.foreach(getColumnListFromExpressionTree(_, list))
- }
- }
-
-
- def getAllColumnListFromExpressionTree(sparkCurrentExp: SparkExpression,
- list: java.util.List[ColumnExpression]): java.util.List[ColumnExpression] = {
- sparkCurrentExp match {
- case carbonBoundRef: CarbonBoundReference => list.add(carbonBoundRef.colExp)
- case _ => sparkCurrentExp.children.foreach(getColumnListFromExpressionTree(_, list))
- }
- list
- }
-
-}
[08/16] carbondata git commit: [CARBONDATA-1597] Remove spark1
integration
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/examples/spark/src/main/scala/org/apache/carbondata/examples/PerfTest.scala
----------------------------------------------------------------------
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/PerfTest.scala b/examples/spark/src/main/scala/org/apache/carbondata/examples/PerfTest.scala
deleted file mode 100644
index 824730f..0000000
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/PerfTest.scala
+++ /dev/null
@@ -1,328 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.examples
-
-import java.io.File
-
-import scala.util.Random
-
-import org.apache.spark.sql.{CarbonContext, DataFrame, Row, SaveMode, SQLContext}
-import org.apache.spark.sql.types.{DataTypes, StructType}
-
-import org.apache.carbondata.examples.PerfTest._
-import org.apache.carbondata.examples.util.ExampleUtils
-
-// scalastyle:off println
-
-/**
- * represent one query
- */
-class Query(val queryType: String, val queryNo: Int, val sqlString: String) {
-
- /**
- * run the query in a batch and calculate average time
- *
- * @param sqlContext context to run the query
- * @param runs run how many time
- * @param datasource datasource to run
- */
- def run(sqlContext: SQLContext, runs: Int, datasource: String): QueryResult = {
- // run repeated and calculate average time elapsed
- require(runs >= 1)
- val sqlToRun = makeSQLString(datasource)
-
- val firstTime = withTime {
- sqlContext.sql(sqlToRun).collect
- }
-
- var totalTime: Long = 0
- var result: Array[Row] = null
- (1 to (runs - 1)).foreach { x =>
- totalTime += withTime {
- result = sqlContext.sql(sqlToRun).collect
- }
- }
-
- val avgTime = totalTime / (runs - 1)
- QueryResult(datasource, result, avgTime, firstTime)
- }
-
- private def makeSQLString(datasource: String): String = {
- sqlString.replaceFirst("tableName", PerfTest.makeTableName(datasource))
- }
-
-}
-
-/**
- * query performance result
- */
-case class QueryResult(datasource: String, result: Array[Row], avgTime: Long, firstTime: Long)
-
-class QueryRunner(sqlContext: SQLContext, dataFrame: DataFrame, datasources: Seq[String]) {
-
- /**
- * run a query on each datasource
- */
- def run(query: Query, runs: Int): Seq[QueryResult] = {
- var results = Seq[QueryResult]()
- datasources.foreach { datasource =>
- val result = query.run(sqlContext, runs, datasource)
- results :+= result
- }
- checkResult(results)
- results
- }
-
- private def checkResult(results: Seq[QueryResult]): Unit = {
- results.foldLeft(results.head) { (last, cur) =>
- if (last.result.sortBy(_.toString()).sameElements(cur.result.sortBy(_.toString()))) cur
- else sys.error(s"result is not the same between " +
- s"${last.datasource} and " +
- s"${cur.datasource}")
- }
- }
-
- private def loadToNative(datasource: String): Unit = {
- val savePath = PerfTest.savePath(datasource)
- println(s"loading data into $datasource, path: $savePath")
- dataFrame.write
- .mode(SaveMode.Overwrite)
- .format(datasource)
- .save(savePath)
- sqlContext.read
- .format(datasource)
- .load(savePath)
- .registerTempTable(PerfTest.makeTableName(datasource))
- }
-
- /**
- * load data to each datasource
- */
- def loadData: Seq[QueryResult] = {
- // load data into all datasources
- var results = Seq[QueryResult]()
- datasources.foreach { datasource =>
- val time = withTime {
- datasource match {
- case "parquet" =>
- dataFrame.sqlContext.setConf(s"spark.sql.$datasource.compression.codec", "snappy")
- loadToNative(datasource)
- case "orc" =>
- dataFrame.sqlContext.sparkContext.hadoopConfiguration.set("orc.compress", "SNAPPY")
- loadToNative(datasource)
- case "carbon" =>
- sqlContext.sql(s"DROP TABLE IF EXISTS ${PerfTest.makeTableName(datasource)}")
- println(s"loading data into $datasource, path: " +
- s"${dataFrame.sqlContext.asInstanceOf[CarbonContext].storePath}")
- dataFrame.write
- .format("org.apache.spark.sql.CarbonSource")
- .option("tableName", PerfTest.makeTableName(datasource))
- .mode(SaveMode.Overwrite)
- .save()
- case _ => sys.error("unsupported data source")
- }
- }
- println(s"load data into $datasource completed, time taken ${time/1000000}ms")
- results :+= QueryResult(datasource, null, time, time)
- }
- results
- }
-
- def shutDown(): Unit = {
- // drop all tables and temp files
- datasources.foreach {
- case datasource @ ("parquet" | "orc") =>
- val f = new File(PerfTest.savePath(datasource))
- if (f.exists()) f.delete()
- case "carbon" =>
- sqlContext.sql(s"DROP TABLE IF EXISTS ${PerfTest.makeTableName("carbon")}")
- case _ => sys.error("unsupported data source")
- }
- }
-}
-
-/**
- * template for table data generation
- *
- * @param dimension number of dimension columns and their cardinality
- * @param measure number of measure columns
- */
-case class TableTemplate(dimension: Seq[(Int, Int)], measure: Int)
-
-/**
- * utility to generate random data according to template
- */
-class TableGenerator(sqlContext: SQLContext) {
-
- /**
- * generate a dataframe from random data
- */
- def genDataFrame(template: TableTemplate, rows: Int): DataFrame = {
- val measures = template.measure
- val dimensions = template.dimension.foldLeft(0) {(x, y) => x + y._1}
- val cardinality = template.dimension.foldLeft(Seq[Int]()) {(x, y) =>
- x ++ (1 to y._1).map(z => y._2)
- }
- print(s"generating data: $rows rows of $dimensions dimensions and $measures measures. ")
- println("cardinality for each dimension: " + cardinality.mkString(", "))
-
- val dimensionFields = (1 to dimensions).map { id =>
- DataTypes.createStructField(s"c$id", DataTypes.StringType, false)
- }
- val measureFields = (dimensions + 1 to dimensions + measures).map { id =>
- DataTypes.createStructField(s"c$id", DataTypes.IntegerType, false)
- }
- val schema = StructType(dimensionFields ++ measureFields)
- val data = sqlContext.sparkContext.parallelize(1 to rows).map { x =>
- val random = new Random()
- val dimSeq = (1 to dimensions).map { y =>
- s"P${y}_${random.nextInt(cardinality(y - 1))}"
- }
- val msrSeq = (1 to measures).map { y =>
- random.nextInt(10)
- }
- Row.fromSeq(dimSeq ++ msrSeq)
- }
- val df = sqlContext.createDataFrame(data, schema)
- df.write.mode(SaveMode.Overwrite).parquet(PerfTest.savePath("temp"))
- sqlContext.parquetFile(PerfTest.savePath("temp"))
- }
-}
-
-object PerfTest {
-
- private val olap: Seq[String] = Seq(
- """SELECT c3, c4, sum(c8) FROM tableName
- |WHERE c1 = 'P1_23' and c2 = 'P2_43'
- |GROUP BY c3, c4""".stripMargin,
-
- """SELECT c2, c3, sum(c9) FROM tableName
- |WHERE c1 = 'P1_432' and c4 = 'P4_3' and c5 = 'P5_2'
- |GROUP by c2, c3 """.stripMargin,
-
- """SELECT c2, count(distinct c1), sum(c8) FROM tableName
- |WHERE c3="P3_4" and c5="P5_4"
- |GROUP BY c2 """.stripMargin,
-
- """SELECT c2, c5, count(distinct c1), sum(c7) FROM tableName
- |WHERE c4="P4_4" and c5="P5_7" and c8>4
- |GROUP BY c2, c5 """.stripMargin
- )
-
- private val point: Seq[String] = Seq(
- """SELECT c4 FROM tableName
- |WHERE c1="P1_43" """.stripMargin,
-
- """SELECT c3 FROM tableName
- |WHERE c1="P1_542" and c2="P2_23" """.stripMargin,
-
- """SELECT c3, c5 FROM tableName
- |WHERE c1="P1_52" and c7=4""".stripMargin,
-
- """SELECT c4, c9 FROM tableName
- |WHERE c1="P1_43" and c8<3""".stripMargin
- )
-
- private val filter: Seq[String] = Seq(
- """SELECT * FROM tableName
- |WHERE c2="P2_43" """.stripMargin,
-
- """SELECT * FROM tableName
- |WHERE c3="P3_3" """.stripMargin,
-
- """SELECT * FROM tableName
- |WHERE c2="P2_32" and c3="P3_23" """.stripMargin,
-
- """SELECT * FROM tableName
- |WHERE c3="P3_28" and c4="P4_3" """.stripMargin
- )
-
- private val scan: Seq[String] = Seq(
- """SELECT sum(c7), sum(c8), avg(c9), max(c10) FROM tableName """.stripMargin,
-
- """SELECT sum(c7) FROM tableName
- |WHERE c2="P2_32" """.stripMargin,
-
- """SELECT sum(c7), sum(c8), sum(9), sum(c10) FROM tableName
- |WHERE c4="P4_4" """.stripMargin,
-
- """SELECT sum(c7), sum(c8), sum(9), sum(c10) FROM tableName
- |WHERE c2="P2_75" and c6<5 """.stripMargin
- )
-
- def main(args: Array[String]) {
- val cc = ExampleUtils.createCarbonContext("PerfTest")
-
- // prepare performance queries
- var workload = Seq[Query]()
- olap.zipWithIndex.foreach(x => workload :+= new Query("OLAP Query", x._2, x._1))
- point.zipWithIndex.foreach(x => workload :+= new Query("Point Query", x._2, x._1))
- filter.zipWithIndex.foreach(x => workload :+= new Query("Filter Query", x._2, x._1))
- scan.zipWithIndex.foreach(x => workload :+= new Query("Scan Query", x._2, x._1))
-
- // prepare data
- val rows = 3 * 1000 * 1000
- val dimension = Seq((1, 1 * 1000), (1, 100), (1, 50), (2, 10)) // cardinality for each column
- val measure = 5 // number of measure
- val template = TableTemplate(dimension, measure)
- val df = new TableGenerator(cc).genDataFrame(template, rows)
- println("generate data completed")
-
- // run all queries against all data sources
- val datasource = Seq("parquet", "orc", "carbon")
- val runner = new QueryRunner(cc, df, datasource)
-
- val results = runner.loadData
- println(s"load performance: ${results.map(_.avgTime / 1000000L).mkString(", ")}")
-
- var parquetTime: Double = 0
- var orcTime: Double = 0
- var carbonTime: Double = 0
-
- println(s"query id: ${datasource.mkString(", ")}, result in millisecond")
- workload.foreach { query =>
- // run 4 times each round, will print performance of first run and avg time of last 3 runs
- print(s"${query.queryType} ${query.queryNo}: ")
- val results = runner.run(query, 4)
- print(s"${results.map(_.avgTime / 1000000L).mkString(", ")} ")
- println(s"[sql: ${query.sqlString.replace('\n', ' ')}]")
- parquetTime += results(0).avgTime
- orcTime += results(1).avgTime
- carbonTime += results(2).avgTime
- }
-
- println(s"Total time: ${parquetTime / 1000000}, ${orcTime / 1000000}, " +
- s"${carbonTime / 1000000} = 1 : ${parquetTime / orcTime} : ${parquetTime / carbonTime}")
- runner.shutDown()
- }
-
- def makeTableName(datasource: String): String = {
- s"${datasource}_perftest_table"
- }
-
- def savePath(datasource: String): String =
- s"${ExampleUtils.currentPath}/target/perftest/${datasource}"
-
- def withTime(body: => Unit): Long = {
- val start = System.nanoTime()
- body
- System.nanoTime() - start
- }
-
-}
-// scalastyle:on println
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/examples/spark/src/main/scala/org/apache/carbondata/examples/util/AllDictionaryUtil.scala
----------------------------------------------------------------------
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/util/AllDictionaryUtil.scala b/examples/spark/src/main/scala/org/apache/carbondata/examples/util/AllDictionaryUtil.scala
deleted file mode 100644
index 3ab61bf..0000000
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/util/AllDictionaryUtil.scala
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.examples.util
-
-import java.io.DataOutputStream
-
-import scala.collection.mutable.{ArrayBuffer, HashSet}
-
-import org.apache.spark.SparkContext
-
-import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.datastore.impl.FileFactory
-
-object AllDictionaryUtil {
- private val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
- def extractDictionary(sc: SparkContext,
- srcData: String,
- outputPath: String,
- fileHeader: String,
- dictCol: String): Unit = {
- val fileHeaderArr = fileHeader.split(",")
- val isDictCol = new Array[Boolean](fileHeaderArr.length)
- for (i <- 0 until fileHeaderArr.length) {
- if (dictCol.contains("|" + fileHeaderArr(i).toLowerCase() + "|")) {
- isDictCol(i) = true
- } else {
- isDictCol(i) = false
- }
- }
- val dictionaryRdd = sc.textFile(srcData).flatMap(x => {
- val tokens = x.split(",")
- val result = new ArrayBuffer[(Int, String)]()
- for (i <- 0 until isDictCol.length) {
- if (isDictCol(i)) {
- try {
- result += ((i, tokens(i)))
- } catch {
- case ex: ArrayIndexOutOfBoundsException =>
- LOGGER.error("Read a bad record: " + x)
- }
- }
- }
- result
- }).groupByKey().flatMap(x => {
- val distinctValues = new HashSet[(Int, String)]()
- for (value <- x._2) {
- distinctValues.add(x._1, value)
- }
- distinctValues
- })
- val dictionaryValues = dictionaryRdd.map(x => x._1 + "," + x._2).collect()
- saveToFile(dictionaryValues, outputPath)
- }
-
- def cleanDictionary(outputPath: String): Unit = {
- try {
- val fileType = FileFactory.getFileType(outputPath)
- val file = FileFactory.getCarbonFile(outputPath, fileType)
- if (file.exists()) {
- file.delete()
- }
- } catch {
- case ex: Exception =>
- LOGGER.error("Clean dictionary catching exception:" + ex)
- }
- }
-
- def saveToFile(contents: Array[String], outputPath: String): Unit = {
- var writer: DataOutputStream = null
- try {
- val fileType = FileFactory.getFileType(outputPath)
- val file = FileFactory.getCarbonFile(outputPath, fileType)
- if (!file.exists()) {
- file.createNewFile()
- }
- writer = FileFactory.getDataOutputStream(outputPath, fileType)
- for (content <- contents) {
- writer.writeBytes(content + "\n")
- }
- } catch {
- case ex: Exception =>
- LOGGER.error("Save dictionary to file catching exception:" + ex)
- } finally {
- if (writer != null) {
- try {
- writer.close()
- } catch {
- case ex: Exception =>
- LOGGER.error("Close output stream catching exception:" + ex)
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/examples/spark/src/main/scala/org/apache/carbondata/examples/util/ExampleUtils.scala
----------------------------------------------------------------------
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/util/ExampleUtils.scala b/examples/spark/src/main/scala/org/apache/carbondata/examples/util/ExampleUtils.scala
deleted file mode 100644
index f98ec3b..0000000
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/util/ExampleUtils.scala
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.examples.util
-
-import java.io.File
-
-import org.apache.spark.{SparkConf, SparkContext}
-import org.apache.spark.sql.{CarbonContext, SaveMode}
-
-import org.apache.carbondata.core.util.CarbonProperties
-
-// scalastyle:off println
-
-object ExampleUtils {
-
- def currentPath: String = new File(this.getClass.getResource("/").getPath + "../../")
- .getCanonicalPath
- val storeLocation = currentPath + "/target/store"
-
- def createCarbonContext(appName: String): CarbonContext = {
- val sc = new SparkContext(new SparkConf()
- .setAppName(appName)
- .setMaster("local[2]"))
- sc.setLogLevel("ERROR")
-
- println(s"Starting $appName using spark version ${sc.version}")
-
- val cc = new CarbonContext(sc, storeLocation, currentPath + "/target/carbonmetastore")
-
- CarbonProperties.getInstance()
- .addProperty("carbon.storelocation", storeLocation)
- cc
- }
-
- /**
- * This func will write a sample CarbonData file containing following schema:
- * c1: String, c2: String, c3: Double
- * Returns table path
- */
- def writeSampleCarbonFile(cc: CarbonContext, tableName: String, numRows: Int = 1000): String = {
- cc.sql(s"DROP TABLE IF EXISTS $tableName")
- writeDataframe(cc, tableName, numRows, SaveMode.Overwrite)
- s"$storeLocation/default/$tableName"
- }
-
- /**
- * This func will append data to the CarbonData file
- * Returns table path
- */
- def appendSampleCarbonFile(cc: CarbonContext, tableName: String, numRows: Int = 1000): String = {
- writeDataframe(cc, tableName, numRows, SaveMode.Append)
- s"$storeLocation/default/$tableName"
- }
-
- /**
- * create a new dataframe and write to CarbonData file, based on save mode
- */
- private def writeDataframe(
- cc: CarbonContext, tableName: String, numRows: Int, mode: SaveMode): Unit = {
- // use CarbonContext to write CarbonData files
- import cc.implicits._
- val sc = cc.sparkContext
- val df = sc.parallelize(1 to numRows, 2)
- .map(x => ("a", "b", x))
- .toDF("c1", "c2", "c3")
-
- // save dataframe directl to carbon file without tempCSV
- df.write
- .format("carbondata")
- .option("tableName", tableName)
- .option("compress", "true")
- .option("tempCSV", "false")
- .mode(mode)
- .save()
- }
-
- def cleanSampleCarbonFile(cc: CarbonContext, tableName: String): Unit = {
- cc.sql(s"DROP TABLE IF EXISTS $tableName")
- }
-}
-// scalastyle:on println
-
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/examples/spark2/pom.xml
----------------------------------------------------------------------
diff --git a/examples/spark2/pom.xml b/examples/spark2/pom.xml
index 94af8ec..af25771 100644
--- a/examples/spark2/pom.xml
+++ b/examples/spark2/pom.xml
@@ -38,20 +38,6 @@
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-spark2</artifactId>
<version>${project.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-hive-thriftserver_2.10</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-repl_2.10</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_2.10</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/hive/pom.xml
----------------------------------------------------------------------
diff --git a/integration/hive/pom.xml b/integration/hive/pom.xml
index 13bd581..e0ad499 100644
--- a/integration/hive/pom.xml
+++ b/integration/hive/pom.xml
@@ -67,20 +67,6 @@
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-spark2</artifactId>
<version>${project.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-hive-thriftserver_2.10</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-repl_2.10</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_2.10</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
@@ -116,12 +102,6 @@
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-hadoop</artifactId>
<version>${project.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_2.10</artifactId>
- </exclusion>
- </exclusions>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/presto/pom.xml
----------------------------------------------------------------------
diff --git a/integration/presto/pom.xml b/integration/presto/pom.xml
index e11ce4c..13d351d 100644
--- a/integration/presto/pom.xml
+++ b/integration/presto/pom.xml
@@ -244,10 +244,6 @@
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</exclusion>
- <exclusion>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_2.10</artifactId>
- </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -371,10 +367,6 @@
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</exclusion>
- <exclusion>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_2.10</artifactId>
- </exclusion>
</exclusions>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark-common-cluster-test/pom.xml
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/pom.xml b/integration/spark-common-cluster-test/pom.xml
index 9728a5c..e529035 100644
--- a/integration/spark-common-cluster-test/pom.xml
+++ b/integration/spark-common-cluster-test/pom.xml
@@ -40,12 +40,6 @@
<artifactId>carbondata-spark-common</artifactId>
<version>${project.version}</version>
<scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-hive-thriftserver_2.10</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
@@ -167,16 +161,6 @@
<artifactId>carbondata-spark</artifactId>
<version>${project.version}</version>
<scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-hive-thriftserver_2.10</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-repl_2.10</artifactId>
- </exclusion>
- </exclusions>
</dependency>
</dependencies>
</profile>
@@ -188,16 +172,6 @@
<artifactId>carbondata-spark</artifactId>
<version>${project.version}</version>
<scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-hive-thriftserver_2.10</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-repl_2.10</artifactId>
- </exclusion>
- </exclusions>
</dependency>
</dependencies>
</profile>
@@ -212,16 +186,6 @@
<artifactId>carbondata-spark2</artifactId>
<version>${project.version}</version>
<scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-hive-thriftserver_2.10</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-repl_2.10</artifactId>
- </exclusion>
- </exclusions>
</dependency>
</dependencies>
</profile>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark-common-test/pom.xml
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/pom.xml b/integration/spark-common-test/pom.xml
index 7c67000..b2ee316 100644
--- a/integration/spark-common-test/pom.xml
+++ b/integration/spark-common-test/pom.xml
@@ -91,12 +91,6 @@
<artifactId>carbondata-spark-common</artifactId>
<version>${project.version}</version>
<scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-hive-thriftserver_2.10</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
@@ -341,16 +335,6 @@
<artifactId>carbondata-spark</artifactId>
<version>${project.version}</version>
<scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-hive-thriftserver_2.10</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-repl_2.10</artifactId>
- </exclusion>
- </exclusions>
</dependency>
</dependencies>
</profile>
@@ -362,16 +346,6 @@
<artifactId>carbondata-spark</artifactId>
<version>${project.version}</version>
<scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-hive-thriftserver_2.10</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-repl_2.10</artifactId>
- </exclusion>
- </exclusions>
</dependency>
</dependencies>
</profile>
@@ -386,16 +360,6 @@
<artifactId>carbondata-spark2</artifactId>
<version>${project.version}</version>
<scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-hive-thriftserver_2.10</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-repl_2.10</artifactId>
- </exclusion>
- </exclusions>
</dependency>
</dependencies>
</profile>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark-common/pom.xml
----------------------------------------------------------------------
diff --git a/integration/spark-common/pom.xml b/integration/spark-common/pom.xml
index 82ff7a4..d40e213 100644
--- a/integration/spark-common/pom.xml
+++ b/integration/spark-common/pom.xml
@@ -38,16 +38,6 @@
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-processing</artifactId>
<version>${project.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-hive-thriftserver_2.10</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_2.10</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.carbondata</groupId>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/CARBON_SPARK_INTERFACELogResource.properties
----------------------------------------------------------------------
diff --git a/integration/spark/CARBON_SPARK_INTERFACELogResource.properties b/integration/spark/CARBON_SPARK_INTERFACELogResource.properties
deleted file mode 100644
index 61856cf..0000000
--- a/integration/spark/CARBON_SPARK_INTERFACELogResource.properties
+++ /dev/null
@@ -1,18 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-carbon.spark.interface = {0}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/pom.xml
----------------------------------------------------------------------
diff --git a/integration/spark/pom.xml b/integration/spark/pom.xml
deleted file mode 100644
index 5060809..0000000
--- a/integration/spark/pom.xml
+++ /dev/null
@@ -1,194 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.carbondata</groupId>
- <artifactId>carbondata-parent</artifactId>
- <version>1.3.0-SNAPSHOT</version>
- <relativePath>../../pom.xml</relativePath>
- </parent>
-
- <artifactId>carbondata-spark</artifactId>
- <name>Apache CarbonData :: Spark</name>
-
- <properties>
- <dev.path>${basedir}/../../dev</dev.path>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.carbondata</groupId>
- <artifactId>carbondata-common</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.carbondata</groupId>
- <artifactId>carbondata-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.carbondata</groupId>
- <artifactId>carbondata-processing</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.carbondata</groupId>
- <artifactId>carbondata-hadoop</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.carbondata</groupId>
- <artifactId>carbondata-spark-common</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-compiler</artifactId>
- </dependency>
- <dependency>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-reflect</artifactId>
- </dependency>
- <dependency>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-library</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-hive-thriftserver_${scala.binary.version}</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-repl_${scala.binary.version}</artifactId>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- </dependency>
- <dependency>
- <groupId>org.scalatest</groupId>
- <artifactId>scalatest_${scala.binary.version}</artifactId>
- <version>2.2.1</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
-
- <build>
- <testSourceDirectory>src/test/scala</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/resources</directory>
- </resource>
- <resource>
- <directory>.</directory>
- <includes>
- <include>CARBON_SPARK_INTERFACELogResource.properties</include>
- </includes>
- </resource>
- </resources>
- <plugins>
- <plugin>
- <groupId>org.scala-tools</groupId>
- <artifactId>maven-scala-plugin</artifactId>
- <version>2.15.2</version>
- <executions>
- <execution>
- <id>compile</id>
- <goals>
- <goal>compile</goal>
- </goals>
- <phase>compile</phase>
- </execution>
- <execution>
- <id>testCompile</id>
- <goals>
- <goal>testCompile</goal>
- </goals>
- <phase>test</phase>
- </execution>
- <execution>
- <phase>process-resources</phase>
- <goals>
- <goal>compile</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <artifactId>maven-compiler-plugin</artifactId>
- <configuration>
- <source>1.7</source>
- <target>1.7</target>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <version>2.18</version>
- <!-- Note config is repeated in scalatest config -->
- <configuration>
- <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
- <argLine>-Xmx3g -XX:MaxPermSize=512m -XX:ReservedCodeCacheSize=512m</argLine>
- <systemProperties>
- <java.awt.headless>true</java.awt.headless>
- </systemProperties>
- <failIfNoTests>false</failIfNoTests>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.scalatest</groupId>
- <artifactId>scalatest-maven-plugin</artifactId>
- <version>1.0</version>
- <!-- Note config is repeated in surefire config -->
- <configuration>
- <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
- <junitxml>.</junitxml>
- <filereports>CarbonTestSuite.txt</filereports>
- <argLine> ${argLine} -ea -Xmx3g -XX:MaxPermSize=512m -XX:ReservedCodeCacheSize=512m
- </argLine>
- <stderr />
- <environmentVariables>
- </environmentVariables>
- <systemProperties>
- <java.awt.headless>true</java.awt.headless>
- </systemProperties>
- </configuration>
- <executions>
- <execution>
- <id>test</id>
- <goals>
- <goal>test</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- <profiles>
- <profile>
- <id>sdvtest</id>
- <properties>
- <maven.test.skip>true</maven.test.skip>
- </properties>
- </profile>
- </profiles>
-
-</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java b/integration/spark/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
deleted file mode 100644
index 92c8402..0000000
--- a/integration/spark/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.spark.readsupport;
-
-import java.io.IOException;
-import java.sql.Date;
-import java.sql.Timestamp;
-
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.metadata.encoder.Encoding;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
-import org.apache.carbondata.core.util.DataTypeUtil;
-import org.apache.carbondata.hadoop.readsupport.impl.DictionaryDecodeReadSupport;
-
-import org.apache.spark.sql.Row;
-import org.apache.spark.sql.catalyst.expressions.GenericRow;
-import org.apache.spark.unsafe.types.UTF8String;
-
-public class SparkRowReadSupportImpl extends DictionaryDecodeReadSupport<Row> {
-
- @Override public void initialize(CarbonColumn[] carbonColumns,
- AbsoluteTableIdentifier absoluteTableIdentifier) throws IOException {
- super.initialize(carbonColumns, absoluteTableIdentifier);
- //can initialize and generate schema here.
- }
-
- @Override public Row readRow(Object[] data) {
- for (int i = 0; i < dictionaries.length; i++) {
- if (data[i] == null) {
- continue;
- }
- if (dictionaries[i] != null) {
- data[i] = DataTypeUtil
- .getDataBasedOnDataType(dictionaries[i].getDictionaryValueForKeyInBytes((int) data[i]),
- (CarbonDimension) carbonColumns[i]);
- if (data[i] == null) {
- continue;
- }
- if (dataTypes[i] == DataTypes.STRING) {
- data[i] = UTF8String.fromString(data[i].toString());
- } else if (dataTypes[i] == DataTypes.TIMESTAMP) {
- data[i] = new Timestamp((long) data[i]);
- } else if (dataTypes[i] == DataTypes.DATE) {
- data[i] = new Date((long) data[i]);
- } else if (dataTypes[i] == DataTypes.LONG) {
- data[i] = data[i];
- }
- }
- else if (carbonColumns[i].hasEncoding(Encoding.DIRECT_DICTIONARY)) {
- //convert the long to timestamp in case of direct dictionary column
- if (DataTypes.TIMESTAMP == carbonColumns[i].getDataType()) {
- data[i] = new Timestamp((long) data[i] / 1000L);
- } else if (DataTypes.DATE == carbonColumns[i].getDataType()) {
- data[i] = new Date((long) data[i]);
- }
- }
- }
- return new GenericRow(data);
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala
deleted file mode 100644
index 7881b93..0000000
--- a/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala
+++ /dev/null
@@ -1,202 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.spark
-
-import org.apache.hadoop.fs.Path
-import org.apache.hadoop.io.compress.GzipCodec
-import org.apache.spark.sql._
-import org.apache.spark.sql.execution.command.LoadTable
-import org.apache.spark.sql.types._
-
-import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.metadata.datatype.{DataTypes => CarbonType}
-
-class CarbonDataFrameWriter(val dataFrame: DataFrame) {
-
- private val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
-
- def saveAsCarbonFile(parameters: Map[String, String] = Map()): Unit = {
- checkContext()
- val cc = CarbonContext.getInstance(dataFrame.sqlContext.sparkContext)
-
- // create a new table using dataframe's schema and write its content into the table
- cc.sql(makeCreateTableString(dataFrame.schema, new CarbonOption(parameters)))
- writeToCarbonFile(parameters)
- }
-
- def appendToCarbonFile(parameters: Map[String, String] = Map()): Unit = {
- // append the data as a new load
- checkContext()
- writeToCarbonFile(parameters)
- }
-
- private def writeToCarbonFile(parameters: Map[String, String] = Map()): Unit = {
- val options = new CarbonOption(parameters)
- val cc = CarbonContext.getInstance(dataFrame.sqlContext.sparkContext)
- if (options.tempCSV) {
- loadTempCSV(options, cc)
- } else {
- loadDataFrame(options, cc)
- }
- }
-
- /**
- * Firstly, saving DataFrame to CSV files
- * Secondly, load CSV files
- * @param options
- * @param cc
- */
- private def loadTempCSV(options: CarbonOption, cc: CarbonContext): Unit = {
- // temporary solution: write to csv file, then load the csv into carbon
- val storePath = CarbonEnv.get.carbonMetastore.storePath
- val tempCSVFolder = new StringBuilder(storePath).append(CarbonCommonConstants.FILE_SEPARATOR)
- .append("tempCSV")
- .append(CarbonCommonConstants.UNDERSCORE).append(options.dbName)
- .append(CarbonCommonConstants.UNDERSCORE).append(options.tableName)
- .append(CarbonCommonConstants.UNDERSCORE).append(System.nanoTime()).toString
- writeToTempCSVFile(tempCSVFolder, options)
-
- val tempCSVPath = new Path(tempCSVFolder)
- val fs = tempCSVPath.getFileSystem(dataFrame.sqlContext.sparkContext.hadoopConfiguration)
-
- def countSize(): Double = {
- var size: Double = 0
- val itor = fs.listFiles(tempCSVPath, true)
- while (itor.hasNext) {
- val f = itor.next()
- if (f.getPath.getName.startsWith("part-")) {
- size += f.getLen
- }
- }
- size
- }
-
- LOGGER.info(s"temporary CSV file size: ${countSize / 1024 / 1024} MB")
-
- try {
- cc.sql(makeLoadString(tempCSVFolder, options))
- } finally {
- fs.delete(tempCSVPath, true)
- }
- }
-
- private def checkContext(): Unit = {
- // To avoid derby problem, dataframe need to be writen and read using CarbonContext
- require(dataFrame.sqlContext.isInstanceOf[CarbonContext],
- "Error in saving dataframe to carbon file, must use CarbonContext to save dataframe"
- )
- }
-
- private def writeToTempCSVFile(tempCSVFolder: String, options: CarbonOption): Unit = {
-
- val strRDD = dataFrame.rdd.mapPartitions { case iter =>
- new Iterator[String] {
- override def hasNext = iter.hasNext
-
- def convertToCSVString(seq: Seq[Any]): String = {
- val build = new java.lang.StringBuilder()
- if (seq.head != null) {
- build.append(seq.head.toString)
- }
- val itemIter = seq.tail.iterator
- while (itemIter.hasNext) {
- build.append(CarbonCommonConstants.COMMA)
- val value = itemIter.next()
- if (value != null) {
- build.append(value.toString)
- }
- }
- build.toString
- }
-
- override def next: String = {
- convertToCSVString(iter.next.toSeq)
- }
- }
- }
-
- if (options.compress) {
- strRDD.saveAsTextFile(tempCSVFolder, classOf[GzipCodec])
- } else {
- strRDD.saveAsTextFile(tempCSVFolder)
- }
- }
-
- /**
- * Loading DataFrame directly without saving DataFrame to CSV files.
- * @param options
- * @param cc
- */
- private def loadDataFrame(options: CarbonOption, cc: CarbonContext): Unit = {
- val header = dataFrame.columns.mkString(",")
- LoadTable(
- Some(options.dbName),
- options.tableName,
- null,
- Seq(),
- Map("fileheader" -> header) ++ options.toMap,
- isOverwriteExist = false,
- null,
- Some(dataFrame),
- None).run(cc)
- }
-
- private def convertToCarbonType(sparkType: DataType): String = {
- sparkType match {
- case StringType => CarbonType.STRING.getName
- case IntegerType => CarbonType.INT.getName
- case ShortType => "smallint"
- case LongType => "bigint"
- case FloatType => CarbonType.DOUBLE.getName
- case DoubleType => CarbonType.DOUBLE.getName
- case TimestampType => CarbonType.TIMESTAMP.getName
- case DateType => CarbonType.DATE.getName
- case decimal: DecimalType => s"${CarbonType.DECIMAL.getName} (${decimal.precision}" +
- s", ${decimal.scale})"
- case other => sys.error(s"unsupported type: $other")
- }
- }
-
- private def makeCreateTableString(schema: StructType, options: CarbonOption): String = {
- val properties = Map(
- "DICTIONARY_INCLUDE" -> options.dictionaryInclude,
- "DICTIONARY_EXCLUDE" -> options.dictionaryExclude
- ).filter(_._2.isDefined).map(p => s"'${p._1}' = '${p._2.get}'").mkString(",")
- val carbonSchema = schema.map { field =>
- s"${ field.name } ${ convertToCarbonType(field.dataType) }"
- }
- s"""
- CREATE TABLE IF NOT EXISTS ${options.dbName}.${options.tableName}
- (${ carbonSchema.mkString(", ") })
- STORED BY '${ CarbonContext.datasourceName }'
- ${ if (properties.nonEmpty) " TBLPROPERTIES (" + properties + ")" else ""}
- """
- }
-
- private def makeLoadString(csvFolder: String, options: CarbonOption): String = {
- s"""
- LOAD DATA INPATH '$csvFolder'
- INTO TABLE ${options.dbName}.${options.tableName}
- OPTIONS ('FILEHEADER' = '${dataFrame.columns.mkString(",")}',
- 'SINGLE_PASS' = '${options.singlePass}')
- """
- }
-
-
-}
[14/16] carbondata git commit: [CARBONDATA-1568] Optimize annotation
of code
Posted by ra...@apache.org.
[CARBONDATA-1568] Optimize annotation of code
There are some improper places in code annotation by IDEA inspect code
We optimize annotation of code:
Fix an error of Javadoc issues
modify the expression of annotation
This closes #1408
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/9326cfdc
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/9326cfdc
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/9326cfdc
Branch: refs/heads/pre-aggregate
Commit: 9326cfdc07bf741b3516c22fae7362088ce1d7ac
Parents: 6abdd97
Author: xubo245 <60...@qq.com>
Authored: Tue Oct 10 20:59:46 2017 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Fri Oct 20 08:47:30 2017 +0800
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 2 +-
.../SegmentUpdateStatusManager.java | 17 ++++----
.../carbondata/core/util/CarbonProperties.java | 18 +++++----
.../core/util/path/CarbonStorePath.java | 4 ++
.../core/util/path/CarbonTablePath.java | 12 +++++-
.../carbondata/presto/server/PrestoServer.scala | 2 +-
.../presto/util/CarbonDataStoreCreator.scala | 4 +-
.../processing/merger/CarbonDataMergerUtil.java | 41 +++++++++++---------
8 files changed, 59 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9326cfdc/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 0510d7f..4ee3001 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -1031,7 +1031,7 @@ public final class CarbonCommonConstants {
public static final String CARBON_DATA_FILE_VERSION = "carbon.data.file.version";
/**
- * property to set is IS_DRIVER_INSTANCE
+ * Index file name will end with this extension when update.
*/
public static final String UPDATE_INDEX_FILE_EXT = ".carbonindex";
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9326cfdc/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
index 20c7ad7..1218231 100644
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
@@ -333,10 +333,11 @@ public class SegmentUpdateStatusManager {
}
/**
+ * check the block whether is valid
*
- * @param segName
- * @param blockName
- * @return
+ * @param segName segment name
+ * @param blockName block name
+ * @return the status of block whether is valid
*/
public boolean isBlockValid(String segName, String blockName) {
@@ -351,11 +352,11 @@ public class SegmentUpdateStatusManager {
/**
* Returns all delta file paths of specified block
*
- * @param blockDir
- * @param blockNameFromTuple
- * @param listOfSegmentUpdateDetailsArray
- * @param extension
- * @return
+ * @param blockDir block directory with CarbonFile format
+ * @param blockNameFromTuple block name from tuple
+ * @param extension the file extension name
+ * @param segment the segment name
+ * @return the list of delete file
*/
private List<String> getDeltaFiles(CarbonFile blockDir, final String blockNameFromTuple,
final String extension,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9326cfdc/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
index 0ab28e2..cdd6183 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
@@ -615,7 +615,7 @@ public final class CarbonProperties {
/**
* gettting the unmerged segment numbers to be merged.
*
- * @return
+ * @return corrected value of unmerged segments to be merged
*/
public int[] getCompactionSegmentLevelCount() {
String commaSeparatedLevels;
@@ -632,8 +632,10 @@ public final class CarbonProperties {
}
/**
- * @param commaSeparatedLevels
- * @return
+ * Separating the count for Number of segments to be merged in levels by comma
+ *
+ * @param commaSeparatedLevels the string format value before separating
+ * @return the int array format value after separating by comma
*/
private int[] getIntArray(String commaSeparatedLevels) {
String[] levels = commaSeparatedLevels.split(",");
@@ -727,11 +729,11 @@ public final class CarbonProperties {
/**
* Validate the restrictions
*
- * @param actual
- * @param max
- * @param min
- * @param defaultVal
- * @return
+ * @param actual the actual value for minor compaction
+ * @param max max value for minor compaction
+ * @param min min value for minor compaction
+ * @param defaultVal default value when the actual is improper
+ * @return corrected Value after validating
*/
public int validate(int actual, int max, int min, int defaultVal) {
if (actual <= max && actual >= min) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9326cfdc/core/src/main/java/org/apache/carbondata/core/util/path/CarbonStorePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonStorePath.java b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonStorePath.java
index 3aece69..201e05f 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonStorePath.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonStorePath.java
@@ -37,6 +37,10 @@ public class CarbonStorePath extends Path {
/**
* gets CarbonTablePath object to manage table paths
+ *
+ * @param storePath the store path of the segment
+ * @param tableIdentifier identifier of carbon table that the segment belong to
+ * @return the store location of the segment
*/
public static CarbonTablePath getCarbonTablePath(String storePath,
CarbonTableIdentifier tableIdentifier) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9326cfdc/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
index 0910afc..7be9c76 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
@@ -55,9 +55,10 @@ public class CarbonTablePath extends Path {
protected CarbonTableIdentifier carbonTableIdentifier;
/**
+ * structure CarbonTablePath object to manage table paths
*
- * @param carbonTableIdentifier
- * @param tablePathString
+ * @param carbonTableIdentifier identifier of carbon table that the segment belong to
+ * @param tablePathString the store path of the segment
*/
public CarbonTablePath(CarbonTableIdentifier carbonTableIdentifier, String tablePathString) {
super(tablePathString);
@@ -65,6 +66,13 @@ public class CarbonTablePath extends Path {
this.tablePath = tablePathString;
}
+ /**
+ * structure CarbonTablePath object to manage table paths
+ *
+ * @param storePath the store path of the segment
+ * @param dbName database name
+ * @param tableName table name
+ */
public CarbonTablePath(String storePath, String dbName, String tableName) {
super(storePath + File.separator + dbName + File.separator + tableName);
this.carbonTableIdentifier = new CarbonTableIdentifier(dbName, tableName, "");
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9326cfdc/integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala
----------------------------------------------------------------------
diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala
index 3497f47..cdf8833 100644
--- a/integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala
+++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala
@@ -50,7 +50,7 @@ object PrestoServer {
/**
* start the presto server
*
- * @param carbonStorePath
+ * @param carbonStorePath the store path of carbon
*/
def startServer(carbonStorePath: String) = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9326cfdc/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
----------------------------------------------------------------------
diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
index 0a811b7..b8031a2 100644
--- a/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
+++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
@@ -403,8 +403,8 @@ object CarbonDataStoreCreator {
/**
* Execute graph which will further load data
*
- * @param loadModel
- * @param storeLocation
+ * @param loadModel Carbon load model
+ * @param storeLocation store location directory
* @throws Exception
*/
private def executeGraph(loadModel: CarbonLoadModel, storeLocation: String): Unit = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9326cfdc/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
index 8631e1d..13972c8 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
@@ -570,11 +570,13 @@ public final class CarbonDataMergerUtil {
/**
* Identify the segments to be merged based on the Size in case of Major compaction.
*
- * @param compactionSize
- * @param listOfSegmentsAfterPreserve
- * @param carbonLoadModel
- * @param storeLocation
- * @return
+ * @param compactionSize compaction size in MB format
+ * @param listOfSegmentsAfterPreserve the segments list after
+ * preserving the configured number of latest loads
+ * @param carbonLoadModel carbon load model
+ * @param storeLocation the store location of the segment
+ * @return the list of segments that need to be merged
+ * based on the Size in case of Major compaction
*/
private static List<LoadMetadataDetails> identifySegmentsToBeMergedBasedOnSize(
long compactionSize, List<LoadMetadataDetails> listOfSegmentsAfterPreserve,
@@ -634,14 +636,14 @@ public final class CarbonDataMergerUtil {
/**
* For calculating the size of the specified segment
- * @param storeLocation
- * @param tableIdentifier
- * @param segId
- * @return
+ * @param storePath the store path of the segment
+ * @param tableIdentifier identifier of table that the segment belong to
+ * @param segId segment id
+ * @return the data size of the segment
*/
- private static long getSizeOfSegment(String storeLocation,
+ private static long getSizeOfSegment(String storePath,
CarbonTableIdentifier tableIdentifier, String segId) {
- String loadPath = getStoreLocation(storeLocation, tableIdentifier, segId);
+ String loadPath = getStoreLocation(storePath, tableIdentifier, segId);
CarbonFile segmentFolder =
FileFactory.getCarbonFile(loadPath, FileFactory.getFileType(loadPath));
return getSizeOfFactFileInLoad(segmentFolder);
@@ -650,10 +652,10 @@ public final class CarbonDataMergerUtil {
/**
* This method will get the store location for the given path, segemnt id and partition id
*
- * @param storePath
- * @param carbonTableIdentifier
- * @param segmentId
- * @return
+ * @param storePath the store path of the segment
+ * @param carbonTableIdentifier identifier of catbon table that the segment belong to
+ * @param segmentId segment id
+ * @return the store location of the segment
*/
private static String getStoreLocation(String storePath,
CarbonTableIdentifier carbonTableIdentifier, String segmentId) {
@@ -666,8 +668,9 @@ public final class CarbonDataMergerUtil {
/**
* Identify the segments to be merged based on the segment count
*
- * @param listOfSegmentsAfterPreserve
- * @return
+ * @param listOfSegmentsAfterPreserve the list of segments after
+ * preserve and before filtering by minor compaction level
+ * @return the list of segments to be merged after filtering by minor compaction level
*/
private static List<LoadMetadataDetails> identifySegmentsToBeMergedBasedOnSegCount(
List<LoadMetadataDetails> listOfSegmentsAfterPreserve) {
@@ -730,8 +733,8 @@ public final class CarbonDataMergerUtil {
/**
* To check if the segment is merged or not.
- * @param segName
- * @return
+ * @param segName segment name
+ * @return the status whether the segment has been Merged
*/
private static boolean isMergedSegment(String segName) {
if (segName.contains(".")) {
[09/16] carbondata git commit: [CARBONDATA-1597] Remove spark1
integration
Posted by ra...@apache.org.
[CARBONDATA-1597] Remove spark1 integration
As voted by community, spark version 1 integration can be removed. This PR removes following profiles and module for spark version 1 integration:1)spark-1.5 profile, 2) spark-1.6 profile, 3) example/spark module, 4) integration/spark module
This closes #1421
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/0bf597d9
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/0bf597d9
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/0bf597d9
Branch: refs/heads/pre-aggregate
Commit: 0bf597d9e5268c74eb38751b2ef26cd4d5e80f07
Parents: 311a5b7
Author: Jacky Li <ja...@qq.com>
Authored: Thu Oct 19 16:11:44 2017 +0800
Committer: chenliang613 <ch...@apache.org>
Committed: Thu Oct 19 16:02:49 2017 +0530
----------------------------------------------------------------------
bin/carbon-spark-sql | 5 +-
.../spark/CARBON_EXAMPLESLogResource.properties | 18 -
examples/spark/pom.xml | 85 --
.../spark/src/main/resources/complexdata.csv | 101 --
examples/spark/src/main/resources/data.csv | 11 -
examples/spark/src/main/resources/dimSample.csv | 21 -
.../spark/src/main/resources/factSample.csv | 51 -
.../examples/AllDictionaryExample.scala | 67 --
.../carbondata/examples/AlluxioExample.scala | 65 --
.../carbondata/examples/CarbonExample.scala | 61 -
.../examples/CarbonPartitionExample.scala | 148 ---
.../examples/CaseClassDataFrameAPIExample.scala | 49 -
.../examples/ComplexTypeExample.scala | 78 --
.../examples/DataFrameAPIExample.scala | 51 -
.../examples/DataManagementExample.scala | 78 --
.../examples/DataUpdateDeleteExample.scala | 185 ---
.../carbondata/examples/DatasourceExample.scala | 44 -
.../carbondata/examples/DirectSQLExample.scala | 47 -
.../examples/GenerateDictionaryExample.scala | 96 --
.../carbondata/examples/HadoopFileExample.scala | 53 -
.../apache/carbondata/examples/PerfTest.scala | 328 ------
.../examples/util/AllDictionaryUtil.scala | 109 --
.../carbondata/examples/util/ExampleUtils.scala | 97 --
examples/spark2/pom.xml | 14 -
integration/hive/pom.xml | 20 -
integration/presto/pom.xml | 8 -
integration/spark-common-cluster-test/pom.xml | 36 -
integration/spark-common-test/pom.xml | 36 -
integration/spark-common/pom.xml | 10 -
...CARBON_SPARK_INTERFACELogResource.properties | 18 -
integration/spark/pom.xml | 194 ----
.../readsupport/SparkRowReadSupportImpl.java | 76 --
.../spark/CarbonDataFrameWriter.scala | 202 ----
.../spark/rdd/CarbonDataRDDFactory.scala | 1088 ------------------
.../spark/thriftserver/CarbonThriftServer.scala | 66 --
.../carbondata/spark/util/CarbonSparkUtil.scala | 45 -
.../carbondata/spark/util/QueryPlanUtil.scala | 56 -
.../org/apache/spark/CarbonInputMetrics.scala | 66 --
.../apache/spark/sql/CarbonBoundReference.scala | 46 -
.../spark/sql/CarbonCatalystOperators.scala | 166 ---
.../org/apache/spark/sql/CarbonContext.scala | 197 ----
.../sql/CarbonDatasourceHadoopRelation.scala | 189 ---
.../spark/sql/CarbonDatasourceRelation.scala | 321 ------
.../spark/sql/CarbonDictionaryDecoder.scala | 259 -----
.../scala/org/apache/spark/sql/CarbonEnv.scala | 56 -
.../org/apache/spark/sql/CarbonSQLConf.scala | 36 -
.../scala/org/apache/spark/sql/CarbonScan.scala | 163 ---
.../org/apache/spark/sql/CarbonSparkUtil.scala | 46 -
.../org/apache/spark/sql/CarbonSqlParser.scala | 589 ----------
.../apache/spark/sql/CodeGenerateFactory.scala | 155 ---
.../sql/CustomDeterministicExpression.scala | 41 -
.../spark/sql/SparkUnknownExpression.scala | 130 ---
.../sql/execution/command/IUDCommands.scala | 842 --------------
.../execution/command/carbonTableSchema.scala | 1019 ----------------
.../spark/sql/hive/CarbonAnalysisRules.scala | 175 ---
.../spark/sql/hive/CarbonHiveMetadataUtil.scala | 58 -
.../apache/spark/sql/hive/CarbonMetastore.scala | 562 ---------
.../spark/sql/hive/CarbonSQLDialect.scala | 44 -
.../spark/sql/hive/CarbonStrategies.scala | 370 ------
.../apache/spark/sql/hive/HiveQlWrapper.scala | 32 -
.../spark/sql/hive/cli/CarbonSQLCLIDriver.scala | 83 --
.../execution/command/CarbonHiveCommands.scala | 55 -
.../spark/sql/optimizer/CarbonFilters.scala | 431 -------
.../spark/sql/optimizer/CarbonOptimizer.scala | 862 --------------
.../spark/sql/test/SparkTestQueryExecutor.scala | 55 -
.../org/apache/spark/util/TaskContextUtil.scala | 29 -
....apache.spark.sql.sources.DataSourceRegister | 17 -
...che.spark.sql.test.TestQueryExecutorRegister | 17 -
.../src/test/resources/badrecords/test2.csv | 4 -
...plexPrimitiveTimestampDirectDictionary.scala | 65 --
.../dataload/SparkDatasourceSuite.scala | 192 ----
.../TestLoadDataWithSingleQuotechar.scala | 57 -
.../allqueries/AllQueriesSpark1TestCase.scala | 60 -
.../InsertIntoCarbonTableSpark1TestCase.scala | 81 --
.../BadRecordLoggerSharedDictionaryTest.scala | 84 --
.../createtable/TestCreateTableSyntax.scala | 186 ---
.../CompactionSystemLockFeatureTest.scala | 143 ---
.../DataCompactionMinorThresholdTest.scala | 103 --
.../DataCompactionNoDictionaryTest.scala | 173 ---
.../datacompaction/DataCompactionTest.scala | 224 ----
.../GrtLtFilterProcessorTestCase.scala | 64 --
.../HadoopFSRelationTestCase.scala | 68 --
.../spark/util/AllDictionaryTestCase.scala | 140 ---
.../spark/util/DictionaryTestCaseUtil.scala | 51 -
.../util/ExternalColumnDictionaryTestCase.scala | 252 ----
...GlobalDictionaryUtilConcurrentTestCase.scala | 183 ---
.../util/GlobalDictionaryUtilTestCase.scala | 210 ----
.../apache/spark/sql/TestCarbonSqlParser.scala | 327 ------
integration/spark2/pom.xml | 10 -
pom.xml | 69 +-
processing/pom.xml | 6 -
91 files changed, 5 insertions(+), 13575 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/bin/carbon-spark-sql
----------------------------------------------------------------------
diff --git a/bin/carbon-spark-sql b/bin/carbon-spark-sql
index 2c799d1..4b927d1 100755
--- a/bin/carbon-spark-sql
+++ b/bin/carbon-spark-sql
@@ -33,10 +33,7 @@ fi
export FWDIR=$SPARK_HOME
export CARBON_SOURCE="$(cd "`dirname "$0"`"/..; pwd)"
-ASSEMBLY_DIR="$CARBON_SOURCE/assembly/target/scala-2.10"
-if [ -d "$CARBON_SOURCE/assembly/target/scala-2.11" ]; then
- ASSEMBLY_DIR="$CARBON_SOURCE/assembly/target/scala-2.11"
-fi
+ASSEMBLY_DIR="$CARBON_SOURCE/assembly/target/scala-2.11"
GREP_OPTIONS=
num_jars="$(ls -1 "$ASSEMBLY_DIR" | grep "^carbondata.*hadoop.*\.jar$" | wc -l)"
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/examples/spark/CARBON_EXAMPLESLogResource.properties
----------------------------------------------------------------------
diff --git a/examples/spark/CARBON_EXAMPLESLogResource.properties b/examples/spark/CARBON_EXAMPLESLogResource.properties
deleted file mode 100644
index 78f873e..0000000
--- a/examples/spark/CARBON_EXAMPLESLogResource.properties
+++ /dev/null
@@ -1,18 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-carbon.examples = {0}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/examples/spark/pom.xml
----------------------------------------------------------------------
diff --git a/examples/spark/pom.xml b/examples/spark/pom.xml
deleted file mode 100644
index cc078ff..0000000
--- a/examples/spark/pom.xml
+++ /dev/null
@@ -1,85 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.carbondata</groupId>
- <artifactId>carbondata-parent</artifactId>
- <version>1.3.0-SNAPSHOT</version>
- <relativePath>../../pom.xml</relativePath>
- </parent>
-
- <artifactId>carbondata-examples-spark</artifactId>
- <name>Apache CarbonData :: Spark Examples</name>
-
- <properties>
- <dev.path>${basedir}/../../dev</dev.path>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.carbondata</groupId>
- <artifactId>carbondata-spark</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
-
- <build>
- <sourceDirectory>src/main/scala</sourceDirectory>
- <resources>
- <resource>
- <directory>.</directory>
- <includes>
- <include>CARBON_EXAMPLESLogResource.properties</include>
- </includes>
- </resource>
- </resources>
- <plugins>
- <plugin>
- <groupId>org.scala-tools</groupId>
- <artifactId>maven-scala-plugin</artifactId>
- <version>2.15.2</version>
- <executions>
- <execution>
- <id>compile</id>
- <goals>
- <goal>compile</goal>
- </goals>
- <phase>compile</phase>
- </execution>
- <execution>
- <phase>process-resources</phase>
- <goals>
- <goal>compile</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <artifactId>maven-compiler-plugin</artifactId>
- <configuration>
- <source>1.7</source>
- <target>1.7</target>
- </configuration>
- </plugin>
- </plugins>
- </build>
-
-</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/examples/spark/src/main/resources/complexdata.csv
----------------------------------------------------------------------
diff --git a/examples/spark/src/main/resources/complexdata.csv b/examples/spark/src/main/resources/complexdata.csv
deleted file mode 100644
index 23a3949..0000000
--- a/examples/spark/src/main/resources/complexdata.csv
+++ /dev/null
@@ -1,101 +0,0 @@
-deviceInformationId,channelsId,ROMSize,purchasedate,mobile,MAC,locationinfo,proddate,gamePointId,contractNumber
-1,109,4ROM size,29-11-2015,1AA1$2BB1,MAC1$MAC2$MAC3,7:Chinese:Hubei Province:yichang:yichang:yichang$7:India:New Delhi:delhi:delhi:delhi,29-11-2015$29-11-2015:29-11-2015,109,2738.562
-10,93,1ROM size,29-11-2015,1AA10$2BB10,MAC4$MAC5$MAC6,7:Chinese:Hubei Province:yichang:yichang:yichang$7:India:New Delhi:delhi:delhi:delhi,30-11-2015$30-11-2015:30-11-2015,93,1714.635
-100,2591,2ROM size,29-11-2015,1AA100$2BB100,MAC7$MAC8$MAC9,4:Chinese:Hunan Province:xiangtan:xiangtan:jianshelu$4:India:Hunan Province:xiangtan:xiangtan:jianshelu,01-12-2015$01-12-2015:01-12-2015,2591,1271
-1000,2531,2ROM size,29-11-2015,1AA1000$2BB1000,MAC10$$MAC12,6:Chinese:Hubei Province:wuhan:hongshan:hongshan$6:India:New Delhi:wuhan:hongshan:hongshan,02-12-2015$02-12-2015:02-12-2015,2531,692
-10000,2408,0ROM size,29-11-2015,1AA10000$2BB10000,MAC13$$MAC15,2:Chinese:Guangdong Province:guangzhou:longhua:mingzhi$2:India:Guangdong Province:guangzhou:longhua:mingzhi,03-12-2015$03-12-2015:03-12-2015,2408,2175
-100000,1815,0ROM size,29-11-2015,1AA100000$2BB100000,MAC16$$MAC18,6:Chinese:Hubei Province:wuhan:hongshan:hongshan$6:India:New Delhi:wuhan:hongshan:hongshan,04-12-2015$04-12-2015:04-12-2015,1815,136
-1000000,2479,4ROM size,29-11-2015,1AA1000000$2BB1000000,MAC19$$MAC21,7:Chinese:Hubei Province:yichang:yichang:yichang$7:India:New Delhi:delhi:delhi:delhi,05-12-2015$05-12-2015:05-12-2015,2479,1600
-100001,1845,7ROM size,29-11-2015,1AA100001$,MAC22$$MAC24,4:Chinese:Hunan Province:xiangtan:xiangtan:jianshelu$4:India:Hunan Province:xiangtan:xiangtan:jianshelu,06-12-2015$06-12-2015:06-12-2015,1845,505
-100002,2008,1ROM size,29-11-2015,1AA100002$,MAC25$$MAC27,3:Chinese:Hunan Province:changsha:yuhua:shazitang$3:India:Hunan Province:changsha:yuhua:shazitang,07-12-2015$07-12-2015:07-12-2015,2008,1341
-100003,1121,5ROM size,29-11-2015,1AA100003$,MAC28$$MAC30,5:Chinese:Hunan Province:zhuzhou:tianyuan:tianyua$5:India:Hunan Province:zhuzhou:tianyuan:tianyua,08-12-2015$08-12-2015:08-12-2015,1121,2239
-100004,1511,8ROM size,29-11-2015,1AA100004$,MAC31$$MAC33,7:Chinese:Hubei Province:yichang:yichang:yichang$7:India:New Delhi:delhi:delhi:delhi,09-12-2015$09-12-2015:09-12-2015,1511,2970
-100005,2759,0ROM size,29-11-2015,1AA100005$,MAC34$$MAC36,7:Chinese:Hubei Province:yichang:yichang:yichang$7:India:New Delhi:delhi:delhi:delhi,10-12-2015$10-12-2015:10-12-2015,2759,2593
-100006,2069,7ROM size,29-11-2015,1AA100006$,MAC37$$MAC39,3:Chinese:Hunan Province:changsha:yuhua:shazitang$3:India:Hunan Province:changsha:yuhua:shazitang,11-12-2015$11-12-2015:11-12-2015,2069,2572
-100007,396,7ROM size,29-11-2015,1AA100007$,MAC40$$MAC42,3:Chinese:Hunan Province:changsha:yuhua:shazitang$3:India:Hunan Province:changsha:yuhua:shazitang,12-12-2015$12-12-2015:12-12-2015,396,1991
-100008,104,2ROM size,29-11-2015,1AA100008$,MAC43$$MAC45,3:Chinese:Hunan Province:changsha:yuhua:shazitang$3:India:Hunan Province:changsha:yuhua:shazitang,13-12-2015$13-12-2015:13-12-2015,104,1442
-100009,477,3ROM size,29-11-2015,1AA100009$,MAC46$$MAC48,7:Chinese:Hubei Province:yichang:yichang:yichang$7:India:New Delhi:delhi:delhi:delhi,14-12-2015$14-12-2015:14-12-2015,477,1841
-10001,546,8ROM size,29-11-2015,1AA10001$2,MAC49$$MAC51,3:Chinese:Hunan Province:changsha:yuhua:shazitang$3:India:Hunan Province:changsha:yuhua:shazitang,15-12-2015$15-12-2015:15-12-2015,546,298
-100010,2696,3ROM size,29-11-2015,1AA100010$2BB100010,MAC52$$MAC54,5:Chinese:Hunan Province:zhuzhou:tianyuan:tianyua$5:India:Hunan Province:zhuzhou:tianyuan:tianyua,16-12-2015$16-12-2015:16-12-2015,2696,79
-100011,466,2ROM size,29-11-2015,1AA100011$2BB100011,MAC55$$MAC57,2:Chinese:Guangdong Province:guangzhou:longhua:mingzhi$2:India:Guangdong Province:guangzhou:longhua:mingzhi,17-12-2015$17-12-2015:17-12-2015,466,202
-100012,2644,2ROM size,29-11-2015,1AA100012$2BB100012,MAC58$$MAC60,4:Chinese:Hunan Province:xiangtan:xiangtan:jianshelu$4:India:Hunan Province:xiangtan:xiangtan:jianshelu,18-12-2015$18-12-2015:18-12-2015,2644,568
-100013,2167,3ROM size,29-11-2015,1AA100013$2BB100013,MAC61$MAC62,3:Chinese:Hunan Province:changsha:yuhua:shazitang$3:India:Hunan Province:changsha:yuhua:shazitang,19-12-2015$19-12-2015:19-12-2015,2167,355
-100014,1069,7ROM size,29-11-2015,1AA100014$2BB100014,MAC64$MAC65,5:Chinese:Hunan Province:zhuzhou:tianyuan:tianyua$5:India:Hunan Province:zhuzhou:tianyuan:tianyua,20-12-2015$20-12-2015:20-12-2015,1069,151
-100015,1447,9ROM size,29-11-2015,1AA100015$2BB100015,MAC67$MAC68,4:Chinese:Hunan Province:xiangtan:xiangtan:jianshelu$4:India:Hunan Province:xiangtan:xiangtan:jianshelu,21-12-2015$21-12-2015:21-12-2015,1447,2863
-100016,2963,3ROM size,29-11-2015,1AA100016$2BB100016,MAC70$MAC71,3:Chinese:Hunan Province:changsha:yuhua:shazitang$3:India:Hunan Province:changsha:yuhua:shazitang,22-12-2015$22-12-2015:22-12-2015,2963,1873
-100017,1580,5ROM size,29-11-2015,1AA100017$2BB100017,MAC73$MAC74,4:Chinese:Hunan Province:xiangtan:xiangtan:jianshelu$4:India:Hunan Province:xiangtan:xiangtan:jianshelu,23-12-2015$23-12-2015:23-12-2015,1580,2205
-100018,446,2ROM size,29-11-2015,1AA100018$2BB100018,MAC76$MAC77,7:Chinese:Hubei Province:yichang:yichang:yichang$7:India:New Delhi:delhi:delhi:delhi,24-12-2015$24-12-2015:24-12-2015,446,441
-100019,2151,7ROM size,29-11-2015,1AA100019$2BB100019,MAC79$MAC80,5:Chinese:Hunan Province:zhuzhou:tianyuan:tianyua$5:India:Hunan Province:zhuzhou:tianyuan:tianyua,25-12-2015$25-12-2015:25-12-2015,2151,2194
-10002,2201,1ROM size,29-11-2015,2BB10002,MAC82$MAC83,6:Chinese:Hubei Province:wuhan:hongshan:hongshan$6:India:New Delhi:wuhan:hongshan:hongshan,26-12-2015$26-12-2015:26-12-2015,2201,2972
-100020,2574,5ROM size,29-11-2015,$2BB100020,MAC85$MAC86,1:Chinese:Guangdong Province:shenzhen:longgang:matishan$1:India:Guangdong Province:shenzhen:longgang:matishan,27-12-2015$27-12-2015:27-12-2015,2574,256
-100021,1734,4ROM size,29-11-2015,$2BB100021,MAC88$MAC89,3:Chinese:Hunan Province:changsha:yuhua:shazitang$3:India:Hunan Province:changsha:yuhua:shazitang,28-12-2015$28-12-2015:28-12-2015,1734,1778
-100022,155,3ROM size,29-11-2015,$2BB100022,MAC91$MAC92,5:Chinese:Hunan Province:zhuzhou:tianyuan:tianyua$5:India:Hunan Province:zhuzhou:tianyuan:tianyua,29-12-2015$29-12-2015:29-12-2015,155,1999
-100023,1386,8ROM size,29-11-2015,$2BB100023,MAC94$MAC95,2:Chinese:Guangdong Province:guangzhou:longhua:mingzhi$2:India:Guangdong Province:guangzhou:longhua:mingzhi,30-12-2015$30-12-2015:30-12-2015,1386,2194
-100024,1017,9ROM size,29-11-2015,$2BB100024,MAC97$MAC98,3:Chinese:Hunan Province:changsha:yuhua:shazitang$3:India:Hunan Province:changsha:yuhua:shazitang,,1017,2483
-100025,47,2ROM size,29-11-2015,$2BB100025,$MAC101$MAC102,2:Chinese:Guangdong Province:guangzhou:longhua:mingzhi$2:India:Guangdong Province:guangzhou:longhua:mingzhi,,47,1724
-100026,2930,7ROM size,29-11-2015,$2BB100026,$MAC104$MAC105,7:Chinese:Hubei Province:yichang:yichang:yichang$7:India:New Delhi:delhi:delhi:delhi,,2930,1768
-100027,2940,0ROM size,29-11-2015,$2BB100027,$MAC107$MAC108,5:Chinese:Hunan Province:zhuzhou:tianyuan:tianyua$5:India:Hunan Province:zhuzhou:tianyuan:tianyua,,2940,2436
-100028,297,5ROM size,29-11-2015,$2BB100028,$MAC110$MAC111,5:Chinese:Hunan Province:zhuzhou:tianyuan:tianyua$5:India:Hunan Province:zhuzhou:tianyuan:tianyua,,297,2849
-100029,1695,2ROM size,29-11-2015,$2BB100029,$MAC113$MAC114,4:Chinese:Hunan Province:xiangtan:xiangtan:jianshelu$4:India:Hunan Province:xiangtan:xiangtan:jianshelu,,1695,1691
-10003,1326,7ROM size,29-11-2015,2BB10003,$MAC116$MAC117,4:Chinese:Hunan Province:xiangtan:xiangtan:jianshelu$4:India:Hunan Province:xiangtan:xiangtan:jianshelu,,1326,2071
-100030,513,7ROM size,29-11-2015,$2BB100030,$MAC119$MAC120,5:Chinese:Hunan Province:zhuzhou:tianyuan:tianyua$5:India:Hunan Province:zhuzhou:tianyuan:tianyua,07-01-2016$07-01-2016:,513,1333
-100031,1741,1ROM size,29-11-2015,$2BB100031,$MAC122$MAC123,7:Chinese:Hubei Province:yichang:yichang:yichang$7:India:New Delhi:delhi:delhi:delhi,08-01-2016$08-01-2016:,1741,1080
-100032,1198,0ROM size,29-11-2015,$2BB100032,$MAC125$MAC126,1:Chinese:Guangdong Province:shenzhen:longgang:matishan$1:India:Guangdong Province:shenzhen:longgang:matishan,09-01-2016$09-01-2016:,1198,1053
-100033,273,9ROM size,29-11-2015,$2BB100033,$MAC128$MAC129,6:Chinese:Hubei Province:wuhan:hongshan:hongshan$6:India:New Delhi:wuhan:hongshan:hongshan,10-01-2016$10-01-2016:,273,760
-100034,1234,6ROM size,29-11-2015,$2BB100034,$MAC131$MAC132,2:Chinese:Guangdong Province:guangzhou:longhua:mingzhi$2:India:Guangdong Province:guangzhou:longhua:mingzhi,11-01-2016$11-01-2016:,1234,2061
-100035,1619,1ROM size,29-11-2015,$2BB100035,$MAC134$MAC135,3:Chinese:Hunan Province:changsha:yuhua:shazitang$3:India:Hunan Province:changsha:yuhua:shazitang,12-01-2016$12-01-2016:,1619,2142
-100036,2415,2ROM size,29-11-2015,$2BB100036,$MAC137$MAC138,3:Chinese:Hunan Province:changsha:yuhua:shazitang$3:India:Hunan Province:changsha:yuhua:shazitang,13-01-2016$13-01-2016:,2415,2224
-100037,2381,2ROM size,29-11-2015,$2BB100037,$MAC140$MAC141,4:Chinese:Hunan Province:xiangtan:xiangtan:jianshelu$4:India:Hunan Province:xiangtan:xiangtan:jianshelu,14-01-2016$14-01-2016:,2381,1015
-100038,872,7ROM size,29-11-2015,1AA100038$2BB100038,$MAC143$MAC144,1:Chinese:Guangdong Province:shenzhen:longgang:matishan$1:India:Guangdong Province:shenzhen:longgang:matishan,15-01-2016$15-01-2016,872,1229
-100039,1835,9ROM size,29-11-2015,1AA100039$2BB100039,$$MAC147,1:Chinese:Guangdong Province:shenzhen:longgang:matishan$1:India:Guangdong Province:shenzhen:longgang:matishan,16-01-2016$16-01-2016,1835,1750
-10004,2597,1ROM size,29-11-2015,1AA10004$2BB10004,$$MAC150,2:Chinese:Guangdong Province:guangzhou:longhua:mingzhi$2:India:Guangdong Province:guangzhou:longhua:mingzhi,17-01-2016$17-01-2016,2597,1717
-100040,1969,9ROM size,29-11-2015,1AA100040$2BB100040,$$MAC153,,18-01-2016$18-01-2016,1969,2078
-100041,2133,8ROM size,29-11-2015,$,$$MAC156,,19-01-2016$19-01-2016,2133,2734
-100042,631,9ROM size,29-11-2015,$,$$MAC159,,20-01-2016$20-01-2016,631,2745
-100043,187,4ROM size,29-11-2015,$,$$MAC162,2:Chinese::guangzhou:longhua:mingzhi$2:India::guangzhou:longhua:mingzhi,21-01-2016$21-01-2016,187,571
-100044,1232,5ROM size,29-11-2015,$,$$MAC165,2::Guangdong Province:guangzhou:longhua:mingzhi$2::Guangdong Province:guangzhou:longhua:mingzhi,22-01-2016$22-01-2016,1232,1697
-100045,1602,6ROM size,29-11-2015,$,$$MAC168,4:Chinese:Hunan Province::xiangtan:jianshelu$4:India:Hunan Province::xiangtan:jianshelu,23-01-2016$23-01-2016,1602,2553
-100046,2319,9ROM size,29-11-2015,$,$$MAC171,2:Chinese:Guangdong Province:guangzhou::mingzhi$2:India:Guangdong Province:guangzhou::mingzhi,24-01-2016$24-01-2016,2319,1077
-100047,839,4ROM size,29-11-2015,$,$$MAC174,5:Chinese:Hunan Province:zhuzhou:tianyuan:$5:India:Hunan Province:zhuzhou:tianyuan:,25-01-2016$25-01-2016,839,1823
-100048,1184,2ROM size,29-11-2015,$,$$MAC177,2:Chinese:Guangdong Province:guangzhou:longhua:mingzhi$2:India:Guangdong Province:guangzhou:longhua:mingzhi,26-01-2016$:,1184,2399
-100049,2705,2ROM size,29-11-2015,$,$$MAC180,2:Chinese:Guangdong Province$2:India:Guangdong Province,27-01-2016$:,2705,2890
-10005,1185,1ROM size,29-11-2015,,$$MAC183,4:Chinese:Hunan Province:xiangtan:xiangtan:jianshelu$4:India:Hunan Province:xiangtan:xiangtan:jianshelu,28-01-2016$:,1185,1608
-100050,2457,9ROM size,29-11-2015,,$$MAC186,7:Chinese:Hubei Province:yichang:yichang:yichang$7:India:New Delhi:delhi:delhi:delhi,29-01-2016$:,2457,29
-100051,2320,8ROM size,29-11-2015,,$$MAC189,2:Chinese:Guangdong Province:guangzhou:longhua:mingzhi$2:India:Guangdong Province:guangzhou:longhua:mingzhi,30-01-2016$:,2320,1407
-100052,2300,0ROM size,29-11-2015,,$$,5:Chinese:Hunan Province:zhuzhou:tianyuan:tianyua$5:India:Hunan Province:zhuzhou:tianyuan:tianyua,31-01-2016$:,2300,845
-100053,1210,4ROM size,29-11-2015,,$$,6:Chinese:Hubei Province:wuhan:hongshan:hongshan$6:India:New Delhi:wuhan:hongshan:hongshan,01-02-2016$:,1210,1655
-100054,1689,8ROM size,29-11-2015,,$$,1:Chinese:Guangdong Province:shenzhen:longgang:matishan$1:India:Guangdong Province:shenzhen:longgang:matishan,02-02-2016$:,1689,1368
-100055,2823,2ROM size,29-11-2015,,$$,7:Chinese:Hubei Province:yichang:yichang:yichang$7:India:New Delhi:delhi:delhi:delhi,$03-02-2016:03-02-2016,2823,1728
-100056,68,6ROM size,29-11-2015,,$$,6:Chinese:Hubei Province:wuhan:hongshan:hongshan$6:India:New Delhi:wuhan:hongshan:hongshan,$04-02-2016:04-02-2016,68,750
-100057,716,0ROM size,29-11-2015,,$$,5:Chinese:Hunan Province:zhuzhou:tianyuan:tianyua$5:India:Hunan Province:zhuzhou:tianyuan:tianyua,$05-02-2016:05-02-2016,716,2288
-100058,864,6ROM size,29-11-2015,,$$,2:Chinese:Guangdong Province:guangzhou:longhua:mingzhi$2:India:Guangdong Province:guangzhou:longhua:mingzhi,$06-02-2016:06-02-2016,864,2635
-100059,499,6ROM size,29-11-2015,,$$,1:Chinese:Guangdong Province:shenzhen:longgang:matishan$1:India:Guangdong Province:shenzhen:longgang:matishan,$07-02-2016:07-02-2016,499,1337
-10006,1429,3ROM size,29-11-2015,,$$,:Chinese:Guangdong Province:guangzhou:longhua:mingzhi$:India:Guangdong Province:guangzhou:longhua:mingzhi,$08-02-2016:08-02-2016,1429,2478
-100060,2176,2ROM size,29-11-2015,,$$,:Chinese:Hunan Province:xiangtan:xiangtan:jianshelu$:India:Hunan Province:xiangtan:xiangtan:jianshelu,$09-02-2016:09-02-2016,2176,538
-100061,2563,7ROM size,29-11-2015,,,3:Chinese:Hunan Province:changsha:yuhua:shazitang$3:India:Hunan Province:changsha:yuhua:shazitang,$10-02-2016:10-02-2016,2563,1407
-100062,2594,3ROM size,29-11-2015,,,7:Chinese:Hubei Province:yichang:yichang:yichang$7:India:New Delhi:delhi:delhi:delhi,$11-02-2016:11-02-2016,2594,2952
-100063,2142,1ROM size,29-11-2015,,,7:Chinese:Hubei Province:yichang:yichang:yichang$7:India:New Delhi:delhi:delhi:delhi,$12-02-2016:12-02-2016,2142,1226
-100064,138,0ROM size,29-11-2015,1AA100064$2BB100064,,5:Chinese:Hunan Province:zhuzhou:tianyuan:tianyua$5:India:Hunan Province:zhuzhou:tianyuan:tianyua,$13-02-2016:13-02-2016,138,865
-100065,1168,6ROM size,29-11-2015,1AA100065$2BB100065,,4:Chinese:Hunan Province:xiangtan:xiangtan:jianshelu$4:India:Hunan Province:xiangtan:xiangtan:jianshelu,$14-02-2016:14-02-2016,1168,901
-100066,2828,5ROM size,29-11-2015,1AA100066$2BB100066,,5:Chinese:Hunan Province:zhuzhou:tianyuan:tianyua$5:India:Hunan Province:zhuzhou:tianyuan:tianyua,$:,2828,1864
-100067,1160,0ROM size,29-11-2015,1AA100067$2BB100067,,6:Chinese:Hubei Province:wuhan:hongshan:hongshan$6:India:New Delhi:wuhan:hongshan:hongshan,$:,1160,572
-100068,1890,6ROM size,29-11-2015,1AA100068$2BB100068,,2:Chinese:Guangdong Province:guangzhou:longhua:mingzhi$2:India:Guangdong Province:guangzhou:longhua:mingzhi,$:,1890,412
-100069,1195,4ROM size,29-11-2015,1AA100069$2BB100069,,4:Chinese:Hunan Province:xiangtan:xiangtan:jianshelu$4:India:Hunan Province:xiangtan:xiangtan:jianshelu,$:,1195,1491
-10007,2797,9ROM size,29-11-2015,1AA10007$2BB10007,,4:Chinese:Hunan Province:xiangtan:xiangtan:jianshelu$4:India:Hunan Province:xiangtan:xiangtan:jianshelu,$:,2797,1350
-100070,44,5ROM size,29-11-2015,1AA100070$2BB100070,,2:Chinese:Guangdong Province:guangzhou:longhua:mingzhi$2:India:Guangdong Province:guangzhou:longhua:mingzhi,$:,44,1567
-100071,1683,6ROM size,29-11-2015,1AA100071$2BB100071,,2:Chinese:Guangdong Province:guangzhou:longhua:mingzhi$2:India:Guangdong Province:guangzhou:longhua:mingzhi,$:,1683,1973
-100072,1085,1ROM size,29-11-2015,1AA100072$2BB100072,,3:Chinese:Hunan Province:changsha:yuhua:shazitang$3:India:Hunan Province:changsha:yuhua:shazitang,22-02-2016$22-02-2016:22-02-2016,1085,448
-100073,776,7ROM size,29-11-2015,1AA100073$2BB100073,,5:Chinese:Hunan Province:zhuzhou:tianyuan:tianyua$5:India:Hunan Province:zhuzhou:tianyuan:tianyua,23-02-2016$23-02-2016:23-02-2016,776,2488
-100074,2074,9ROM size,29-11-2015,1AA100074$2BB100074,MAC262$MAC263$,6:Chinese:Hubei Province:wuhan:hongshan:hongshan$6:India:New Delhi:wuhan:hongshan:hongshan,24-02-2016$24-02-2016:24-02-2016,2074,907
-100075,1062,2ROM size,29-11-2015,1AA100075$2BB100075,MAC265$MAC266$,1:Chinese:Guangdong Province:shenzhen:longgang:matishan$1:India:Guangdong Province:shenzhen:longgang:matishan,25-02-2016$25-02-2016:25-02-2016,1062,2507
-100076,987,7ROM size,29-11-2015,1AA100076$2BB100076,MAC268$MAC269$,6:Chinese:Hubei Province:wuhan:hongshan:hongshan$6:India:New Delhi:wuhan:hongshan:hongshan,26-02-2016$26-02-2016:26-02-2016,987,732
-100077,2799,9ROM size,29-11-2015,1AA100077$2BB100077,MAC271$MAC272$,7:Chinese:Hubei Province:yichang:yichang:yichang$7:India:New Delhi:delhi:delhi:delhi,27-02-2016$27-02-2016:27-02-2016,2799,2077
-100078,2765,1ROM size,29-11-2015,1AA100078$2BB100078,MAC274$MAC275$,7:Chinese:Hubei Province:yichang:yichang:yichang$7:India:New Delhi:delhi:delhi:delhi,28-02-2016$28-02-2016:28-02-2016,2765,1434
-100079,2164,1ROM size,29-11-2015,1AA100079$2BB100079,MAC277$MAC278$,4:Chinese:Hunan Province:xiangtan:xiangtan:jianshelu$4:India:Hunan Province:xiangtan:xiangtan:jianshelu,29-02-2016$29-02-2016:29-02-2016,2164,1098
-10008,1624,6ROM size,29-11-2015,1AA10008$2BB10008,MAC280$MAC281$,1:Chinese:Guangdong Province:shenzhen:longgang:matishan$1:India:Guangdong Province:shenzhen:longgang:matishan,01-03-2016$01-03-2016:01-03-2016,1624,813
-100080,2355,1ROM size,29-11-2015,1AA100080$2BB100080,MAC283$MAC284$MAC285,1:Chinese:Guangdong Province:shenzhen:longgang:matishan$1:India:Guangdong Province:shenzhen:longgang:matishan,02-03-2016$02-03-2016:02-03-2016,2355,954
-100081,1650,6ROM size,29-11-2015,1AA100081$2BB100081,MAC286$MAC287$MAC288,1:Chinese:Guangdong Province:shenzhen:longgang:matishan$1:India:Guangdong Province:shenzhen:longgang:matishan,03-03-2016$03-03-2016:03-03-2016,1650,613
-100082,2761,3ROM size,29-11-2015,1AA100082$2BB100082,MAC289$MAC290$MAC291,4:Chinese:Hunan Province:xiangtan:xiangtan:jianshelu$4:India:Hunan Province:xiangtan:xiangtan:jianshelu,04-03-2016$04-03-2016:04-03-2016,2761,2348
-100083,1856,3ROM size,29-11-2015,1AA100083$2BB100083,MAC292$MAC293$MAC294,5:Chinese:Hunan Province:zhuzhou:tianyuan:tianyua$5:India:Hunan Province:zhuzhou:tianyuan:tianyua,05-03-2016$05-03-2016:05-03-2016,1856,2192
-100084,1841,7ROM size,29-11-2015,1AA100084$2BB100084,MAC295$MAC296$MAC297,2:Chinese:Guangdong Province:guangzhou:longhua:mingzhi$2:India:Guangdong Province:guangzhou:longhua:mingzhi,06-03-2016$06-03-2016:06-03-2016,1841,2826
-100085,1841,7ROM size,29-11-2015,1AA100084$2BB100084,MAC295$MAC296$MAC297,2:Chinese:Guangdong Province:guangzhou:longhua:mingzhi$2:India:Guangdong Province:guangzhou:longhua:mingzhi,06-03-2016$06-03-2016:06-03-2016,1841,2826
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/examples/spark/src/main/resources/data.csv
----------------------------------------------------------------------
diff --git a/examples/spark/src/main/resources/data.csv b/examples/spark/src/main/resources/data.csv
deleted file mode 100644
index 5696978..0000000
--- a/examples/spark/src/main/resources/data.csv
+++ /dev/null
@@ -1,11 +0,0 @@
-ID,date,country,name,phonetype,serialname,salary,floatField
-1,2015/7/23,china,aaa1,phone197,ASD69643,15000,2.34
-2,2015/7/24,china,aaa2,phone756,ASD42892,15001,2.34
-3,2015/7/25,china,aaa3,phone1904,ASD37014,15002,2.34
-4,2015/7/26,china,aaa4,phone2435,ASD66902,15003,2.34
-5,2015/7/27,china,aaa5,phone2441,ASD90633,15004,2.34
-6,2015/7/28,china,aaa6,phone294,ASD59961,15005,3.5
-7,2015/7/29,china,aaa7,phone610,ASD14875,15006,2.34
-8,2015/7/30,china,aaa8,phone1848,ASD57308,15007,2.34
-9,2015/7/18,china,aaa9,phone706,ASD86717,15008,2.34
-10,2015/7/19,usa,aaa10,phone685,ASD30505,15009,2.34
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/examples/spark/src/main/resources/dimSample.csv
----------------------------------------------------------------------
diff --git a/examples/spark/src/main/resources/dimSample.csv b/examples/spark/src/main/resources/dimSample.csv
deleted file mode 100644
index 0c8f27a..0000000
--- a/examples/spark/src/main/resources/dimSample.csv
+++ /dev/null
@@ -1,21 +0,0 @@
-id,name,city
-1,David,Beijing
-2,Mark,Paris
-3,Bill,NewYork
-4,Sara,Tokyo
-5,John,Beijing
-6,Michel,Chicago
-7,Robert,Houston
-8,Sunny,Boston
-9,Mary,Tokyo
-10,Edward,Paris
-11,James,Washington
-12,Maria,Berlin
-13,Adam,Athens
-14,Peter,Boston
-15,George,Paris
-16,Paul,Shanghai
-17,Lisa,Hangzhou
-18,Angel,Beijing
-19,Emily,Bangalore
-20,Kevin,Singapore
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/examples/spark/src/main/resources/factSample.csv
----------------------------------------------------------------------
diff --git a/examples/spark/src/main/resources/factSample.csv b/examples/spark/src/main/resources/factSample.csv
deleted file mode 100644
index 9693156..0000000
--- a/examples/spark/src/main/resources/factSample.csv
+++ /dev/null
@@ -1,51 +0,0 @@
-id,name,city,salary
-1,David,Beijing,15000
-1,David,Tokyo,20000
-1,David,Hangzhou,18000
-2,Mark,Paris,12000
-2,Mark,Boston,15000
-2,Mark,Chicago,18000
-3,Bill,NewYork,20000
-3,Bill,Boston,23000
-4,Sara,Tokyo,11000
-4,Sara,Paris,15000
-4,Sara,Chicago,21000
-4,Sara,Hangzhou,17000
-5,John,Beijing,15000
-5,John,Shanghai,16000
-6,Michel,Chicago,11000
-6,Michel,Boston,12000
-6,Michel,Tokyo,11000
-8,Sunny,Boston,14000
-8,Sunny,Beijing,22000
-8,Sunny,Tokyo,20000
-9,Mary,Tokyo,13000
-9,Mary,NewYork,18000
-9,Mary,Paris,16000
-9,Mary,Washington,20000
-9,Mary,Boston,17000
-10,Edward,Paris,20000
-10,Edward,Beijing,12000
-10,Edward,Berlin,15000
-11,James,Washington,16000
-12,Maria,Berlin,15000
-12,Maria,Beijing,16000
-13,Adam,Athens,21000
-13,Adam,Berlin,18000
-13,Adam,Hangzhou,17000
-14,Peter,Boston,20000
-14,Peter,Berlin,21000
-14,Peter,Shanghai,18000
-15,George,Paris,17000
-15,George,Tokyo,12000
-15,George,Beijing,15000
-15,George,Berlin,18000
-16,Paul,Shanghai,22000
-16,Paul,Tokyo,19000
-16,Paul,Paris,24000
-16,Paul,Hangzhou,22000
-18,Angel,Beijing,22000
-18,Angel,NewYork,25000
-18,Angel,Tokyo,22000
-20,Kevin,Singapore,18000
-20,Kevin,Bangalore,16000
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/examples/spark/src/main/scala/org/apache/carbondata/examples/AllDictionaryExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/AllDictionaryExample.scala b/examples/spark/src/main/scala/org/apache/carbondata/examples/AllDictionaryExample.scala
deleted file mode 100644
index 9012dcf..0000000
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/AllDictionaryExample.scala
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.examples
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.examples.util.{AllDictionaryUtil, ExampleUtils}
-
-object AllDictionaryExample {
-
- def main(args: Array[String]) {
- val cc = ExampleUtils.createCarbonContext("AllDictionaryExample")
- val testData = ExampleUtils.currentPath + "/src/main/resources/data.csv"
- val csvHeader = "ID,date,country,name,phonetype,serialname,salary"
- val dictCol = "|date|country|name|phonetype|serialname|"
- val allDictFile = ExampleUtils.currentPath + "/src/main/resources/data.dictionary"
- // extract all dictionary files from source data
- AllDictionaryUtil.extractDictionary(cc.sparkContext,
- testData, allDictFile, csvHeader, dictCol)
- // Specify date format based on raw data
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "yyyy/MM/dd")
-
- cc.sql("DROP TABLE IF EXISTS t3")
-
- cc.sql("""
- CREATE TABLE IF NOT EXISTS t3
- (ID Int, date Date, country String,
- name String, phonetype String, serialname String, salary Int,floatField float)
- STORED BY 'carbondata'
- """)
-
- cc.sql(s"""
- LOAD DATA LOCAL INPATH '$testData' into table t3
- options('ALL_DICTIONARY_PATH'='$allDictFile')
- """)
-
- cc.sql("""
- SELECT * FROM t3
- """).show()
-
- cc.sql("""
- SELECT * FROM t3 where floatField=3.5
- """).show()
-
- cc.sql("DROP TABLE IF EXISTS t3")
-
- // clean local dictionary files
- AllDictionaryUtil.cleanDictionary(allDictFile)
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/examples/spark/src/main/scala/org/apache/carbondata/examples/AlluxioExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/AlluxioExample.scala b/examples/spark/src/main/scala/org/apache/carbondata/examples/AlluxioExample.scala
deleted file mode 100644
index 12901b5..0000000
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/AlluxioExample.scala
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.examples
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datastore.impl.FileFactory
-import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.examples.util.ExampleUtils
-
-/**
- * configure alluxio:
- * 1.start alluxio
- * 2.upload the jar :"/alluxio_path/core/client/target/
- * alluxio-core-client-YOUR-VERSION-jar-with-dependencies.jar"
- * 3.Get more detail at:http://www.alluxio.org/docs/master/en/Running-Spark-on-Alluxio.html
- */
-
-object AlluxioExample {
- def main(args: Array[String]) {
- val cc = ExampleUtils.createCarbonContext("AlluxioExample")
- cc.sparkContext.hadoopConfiguration.set("fs.alluxio.impl", "alluxio.hadoop.FileSystem")
- FileFactory.getConfiguration.set("fs.alluxio.impl", "alluxio.hadoop.FileSystem")
-
- // Specify date format based on raw data
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "yyyy/MM/dd")
-
- cc.sql("DROP TABLE IF EXISTS t3")
-
- cc.sql("""
- CREATE TABLE IF NOT EXISTS t3
- (ID Int, date Date, country String,
- name String, phonetype String, serialname String, salary Int)
- STORED BY 'carbondata'
- """)
-
- cc.sql(s"""
- LOAD DATA LOCAL INPATH 'alluxio://localhost:19998/data.csv' into table t3
- """)
-
- cc.sql("""
- SELECT country, count(salary) AS amount
- FROM t3
- WHERE country IN ('china','france')
- GROUP BY country
- """).show()
-
- cc.sql("DROP TABLE IF EXISTS t3")
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/examples/spark/src/main/scala/org/apache/carbondata/examples/CarbonExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/CarbonExample.scala b/examples/spark/src/main/scala/org/apache/carbondata/examples/CarbonExample.scala
deleted file mode 100644
index 36013a8..0000000
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/CarbonExample.scala
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.examples
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.examples.util.ExampleUtils
-
-object CarbonExample {
-
- def main(args: Array[String]) {
- val cc = ExampleUtils.createCarbonContext("CarbonExample")
- val testData = ExampleUtils.currentPath + "/src/main/resources/data.csv"
-
- // Specify date format based on raw data
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "yyyy/MM/dd")
-
- cc.sql("DROP TABLE IF EXISTS t3")
-
- // Create table, 6 dimensions, 1 measure
- cc.sql("""
- CREATE TABLE IF NOT EXISTS t3
- (ID Int, date Date, country String,
- name String, phonetype String, serialname char(10), salary Int)
- STORED BY 'carbondata'
- """)
-
- // Load data
- cc.sql(s"""
- LOAD DATA LOCAL INPATH '$testData' into table t3
- """)
-
- // Perform a query
- cc.sql("""
- SELECT country, count(salary) AS amount
- FROM t3
- WHERE country IN ('china','france')
- GROUP BY country
- """).show()
-
- // Drop table
- cc.sql("DROP TABLE IF EXISTS t3")
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/examples/spark/src/main/scala/org/apache/carbondata/examples/CarbonPartitionExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/CarbonPartitionExample.scala b/examples/spark/src/main/scala/org/apache/carbondata/examples/CarbonPartitionExample.scala
deleted file mode 100644
index 9ceadea..0000000
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/CarbonPartitionExample.scala
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.examples
-
-import scala.collection.mutable.LinkedHashMap
-
-import org.apache.spark.sql.AnalysisException
-
-import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.examples.util.ExampleUtils
-
-object CarbonPartitionExample {
-
- def main(args: Array[String]) {
- val cc = ExampleUtils.createCarbonContext("CarbonPartitionExample")
- val testData = ExampleUtils.currentPath + "/src/main/resources/data.csv"
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
- val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
- // none partition table
- cc.sql("DROP TABLE IF EXISTS t0")
- cc.sql("""
- | CREATE TABLE IF NOT EXISTS t0
- | (
- | vin String,
- | logdate Timestamp,
- | phonenumber Int,
- | country String,
- | area String
- | )
- | STORED BY 'carbondata'
- """.stripMargin)
-
- // range partition
- cc.sql("DROP TABLE IF EXISTS t1")
- cc.sql("""
- | CREATE TABLE IF NOT EXISTS t1(
- | vin STRING,
- | phonenumber INT,
- | country STRING,
- | area STRING
- | )
- | PARTITIONED BY (logdate TIMESTAMP)
- | STORED BY 'carbondata'
- | TBLPROPERTIES('PARTITION_TYPE'='RANGE',
- | 'RANGE_INFO'='2014/01/01,2015/01/01,2016/01/01')
- """.stripMargin)
-
- // hash partition
- cc.sql("""
- | CREATE TABLE IF NOT EXISTS t3(
- | logdate Timestamp,
- | phonenumber Int,
- | country String,
- | area String
- | )
- | PARTITIONED BY (vin String)
- | STORED BY 'carbondata'
- | TBLPROPERTIES('PARTITION_TYPE'='HASH','NUM_PARTITIONS'='5')
- """.stripMargin)
-
- // list partition
- cc.sql("DROP TABLE IF EXISTS t5")
- cc.sql("""
- | CREATE TABLE IF NOT EXISTS t5(
- | vin String,
- | logdate Timestamp,
- | phonenumber Int,
- | area String
- | )
- | PARTITIONED BY (country string)
- | STORED BY 'carbondata'
- | TBLPROPERTIES('PARTITION_TYPE'='LIST',
- | 'LIST_INFO'='(China,United States),UK ,japan,(Canada,Russia), South Korea ')
- """.stripMargin)
-
- cc.sql(s"DROP TABLE IF EXISTS partitionDB.t9")
- cc.sql(s"DROP DATABASE IF EXISTS partitionDB")
- cc.sql(s"CREATE DATABASE partitionDB")
- cc.sql(s"""
- | CREATE TABLE IF NOT EXISTS partitionDB.t9(
- | logdate Timestamp,
- | phonenumber Int,
- | country String,
- | area String
- | )
- | PARTITIONED BY (vin String)
- | STORED BY 'carbondata'
- | TBLPROPERTIES('PARTITION_TYPE'='HASH','NUM_PARTITIONS'='5')
- """.stripMargin)
- // hive partition table
- cc.sql("DROP TABLE IF EXISTS t7")
- cc.sql("""
- | create table t7(id int, name string) partitioned by (city string)
- | row format delimited fields terminated by ','
- """.stripMargin)
- cc.sql("alter table t7 add partition (city = 'Hangzhou')")
- // hive partition table
- cc.sql(s"DROP TABLE IF EXISTS hiveDB.t7")
- cc.sql(s"CREATE DATABASE IF NOT EXISTS hiveDB")
- cc.sql("""
- | create table hiveDB.t7(id int, name string) partitioned by (city string)
- | row format delimited fields terminated by ','
- """.stripMargin)
- cc.sql("alter table hiveDB.t7 add partition (city = 'Shanghai')")
- // show partitions
- try {
- cc.sql("SHOW PARTITIONS t0").show(100, false)
- } catch {
- case ex: AnalysisException => LOGGER.error(ex.getMessage())
- }
- cc.sql("SHOW PARTITIONS t1").show(100, false)
- cc.sql("SHOW PARTITIONS t3").show(100, false)
- cc.sql("SHOW PARTITIONS t5").show(100, false)
- cc.sql("SHOW PARTITIONS t7").show(100, false)
- cc.sql("use hiveDB").show()
- cc.sql("SHOW PARTITIONS t7").show(100, false)
- cc.sql("use default").show()
- cc.sql("SHOW PARTITIONS partitionDB.t9").show(100, false)
-
- cc.sql("DROP TABLE IF EXISTS t0")
- cc.sql("DROP TABLE IF EXISTS t1")
- cc.sql("DROP TABLE IF EXISTS t3")
- cc.sql("DROP TABLE IF EXISTS t5")
- cc.sql("DROP TABLE IF EXISTS t7")
- cc.sql(s"DROP TABLE IF EXISTS hiveDb.t7")
- cc.sql(s"DROP TABLE IF EXISTS partitionDB.t9")
- cc.sql(s"DROP DATABASE IF EXISTS partitionDB")
-
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/examples/spark/src/main/scala/org/apache/carbondata/examples/CaseClassDataFrameAPIExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/CaseClassDataFrameAPIExample.scala b/examples/spark/src/main/scala/org/apache/carbondata/examples/CaseClassDataFrameAPIExample.scala
deleted file mode 100644
index 21bd002..0000000
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/CaseClassDataFrameAPIExample.scala
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.examples
-
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.{DataFrame, SaveMode}
-
-import org.apache.carbondata.examples.util.ExampleUtils
-
-case class People(name: String, occupation: String, id: Int)
-
-object CaseClassDataFrameAPIExample {
-
- def main(args: Array[String]) {
- val cc = ExampleUtils.createCarbonContext("CaseClassDataFrameAPIExample")
- import cc.implicits._
-
- val people = List(People("sangeeta", "engineer", 1), People("pallavi", "consultant", 2))
- val peopleRDD: RDD[People] = cc.sc.parallelize(people)
- val peopleDF: DataFrame = peopleRDD.toDF("name", "occupation", "id")
-
- // writing data to carbon table
- peopleDF.write
- .format("carbondata")
- .option("tableName", "carbon2")
- .option("compress", "true")
- .mode(SaveMode.Overwrite)
- .save()
-
- cc.sql("SELECT * FROM carbon2").show()
-
- cc.sql("DROP TABLE IF EXISTS carbon2")
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/examples/spark/src/main/scala/org/apache/carbondata/examples/ComplexTypeExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/ComplexTypeExample.scala b/examples/spark/src/main/scala/org/apache/carbondata/examples/ComplexTypeExample.scala
deleted file mode 100644
index 992c3f9..0000000
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/ComplexTypeExample.scala
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.examples
-
-import org.apache.carbondata.examples.util.ExampleUtils
-
-/**
- * Carbon supports the complex types ARRAY and STRUCT.
- * The complex type columns can be used with all SQL clauses.
- */
-object ComplexTypeExample {
-
- def main(args: Array[String]) {
- val cc = ExampleUtils.createCarbonContext("ComplexTypeExample")
- val dataPath = ExampleUtils.currentPath + "/src/main/resources/complexdata.csv"
- val tableName = "complexTypeTable"
-
- cc.sql(s"DROP TABLE IF EXISTS $tableName")
- cc.sql(s"""CREATE TABLE $tableName (
- deviceInformationId int,
- channelsId string,
- ROMSize string,
- purchasedate string,
- mobile struct<imei:string,
- imsi:string>,
- MAC array<string>,
- locationinfo array<struct<ActiveAreaId:int,
- ActiveCountry:string,
- ActiveProvince:string,
- Activecity:string,
- ActiveDistrict:string,
- ActiveStreet:string>>,
- proddate struct<productionDate: string,
- activeDeactivedate: array<string>>,
- gamePointId double,
- contractNumber double)
- STORED BY 'org.apache.carbondata.format' """)
-
- cc.sql(s"load data local inpath '$dataPath' into table $tableName " +
- "options ('COMPLEX_DELIMITER_LEVEL_1'='$', 'COMPLEX_DELIMITER_LEVEL_2'=':')")
-
- // filter on complex ARRAY type with index filter
- cc.sql(s"SELECT mobile, proddate.activeDeactivedate, MAC[0] FROM $tableName " +
- "WHERE MAC[0] LIKE 'MAC1%'").show
-
- // filter on complex STRUCT type
- cc.sql(s"SELECT mobile, proddate.activeDeactivedate FROM $tableName " +
- "WHERE mobile.imei = '1AA1' or mobile.imsi = ''").show
-
- // filter on complex STRUCT<ARRAY>
- cc.sql(s"SELECT mobile, proddate.activeDeactivedate[0] FROM $tableName " +
- "WHERE proddate.activeDeactivedate[0] = '29-11-2015'").show
-
- // filter on complex ARRAY<STRUCT>
- cc.sql(s"SELECT mobile, locationinfo[0] FROM $tableName " +
- "WHERE locationinfo[0].ActiveCountry = 'Chinese'").show
-
- // complex type aggregation and group by complex type
- cc.sql(s"SELECT mobile, count(proddate) FROM $tableName GROUP BY mobile").show
-
- cc.sql(s"DROP TABLE IF EXISTS $tableName")
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/examples/spark/src/main/scala/org/apache/carbondata/examples/DataFrameAPIExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/DataFrameAPIExample.scala b/examples/spark/src/main/scala/org/apache/carbondata/examples/DataFrameAPIExample.scala
deleted file mode 100644
index db5def9..0000000
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/DataFrameAPIExample.scala
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.examples
-
-import org.apache.carbondata.examples.util.ExampleUtils
-
-// scalastyle:off println
-object DataFrameAPIExample {
-
- def main(args: Array[String]) {
- val cc = ExampleUtils.createCarbonContext("DataFrameAPIExample")
- ExampleUtils.writeSampleCarbonFile(cc, "carbon1", 1000)
-
- // use datasource api to read
- val in = cc.read
- .format("carbondata")
- .option("tableName", "carbon1")
- .load()
-
- import cc.implicits._
- var count = in.where($"c3" > 500).select($"*").count()
- println(s"count after 1 load: $count")
-
- // append new data, query answer should be 1000
- ExampleUtils.appendSampleCarbonFile(cc, "carbon1")
- count = in.where($"c3" > 500).select($"*").count()
- println(s"count after 2 load: $count")
-
- // use SQL to read
- cc.sql("SELECT c1, count(c3) FROM carbon1 where c3 > 500 group by c1 limit 10").show
-
- // delete carbondata file
- ExampleUtils.cleanSampleCarbonFile(cc, "carbon1")
- }
-}
-// scalastyle:on println
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/examples/spark/src/main/scala/org/apache/carbondata/examples/DataManagementExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/DataManagementExample.scala b/examples/spark/src/main/scala/org/apache/carbondata/examples/DataManagementExample.scala
deleted file mode 100644
index 551a008..0000000
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/DataManagementExample.scala
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.examples
-
-import org.apache.carbondata.examples.util.ExampleUtils
-
-object DataManagementExample {
- def main(args: Array[String]) {
- val cc = ExampleUtils.createCarbonContext("DataManagementExample")
-
- cc.sql("DROP TABLE IF EXISTS t3")
-
- // create a table using CarbonData
- cc.sql(
- """
- CREATE TABLE IF NOT EXISTS t3
- (ID Int, date Timestamp, country String,
- name String, phonetype String, serialname String, salary Int)
- STORED BY 'carbondata'
- """
- )
-
- // data.csv has 1000 lines
- val testData = ExampleUtils.currentPath + "/src/main/resources/data.csv"
-
- // load data 5 times, each load of data is called a segment in CarbonData
- (1 to 5).map { i =>
- cc.sql(s"LOAD DATA LOCAL INPATH '$testData' into table t3")
- }
- cc.sql("SHOW SEGMENTS FOR TABLE t3 ").show
-
- // delete the first segment
- cc.sql("DELETE FROM TABLE T3 WHERE SEGMENT.ID IN (0)")
- cc.sql("SHOW SEGMENTS FOR TABLE t3 LIMIT 10").show
-
- // this query will be executed on last 4 segments, it should return 4000 rows
- cc.sql("SELECT count(*) AS amount FROM t3").show
-
- // force a major compaction to compact all segments into one
- cc.sql("ALTER TABLE t3 COMPACT 'MAJOR' ")
- cc.sql("SHOW SEGMENTS FOR TABLE t3 LIMIT 10").show
-
- // load again, add another 1000 rows
- cc.sql(s"LOAD DATA LOCAL INPATH '$testData' into table t3")
- cc.sql("SHOW SEGMENTS FOR TABLE t3 LIMIT 10").show
-
- // this query will be executed on 2 segments, it should return 5000 rows
- cc.sql("SELECT count(*) AS amount FROM t3").show
-
- // delete all segments whose loading time is before '2099-01-01 01:00:00'
- cc.sql("DELETE FROM TABLE T3 WHERE SEGMENT.STARTTIME BEFORE '2099-01-01 01:00:00'")
- cc.sql("SHOW SEGMENTS FOR TABLE t3 ").show
-
- // this query will be executed on 0 segments, it should return 0 rows
- cc.sql("SELECT count(*) AS amount FROM t3").show
-
- // force clean up all 'MARKED_FOR_DELETE' and 'COMPACTED' segments immediately
- cc.sql("CLEAN FILES FOR TABLE t3")
- cc.sql("SHOW SEGMENTS FOR TABLE t3").show
-
- cc.sql("DROP TABLE IF EXISTS t3")
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/examples/spark/src/main/scala/org/apache/carbondata/examples/DataUpdateDeleteExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/DataUpdateDeleteExample.scala b/examples/spark/src/main/scala/org/apache/carbondata/examples/DataUpdateDeleteExample.scala
deleted file mode 100644
index 830a819..0000000
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/DataUpdateDeleteExample.scala
+++ /dev/null
@@ -1,185 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.examples
-
-import java.io.File
-import java.text.SimpleDateFormat
-
-import org.apache.spark.{SparkConf, SparkContext}
-import org.apache.spark.sql.{CarbonContext, DataFrame, Row, SaveMode, SQLContext}
-import org.apache.spark.sql.types.{DataTypes, StructField, StructType}
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.examples.util.ExampleUtils
-
-object DataUpdateDeleteExample {
-
- def main(args: Array[String]) {
- val cc = ExampleUtils.createCarbonContext("DataUpdateDeleteExample")
-
- // for local files
- var rootPath = ExampleUtils.currentPath
- // for hdfs files
- // var rootPath = "hdfs://hdfs-host/carbon"
-
- val testData = rootPath + "/src/main/resources/data.csv"
-
- // Specify date format based on raw data
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "yyyy/MM/dd")
-
- cc.sql("DROP TABLE IF EXISTS t3")
- cc.sql("DROP TABLE IF EXISTS t5")
-
- // Create table, 6 dimensions, 1 measure
- cc.sql("""
- CREATE TABLE IF NOT EXISTS t3
- (id Int, date Date, country String,
- name String, phonetype String, serialname char(10), salary Int)
- STORED BY 'carbondata'
- """)
-
- cc.sql(s"""
- LOAD DATA LOCAL INPATH '$testData' INTO TABLE t3
- """)
-
- // Specify date format based on raw data
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "yyyy-MM-dd")
-
- // Simulate data and write to table t5
- var fields = Seq[StructField]()
- fields = fields :+ DataTypes.createStructField("t5_id", DataTypes.IntegerType, false)
- fields = fields :+ DataTypes.createStructField("t5_date", DataTypes.DateType, false)
- fields = fields :+ DataTypes.createStructField("t5_country", DataTypes.StringType, false)
- fields = fields :+ DataTypes.createStructField("t5_name", DataTypes.StringType, false)
- fields = fields :+ DataTypes.createStructField("t5_phonetype", DataTypes.StringType, false)
- fields = fields :+ DataTypes.createStructField("t5_serialname", DataTypes.StringType, false)
- fields = fields :+ DataTypes.createStructField("t5_salary", DataTypes.IntegerType, false)
- var schema = StructType(fields)
- var sdf = new SimpleDateFormat("yyyy-MM-dd")
- var data = cc.sparkContext.parallelize(1 to 10).map { x =>
- val day = x % 20 + 1
- var dateStr = ""
- if (day >= 10) {
- dateStr = "2017-07-" + day
- } else {
- dateStr = "2017-07-0" + day
- }
- val dt = new java.sql.Date(sdf.parse(dateStr).getTime);
- var row = Seq[Any]()
- row = row :+ x
- row = row :+ dt
- row = row :+ "china"
- row = row :+ "bbb" + x
- row = row :+ "phone" + 100 * x
- row = row :+ "ASD" + (1000 * x - x)
- row = row :+ (25000 + x)
- Row.fromSeq(row)
- }
- var df = cc.createDataFrame(data, schema)
- df.write
- .format("carbondata")
- .option("tableName", "t5")
- .option("tempCSV", "true")
- .option("compress", "true")
- .mode(SaveMode.Overwrite)
- .save()
- cc.sql("""
- SELECT * FROM t5 ORDER BY t5_id
- """).show()
-
- // 1.Update data with simple SET
- cc.sql("""
- SELECT * FROM t3 ORDER BY t3.id
- """).show()
-
- // Update data where salary < 15003
- val dateStr = "2018-08-08"
- cc.sql(s"""
- UPDATE t3 SET (t3.date, t3.country) = ('$dateStr', 'india') WHERE t3.salary < 15003
- """).show()
- // Query data again after the above update
- cc.sql("""
- SELECT * FROM t3 ORDER BY t3.id
- """).show()
-
- cc.sql("""
- UPDATE t3 SET (t3.salary) = (t3.salary + 9) WHERE t3.name = 'aaa1'
- """).show()
- // Query data again after the above update
- cc.sql("""
- SELECT * FROM t3 ORDER BY t3.id
- """).show()
-
- // 2.Update data with subquery result SET
- cc.sql("""
- UPDATE t3
- SET (t3.country, t3.name) = (SELECT t5_country, t5_name FROM t5 WHERE t5_id = 5)
- WHERE t3.id < 5""").show()
- cc.sql("""
- UPDATE t3
- SET (t3.date, t3.serialname, t3.salary) =
- (SELECT '2099-09-09', t5_serialname, '9999' FROM t5 WHERE t5_id = 5)
- WHERE t3.id < 5""").show()
-
- // Query data again after the above update
- cc.sql("""
- SELECT * FROM t3 ORDER BY t3.id
- """).show()
-
- // 3.Update data with join query result SET
- cc.sql("""
- UPDATE t3
- SET (t3.country, t3.salary) =
- (SELECT t5_country, t5_salary FROM t5 FULL JOIN t3 u
- WHERE u.id = t5_id and t5_id=6) WHERE t3.id >6""").show()
-
- // Query data again after the above update
- cc.sql("""
- SELECT * FROM t3 ORDER BY t3.id
- """).show()
-
- // 4.Delete data where salary > 15005
- cc.sql("""
- DELETE FROM t3 WHERE t3.salary > 15005
- """).show()
-
- // Query data again after delete data
- cc.sql("""
- SELECT * FROM t3 ORDER BY t3.id
- """).show()
-
- // 5.Delete data WHERE id in (1, 2, $key)
- var key = 3
- cc.sql(s"""
- DELETE FROM t3 WHERE t3.id in (1, 2, $key)
- """).show()
-
- // Query data again after delete data
- cc.sql("""
- SELECT * FROM t3 ORDER BY t3.id
- """).show()
-
- // Drop table
- cc.sql("DROP TABLE IF EXISTS t3")
- cc.sql("DROP TABLE IF EXISTS t5")
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/examples/spark/src/main/scala/org/apache/carbondata/examples/DatasourceExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/DatasourceExample.scala b/examples/spark/src/main/scala/org/apache/carbondata/examples/DatasourceExample.scala
deleted file mode 100644
index a3af2c3..0000000
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/DatasourceExample.scala
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.examples
-
-import org.apache.spark.sql.{SaveMode, SQLContext}
-
-import org.apache.carbondata.examples.util.ExampleUtils
-
-object DatasourceExample {
-
- def main(args: Array[String]) {
- // use CarbonContext to write CarbonData files
- val cc = ExampleUtils.createCarbonContext("DatasourceExample")
- ExampleUtils.writeSampleCarbonFile(cc, "table1")
-
- // Use SQLContext to read CarbonData files
- val sqlContext = new SQLContext(cc.sparkContext)
- sqlContext.sql(
- s"""
- | CREATE TEMPORARY TABLE source
- | USING org.apache.spark.sql.CarbonSource
- | OPTIONS (path '${cc.storePath}/default/table1')
- """.stripMargin)
- sqlContext.sql("SELECT c1, c2, count(*) FROM source WHERE c3 > 100 GROUP BY c1, c2").show
-
- // delete carbondata file
- ExampleUtils.cleanSampleCarbonFile(cc, "table1")
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/examples/spark/src/main/scala/org/apache/carbondata/examples/DirectSQLExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/DirectSQLExample.scala b/examples/spark/src/main/scala/org/apache/carbondata/examples/DirectSQLExample.scala
deleted file mode 100644
index 6a66b93..0000000
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/DirectSQLExample.scala
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.examples
-
-import org.apache.spark.sql.SQLContext
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.examples.util.ExampleUtils
-
-/**
- * This example needs Spark 1.6 or later version to run
- */
-object DirectSQLExample {
-
- def main(args: Array[String]) {
- val cc = ExampleUtils.createCarbonContext("DirectSQLExample")
- ExampleUtils.writeSampleCarbonFile(cc, "table1")
-
- // Use SQLContext to read CarbonData files without creating table
- val sqlContext = new SQLContext(cc.sparkContext)
- sqlContext.sql(
- s"""
- | SELECT c1, c2, count(*)
- | FROM carbondata.`${cc.storePath}/${CarbonCommonConstants.DATABASE_DEFAULT_NAME}/table1`
- | WHERE c3 > 100
- | GROUP BY c1, c2
- """.stripMargin).show
-
- // delete carbondata file
- ExampleUtils.cleanSampleCarbonFile(cc, "table1")
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/examples/spark/src/main/scala/org/apache/carbondata/examples/GenerateDictionaryExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/GenerateDictionaryExample.scala b/examples/spark/src/main/scala/org/apache/carbondata/examples/GenerateDictionaryExample.scala
deleted file mode 100644
index 94d35b0..0000000
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/GenerateDictionaryExample.scala
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.examples
-
-import org.apache.spark.sql.{CarbonContext, CarbonEnv, CarbonRelation}
-
-import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
-import org.apache.carbondata.core.util.path.CarbonStorePath
-import org.apache.carbondata.examples.util.ExampleUtils
-import org.apache.carbondata.processing.util.CarbonLoaderUtil
-
-/**
- * example for global dictionary generation
- * pls check files under directory of target/store/default/dictSample/Metadata
- * and verify global dictionary values
- */
-object GenerateDictionaryExample {
-
- def main(args: Array[String]) {
- val cc = ExampleUtils.createCarbonContext("GenerateDictionaryExample")
- val factFilePath = ExampleUtils.currentPath + "/src/main/resources/factSample.csv"
- val carbonTablePath = CarbonStorePath.getCarbonTablePath(ExampleUtils.storeLocation,
- new CarbonTableIdentifier(CarbonCommonConstants.DATABASE_DEFAULT_NAME, "dictSample", "1"))
- val dictFolderPath = carbonTablePath.getMetadataDirectoryPath
-
- // execute sql statement
- cc.sql("DROP TABLE IF EXISTS dictSample")
-
- cc.sql("""
- CREATE TABLE IF NOT EXISTS dictSample(id Int, name String, city String, salary Int)
- STORED BY 'org.apache.carbondata.format'
- """)
-
- cc.sql(s"""
- LOAD DATA LOCAL INPATH '$factFilePath' INTO TABLE dictSample
- """)
-
- // check generated dictionary
- val tableIdentifier =
- new CarbonTableIdentifier(CarbonCommonConstants.DATABASE_DEFAULT_NAME, "dictSample", "1")
- printDictionary(cc, tableIdentifier, dictFolderPath)
- }
-
- def printDictionary(cc: CarbonContext, carbonTableIdentifier: CarbonTableIdentifier,
- dictFolderPath: String) {
- val dataBaseName = carbonTableIdentifier.getDatabaseName
- val tableName = carbonTableIdentifier.getTableName
- val carbonRelation = CarbonEnv.get.carbonMetastore.lookupRelation1(Option(dataBaseName),
- tableName)(cc).asInstanceOf[CarbonRelation]
- val carbonTable = carbonRelation.tableMeta.carbonTable
- val dimensions = carbonTable.getDimensionByTableName(tableName.toLowerCase())
- .toArray.map(_.asInstanceOf[CarbonDimension])
- // scalastyle:off println
- // print dictionary information
- println("**********************************************************************************")
- println(s"table:$tableName in " + s"database:$dataBaseName")
- for (dimension <- dimensions) {
- println("**********************************************************************************")
- println(s"dictionary of dimension: ${dimension.getColName}")
- println(s"Key\t\t\tValue")
- val columnIdentifier = new DictionaryColumnUniqueIdentifier(carbonTableIdentifier,
- dimension.getColumnIdentifier, dimension.getDataType,
- CarbonStorePath
- .getCarbonTablePath(carbonTable.getStorePath, carbonTable.getCarbonTableIdentifier))
- val dict = CarbonLoaderUtil.getDictionary(columnIdentifier, cc.storePath)
- var index: Int = 1
- var distinctValue = dict.getDictionaryValueForKey(index)
- while (distinctValue != null) {
- println(index + s"\t\t\t" + distinctValue)
- index += 1
- distinctValue = dict.getDictionaryValueForKey(index)
- }
- }
- println("**********************************************************************************")
- // scalastyle:on println
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/examples/spark/src/main/scala/org/apache/carbondata/examples/HadoopFileExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/HadoopFileExample.scala b/examples/spark/src/main/scala/org/apache/carbondata/examples/HadoopFileExample.scala
deleted file mode 100644
index d471ab6..0000000
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/HadoopFileExample.scala
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.examples
-
-import org.apache.hadoop.conf.Configuration
-
-import org.apache.carbondata.examples.util.ExampleUtils
-import org.apache.carbondata.hadoop.{CarbonInputFormat, CarbonProjection}
-
-// scalastyle:off println
-object HadoopFileExample {
-
- def main(args: Array[String]): Unit = {
- val cc = ExampleUtils.createCarbonContext("HadoopFileExample")
- ExampleUtils.writeSampleCarbonFile(cc, "carbon1")
-
- // read two columns
- val projection = new CarbonProjection
- projection.addColumn("c1") // column c1
- projection.addColumn("c3") // column c3
- val conf = new Configuration()
- CarbonInputFormat.setColumnProjection(conf, projection)
-
- val sc = cc.sparkContext
- val input = sc.newAPIHadoopFile(s"${cc.storePath}/default/carbon1",
- classOf[CarbonInputFormat[Array[Object]]],
- classOf[Void],
- classOf[Array[Object]],
- conf)
- val result = input.map(x => x._2.toList).collect
- result.foreach(x => println(x.mkString(", ")))
-
- // delete carbondata file
- ExampleUtils.cleanSampleCarbonFile(cc, "carbon1")
- }
-}
-// scalastyle:on println
-
[15/16] carbondata git commit: [CARBONDATA-1512] Fixed bug for
failing of concurrently executing sql queries in presto
Posted by ra...@apache.org.
[CARBONDATA-1512] Fixed bug for failing of concurrently executing sql queries in presto
Resolved concurrent query execution failure
This closes #1388
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/7036696e
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/7036696e
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/7036696e
Branch: refs/heads/pre-aggregate
Commit: 7036696e26973f7bb5b79ca122535efc757dc3e9
Parents: 9326cfd
Author: Geetika Gupta <ge...@knoldus.in>
Authored: Wed Sep 27 15:04:58 2017 +0530
Committer: Jacky Li <ja...@qq.com>
Committed: Fri Oct 20 15:10:56 2017 +0800
----------------------------------------------------------------------
.../carbondata/presto/impl/CarbonTableReader.java | 18 ++++++++++--------
1 file changed, 10 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7036696e/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
index 5c00026..0fa7684 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
@@ -99,6 +99,7 @@ public class CarbonTableReader {
@Inject public CarbonTableReader(CarbonTableConfig config) {
this.config = requireNonNull(config, "CarbonTableConfig is null");
this.cc = new ConcurrentHashMap<>();
+ tableList = new LinkedList<>();
}
/**
@@ -121,7 +122,7 @@ public class CarbonTableReader {
}
}
}
- updateSchemaTables();
+ updateSchemaTables(table);
parseCarbonMetadata(table);
}
@@ -207,7 +208,7 @@ public class CarbonTableReader {
*/
public CarbonTable getTable(SchemaTableName schemaTableName) {
try {
- updateSchemaTables();
+ updateSchemaTables(schemaTableName);
} catch (Exception e) {
throw new RuntimeException(e);
}
@@ -223,16 +224,17 @@ public class CarbonTableReader {
* and cache all the table names in this.tableList. Notice that whenever this method
* is called, it clears this.tableList and populate the list by reading the files.
*/
- private void updateSchemaTables() {
+ private void updateSchemaTables(SchemaTableName schemaTableName) {
// update logic determine later
if (carbonFileList == null) {
updateSchemaList();
}
- tableList = new LinkedList<>();
- for (CarbonFile cf : carbonFileList.listFiles()) {
- if (!cf.getName().endsWith(".mdt")) {
- for (CarbonFile table : cf.listFiles()) {
- tableList.add(new SchemaTableName(cf.getName(), table.getName()));
+ if(!tableList.contains(schemaTableName)) {
+ for (CarbonFile cf : carbonFileList.listFiles()) {
+ if (!cf.getName().endsWith(".mdt")) {
+ for (CarbonFile table : cf.listFiles()) {
+ tableList.add(new SchemaTableName(cf.getName(), table.getName()));
+ }
}
}
}
[05/16] carbondata git commit: [CARBONDATA-1597] Remove spark1
integration
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala
deleted file mode 100644
index cb35960..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala
+++ /dev/null
@@ -1,842 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.execution.command
-
-import java.util
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable.ListBuffer
-
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql._
-import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Project}
-import org.apache.spark.sql.execution.RunnableCommand
-import org.apache.spark.sql.execution.datasources.LogicalRelation
-import org.apache.spark.storage.StorageLevel
-
-import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datastore.impl.FileFactory
-import org.apache.carbondata.core.locks.{CarbonLockFactory, CarbonLockUtil, LockUsage}
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.mutate.{CarbonUpdateUtil, DeleteDeltaBlockDetails, SegmentUpdateDetails, TupleIdEnum}
-import org.apache.carbondata.core.mutate.data.RowCountDetailsVO
-import org.apache.carbondata.core.statusmanager.{SegmentStatusManager, SegmentUpdateStatusManager}
-import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
-import org.apache.carbondata.core.writer.CarbonDeleteDeltaWriterImpl
-import org.apache.carbondata.processing.exception.MultipleMatchingException
-import org.apache.carbondata.processing.loading.FailureCauses
-import org.apache.carbondata.processing.merger.{CarbonDataMergerUtil, CarbonDataMergerUtilResult, CompactionType}
-import org.apache.carbondata.spark.DeleteDelataResultImpl
-import org.apache.carbondata.spark.util.QueryPlanUtil
-
-
-/**
- * IUD update delete and compaction framework.
- *
- */
-
-private[sql] case class ProjectForDeleteCommand(
- plan: LogicalPlan,
- identifier: Seq[String],
- timestamp: String) extends RunnableCommand {
-
- val LOG = LogServiceFactory.getLogService(this.getClass.getName)
- var horizontalCompactionFailed = false
-
- override def run(sqlContext: SQLContext): Seq[Row] = {
-
- val dataFrame = DataFrame(sqlContext, plan)
- val dataRdd = dataFrame.rdd
-
- val relation = CarbonEnv.get.carbonMetastore
- .lookupRelation1(deleteExecution.getTableIdentifier(identifier))(sqlContext).
- asInstanceOf[CarbonRelation]
- val carbonTable = relation.tableMeta.carbonTable
- val metadataLock = CarbonLockFactory
- .getCarbonLockObj(carbonTable.getAbsoluteTableIdentifier.getCarbonTableIdentifier,
- LockUsage.METADATA_LOCK)
- var lockStatus = false
- try {
- lockStatus = metadataLock.lockWithRetries()
- LOG.audit(s" Delete data request has been received " +
- s"for ${ relation.databaseName }.${ relation.tableName }.")
- if (lockStatus) {
- LOG.info("Successfully able to get the table metadata file lock")
- }
- else {
- throw new Exception("Table is locked for deletion. Please try after some time")
- }
- val tablePath = CarbonStorePath.getCarbonTablePath(
- carbonTable.getStorePath,
- carbonTable.getAbsoluteTableIdentifier.getCarbonTableIdentifier)
- var executorErrors = new ExecutionErrors(FailureCauses.NONE, "")
-
- // handle the clean up of IUD.
- CarbonUpdateUtil.cleanUpDeltaFiles(carbonTable, false)
-
- if (deleteExecution
- .deleteDeltaExecution(identifier, sqlContext, dataRdd, timestamp, relation,
- false, executorErrors)) {
- // call IUD Compaction.
- IUDCommon.tryHorizontalCompaction(sqlContext, relation, isUpdateOperation = false)
- }
- } catch {
- case e: HorizontalCompactionException =>
- LOG.error("Delete operation passed. Exception in Horizontal Compaction." +
- " Please check logs. " + e.getMessage)
- CarbonUpdateUtil.cleanStaleDeltaFiles(carbonTable, e.compactionTimeStamp.toString)
-
- case e: Exception =>
- LOG.error("Exception in Delete data operation " + e.getMessage)
- // ****** start clean up.
- // In case of failure , clean all related delete delta files
- CarbonUpdateUtil.cleanStaleDeltaFiles(carbonTable, timestamp)
-
- // clean up. Null check is required as for executor error some times message is null
- if (null != e.getMessage) {
- sys.error("Delete data operation is failed. " + e.getMessage)
- }
- else {
- sys.error("Delete data operation is failed. Please check logs.")
- }
- } finally {
- if (lockStatus) {
- CarbonLockUtil.fileUnlock(metadataLock, LockUsage.METADATA_LOCK)
- }
- }
- Seq.empty
- }
-}
-
-private[sql] case class ProjectForUpdateCommand(
- plan: LogicalPlan, tableIdentifier: Seq[String]) extends RunnableCommand {
- val LOGGER = LogServiceFactory.getLogService(ProjectForUpdateCommand.getClass.getName)
-
- override def run(sqlContext: SQLContext): Seq[Row] = {
-
- val res = plan find {
- case relation: LogicalRelation if (relation.relation
- .isInstanceOf[CarbonDatasourceRelation]) =>
- true
- case _ => false
- }
-
- if (!res.isDefined) {
- return Seq.empty
- }
-
- val relation = CarbonEnv.get.carbonMetastore
- .lookupRelation1(deleteExecution.getTableIdentifier(tableIdentifier))(sqlContext).
- asInstanceOf[CarbonRelation]
- val carbonTable = relation.tableMeta.carbonTable
- val metadataLock = CarbonLockFactory
- .getCarbonLockObj(carbonTable.getAbsoluteTableIdentifier.getCarbonTableIdentifier,
- LockUsage.METADATA_LOCK)
- var lockStatus = false
- // get the current time stamp which should be same for delete and update.
- val currentTime = CarbonUpdateUtil.readCurrentTime
- var dataFrame: DataFrame = null
- val isPersistEnabledUserValue = CarbonProperties.getInstance
- .getProperty(CarbonCommonConstants.isPersistEnabled,
- CarbonCommonConstants.defaultValueIsPersistEnabled)
- var isPersistEnabled = CarbonCommonConstants.defaultValueIsPersistEnabled.toBoolean
- if (isPersistEnabledUserValue.equalsIgnoreCase("false")) {
- isPersistEnabled = false
- }
- else if (isPersistEnabledUserValue.equalsIgnoreCase("true")) {
- isPersistEnabled = true
- }
- try {
- lockStatus = metadataLock.lockWithRetries()
- if (lockStatus) {
- logInfo("Successfully able to get the table metadata file lock")
- }
- else {
- throw new Exception("Table is locked for updation. Please try after some time")
- }
- val tablePath = CarbonStorePath.getCarbonTablePath(
- carbonTable.getStorePath,
- carbonTable.getAbsoluteTableIdentifier.getCarbonTableIdentifier)
- // Get RDD.
- dataFrame = if (isPersistEnabled) {
- DataFrame(sqlContext, plan)
- .persist(StorageLevel.MEMORY_AND_DISK)
- }
- else {
- DataFrame(sqlContext, plan)
- }
- var executionErrors = new ExecutionErrors(FailureCauses.NONE, "")
-
-
- // handle the clean up of IUD.
- CarbonUpdateUtil.cleanUpDeltaFiles(carbonTable, false)
-
- // do delete operation.
- deleteExecution.deleteDeltaExecution(tableIdentifier, sqlContext, dataFrame.rdd,
- currentTime + "",
- relation, isUpdateOperation = true, executionErrors)
-
- if(executionErrors.failureCauses != FailureCauses.NONE) {
- throw new Exception(executionErrors.errorMsg)
- }
-
- // do update operation.
- UpdateExecution.performUpdate(dataFrame, tableIdentifier, plan,
- sqlContext, currentTime, executionErrors)
-
- if(executionErrors.failureCauses != FailureCauses.NONE) {
- throw new Exception(executionErrors.errorMsg)
- }
-
- // Do IUD Compaction.
- IUDCommon.tryHorizontalCompaction(sqlContext, relation, isUpdateOperation = true)
- }
-
- catch {
- case e: HorizontalCompactionException =>
- LOGGER.error(
- "Update operation passed. Exception in Horizontal Compaction. Please check logs." + e)
- // In case of failure , clean all related delta files
- CarbonUpdateUtil.cleanStaleDeltaFiles(carbonTable, e.compactionTimeStamp.toString)
-
- case e: Exception =>
- LOGGER.error("Exception in update operation" + e)
- // ****** start clean up.
- // In case of failure , clean all related delete delta files
- CarbonUpdateUtil.cleanStaleDeltaFiles(carbonTable, currentTime + "")
-
- // *****end clean up.
- if (null != e.getMessage) {
- sys.error("Update operation failed. " + e.getMessage)
- }
- if (null != e.getCause && null != e.getCause.getMessage) {
- sys.error("Update operation failed. " + e.getCause.getMessage)
- }
- sys.error("Update operation failed. please check logs.")
- }
- finally {
- if (null != dataFrame && isPersistEnabled) {
- dataFrame.unpersist()
- }
- if (lockStatus) {
- CarbonLockUtil.fileUnlock(metadataLock, LockUsage.METADATA_LOCK)
- }
- }
- Seq.empty
- }
-}
-
-object IUDCommon {
-
- val LOG = LogServiceFactory.getLogService(this.getClass.getName)
-
- /**
- * The method does horizontal compaction. After Update and Delete completion
- * tryHorizontal compaction will be called. In case this method is called after
- * Update statement then Update Compaction followed by Delete Compaction will be
- * processed whereas for tryHorizontalCompaction called after Delete statement
- * then only Delete Compaction will be processed.
- *
- * @param sqlContext
- * @param carbonRelation
- * @param isUpdateOperation
- */
- def tryHorizontalCompaction(sqlContext: SQLContext,
- carbonRelation: CarbonRelation,
- isUpdateOperation: Boolean): Unit = {
-
- var ishorizontalCompaction = CarbonDataMergerUtil.isHorizontalCompactionEnabled()
-
- if (ishorizontalCompaction == false) {
- return
- }
-
- var compactionTypeIUD = CompactionType.IUD_UPDDEL_DELTA_COMPACTION
- val carbonTable = carbonRelation.tableMeta.carbonTable
- val (db, table) = (carbonTable.getDatabaseName, carbonTable.getFactTableName)
- val absTableIdentifier = carbonTable.getAbsoluteTableIdentifier
- val updateTimeStamp = System.currentTimeMillis()
- // To make sure that update and delete timestamps are not same,
- // required to commit to status metadata and cleanup
- val deleteTimeStamp = updateTimeStamp + 1
-
- // get the valid segments
- var segLists = CarbonDataMergerUtil.getValidSegmentList(absTableIdentifier)
-
- if (segLists == null || segLists.size() == 0) {
- return
- }
-
- // Should avoid reading Table Status file from Disk every time. Better to load it
- // in-memory at the starting and pass it along the routines. The constructor of
- // SegmentUpdateStatusManager reads the Table Status File and Table Update Status
- // file and save the content in segmentDetails and updateDetails respectively.
- val segmentUpdateStatusManager: SegmentUpdateStatusManager = new SegmentUpdateStatusManager(
- absTableIdentifier)
-
- if (isUpdateOperation == true) {
-
- // This is only update operation, perform only update compaction.
- compactionTypeIUD = CompactionType.IUD_UPDDEL_DELTA_COMPACTION
- performUpdateDeltaCompaction(sqlContext,
- compactionTypeIUD,
- carbonTable,
- absTableIdentifier,
- segmentUpdateStatusManager,
- updateTimeStamp,
- segLists)
- }
-
- // After Update Compaction perform delete compaction
- compactionTypeIUD = CompactionType.IUD_DELETE_DELTA_COMPACTION
- segLists = CarbonDataMergerUtil.getValidSegmentList(absTableIdentifier)
- if (segLists == null || segLists.size() == 0) {
- return
- }
-
- // Delete Compaction
- performDeleteDeltaCompaction(sqlContext,
- compactionTypeIUD,
- carbonTable,
- absTableIdentifier,
- segmentUpdateStatusManager,
- deleteTimeStamp,
- segLists)
- }
-
- /**
- * Update Delta Horizontal Compaction.
- *
- * @param sqlContext
- * @param compactionTypeIUD
- * @param carbonTable
- * @param absTableIdentifier
- * @param segLists
- */
- private def performUpdateDeltaCompaction(sqlContext: SQLContext,
- compactionTypeIUD: CompactionType,
- carbonTable: CarbonTable,
- absTableIdentifier: AbsoluteTableIdentifier,
- segmentUpdateStatusManager: SegmentUpdateStatusManager,
- factTimeStamp: Long,
- segLists: util.List[String]): Unit = {
- val db = carbonTable.getDatabaseName
- val table = carbonTable.getFactTableName
- // get the valid segments qualified for update compaction.
- val validSegList = CarbonDataMergerUtil.getSegListIUDCompactionQualified(segLists,
- absTableIdentifier,
- segmentUpdateStatusManager,
- compactionTypeIUD)
-
- if (validSegList.size() == 0) {
- return
- }
-
- LOG.info(s"Horizontal Update Compaction operation started for [${db}.${table}].")
- LOG.audit(s"Horizontal Update Compaction operation started for [${db}.${table}].")
-
- try {
- // Update Compaction.
- val altertablemodel = AlterTableModel(Option(carbonTable.getDatabaseName),
- carbonTable.getFactTableName,
- Some(segmentUpdateStatusManager),
- CompactionType.IUD_UPDDEL_DELTA_COMPACTION.toString,
- Some(factTimeStamp),
- "")
-
- AlterTableCompaction(altertablemodel).run(sqlContext)
- }
- catch {
- case e: Exception =>
- val msg = if (null != e.getMessage) {
- e.getMessage
- } else {
- "Please check logs for more info"
- }
- throw new HorizontalCompactionException(
- s"Horizontal Update Compaction Failed for [${ db }.${ table }]. " + msg, factTimeStamp)
- }
- LOG.info(s"Horizontal Update Compaction operation completed for [${ db }.${ table }].")
- LOG.audit(s"Horizontal Update Compaction operation completed for [${ db }.${ table }].")
- }
-
- /**
- * Delete Delta Horizontal Compaction.
- *
- * @param sqlContext
- * @param compactionTypeIUD
- * @param carbonTable
- * @param absTableIdentifier
- * @param segLists
- */
- private def performDeleteDeltaCompaction(sqlContext: SQLContext,
- compactionTypeIUD: CompactionType,
- carbonTable: CarbonTable,
- absTableIdentifier: AbsoluteTableIdentifier,
- segmentUpdateStatusManager: SegmentUpdateStatusManager,
- factTimeStamp: Long,
- segLists: util.List[String]): Unit = {
-
- val db = carbonTable.getDatabaseName
- val table = carbonTable.getFactTableName
- val deletedBlocksList = CarbonDataMergerUtil.getSegListIUDCompactionQualified(segLists,
- absTableIdentifier,
- segmentUpdateStatusManager,
- compactionTypeIUD)
-
- if (deletedBlocksList.size() == 0) {
- return
- }
-
- LOG.info(s"Horizontal Delete Compaction operation started for [${db}.${table}].")
- LOG.audit(s"Horizontal Delete Compaction operation started for [${db}.${table}].")
-
- try {
-
- // Delete Compaction RDD
- val rdd1 = sqlContext.sparkContext
- .parallelize(deletedBlocksList.asScala.toSeq, deletedBlocksList.size())
-
- val timestamp = factTimeStamp
- val updateStatusDetails = segmentUpdateStatusManager.getUpdateStatusDetails
- val result = rdd1.mapPartitions(iter =>
- new Iterator[Seq[CarbonDataMergerUtilResult]] {
- override def hasNext: Boolean = iter.hasNext
-
- override def next(): Seq[CarbonDataMergerUtilResult] = {
- val segmentAndBlocks = iter.next
- val segment = segmentAndBlocks.substring(0, segmentAndBlocks.lastIndexOf("/"))
- val blockName = segmentAndBlocks
- .substring(segmentAndBlocks.lastIndexOf("/") + 1, segmentAndBlocks.length)
-
- val result = CarbonDataMergerUtil.compactBlockDeleteDeltaFiles(segment, blockName,
- absTableIdentifier,
- updateStatusDetails,
- timestamp)
-
- result.asScala.toList
-
- }
- }).collect
-
- val resultList = ListBuffer[CarbonDataMergerUtilResult]()
- result.foreach(x => {
- x.foreach(y => {
- resultList += y
- })
- })
-
- val updateStatus = CarbonDataMergerUtil.updateStatusFile(resultList.toList.asJava,
- carbonTable,
- timestamp.toString,
- segmentUpdateStatusManager)
- if (updateStatus == false) {
- LOG.audit(s"Delete Compaction data operation is failed for [${db}.${table}].")
- LOG.error("Delete Compaction data operation is failed.")
- throw new HorizontalCompactionException(
- s"Horizontal Delete Compaction Failed for [${db}.${table}] ." +
- s" Please check logs for more info.", factTimeStamp)
- }
- else {
- LOG.info(s"Horizontal Delete Compaction operation completed for [${db}.${table}].")
- LOG.audit(s"Horizontal Delete Compaction operation completed for [${db}.${table}].")
- }
- }
- catch {
- case e: Exception =>
- val msg = if (null != e.getMessage) {
- e.getMessage
- } else {
- "Please check logs for more info"
- }
- throw new HorizontalCompactionException(
- s"Horizontal Delete Compaction Failed for [${ db }.${ table }]. " + msg, factTimeStamp)
- }
- }
-}
-
-class HorizontalCompactionException(
- message: String,
- // required for cleanup
- val compactionTimeStamp: Long) extends RuntimeException(message) {
-}
-
-object deleteExecution {
- val LOGGER = LogServiceFactory.getLogService(this.getClass.getName)
-
- def getTableIdentifier(tableIdentifier: Seq[String]): TableIdentifier = {
- if (tableIdentifier.size > 1) {
- TableIdentifier(tableIdentifier(1), Some(tableIdentifier(0)))
- } else {
- TableIdentifier(tableIdentifier(0), None)
- }
- }
-
- def deleteDeltaExecution(identifier: Seq[String],
- sqlContext: SQLContext,
- dataRdd: RDD[Row],
- timestamp: String, relation: CarbonRelation, isUpdateOperation: Boolean,
- executorErrors: ExecutionErrors): Boolean = {
-
- var res: Array[List[(String, (SegmentUpdateDetails, ExecutionErrors))]] = null
- val tableName = getTableIdentifier(identifier).table
- val database = getDB.getDatabaseName(getTableIdentifier(identifier).database, sqlContext)
- val relation = CarbonEnv.get.carbonMetastore
- .lookupRelation1(getTableIdentifier(identifier))(sqlContext).
- asInstanceOf[CarbonRelation]
-
- val storeLocation = relation.tableMeta.storePath
- val absoluteTableIdentifier: AbsoluteTableIdentifier = new
- AbsoluteTableIdentifier(storeLocation,
- relation.tableMeta.carbonTableIdentifier)
- var tablePath = CarbonStorePath
- .getCarbonTablePath(storeLocation,
- absoluteTableIdentifier.getCarbonTableIdentifier())
- var tableUpdateStatusPath = tablePath.getTableUpdateStatusFilePath
- val totalSegments =
- SegmentStatusManager.readLoadMetadata(tablePath.getMetadataDirectoryPath).length
- var factPath = tablePath.getFactDir
-
- var carbonTable = relation.tableMeta.carbonTable
- var deleteStatus = true
- val deleteRdd = if (isUpdateOperation) {
- val schema =
- org.apache.spark.sql.types.StructType(Seq(org.apache.spark.sql.types.StructField(
- CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID,
- org.apache.spark.sql.types.StringType)))
- val rdd = dataRdd
- .map(row => Row(row.get(row.fieldIndex(
- CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID))))
- sqlContext.createDataFrame(rdd, schema).rdd
- } else {
- dataRdd
- }
-
- val (carbonInputFormat, job) =
- QueryPlanUtil.createCarbonInputFormat(absoluteTableIdentifier)
-
- val keyRdd = deleteRdd.map({ row =>
- val tupleId: String = row
- .getString(row.fieldIndex(CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID))
- val key = CarbonUpdateUtil.getSegmentWithBlockFromTID(tupleId)
- (key, row)
- }).groupByKey()
-
- // if no loads are present then no need to do anything.
- if (keyRdd.partitions.size == 0) {
- return true
- }
-
- var blockMappingVO = carbonInputFormat.getBlockRowCount(job, absoluteTableIdentifier)
- val segmentUpdateStatusMngr = new SegmentUpdateStatusManager(absoluteTableIdentifier)
- CarbonUpdateUtil
- .createBlockDetailsMap(blockMappingVO, segmentUpdateStatusMngr)
-
- val rowContRdd = sqlContext.sparkContext
- .parallelize(blockMappingVO.getCompleteBlockRowDetailVO.asScala.toSeq,
- keyRdd.partitions.size)
-
- val rdd = rowContRdd.join(keyRdd)
-
- res = rdd.mapPartitionsWithIndex(
- (index: Int, records: Iterator[((String), (RowCountDetailsVO, Iterable[Row]))]) =>
- Iterator[List[(String, (SegmentUpdateDetails, ExecutionErrors))]] {
-
- var result = List[(String, (SegmentUpdateDetails, ExecutionErrors))]()
- while (records.hasNext) {
- val ((key), (rowCountDetailsVO, groupedRows)) = records.next
- result = result ++
- deleteDeltaFunc(index,
- key,
- groupedRows.toIterator,
- timestamp,
- rowCountDetailsVO)
-
- }
- result
- }
- ).collect()
-
- // if no loads are present then no need to do anything.
- if (res.isEmpty) {
- return true
- }
-
- // update new status file
- checkAndUpdateStatusFiles
-
- // all or none : update status file, only if complete delete opeartion is successfull.
- def checkAndUpdateStatusFiles: Unit = {
- val blockUpdateDetailsList = new util.ArrayList[SegmentUpdateDetails]()
- val segmentDetails = new util.HashSet[String]()
- res.foreach(resultOfSeg => resultOfSeg.foreach(
- resultOfBlock => {
- if (resultOfBlock._1.equalsIgnoreCase(CarbonCommonConstants.STORE_LOADSTATUS_SUCCESS)) {
- blockUpdateDetailsList.add(resultOfBlock._2._1)
- segmentDetails.add(resultOfBlock._2._1.getSegmentName)
- // if this block is invalid then decrement block count in map.
- if (CarbonUpdateUtil.isBlockInvalid(resultOfBlock._2._1.getStatus)) {
- CarbonUpdateUtil.decrementDeletedBlockCount(resultOfBlock._2._1,
- blockMappingVO.getSegmentNumberOfBlockMapping)
- }
- }
- else {
- deleteStatus = false
- // In case of failure , clean all related delete delta files
- CarbonUpdateUtil.cleanStaleDeltaFiles(carbonTable, timestamp)
- LOGGER.audit(s"Delete data operation is failed for ${ database }.${ tableName }")
- val errorMsg =
- "Delete data operation is failed due to failure in creating delete delta file for " +
- "segment : " + resultOfBlock._2._1.getSegmentName + " block : " +
- resultOfBlock._2._1.getBlockName
- executorErrors.failureCauses = resultOfBlock._2._2.failureCauses
- executorErrors.errorMsg = resultOfBlock._2._2.errorMsg
-
- if (executorErrors.failureCauses == FailureCauses.NONE) {
- executorErrors.failureCauses = FailureCauses.EXECUTOR_FAILURE
- executorErrors.errorMsg = errorMsg
- }
- LOGGER.error(errorMsg)
- return
- }
- }
- )
- )
-
- val listOfSegmentToBeMarkedDeleted = CarbonUpdateUtil
- .getListOfSegmentsToMarkDeleted(blockMappingVO.getSegmentNumberOfBlockMapping)
-
-
-
- // this is delete flow so no need of putting timestamp in the status file.
- if (CarbonUpdateUtil
- .updateSegmentStatus(blockUpdateDetailsList, carbonTable, timestamp, false) &&
- CarbonUpdateUtil
- .updateTableMetadataStatus(segmentDetails,
- carbonTable,
- timestamp,
- !isUpdateOperation,
- listOfSegmentToBeMarkedDeleted)
- ) {
- LOGGER.info(s"Delete data operation is successful for ${ database }.${ tableName }")
- LOGGER.audit(s"Delete data operation is successful for ${ database }.${ tableName }")
- }
- else {
- // In case of failure , clean all related delete delta files
- CarbonUpdateUtil.cleanStaleDeltaFiles(carbonTable, timestamp)
-
- val errorMessage = "Delete data operation is failed due to failure " +
- "in table status updation."
- LOGGER.audit(s"Delete data operation is failed for ${ database }.${ tableName }")
- LOGGER.error("Delete data operation is failed due to failure in table status updation.")
- executorErrors.failureCauses = FailureCauses.STATUS_FILE_UPDATION_FAILURE
- executorErrors.errorMsg = errorMessage
- // throw new Exception(errorMessage)
- }
- }
-
- def deleteDeltaFunc(index: Int,
- key: String,
- iter: Iterator[Row],
- timestamp: String,
- rowCountDetailsVO: RowCountDetailsVO):
- Iterator[(String, (SegmentUpdateDetails, ExecutionErrors))] = {
-
- val result = new DeleteDelataResultImpl()
- var deleteStatus = CarbonCommonConstants.STORE_LOADSTATUS_FAILURE
- val LOGGER = LogServiceFactory.getLogService(this.getClass.getName)
- // here key = segment/blockName
- val blockName = CarbonUpdateUtil
- .getBlockName(
- CarbonTablePath.addDataPartPrefix(key.split(CarbonCommonConstants.FILE_SEPARATOR)(1)))
- val segmentId = key.split(CarbonCommonConstants.FILE_SEPARATOR)(0)
- var deleteDeltaBlockDetails: DeleteDeltaBlockDetails = new DeleteDeltaBlockDetails(blockName)
- val resultIter = new Iterator[(String, (SegmentUpdateDetails, ExecutionErrors))] {
- val segmentUpdateDetails = new SegmentUpdateDetails()
- var TID = ""
- var countOfRows = 0
- try {
- while (iter.hasNext) {
- val oneRow = iter.next
- TID = oneRow
- .get(oneRow.fieldIndex(CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID)).toString
- val offset = CarbonUpdateUtil.getRequiredFieldFromTID(TID, TupleIdEnum.OFFSET)
- val blockletId = CarbonUpdateUtil
- .getRequiredFieldFromTID(TID, TupleIdEnum.BLOCKLET_ID)
- val pageId = Integer.parseInt(CarbonUpdateUtil
- .getRequiredFieldFromTID(TID, TupleIdEnum.PAGE_ID))
- val IsValidOffset = deleteDeltaBlockDetails.addBlocklet(blockletId, offset, pageId)
- // stop delete operation
- if(!IsValidOffset) {
- executorErrors.failureCauses = FailureCauses.MULTIPLE_INPUT_ROWS_MATCHING
- executorErrors.errorMsg = "Multiple input rows matched for same row."
- throw new MultipleMatchingException("Multiple input rows matched for same row.")
- }
- countOfRows = countOfRows + 1
- }
-
- val blockPath = CarbonUpdateUtil.getTableBlockPath(TID, factPath)
- val completeBlockName = CarbonTablePath
- .addDataPartPrefix(CarbonUpdateUtil.getRequiredFieldFromTID(TID, TupleIdEnum.BLOCK_ID) +
- CarbonCommonConstants.FACT_FILE_EXT)
- val deleteDeletaPath = CarbonUpdateUtil
- .getDeleteDeltaFilePath(blockPath, blockName, timestamp)
- val carbonDeleteWriter = new CarbonDeleteDeltaWriterImpl(deleteDeletaPath,
- FileFactory.getFileType(deleteDeletaPath))
-
-
-
- segmentUpdateDetails.setBlockName(blockName)
- segmentUpdateDetails.setActualBlockName(completeBlockName)
- segmentUpdateDetails.setSegmentName(segmentId)
- segmentUpdateDetails.setDeleteDeltaEndTimestamp(timestamp)
- segmentUpdateDetails.setDeleteDeltaStartTimestamp(timestamp)
-
- val alreadyDeletedRows: Long = rowCountDetailsVO.getDeletedRowsInBlock
- val totalDeletedRows: Long = alreadyDeletedRows + countOfRows
- segmentUpdateDetails.setDeletedRowsInBlock(totalDeletedRows.toString)
- if (totalDeletedRows == rowCountDetailsVO.getTotalNumberOfRows) {
- segmentUpdateDetails.setStatus(CarbonCommonConstants.MARKED_FOR_DELETE)
- }
- else {
- // write the delta file
- carbonDeleteWriter.write(deleteDeltaBlockDetails)
- }
-
- deleteStatus = CarbonCommonConstants.STORE_LOADSTATUS_SUCCESS
- } catch {
- case e : MultipleMatchingException =>
- LOGGER.audit(e.getMessage)
- LOGGER.error(e.getMessage)
- // dont throw exception here.
- case e: Exception =>
- val errorMsg = s"Delete data operation is failed for ${ database }.${ tableName }."
- LOGGER.audit(errorMsg)
- LOGGER.error(errorMsg + e.getMessage)
- throw e
- }
-
-
- var finished = false
-
- override def hasNext: Boolean = {
- if (!finished) {
- finished = true
- finished
- }
- else {
- !finished
- }
- }
-
- override def next(): (String, (SegmentUpdateDetails, ExecutionErrors)) = {
- finished = true
- result.getKey(deleteStatus, (segmentUpdateDetails, executorErrors))
- }
- }
- resultIter
- }
- true
- }
-}
-
-
-
-object UpdateExecution {
-
- def performUpdate(
- dataFrame: DataFrame,
- tableIdentifier: Seq[String],
- plan: LogicalPlan,
- sqlContext: SQLContext,
- currentTime: Long,
- executorErrors: ExecutionErrors): Unit = {
-
- def isDestinationRelation(relation: CarbonDatasourceRelation): Boolean = {
-
- val tableName = relation.getTable()
- val dbName = relation.getDatabaseName()
- (tableIdentifier.size > 1 &&
- tableIdentifier(0) == dbName &&
- tableIdentifier(1) == tableName) ||
- (tableIdentifier(0) == tableName)
- }
- def getHeader(relation: CarbonDatasourceRelation, plan: LogicalPlan): String = {
- var header = ""
- var found = false
-
- plan match {
- case Project(pList, _) if (!found) =>
- found = true
- header = pList
- .filter(field => !field.name
- .equalsIgnoreCase(CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID))
- .map(col => if (col.name.endsWith(CarbonCommonConstants.UPDATED_COL_EXTENSION)) {
- col.name
- .substring(0, col.name.lastIndexOf(CarbonCommonConstants.UPDATED_COL_EXTENSION))
- }
- else {
- col.name
- }).mkString(",")
- }
- header
- }
- val ex = dataFrame.queryExecution.analyzed
- val res = ex find {
- case relation: LogicalRelation if (relation.relation.isInstanceOf[CarbonDatasourceRelation] &&
- isDestinationRelation(relation.relation
- .asInstanceOf[CarbonDatasourceRelation])) =>
- true
- case _ => false
- }
- val carbonRelation: CarbonDatasourceRelation = res match {
- case Some(relation: LogicalRelation) =>
- relation.relation.asInstanceOf[CarbonDatasourceRelation]
- case _ => sys.error("")
- }
-
- val updateTableModel = UpdateTableModel(true, currentTime, executorErrors)
-
- val header = getHeader(carbonRelation, plan)
-
-
-
- LoadTable(
- Some(carbonRelation.getDatabaseName()),
- carbonRelation.getTable(),
- null,
- Seq(),
- Map(("fileheader" -> header)),
- false,
- null,
- Some(dataFrame),
- Some(updateTableModel)).run(sqlContext)
-
-
- executorErrors.errorMsg = updateTableModel.executorErrors.errorMsg
- executorErrors.failureCauses = updateTableModel.executorErrors.failureCauses
-
- Seq.empty
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
deleted file mode 100644
index 9814cc2..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ /dev/null
@@ -1,1019 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.execution.command
-
-import java.io.File
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable.ListBuffer
-import scala.language.implicitConversions
-
-import org.apache.commons.lang3.StringUtils
-import org.apache.spark.scheduler.{SparkListener, SparkListenerApplicationEnd}
-import org.apache.spark.sql._
-import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute
-import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference}
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.execution.{RunnableCommand, SparkPlan}
-import org.apache.spark.sql.hive.CarbonMetastore
-import org.apache.spark.sql.types.TimestampType
-import org.apache.spark.util.{CausedBy, FileUtils}
-import org.codehaus.jackson.map.ObjectMapper
-
-import org.apache.carbondata.api.CarbonStore
-import org.apache.carbondata.common.constants.LoggerAction
-import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.constants.{CarbonCommonConstants, CarbonLoadOptionConstants}
-import org.apache.carbondata.core.datastore.impl.FileFactory
-import org.apache.carbondata.core.dictionary.server.DictionaryServer
-import org.apache.carbondata.core.exception.InvalidConfigurationException
-import org.apache.carbondata.core.locks.{CarbonLockFactory, CarbonLockUtil, ICarbonLock, LockUsage}
-import org.apache.carbondata.core.metadata.{CarbonMetadata, CarbonTableIdentifier}
-import org.apache.carbondata.core.metadata.encoder.Encoding
-import org.apache.carbondata.core.metadata.schema.table.TableInfo
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
-import org.apache.carbondata.core.mutate.{CarbonUpdateUtil, TupleIdEnum}
-import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
-import org.apache.carbondata.core.util.path.CarbonStorePath
-import org.apache.carbondata.processing.exception.DataLoadingException
-import org.apache.carbondata.processing.loading.constants.DataLoadProcessorConstants
-import org.apache.carbondata.processing.loading.exception.NoRetryException
-import org.apache.carbondata.processing.loading.model.{CarbonDataLoadSchema, CarbonLoadModel}
-import org.apache.carbondata.processing.util.TableOptionConstant
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
-import org.apache.carbondata.spark.load.ValidateUtil
-import org.apache.carbondata.spark.rdd.{CarbonDataRDDFactory, DataManagementFunc, DictionaryLoadModel}
-import org.apache.carbondata.spark.util.{CommonUtil, GlobalDictionaryUtil}
-
-object Checker {
- def validateTableExists(
- dbName: Option[String],
- tableName: String,
- sqlContext: SQLContext): Unit = {
- val identifier = TableIdentifier(tableName, dbName)
- if (!CarbonEnv.get.carbonMetastore.tableExists(identifier)(sqlContext)) {
- val err = s"table $dbName.$tableName not found"
- LogServiceFactory.getLogService(this.getClass.getName).error(err)
- throw new IllegalArgumentException(err)
- }
- }
-}
-
-/**
- * Command for show table partitions Command
- *
- * @param tableIdentifier
- */
-private[sql] case class ShowCarbonPartitionsCommand(
- tableIdentifier: TableIdentifier) extends RunnableCommand {
- val LOGGER = LogServiceFactory.getLogService(ShowCarbonPartitionsCommand.getClass.getName)
- override val output = CommonUtil.partitionInfoOutput
- override def run(sqlContext: SQLContext): Seq[Row] = {
- val relation = CarbonEnv.get.carbonMetastore
- .lookupRelation1(tableIdentifier)(sqlContext).
- asInstanceOf[CarbonRelation]
- val carbonTable = relation.tableMeta.carbonTable
- var tableName = carbonTable.getFactTableName
- var partitionInfo = carbonTable.getPartitionInfo(
- carbonTable.getAbsoluteTableIdentifier.getCarbonTableIdentifier.getTableName)
- if (partitionInfo == null) {
- throw new AnalysisException(
- s"SHOW PARTITIONS is not allowed on a table that is not partitioned: $tableName")
- }
- var partitionType = partitionInfo.getPartitionType
- var columnName = partitionInfo.getColumnSchemaList.get(0).getColumnName
- LOGGER.info("partition column name:" + columnName)
- CommonUtil.getPartitionInfo(columnName, partitionType, partitionInfo)
- }
-}
-
-/**
- * Command for the compaction in alter table command
- *
- * @param alterTableModel
- */
-private[sql] case class AlterTableCompaction(alterTableModel: AlterTableModel) extends
- RunnableCommand {
-
- def run(sqlContext: SQLContext): Seq[Row] = {
- // TODO : Implement it.
- val tableName = alterTableModel.tableName
- val databaseName = getDB.getDatabaseName(alterTableModel.dbName, sqlContext)
- if (null == CarbonMetadata.getInstance.getCarbonTable(databaseName + "_" + tableName)) {
- logError(s"alter table failed. table not found: $databaseName.$tableName")
- sys.error(s"alter table failed. table not found: $databaseName.$tableName")
- }
-
- val relation =
- CarbonEnv.get.carbonMetastore
- .lookupRelation1(Option(databaseName), tableName)(sqlContext)
- .asInstanceOf[CarbonRelation]
- if (relation == null) {
- sys.error(s"Table $databaseName.$tableName does not exist")
- }
- val carbonLoadModel = new CarbonLoadModel()
-
-
- val table = relation.tableMeta.carbonTable
- carbonLoadModel.setTableName(table.getFactTableName)
- val dataLoadSchema = new CarbonDataLoadSchema(table)
- // Need to fill dimension relation
- carbonLoadModel.setCarbonDataLoadSchema(dataLoadSchema)
- carbonLoadModel.setTableName(relation.tableMeta.carbonTableIdentifier.getTableName)
- carbonLoadModel.setDatabaseName(relation.tableMeta.carbonTableIdentifier.getDatabaseName)
- carbonLoadModel.setStorePath(relation.tableMeta.storePath)
-
- var storeLocation = CarbonProperties.getInstance
- .getProperty(CarbonCommonConstants.STORE_LOCATION_TEMP_PATH,
- System.getProperty("java.io.tmpdir")
- )
- storeLocation = storeLocation + "/carbonstore/" + System.nanoTime()
- try {
- CarbonDataRDDFactory.alterTableForCompaction(sqlContext,
- alterTableModel,
- carbonLoadModel,
- relation.tableMeta.storePath,
- storeLocation
- )
- } catch {
- case e: Exception =>
- if (null != e.getMessage) {
- sys.error(s"Compaction failed. Please check logs for more info. ${ e.getMessage }")
- } else {
- sys.error("Exception in compaction. Please check logs for more info.")
- }
- }
- Seq.empty
- }
-}
-
-case class CreateTable(cm: TableModel) extends RunnableCommand {
-
- def run(sqlContext: SQLContext): Seq[Row] = {
- val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
- cm.databaseName = getDB.getDatabaseName(cm.databaseNameOp, sqlContext)
- val tbName = cm.tableName
- val dbName = cm.databaseName
- LOGGER.audit(s"Creating Table with Database name [$dbName] and Table name [$tbName]")
-
- val tableInfo: TableInfo = TableNewProcessor(cm)
-
- // Add validation for sort scope when create table
- val sortScope = tableInfo.getFactTable.getTableProperties
- .getOrDefault("sort_scope", CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT)
- if (!CarbonUtil.isValidSortOption(sortScope)) {
- throw new InvalidConfigurationException(s"Passing invalid SORT_SCOPE '$sortScope'," +
- s" valid SORT_SCOPE are 'NO_SORT', 'BATCH_SORT', 'LOCAL_SORT' and 'GLOBAL_SORT' ")
- }
-
- if (tableInfo.getFactTable.getListOfColumns.isEmpty) {
- sys.error("No Dimensions found. Table should have at least one dimesnion !")
- }
-
- if (sqlContext.tableNames(dbName).exists(_.equalsIgnoreCase(tbName))) {
- if (!cm.ifNotExistsSet) {
- LOGGER.audit(
- s"Table creation with Database name [$dbName] and Table name [$tbName] failed. " +
- s"Table [$tbName] already exists under database [$dbName]")
- sys.error(s"Table [$tbName] already exists under database [$dbName]")
- }
- } else {
- // Add Database to catalog and persist
- val catalog = CarbonEnv.get.carbonMetastore
- // Need to fill partitioner class when we support partition
- val tablePath = catalog.createTableFromThrift(tableInfo, dbName, tbName, null)(sqlContext)
- try {
- sqlContext.sql(
- s"""CREATE TABLE $dbName.$tbName USING carbondata""" +
- s""" OPTIONS (tableName "$dbName.$tbName", tablePath "$tablePath") """)
- .collect
- } catch {
- case e: Exception =>
- val identifier: TableIdentifier = TableIdentifier(tbName, Some(dbName))
- // call the drop table to delete the created table.
-
- CarbonEnv.get.carbonMetastore
- .dropTable(catalog.storePath, identifier)(sqlContext)
-
- LOGGER.audit(s"Table creation with Database name [$dbName] " +
- s"and Table name [$tbName] failed")
- throw e
- }
-
- LOGGER.audit(s"Table created with Database name [$dbName] and Table name [$tbName]")
- }
-
- Seq.empty
- }
-
- def setV(ref: Any, name: String, value: Any): Unit = {
- ref.getClass.getFields.find(_.getName == name).get
- .set(ref, value.asInstanceOf[AnyRef])
- }
-}
-
-private[sql] case class DeleteLoadsById(
- loadids: Seq[String],
- databaseNameOp: Option[String],
- tableName: String) extends RunnableCommand {
-
- val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
-
- def run(sqlContext: SQLContext): Seq[Row] = {
- Checker.validateTableExists(databaseNameOp, tableName, sqlContext)
- val carbonTable = CarbonEnv.get.carbonMetastore.lookupRelation1(databaseNameOp,
- tableName)(sqlContext).asInstanceOf[CarbonRelation].tableMeta.carbonTable
- CarbonStore.deleteLoadById(
- loadids,
- getDB.getDatabaseName(databaseNameOp, sqlContext),
- tableName,
- carbonTable
- )
- Seq.empty
-
- }
-
- // validates load ids
- private def validateLoadIds: Unit = {
- if (loadids.isEmpty) {
- val errorMessage = "Error: Segment id(s) should not be empty."
- throw new MalformedCarbonCommandException(errorMessage)
-
- }
- }
-}
-
-private[sql] case class DeleteLoadsByLoadDate(
- databaseNameOp: Option[String],
- tableName: String,
- dateField: String,
- loadDate: String) extends RunnableCommand {
-
- val LOGGER = LogServiceFactory.getLogService("org.apache.spark.sql.tablemodel.tableSchema")
-
- def run(sqlContext: SQLContext): Seq[Row] = {
- Checker.validateTableExists(databaseNameOp, tableName, sqlContext)
- val carbonTable = CarbonEnv.get.carbonMetastore.lookupRelation1(databaseNameOp,
- tableName)(sqlContext).asInstanceOf[CarbonRelation].tableMeta.carbonTable
- CarbonStore.deleteLoadByDate(
- loadDate,
- getDB.getDatabaseName(databaseNameOp, sqlContext),
- tableName,
- carbonTable
- )
- Seq.empty
-
- }
-
-}
-
-object LoadTable {
-
- def updateTableMetadata(carbonLoadModel: CarbonLoadModel,
- sqlContext: SQLContext,
- model: DictionaryLoadModel,
- noDictDimension: Array[CarbonDimension]): Unit = {
-
- val carbonTablePath = CarbonStorePath.getCarbonTablePath(model.hdfsLocation,
- model.table)
- val schemaFilePath = carbonTablePath.getSchemaFilePath
-
- // read TableInfo
- val tableInfo = CarbonMetastore.readSchemaFileToThriftTable(schemaFilePath)
-
- // modify TableInfo
- val columns = tableInfo.getFact_table.getTable_columns
- for (i <- 0 until columns.size) {
- if (noDictDimension.exists(x => columns.get(i).getColumn_id.equals(x.getColumnId))) {
- columns.get(i).encoders.remove(org.apache.carbondata.format.Encoding.DICTIONARY)
- }
- }
-
- // write TableInfo
- CarbonMetastore.writeThriftTableToSchemaFile(schemaFilePath, tableInfo)
-
- // update Metadata
- val catalog = CarbonEnv.get.carbonMetastore
- catalog.updateMetadataByThriftTable(schemaFilePath, tableInfo,
- model.table.getDatabaseName, model.table.getTableName, carbonLoadModel.getStorePath)
-
- // update CarbonDataLoadSchema
- val carbonTable = catalog.lookupRelation1(Option(model.table.getDatabaseName),
- model.table.getTableName)(sqlContext).asInstanceOf[CarbonRelation].tableMeta.carbonTable
- carbonLoadModel.setCarbonDataLoadSchema(new CarbonDataLoadSchema(carbonTable))
- }
-
-}
-
-private[sql] case class LoadTableByInsert(relation: CarbonDatasourceRelation,
- child: LogicalPlan, isOverwriteExist: Boolean) extends RunnableCommand {
- val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
- def run(sqlContext: SQLContext): Seq[Row] = {
- val df = new DataFrame(sqlContext, child)
- val header = relation.carbonRelation.output.map(_.name).mkString(",")
- val load = LoadTable(
- Some(relation.carbonRelation.databaseName),
- relation.carbonRelation.tableName,
- null,
- Seq(),
- scala.collection.immutable.Map("fileheader" -> header),
- isOverwriteExist,
- null,
- Some(df)).run(sqlContext)
- // updating relation metadata. This is in case of auto detect high cardinality
- relation.carbonRelation.metaData =
- CarbonSparkUtil.createSparkMeta(relation.carbonRelation.tableMeta.carbonTable)
- load
- }
-}
-case class LoadTable(
- databaseNameOp: Option[String],
- tableName: String,
- factPathFromUser: String,
- dimFilesPath: Seq[DataLoadTableFileMapping],
- options: scala.collection.immutable.Map[String, String],
- isOverwriteExist: Boolean,
- var inputSqlString: String = null,
- dataFrame: Option[DataFrame] = None,
- updateModel: Option[UpdateTableModel] = None) extends RunnableCommand {
-
- val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
-
- private def checkDefaultValue(value: String, default: String) = if (StringUtils.isEmpty(value)) {
- default
- } else {
- value
- }
-
- def run(sqlContext: SQLContext): Seq[Row] = {
- if (dataFrame.isDefined && !updateModel.isDefined) {
- val rdd = dataFrame.get.rdd
- if (rdd.partitions == null || rdd.partitions.length == 0) {
- LOGGER.warn("DataLoading finished. No data was loaded.")
- return Seq.empty
- }
- }
-
- val dbName = getDB.getDatabaseName(databaseNameOp, sqlContext)
- if (null == CarbonMetadata.getInstance.getCarbonTable(dbName + "_" + tableName)) {
- logError(s"Data loading failed. table not found: $dbName.$tableName")
- LOGGER.audit(s"Data loading failed. table not found: $dbName.$tableName")
- sys.error(s"Data loading failed. table not found: $dbName.$tableName")
- }
-
- val relation = CarbonEnv.get.carbonMetastore
- .lookupRelation1(Option(dbName), tableName)(sqlContext)
- .asInstanceOf[CarbonRelation]
- if (relation == null) {
- sys.error(s"Table $dbName.$tableName does not exist")
- }
- CarbonProperties.getInstance().addProperty("zookeeper.enable.lock", "false")
- val carbonLock = CarbonLockFactory
- .getCarbonLockObj(relation.tableMeta.carbonTable.getAbsoluteTableIdentifier
- .getCarbonTableIdentifier,
- LockUsage.METADATA_LOCK
- )
- try {
- // take lock only in case of normal data load.
- if (!updateModel.isDefined) {
- if (carbonLock.lockWithRetries()) {
- logInfo("Successfully able to get the table metadata file lock")
- } else {
- sys.error("Table is locked for updation. Please try after some time")
- }
- }
-
- val factPath = if (dataFrame.isDefined) {
- ""
- } else {
- FileUtils.getPaths(
- CarbonUtil.checkAndAppendHDFSUrl(factPathFromUser))
- }
- val carbonLoadModel = new CarbonLoadModel()
- carbonLoadModel.setTableName(relation.tableMeta.carbonTableIdentifier.getTableName)
- carbonLoadModel.setDatabaseName(relation.tableMeta.carbonTableIdentifier.getDatabaseName)
- carbonLoadModel.setStorePath(relation.tableMeta.storePath)
-
- val table = relation.tableMeta.carbonTable
- carbonLoadModel.setTableName(table.getFactTableName)
- val dataLoadSchema = new CarbonDataLoadSchema(table)
- // Need to fill dimension relation
- carbonLoadModel.setCarbonDataLoadSchema(dataLoadSchema)
-
- val partitionLocation = relation.tableMeta.storePath + "/partition/" +
- relation.tableMeta.carbonTableIdentifier.getDatabaseName + "/" +
- relation.tableMeta.carbonTableIdentifier.getTableName + "/"
-
-
- val columnar = sqlContext.getConf("carbon.is.columnar.storage", "true").toBoolean
-
- val delimiter = options.getOrElse("delimiter", ",")
- val quoteChar = options.getOrElse("quotechar", "\"")
- var fileHeader = options.getOrElse("fileheader", "")
- val escapeChar = options.getOrElse("escapechar", "\\")
- val commentchar = options.getOrElse("commentchar", "#")
- val columnDict = options.getOrElse("columndict", null)
- val serializationNullFormat = options.getOrElse("serialization_null_format", "\\N")
- val badRecordsLoggerEnable = options.getOrElse("bad_records_logger_enable", "false")
- val badRecordActionValue = CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION,
- CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION_DEFAULT)
- val badRecordsAction = options.getOrElse("bad_records_action", badRecordActionValue)
- val isEmptyDataBadRecord = options.getOrElse("is_empty_data_bad_record", "false")
- val allDictionaryPath = options.getOrElse("all_dictionary_path", "")
- val complex_delimiter_level_1 = options.getOrElse("complex_delimiter_level_1", "\\$")
- val complex_delimiter_level_2 = options.getOrElse("complex_delimiter_level_2", "\\:")
- val dateFormat = options.getOrElse("dateformat", null)
- ValidateUtil.validateDateFormat(dateFormat, table, tableName)
- val maxColumns = options.getOrElse("maxcolumns", null)
- val tableProperties = table.getTableInfo.getFactTable.getTableProperties
- val sortScopeDefault = CarbonProperties.getInstance().
- getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_SORT_SCOPE,
- CarbonProperties.getInstance().getProperty(CarbonCommonConstants.LOAD_SORT_SCOPE,
- CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT))
- val sortScope = if (null == tableProperties) {
- sortScopeDefault
- } else {
- tableProperties.getOrDefault("sort_scope", sortScopeDefault)
- }
-
- ValidateUtil.validateSortScope(table, sortScope)
- val carbonProperty: CarbonProperties = CarbonProperties.getInstance()
- val batchSortSizeInMB = options.getOrElse("batch_sort_size_inmb", carbonProperty
- .getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_BATCH_SORT_SIZE_INMB,
- carbonProperty.getProperty(CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB,
- CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB_DEFAULT)))
- val globalSortPartitions = options.getOrElse("global_sort_partitions", carbonProperty
- .getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_GLOBAL_SORT_PARTITIONS, null))
- ValidateUtil.validateGlobalSortPartitions(globalSortPartitions)
-
- // if there isn't file header in csv file and load sql doesn't provide FILEHEADER option,
- // we should use table schema to generate file header.
- val headerOption = options.get("header")
- if (headerOption.isDefined) {
- // whether the csv file has file header
- // the default value is true
- val header = try {
- headerOption.get.toBoolean
- } catch {
- case ex: IllegalArgumentException =>
- throw new MalformedCarbonCommandException(
- "'header' option should be either 'true' or 'false'. " + ex.getMessage)
- }
- header match {
- case true =>
- if (fileHeader.nonEmpty) {
- throw new MalformedCarbonCommandException(
- "When 'header' option is true, 'fileheader' option is not required.")
- }
- case false =>
- // generate file header
- if (fileHeader.isEmpty) {
- fileHeader = table.getCreateOrderColumn(table.getFactTableName)
- .asScala.map(_.getColName).mkString(",")
- }
- }
- }
-
- val bad_record_path = options.getOrElse("bad_record_path",
- CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC,
- CarbonCommonConstants.CARBON_BADRECORDS_LOC_DEFAULT_VAL))
- if (badRecordsLoggerEnable.toBoolean ||
- LoggerAction.REDIRECT.name().equalsIgnoreCase(badRecordsAction)) {
- if (!CarbonUtil.isValidBadStorePath(bad_record_path)) {
- sys.error("Invalid bad records location.")
- }
- }
- carbonLoadModel.setBadRecordsLocation(bad_record_path)
- carbonLoadModel.setEscapeChar(checkDefaultValue(escapeChar, "\\"))
- carbonLoadModel.setQuoteChar(checkDefaultValue(quoteChar, "\""))
- carbonLoadModel.setCommentChar(checkDefaultValue(commentchar, "#"))
- carbonLoadModel.setDateFormat(dateFormat)
- carbonLoadModel.setDefaultTimestampFormat(CarbonProperties.getInstance().getProperty(
- CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
- CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT))
- carbonLoadModel.setDefaultDateFormat(CarbonProperties.getInstance().getProperty(
- CarbonCommonConstants.CARBON_DATE_FORMAT,
- CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT))
- carbonLoadModel
- .setSerializationNullFormat(
- TableOptionConstant.SERIALIZATION_NULL_FORMAT.getName + "," + serializationNullFormat)
- carbonLoadModel
- .setBadRecordsLoggerEnable(
- TableOptionConstant.BAD_RECORDS_LOGGER_ENABLE.getName + "," + badRecordsLoggerEnable)
- carbonLoadModel
- .setBadRecordsAction(
- TableOptionConstant.BAD_RECORDS_ACTION.getName + "," + badRecordsAction)
- carbonLoadModel
- .setIsEmptyDataBadRecord(
- DataLoadProcessorConstants.IS_EMPTY_DATA_BAD_RECORD + "," + isEmptyDataBadRecord)
- carbonLoadModel.setSortScope(sortScope)
- carbonLoadModel.setBatchSortSizeInMb(batchSortSizeInMB)
- carbonLoadModel.setGlobalSortPartitions(globalSortPartitions)
- // when single_pass=true, and not use all dict
- val useOnePass = options.getOrElse("single_pass", "false").trim.toLowerCase match {
- case "true" =>
- true
- case "false" =>
- // when single_pass = false and if either alldictionary
- // or columnDict is configured the do not allow load
- if (StringUtils.isNotEmpty(allDictionaryPath) || StringUtils.isNotEmpty(columnDict)) {
- throw new MalformedCarbonCommandException(
- "Can not use all_dictionary_path or columndict without single_pass.")
- } else {
- false
- }
- case illegal =>
- LOGGER.error(s"Can't use single_pass, because illegal syntax found: [" + illegal + "] " +
- "Please set it as 'true' or 'false'")
- false
- }
- carbonLoadModel.setUseOnePass(useOnePass)
-
- if (delimiter.equalsIgnoreCase(complex_delimiter_level_1) ||
- complex_delimiter_level_1.equalsIgnoreCase(complex_delimiter_level_2) ||
- delimiter.equalsIgnoreCase(complex_delimiter_level_2)) {
- sys.error(s"Field Delimiter & Complex types delimiter are same")
- }
- else {
- carbonLoadModel.setComplexDelimiterLevel1(
- CarbonUtil.delimiterConverter(complex_delimiter_level_1))
- carbonLoadModel.setComplexDelimiterLevel2(
- CarbonUtil.delimiterConverter(complex_delimiter_level_2))
- }
- // set local dictionary path, and dictionary file extension
- carbonLoadModel.setAllDictPath(allDictionaryPath)
-
- val partitionStatus = CarbonCommonConstants.STORE_LOADSTATUS_SUCCESS
-
- try {
- // First system has to partition the data first and then call the load data
- LOGGER.info(s"Initiating Direct Load for the Table : ($dbName.$tableName)")
- carbonLoadModel.setFactFilePath(factPath)
- carbonLoadModel.setCsvDelimiter(CarbonUtil.unescapeChar(delimiter))
- carbonLoadModel.setCsvHeader(fileHeader)
- carbonLoadModel.setColDictFilePath(columnDict)
- carbonLoadModel.setDirectLoad(true)
- carbonLoadModel.setCsvHeaderColumns(CommonUtil.getCsvHeaderColumns(carbonLoadModel))
- val validatedMaxColumns = CommonUtil.validateMaxColumns(carbonLoadModel.getCsvHeaderColumns,
- maxColumns)
- carbonLoadModel.setMaxColumns(validatedMaxColumns.toString)
- GlobalDictionaryUtil.updateTableMetadataFunc = updateTableMetadata
- val storePath = relation.tableMeta.storePath
- val carbonTable = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
- val carbonTableIdentifier = carbonTable.getAbsoluteTableIdentifier
- .getCarbonTableIdentifier
- val carbonTablePath = CarbonStorePath
- .getCarbonTablePath(storePath, carbonTableIdentifier)
- val dictFolderPath = carbonTablePath.getMetadataDirectoryPath
- val dimensions = carbonTable.getDimensionByTableName(
- carbonTable.getFactTableName).asScala.toArray
- // add the start entry for the new load in the table status file
- if (!updateModel.isDefined) {
- CommonUtil.
- readAndUpdateLoadProgressInTableMeta(carbonLoadModel, storePath, isOverwriteExist)
- }
- if (isOverwriteExist) {
- LOGGER.info(s"Overwrite is in progress for carbon table with $dbName.$tableName")
- }
- if (null == carbonLoadModel.getLoadMetadataDetails) {
- CommonUtil.readLoadMetadataDetails(carbonLoadModel)
- }
- if (carbonLoadModel.getLoadMetadataDetails.isEmpty && carbonLoadModel.getUseOnePass &&
- StringUtils.isEmpty(columnDict) && StringUtils.isEmpty(allDictionaryPath)) {
- LOGGER.info(s"Cannot use single_pass=true for $dbName.$tableName during the first load")
- LOGGER.audit(s"Cannot use single_pass=true for $dbName.$tableName during the first load")
- carbonLoadModel.setUseOnePass(false)
- }
- if (carbonLoadModel.getUseOnePass) {
- val colDictFilePath = carbonLoadModel.getColDictFilePath
- if (!StringUtils.isEmpty(colDictFilePath)) {
- carbonLoadModel.initPredefDictMap()
- // generate predefined dictionary
- GlobalDictionaryUtil
- .generatePredefinedColDictionary(colDictFilePath, carbonTableIdentifier,
- dimensions, carbonLoadModel, sqlContext, storePath, dictFolderPath)
- }
- val allDictPath: String = carbonLoadModel.getAllDictPath
- if(!StringUtils.isEmpty(allDictPath)) {
- carbonLoadModel.initPredefDictMap()
- GlobalDictionaryUtil
- .generateDictionaryFromDictionaryFiles(sqlContext,
- carbonLoadModel,
- storePath,
- carbonTableIdentifier,
- dictFolderPath,
- dimensions,
- allDictionaryPath)
- }
- // dictionaryServerClient dictionary generator
- val dictionaryServerPort = CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.DICTIONARY_SERVER_PORT,
- CarbonCommonConstants.DICTIONARY_SERVER_PORT_DEFAULT)
- val sparkDriverHost = sqlContext.sparkContext.getConf.get("spark.driver.host")
- carbonLoadModel.setDictionaryServerHost(sparkDriverHost)
- // start dictionary server when use one pass load and dimension with DICTIONARY
- // encoding is present.
- val allDimensions = table.getAllDimensions.asScala.toList
- val createDictionary = allDimensions.exists {
- carbonDimension => carbonDimension.hasEncoding(Encoding.DICTIONARY) &&
- !carbonDimension.hasEncoding(Encoding.DIRECT_DICTIONARY)
- }
- val server: Option[DictionaryServer] = if (createDictionary) {
- val dictionaryServer = DictionaryServer
- .getInstance(dictionaryServerPort.toInt, carbonTable)
- carbonLoadModel.setDictionaryServerPort(dictionaryServer.getPort)
- sqlContext.sparkContext.addSparkListener(new SparkListener() {
- override def onApplicationEnd(applicationEnd: SparkListenerApplicationEnd) {
- dictionaryServer.shutdown()
- }
- })
- Some(dictionaryServer)
- } else {
- None
- }
- CarbonDataRDDFactory.loadCarbonData(sqlContext,
- carbonLoadModel,
- relation.tableMeta.storePath,
- columnar,
- partitionStatus,
- server,
- isOverwriteExist,
- dataFrame,
- updateModel)
- } else {
- val (dictionaryDataFrame, loadDataFrame) = if (updateModel.isDefined) {
- val fields = dataFrame.get.schema.fields
- import org.apache.spark.sql.functions.udf
- // extracting only segment from tupleId
- val getSegIdUDF = udf((tupleId: String) =>
- CarbonUpdateUtil.getRequiredFieldFromTID(tupleId, TupleIdEnum.SEGMENT_ID))
- // getting all fields except tupleId field as it is not required in the value
- var otherFields = fields.toSeq
- .filter(field => !field.name
- .equalsIgnoreCase(CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID))
- .map(field => {
- if (field.name.endsWith(CarbonCommonConstants.UPDATED_COL_EXTENSION) && false) {
- new Column(field.name
- .substring(0,
- field.name.lastIndexOf(CarbonCommonConstants.UPDATED_COL_EXTENSION)))
- } else {
-
- new Column(field.name)
- }
- })
-
- // extract tupleId field which will be used as a key
- val segIdColumn = getSegIdUDF(new Column(UnresolvedAttribute
- .quotedString(CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID))).
- as(CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_SEGMENTID)
- // use dataFrameWithoutTupleId as dictionaryDataFrame
- val dataFrameWithoutTupleId = dataFrame.get.select(otherFields: _*)
- otherFields = otherFields :+ segIdColumn
- // use dataFrameWithTupleId as loadDataFrame
- val dataFrameWithTupleId = dataFrame.get.select(otherFields: _*)
- (Some(dataFrameWithoutTupleId), Some(dataFrameWithTupleId))
- } else {
- (dataFrame, dataFrame)
- }
- GlobalDictionaryUtil
- .generateGlobalDictionary(sqlContext, carbonLoadModel, relation.tableMeta.storePath,
- dictionaryDataFrame)
- CarbonDataRDDFactory.loadCarbonData(sqlContext,
- carbonLoadModel,
- relation.tableMeta.storePath,
- columnar,
- partitionStatus,
- None,
- isOverwriteExist,
- loadDataFrame,
- updateModel)
- }
- } catch {
- case CausedBy(ex: NoRetryException) =>
- LOGGER.error(ex, s"Dataload failure for $dbName.$tableName")
- throw new RuntimeException(s"Dataload failure for $dbName.$tableName, ${ex.getMessage}")
- case ex: Exception =>
- LOGGER.error(ex)
- LOGGER.audit(s"Dataload failure for $dbName.$tableName. Please check the logs")
- throw ex
- } finally {
- // Once the data load is successful delete the unwanted partition files
- try {
- val fileType = FileFactory.getFileType(partitionLocation)
- if (FileFactory.isFileExist(partitionLocation, fileType)) {
- val file = FileFactory
- .getCarbonFile(partitionLocation, fileType)
- CarbonUtil.deleteFoldersAndFiles(file)
- }
- } catch {
- case ex: Exception =>
- LOGGER.error(ex)
- LOGGER.audit(s"Dataload failure for $dbName.$tableName. " +
- "Problem deleting the partition folder")
- throw ex
- }
-
- }
- } catch {
- case dle: DataLoadingException =>
- LOGGER.audit(s"Dataload failed for $dbName.$tableName. " + dle.getMessage)
- throw dle
- case mce: MalformedCarbonCommandException =>
- LOGGER.audit(s"Dataload failed for $dbName.$tableName. " + mce.getMessage)
- throw mce
- } finally {
- if (carbonLock != null) {
- if (carbonLock.unlock()) {
- logInfo("Table MetaData Unlocked Successfully after data load")
- } else {
- logError("Unable to unlock Table MetaData")
- }
- }
- }
- Seq.empty
- }
-
- private def updateTableMetadata(carbonLoadModel: CarbonLoadModel,
- sqlContext: SQLContext,
- model: DictionaryLoadModel,
- noDictDimension: Array[CarbonDimension]): Unit = {
-
- val carbonTablePath = CarbonStorePath.getCarbonTablePath(model.hdfsLocation,
- model.table)
- val schemaFilePath = carbonTablePath.getSchemaFilePath
-
- // read TableInfo
- val tableInfo = CarbonMetastore.readSchemaFileToThriftTable(schemaFilePath)
-
- // modify TableInfo
- val columns = tableInfo.getFact_table.getTable_columns
- for (i <- 0 until columns.size) {
- if (noDictDimension.exists(x => columns.get(i).getColumn_id.equals(x.getColumnId))) {
- columns.get(i).encoders.remove(org.apache.carbondata.format.Encoding.DICTIONARY)
- }
- }
-
- // write TableInfo
- CarbonMetastore.writeThriftTableToSchemaFile(schemaFilePath, tableInfo)
-
-
- val catalog = CarbonEnv.get.carbonMetastore
-
- // upate the schema modified time
- catalog.updateSchemasUpdatedTime(catalog.touchSchemaFileSystemTime(
- carbonLoadModel.getDatabaseName,
- carbonLoadModel.getTableName))
-
- // update Metadata
- catalog.updateMetadataByThriftTable(schemaFilePath, tableInfo,
- model.table.getDatabaseName, model.table.getTableName, carbonLoadModel.getStorePath)
-
- // update CarbonDataLoadSchema
- val carbonTable = catalog.lookupRelation1(Option(model.table.getDatabaseName),
- model.table.getTableName)(sqlContext).asInstanceOf[CarbonRelation].tableMeta.carbonTable
- carbonLoadModel.setCarbonDataLoadSchema(new CarbonDataLoadSchema(carbonTable))
- }
-
-}
-
-private[sql] case class DropTableCommand(ifExistsSet: Boolean, databaseNameOp: Option[String],
- tableName: String)
- extends RunnableCommand {
-
- def run(sqlContext: SQLContext): Seq[Row] = {
- val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
- val dbName = getDB.getDatabaseName(databaseNameOp, sqlContext)
- val identifier = TableIdentifier(tableName, Option(dbName))
- val carbonTableIdentifier = new CarbonTableIdentifier(dbName, tableName, "")
- val locksToBeAcquired = List(LockUsage.METADATA_LOCK, LockUsage.DROP_TABLE_LOCK)
- val carbonLocks: scala.collection.mutable.ListBuffer[ICarbonLock] = ListBuffer()
- val catalog = CarbonEnv.get.carbonMetastore
- val storePath = catalog.storePath
- try {
- locksToBeAcquired foreach {
- lock => carbonLocks += CarbonLockUtil.getLockObject(carbonTableIdentifier, lock)
- }
- LOGGER.audit(s"Deleting table [$tableName] under database [$dbName]")
- CarbonEnv.get.carbonMetastore.dropTable(storePath, identifier)(sqlContext)
- LOGGER.audit(s"Deleted table [$tableName] under database [$dbName]")
- } catch {
- case ex: Exception =>
- LOGGER.error(ex, s"Dropping table $dbName.$tableName failed")
- sys.error(s"Dropping table $dbName.$tableName failed: ${ex.getMessage}")
- } finally {
- if (carbonLocks.nonEmpty) {
- val unlocked = carbonLocks.forall(_.unlock())
- if (unlocked) {
- logInfo("Table MetaData Unlocked Successfully")
- // deleting any remaining files.
- val metadataFilePath = CarbonStorePath
- .getCarbonTablePath(storePath, carbonTableIdentifier).getMetadataDirectoryPath
- val fileType = FileFactory.getFileType(metadataFilePath)
- if (FileFactory.isFileExist(metadataFilePath, fileType)) {
- val file = FileFactory.getCarbonFile(metadataFilePath, fileType)
- CarbonUtil.deleteFoldersAndFiles(file.getParentFile)
- }
- }
- }
- }
- Seq.empty
- }
-}
-
-private[sql] case class ShowLoads(
- databaseNameOp: Option[String],
- tableName: String,
- limit: Option[String],
- override val output: Seq[Attribute]) extends RunnableCommand {
-
- override def run(sqlContext: SQLContext): Seq[Row] = {
- Checker.validateTableExists(databaseNameOp, tableName, sqlContext)
- val carbonTable = CarbonEnv.get.carbonMetastore.lookupRelation1(databaseNameOp,
- tableName)(sqlContext).asInstanceOf[CarbonRelation].tableMeta.carbonTable
- CarbonStore.showSegments(
- getDB.getDatabaseName(databaseNameOp, sqlContext),
- tableName,
- limit,
- carbonTable.getMetaDataFilepath
- )
- }
-}
-
-private[sql] case class DescribeCommandFormatted(
- child: SparkPlan,
- override val output: Seq[Attribute],
- tblIdentifier: TableIdentifier)
- extends RunnableCommand {
-
- override def run(sqlContext: SQLContext): Seq[Row] = {
- val relation = CarbonEnv.get.carbonMetastore
- .lookupRelation1(tblIdentifier)(sqlContext).asInstanceOf[CarbonRelation]
- val mapper = new ObjectMapper()
- val colProps = StringBuilder.newBuilder
- var results: Seq[(String, String, String)] = child.schema.fields.map { field =>
- val comment = if (relation.metaData.dims.contains(field.name)) {
- val dimension = relation.metaData.carbonTable.getDimensionByName(
- relation.tableMeta.carbonTableIdentifier.getTableName,
- field.name)
- if (null != dimension.getColumnProperties && dimension.getColumnProperties.size() > 0) {
- colProps.append(field.name).append(".")
- .append(mapper.writeValueAsString(dimension.getColumnProperties))
- .append(",")
- }
- if (dimension.hasEncoding(Encoding.DICTIONARY)) {
- "DICTIONARY, KEY COLUMN" + (dimension.hasEncoding(Encoding.INVERTED_INDEX) match {
- case false => ",NOINVERTEDINDEX"
- case _ => ""
- })
- } else {
- "KEY COLUMN" + (dimension.hasEncoding(Encoding.INVERTED_INDEX) match {
- case false => ",NOINVERTEDINDEX"
- case _ => ""
- })
- }
- } else {
- "MEASURE"
- }
- (field.name, field.dataType.simpleString, comment)
- }
- val colPropStr = if (colProps.toString().trim().length() > 0) {
- // drops additional comma at endpom
- colProps.toString().dropRight(1)
- } else {
- colProps.toString()
- }
- results ++= Seq(("", "", ""), ("##Detailed Table Information", "", ""))
- results ++= Seq(("Database Name: ", relation.tableMeta.carbonTableIdentifier
- .getDatabaseName, "")
- )
- results ++= Seq(("Table Name: ", relation.tableMeta.carbonTableIdentifier.getTableName, ""))
- results ++= Seq(("CARBON Store Path: ", relation.tableMeta.storePath, ""))
- val carbonTable = relation.tableMeta.carbonTable
- results ++= Seq(("Table Block Size : ", carbonTable.getBlockSizeInMB + " MB", ""))
- results ++= Seq(("SORT_SCOPE", carbonTable.getTableInfo.getFactTable
- .getTableProperties.getOrDefault("sort_scope", CarbonCommonConstants
- .LOAD_SORT_SCOPE_DEFAULT), CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT))
- results ++= Seq(("", "", ""), ("##Detailed Column property", "", ""))
- if (colPropStr.length() > 0) {
- results ++= Seq((colPropStr, "", ""))
- } else {
- results ++= Seq(("ADAPTIVE", "", ""))
- }
- results ++= Seq(("SORT_COLUMNS", relation.metaData.carbonTable.getSortColumns(
- relation.tableMeta.carbonTableIdentifier.getTableName).asScala
- .map(column => column).mkString(","), ""))
- val dimension = carbonTable
- .getDimensionByTableName(relation.tableMeta.carbonTableIdentifier.getTableName)
- results ++= getColumnGroups(dimension.asScala.toList)
- if (carbonTable.getPartitionInfo(carbonTable.getFactTableName) != null) {
- results ++=
- Seq(("Partition Columns: ", carbonTable.getPartitionInfo(carbonTable.getFactTableName)
- .getColumnSchemaList.asScala.map(_.getColumnName).mkString(","), ""))
- }
- results.map { case (name, dataType, comment) =>
- Row(f"$name%-36s $dataType%-80s $comment%-72s")
- }
- }
-
- private def getColumnGroups(dimensions: List[CarbonDimension]): Seq[(String, String, String)] = {
- var results: Seq[(String, String, String)] =
- Seq(("", "", ""), ("##Column Group Information", "", ""))
- val groupedDimensions = dimensions.groupBy(x => x.columnGroupId()).filter {
- case (groupId, _) => groupId != -1
- }.toSeq.sortBy(_._1)
- val groups = groupedDimensions.map(colGroups => {
- colGroups._2.map(dim => dim.getColName).mkString(", ")
- })
- var index = 1
- groups.foreach { x =>
- results = results :+ (s"Column Group $index", x, "")
- index = index + 1
- }
- results
- }
-}
-
-private[sql] case class DeleteLoadByDate(
- databaseNameOp: Option[String],
- tableName: String,
- dateField: String,
- dateValue: String
-) extends RunnableCommand {
-
- val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
-
- def run(sqlContext: SQLContext): Seq[Row] = {
- val dbName = getDB.getDatabaseName(databaseNameOp, sqlContext)
- LOGGER.audit(s"The delete load by date request has been received for $dbName.$tableName")
- val identifier = TableIdentifier(tableName, Option(dbName))
- val relation = CarbonEnv.get.carbonMetastore
- .lookupRelation1(identifier)(sqlContext).asInstanceOf[CarbonRelation]
- var level: String = ""
- val carbonTable = CarbonMetadata.getInstance().getCarbonTable(dbName + '_' + tableName)
- if (relation == null) {
- LOGGER.audit(s"The delete load by date is failed. Table $dbName.$tableName does not exist")
- sys.error(s"Table $dbName.$tableName does not exist")
- }
- val matches: Seq[AttributeReference] = relation.dimensionsAttr.filter(
- filter => filter.name.equalsIgnoreCase(dateField) &&
- filter.dataType.isInstanceOf[TimestampType]).toList
- if (matches.isEmpty) {
- LOGGER.audit("The delete load by date is failed. " +
- s"Table $dbName.$tableName does not contain date field: $dateField")
- sys.error(s"Table $dbName.$tableName does not contain date field $dateField")
- } else {
- level = matches.asJava.get(0).name
- }
- val actualColName = relation.metaData.carbonTable.getDimensionByName(tableName, level)
- .getColName
- DataManagementFunc.deleteLoadByDate(
- sqlContext,
- new CarbonDataLoadSchema(carbonTable),
- dbName,
- tableName,
- CarbonEnv.get.carbonMetastore.storePath,
- level,
- actualColName,
- dateValue)
- LOGGER.audit(s"The delete load by date $dateValue is successful for $dbName.$tableName.")
- Seq.empty
- }
-
-}
-
-private[sql] case class CleanFiles(
- databaseNameOp: Option[String],
- tableName: String) extends RunnableCommand {
-
- val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
-
- def run(sqlContext: SQLContext): Seq[Row] = {
- Checker.validateTableExists(databaseNameOp, tableName, sqlContext)
- val carbonTable = CarbonEnv.get.carbonMetastore.lookupRelation1(databaseNameOp,
- tableName)(sqlContext).asInstanceOf[CarbonRelation].tableMeta.carbonTable
- CarbonStore.cleanFiles(
- getDB.getDatabaseName(databaseNameOp, sqlContext),
- tableName,
- sqlContext.asInstanceOf[CarbonContext].storePath,
- carbonTable,
- false)
- Seq.empty
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonAnalysisRules.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonAnalysisRules.scala b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonAnalysisRules.scala
deleted file mode 100644
index d23b18f..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonAnalysisRules.scala
+++ /dev/null
@@ -1,175 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.sql.hive
-
-import org.apache.spark.sql._
-import org.apache.spark.sql.catalyst.CarbonTableIdentifierImplicit
-import org.apache.spark.sql.catalyst.CarbonTableIdentifierImplicit._
-import org.apache.spark.sql.catalyst.analysis.{UnresolvedAlias, UnresolvedFunction, UnresolvedRelation, UnresolvedStar}
-import org.apache.spark.sql.catalyst.expressions.Alias
-import org.apache.spark.sql.catalyst.plans.Inner
-import org.apache.spark.sql.catalyst.plans.logical._
-import org.apache.spark.sql.catalyst.rules._
-import org.apache.spark.sql.execution.command.ProjectForDeleteCommand
-import org.apache.spark.sql.execution.datasources.LogicalRelation
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-
-/**
- * Insert into carbon table from other source
- */
-object CarbonPreInsertionCasts extends Rule[LogicalPlan] {
- def apply(plan: LogicalPlan): LogicalPlan = plan.transform {
- // Wait until children are resolved.
- case p: LogicalPlan if !p.childrenResolved => p
-
- case p @ InsertIntoTable(relation: LogicalRelation, _, child, _, _)
- if relation.relation.isInstanceOf[CarbonDatasourceRelation] =>
- castChildOutput(p, relation.relation.asInstanceOf[CarbonDatasourceRelation], child)
- }
-
- def castChildOutput(p: InsertIntoTable, relation: CarbonDatasourceRelation, child: LogicalPlan)
- : LogicalPlan = {
- if (relation.carbonRelation.output.size > CarbonCommonConstants
- .DEFAULT_MAX_NUMBER_OF_COLUMNS) {
- sys
- .error("Maximum supported column by carbon is:" + CarbonCommonConstants
- .DEFAULT_MAX_NUMBER_OF_COLUMNS
- )
- }
- if (child.output.size >= relation.carbonRelation.output.size ) {
- InsertIntoCarbonTable(relation, p.partition, p.child, p.overwrite, p.ifNotExists)
- } else {
- sys.error("Cannot insert into target table because column number are different")
- }
- }
-}
-
-
-object CarbonIUDAnalysisRule extends Rule[LogicalPlan] {
-
- var sqlContext: SQLContext = _
-
- def init(sqlContext: SQLContext) {
- this.sqlContext = sqlContext
- }
-
- private def processUpdateQuery(
- table: UnresolvedRelation,
- columns: List[String],
- selectStmt: String,
- filter: String): LogicalPlan = {
- var includedDestColumns = false
- var includedDestRelation = false
- var addedTupleId = false
-
- def prepareTargetReleation(relation: UnresolvedRelation): Subquery = {
- val tupleId = UnresolvedAlias(Alias(UnresolvedFunction("getTupleId",
- Seq.empty, isDistinct = false), "tupleId")())
- val projList = Seq(
- UnresolvedAlias(UnresolvedStar(table.alias)), tupleId)
- // include tuple id and rest of the required columns in subqury
- Subquery(table.alias.getOrElse(""), Project(projList, relation))
- }
- // get the un-analyzed logical plan
- val targetTable = prepareTargetReleation(table)
- val selectPlan = org.apache.spark.sql.SQLParser.parse(selectStmt, sqlContext) transform {
- case Project(projectList, child) if (!includedDestColumns) =>
- includedDestColumns = true
- if (projectList.size != columns.size) {
- sys.error("Number of source and destination columns are not matching")
- }
- val renamedProjectList = projectList.zip(columns).map{ case(attr, col) =>
- attr match {
- case UnresolvedAlias(child) =>
- UnresolvedAlias(Alias(child, col + "-updatedColumn")())
- case _ => attr
- }
- }
- val list = Seq(
- UnresolvedAlias(UnresolvedStar(table.alias))) ++ renamedProjectList
- Project(list, child)
- case Filter(cond, child) if (!includedDestRelation) =>
- includedDestRelation = true
- Filter(cond, Join(child, targetTable, Inner, None))
- case r @ UnresolvedRelation(t, a) if (!includedDestRelation &&
- t != table.tableIdentifier) =>
- includedDestRelation = true
- Join(r, targetTable, Inner, None)
- }
- val updatedSelectPlan = if (!includedDestRelation) {
- // special case to handle self join queries
- // Eg. update tableName SET (column1) = (column1+1)
- selectPlan transform {
- case relation: UnresolvedRelation if (table.tableIdentifier == relation.tableIdentifier &&
- addedTupleId == false) =>
- addedTupleId = true
- targetTable
- }
- } else {
- selectPlan
- }
- val finalPlan = if (filter.length > 0) {
- val alias = table.alias.getOrElse("")
- var transformed: Boolean = false
- // Create a dummy projection to include filter conditions
- SQLParser.parse("select * from " +
- table.tableIdentifier.mkString(".") + " " + alias + " " + filter, sqlContext) transform {
- case UnresolvedRelation(t, Some(a)) if (
- !transformed && t == table.tableIdentifier && a == alias) =>
- transformed = true
- // Add the filter condition of update statement on destination table
- Subquery(alias, updatedSelectPlan)
- }
- } else {
- updatedSelectPlan
- }
- val tid = CarbonTableIdentifierImplicit.toTableIdentifier(table.tableIdentifier)
- val tidSeq = Seq(getDB.getDatabaseName(tid.database, sqlContext), tid.table)
- val destinationTable = UnresolvedRelation(tidSeq, table.alias)
- ProjectForUpdate(destinationTable, columns, Seq(finalPlan))
- }
-
- def processDeleteRecordsQuery(selectStmt: String, table: UnresolvedRelation): LogicalPlan = {
- val tid = CarbonTableIdentifierImplicit.toTableIdentifier(table.tableIdentifier)
- val tidSeq = Seq(getDB.getDatabaseName(tid.database, sqlContext), tid.table)
- var addedTupleId = false
- val selectPlan = SQLParser.parse(selectStmt, sqlContext) transform {
- case relation: UnresolvedRelation if (table.tableIdentifier == relation.tableIdentifier &&
- addedTupleId == false) =>
- addedTupleId = true
- val tupleId = UnresolvedAlias(Alias(UnresolvedFunction("getTupleId",
- Seq.empty, isDistinct = false), "tupleId")())
- val projList = Seq(
- UnresolvedAlias(UnresolvedStar(table.alias)), tupleId)
- // include tuple id in subqury
- Project(projList, relation)
- }
- ProjectForDeleteCommand(
- selectPlan,
- tidSeq,
- System.currentTimeMillis().toString)
- }
-
- override def apply(logicalplan: LogicalPlan): LogicalPlan = {
-
- logicalplan transform {
- case UpdateTable(t, cols, sel, where) => processUpdateQuery(t, cols, sel, where)
- case DeleteRecords(statement, table) => processDeleteRecordsQuery(statement, table)
- }
- }
-}
[02/16] carbondata git commit: [CARBONDATA-1597] Remove spark1
integration
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSystemLockFeatureTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSystemLockFeatureTest.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSystemLockFeatureTest.scala
deleted file mode 100644
index ab75d6e..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSystemLockFeatureTest.scala
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.spark.testsuite.datacompaction
-
-import scala.collection.JavaConverters._
-
-import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.BeforeAndAfterAll
-
-import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
-import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier}
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datastore.impl.FileFactory
-import org.apache.carbondata.core.statusmanager.SegmentStatusManager
-import org.apache.carbondata.core.util.CarbonProperties
-
-/**
- * FT for compaction scenario where major segment should not be included in minor.
- */
-class CompactionSystemLockFeatureTest extends QueryTest with BeforeAndAfterAll {
-
- override def beforeAll {
- CarbonProperties.getInstance().addProperty("carbon.compaction.level.threshold", "2,2")
- sql("drop table if exists table1")
- sql("drop table if exists table2")
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "mm/dd/yyyy")
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.ENABLE_CONCURRENT_COMPACTION, "false")
- sql(
- "CREATE TABLE IF NOT EXISTS table1 (country String, ID Int, date Timestamp, name " +
- "String, " +
- "phonetype String, serialname String, salary Int) STORED BY 'org.apache.carbondata" +
- ".format'"
- )
- sql(
- "CREATE TABLE IF NOT EXISTS table2 (country String, ID Int, date Timestamp, name " +
- "String, " +
- "phonetype String, serialname String, salary Int) STORED BY 'org.apache.carbondata" +
- ".format'"
- )
-
-
- val csvFilePath1 = s"$resourcesPath/compaction/compaction1.csv"
-
- val csvFilePath2 = s"$resourcesPath/compaction/compaction2.csv"
- val csvFilePath3 = s"$resourcesPath/compaction/compaction3.csv"
-
- // load table1
- sql("LOAD DATA LOCAL INPATH '" + csvFilePath1 + "' INTO TABLE table1 OPTIONS" +
- "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
- )
- sql("LOAD DATA LOCAL INPATH '" + csvFilePath2 + "' INTO TABLE table1 OPTIONS" +
- "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
- )
-
- // load table2
- sql("LOAD DATA LOCAL INPATH '" + csvFilePath1 + "' INTO TABLE table2 OPTIONS" +
- "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
- )
- sql("LOAD DATA LOCAL INPATH '" + csvFilePath2 + "' INTO TABLE table2 OPTIONS" +
- "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
- )
-
- // create a file in table 2 so that it will also be compacted.
- val absoluteTableIdentifier = new
- AbsoluteTableIdentifier(
- CarbonProperties.getInstance.getProperty(CarbonCommonConstants.STORE_LOCATION),
- new CarbonTableIdentifier(CarbonCommonConstants.DATABASE_DEFAULT_NAME, "table2", "rrr")
- )
- val carbonTablePath: CarbonTablePath = CarbonStorePath
- .getCarbonTablePath(absoluteTableIdentifier.getStorePath,
- absoluteTableIdentifier.getCarbonTableIdentifier
- )
-
- val file = carbonTablePath.getMetadataDirectoryPath + CarbonCommonConstants
- .FILE_SEPARATOR + CarbonCommonConstants.majorCompactionRequiredFile
-
- FileFactory.createNewFile(file, FileFactory.getFileType(file))
-
- // compaction will happen here.
- sql("alter table table1 compact 'major'"
- )
-
- }
-
- /**
- * Test whether major compaction is done for both.
- */
- test("check for compaction in both tables") {
- // delete merged segments
- sql("clean files for table table1")
- sql("clean files for table table2")
-
- // check for table 1.
- val segmentStatusManager: SegmentStatusManager = new SegmentStatusManager(new
- AbsoluteTableIdentifier(
- CarbonProperties.getInstance.getProperty(CarbonCommonConstants.STORE_LOCATION),
- new CarbonTableIdentifier("default", "table1", "rrr")
- )
- )
- // merged segment should not be there
- val segments = segmentStatusManager.getValidAndInvalidSegments.getValidSegments.asScala.toList
- assert(segments.contains("0.1"))
- assert(!segments.contains("0"))
- assert(!segments.contains("1"))
- // check for table 2.
- val segmentStatusManager2: SegmentStatusManager = new SegmentStatusManager(new
- AbsoluteTableIdentifier(
- CarbonProperties.getInstance.getProperty(CarbonCommonConstants.STORE_LOCATION),
- new CarbonTableIdentifier("default", "table2", "rrr1")
- )
- )
- // merged segment should not be there
- val segments2 = segmentStatusManager2.getValidAndInvalidSegments.getValidSegments.asScala.toList
- assert(segments2.contains("0.1"))
- assert(!segments2.contains("0"))
- assert(!segments2.contains("1"))
-
- }
-
- override def afterAll {
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
- sql("drop table if exists table1")
- sql("drop table if exists table2")
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionMinorThresholdTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionMinorThresholdTest.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionMinorThresholdTest.scala
deleted file mode 100644
index 827b4f6..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionMinorThresholdTest.scala
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.spark.testsuite.datacompaction
-
-import scala.collection.JavaConverters._
-
-import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.BeforeAndAfterAll
-
-import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier}
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.statusmanager.SegmentStatusManager
-import org.apache.carbondata.core.util.CarbonProperties
-
-/**
- * FT for data compaction Minor threshold verification.
- */
-class DataCompactionMinorThresholdTest extends QueryTest with BeforeAndAfterAll {
- val carbonTableIdentifier: CarbonTableIdentifier =
- new CarbonTableIdentifier("default", "minorthreshold".toLowerCase(), "1")
-
- val identifier = new AbsoluteTableIdentifier(storeLocation, carbonTableIdentifier)
-
- override def beforeAll {
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.COMPACTION_SEGMENT_LEVEL_THRESHOLD, "2,2")
- sql("drop table if exists minorthreshold")
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "mm/dd/yyyy")
- sql(
- "CREATE TABLE IF NOT EXISTS minorthreshold (country String, ID Int, date " +
- "Timestamp, name " +
- "String, " +
- "phonetype String, serialname String, salary Int) STORED BY 'org.apache.carbondata" +
- ".format'"
- )
-
- val csvFilePath1 = s"$resourcesPath/compaction/compaction1.csv"
- val csvFilePath2 = s"$resourcesPath/compaction/compaction2.csv"
- val csvFilePath3 = s"$resourcesPath/compaction/compaction3.csv"
-
- sql("LOAD DATA LOCAL INPATH '" + csvFilePath1 + "' INTO TABLE minorthreshold " +
- "OPTIONS" +
- "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
- )
- sql("LOAD DATA LOCAL INPATH '" + csvFilePath2 + "' INTO TABLE minorthreshold " +
- "OPTIONS" +
- "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
- )
- sql("LOAD DATA LOCAL INPATH '" + csvFilePath3 + "' INTO TABLE minorthreshold " +
- "OPTIONS" +
- "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
- )
- sql("LOAD DATA LOCAL INPATH '" + csvFilePath3 + "' INTO TABLE minorthreshold " +
- "OPTIONS" +
- "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
- )
- // compaction should happen here.
- sql("alter table minorthreshold compact 'minor'")
- }
-
- /**
- * Compaction should be completed correctly for minor compaction.
- */
- test("check if compaction is completed correctly for minor.") {
-
- sql("clean files for table minorthreshold")
-
- val segmentStatusManager: SegmentStatusManager = new SegmentStatusManager(identifier)
- val segments = segmentStatusManager.getValidAndInvalidSegments.getValidSegments.asScala.toList
-
- assert(segments.contains("0.2"))
- assert(!segments.contains("0.1"))
- assert(!segments.contains("0"))
- assert(!segments.contains("1"))
- assert(!segments.contains("2"))
- assert(!segments.contains("3"))
- }
-
- override def afterAll {
- sql("drop table if exists minorthreshold")
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.COMPACTION_SEGMENT_LEVEL_THRESHOLD,
- CarbonCommonConstants.DEFAULT_SEGMENT_LEVEL_THRESHOLD)
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionNoDictionaryTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionNoDictionaryTest.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionNoDictionaryTest.scala
deleted file mode 100644
index 053ad44..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionNoDictionaryTest.scala
+++ /dev/null
@@ -1,173 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.spark.testsuite.datacompaction
-
-import scala.collection.JavaConverters._
-
-import org.apache.spark.sql.Row
-import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.BeforeAndAfterAll
-
-import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier}
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.statusmanager.SegmentStatusManager
-import org.apache.carbondata.core.util.CarbonProperties
-
-/**
- * FT for data compaction scenario.
- */
-class DataCompactionNoDictionaryTest extends QueryTest with BeforeAndAfterAll {
-
- // return segment details
- def getSegments(databaseName : String, tableName : String, tableId : String): List[String] = {
- val identifier = new AbsoluteTableIdentifier(
- CarbonProperties.getInstance.getProperty(CarbonCommonConstants.STORE_LOCATION),
- new CarbonTableIdentifier(databaseName, tableName.toLowerCase , tableId))
-
- val segmentStatusManager: SegmentStatusManager = new SegmentStatusManager(identifier)
- segmentStatusManager.getValidAndInvalidSegments.getValidSegments.asScala.toList
- }
-
- var csvFilePath1 = s"$resourcesPath/compaction/compaction1.csv"
- var csvFilePath2 = s"$resourcesPath/compaction/compaction2.csv"
- var csvFilePath3 = s"$resourcesPath/compaction/compaction3.csv"
-
- override def beforeAll {
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "mm/dd/yyyy")
- sql("DROP TABLE IF EXISTS nodictionaryCompaction")
- sql(
- "CREATE TABLE nodictionaryCompaction (country String, ID Int, date Timestamp, name " +
- "String, " +
- "phonetype String, serialname String, salary Int) STORED BY 'org.apache.carbondata" +
- ".format' TBLPROPERTIES('DICTIONARY_EXCLUDE'='country')"
- )
-
-
-
- sql("LOAD DATA LOCAL INPATH '" + csvFilePath1 + "' INTO TABLE nodictionaryCompaction " +
- "OPTIONS('DELIMITER' = ',')"
- )
- sql("LOAD DATA LOCAL INPATH '" + csvFilePath2 + "' INTO TABLE nodictionaryCompaction " +
- "OPTIONS('DELIMITER' = ',')"
- )
- sql("LOAD DATA LOCAL INPATH '" + csvFilePath3 + "' INTO TABLE nodictionaryCompaction " +
- "OPTIONS('DELIMITER' = ',')"
- )
- // compaction will happen here.
- sql("alter table nodictionaryCompaction compact 'major'"
- )
-
- // wait for compaction to finish.
- Thread.sleep(1000)
- }
-
- // check for 15 seconds if the compacted segments has come or not .
- // if not created after 15 seconds then testcase will fail.
-
- test("check if compaction is completed or not.") {
- var status = true
- var noOfRetries = 0
- while (status && noOfRetries < 10) {
-
- val segments: List[String] = getSegments(
- CarbonCommonConstants.DATABASE_DEFAULT_NAME, "nodictionaryCompaction", "uni21")
-
- if (!segments.contains("0.1")) {
- // wait for 2 seconds for compaction to complete.
- Thread.sleep(2000)
- noOfRetries += 1
- }
- else {
- status = false
- }
- }
- }
-
- test("select country from nodictionaryCompaction") {
- // check answers after compaction.
- checkAnswer(
- sql("select country from nodictionaryCompaction"),
- Seq(Row("america"),
- Row("canada"),
- Row("chile"),
- Row("china"),
- Row("england"),
- Row("burma"),
- Row("butan"),
- Row("mexico"),
- Row("newzealand"),
- Row("westindies"),
- Row("china"),
- Row("india"),
- Row("iran"),
- Row("iraq"),
- Row("ireland")
- )
- )
- }
-
- test("delete merged folder and execute query") {
- // delete merged segments
- sql("clean files for table nodictionaryCompaction")
-
- // merged segment should not be there
- val segments =
- getSegments(CarbonCommonConstants.DATABASE_DEFAULT_NAME, "nodictionaryCompaction", "uni21")
- assert(!segments.contains("0"))
- assert(!segments.contains("1"))
- assert(!segments.contains("2"))
- assert(segments.contains("0.1"))
-
- // now check the answers it should be same.
- checkAnswer(
- sql("select country from nodictionaryCompaction"),
- Seq(Row("america"),
- Row("canada"),
- Row("chile"),
- Row("china"),
- Row("england"),
- Row("burma"),
- Row("butan"),
- Row("mexico"),
- Row("newzealand"),
- Row("westindies"),
- Row("china"),
- Row("india"),
- Row("iran"),
- Row("iraq"),
- Row("ireland")
- )
- )
- sql("LOAD DATA LOCAL INPATH '" + csvFilePath1 + "' INTO TABLE nodictionaryCompaction " +
- "OPTIONS('DELIMITER' = ',')"
- )
- sql("delete from table nodictionaryCompaction where segment.id in (0.1,3)")
- checkAnswer(
- sql("select country from nodictionaryCompaction"),
- Seq()
- )
- }
-
- override def afterAll {
- sql("DROP TABLE IF EXISTS nodictionaryCompaction")
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
- CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE, "false")
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionTest.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionTest.scala
deleted file mode 100644
index 8a0279f..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionTest.scala
+++ /dev/null
@@ -1,224 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.spark.testsuite.datacompaction
-
-import scala.collection.JavaConverters._
-
-import org.apache.spark.sql.Row
-import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.BeforeAndAfterAll
-
-import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier}
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.statusmanager.SegmentStatusManager
-import org.apache.carbondata.core.util.CarbonProperties
-
-/**
- * FT for data compaction scenario.
- */
-class DataCompactionTest extends QueryTest with BeforeAndAfterAll {
-
- override def beforeAll {
- CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE, "true")
- sql("drop table if exists normalcompaction")
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "mm/dd/yyyy")
- sql(
- "CREATE TABLE IF NOT EXISTS normalcompaction (country String, ID Int, date Timestamp, name " +
- "String, " +
- "phonetype String, serialname String, salary Int) STORED BY 'org.apache.carbondata" +
- ".format'"
- )
-
- val csvFilePath1 = s"$resourcesPath/compaction/compaction1.csv"
- val csvFilePath2 = s"$resourcesPath/compaction/compaction2.csv"
- val csvFilePath3 = s"$resourcesPath/compaction/compaction3.csv"
-
- sql("LOAD DATA LOCAL INPATH '" + csvFilePath1 + "' INTO TABLE normalcompaction OPTIONS" +
- "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
- )
- sql("LOAD DATA LOCAL INPATH '" + csvFilePath2 + "' INTO TABLE normalcompaction OPTIONS" +
- "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
- )
- // compaction will happen here.
- sql("LOAD DATA LOCAL INPATH '" + csvFilePath3 + "' INTO TABLE normalcompaction OPTIONS" +
- "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
- )
- // compaction will happen here.
- sql("alter table normalcompaction compact 'major'"
- )
-
- }
-
- test("check if compaction is completed or not.") {
- var status = true
- var noOfRetries = 0
- while (status && noOfRetries < 10) {
-
- val identifier = new AbsoluteTableIdentifier(
- CarbonProperties.getInstance.getProperty(CarbonCommonConstants.STORE_LOCATION),
- new CarbonTableIdentifier(CarbonCommonConstants.DATABASE_DEFAULT_NAME, "normalcompaction", "1")
- )
-
- val segmentStatusManager: SegmentStatusManager = new SegmentStatusManager(identifier)
-
- val segments = segmentStatusManager.getValidAndInvalidSegments.getValidSegments.asScala.toList
-
- if (!segments.contains("0.1")) {
- // wait for 2 seconds for compaction to complete.
- Thread.sleep(2000)
- noOfRetries += 1
- }
- else {
- status = false
- }
- }
- }
-
-
- test("select country from normalcompaction") {
- // check answers after compaction.
- checkAnswer(
- sql("select country from normalcompaction"),
- Seq(Row("america"),
- Row("canada"),
- Row("chile"),
- Row("china"),
- Row("england"),
- Row("burma"),
- Row("butan"),
- Row("mexico"),
- Row("newzealand"),
- Row("westindies"),
- Row("china"),
- Row("india"),
- Row("iran"),
- Row("iraq"),
- Row("ireland")
- )
- )
- }
-
- test("delete merged folder and execute query") {
- // delete merged segments
- sql("clean files for table normalcompaction")
-
- val identifier = new AbsoluteTableIdentifier(
- CarbonProperties.getInstance.getProperty(CarbonCommonConstants.STORE_LOCATION),
- new CarbonTableIdentifier(
- CarbonCommonConstants.DATABASE_DEFAULT_NAME, "normalcompaction", "uniqueid")
- )
-
- val segmentStatusManager: SegmentStatusManager = new SegmentStatusManager(identifier)
-
- // merged segment should not be there
- val segments = segmentStatusManager.getValidAndInvalidSegments.getValidSegments.asScala.toList
- assert(!segments.contains("0"))
- assert(!segments.contains("1"))
- assert(!segments.contains("2"))
- assert(segments.contains("0.1"))
-
- // now check the answers it should be same.
- checkAnswer(
- sql("select country from normalcompaction"),
- Seq(Row("america"),
- Row("canada"),
- Row("chile"),
- Row("china"),
- Row("england"),
- Row("burma"),
- Row("butan"),
- Row("mexico"),
- Row("newzealand"),
- Row("westindies"),
- Row("china"),
- Row("india"),
- Row("iran"),
- Row("iraq"),
- Row("ireland")
- )
- )
- }
-
-
- test("check if compaction with Updates") {
-
- CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE, "false")
- sql("drop table if exists cardinalityUpdatetest")
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "mm/dd/yyyy")
-
- sql(
- "CREATE TABLE IF NOT EXISTS cardinalityUpdateTest (FirstName String, LastName String, date Timestamp," +
- "phonetype String, serialname String, ID int, salary Int) STORED BY 'org.apache.carbondata" +
- ".format'"
- )
-
- val csvFilePath1 = s"$resourcesPath/compaction/compactionIUD1.csv"
- val csvFilePath2 = s"$resourcesPath/compaction/compactionIUD2.csv"
- val csvFilePath3 = s"$resourcesPath/compaction/compactionIUD3.csv"
- val csvFilePath4 = s"$resourcesPath/compaction/compactionIUD4.csv"
-
- sql("LOAD DATA LOCAL INPATH '" + csvFilePath1 + "' INTO TABLE cardinalityUpdateTest OPTIONS" +
- "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
- )
- sql("LOAD DATA LOCAL INPATH '" + csvFilePath2 + "' INTO TABLE cardinalityUpdateTest OPTIONS" +
- "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
- )
- sql("LOAD DATA LOCAL INPATH '" + csvFilePath3 + "' INTO TABLE cardinalityUpdateTest OPTIONS" +
- "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
- )
- sql("LOAD DATA LOCAL INPATH '" + csvFilePath4 + "' INTO TABLE cardinalityUpdateTest OPTIONS" +
- "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
- )
-
- // update the first segment
- sql("update cardinalityUpdateTest set (FirstName) = ('FirstTwentyOne') where ID = 2").show()
-
- // alter table.
- sql("alter table cardinalityUpdateTest compact 'major'")
-
- // Verify the new updated value in compacted segment.
- // now check the answers it should be same.
- checkAnswer(
- sql("select FirstName from cardinalityUpdateTest where FirstName = ('FirstTwentyOne')"),
- Seq(Row("FirstTwentyOne")
- )
- )
-
- checkAnswer(
- sql("select count(*) from cardinalityUpdateTest where FirstName = ('FirstTwentyOne')"),
- Seq(Row(1)
- )
- )
-
- checkAnswer(
- sql("select count(*) from cardinalityUpdateTest"),
- Seq(Row(20)
- )
- )
- }
-
- override def afterAll {
- sql("drop table if exists normalcompaction")
- sql("drop table if exists cardinalityUpdatetest")
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
- CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE, "false")
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/filterexpr/GrtLtFilterProcessorTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/filterexpr/GrtLtFilterProcessorTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/filterexpr/GrtLtFilterProcessorTestCase.scala
deleted file mode 100644
index 250ec06..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/filterexpr/GrtLtFilterProcessorTestCase.scala
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.spark.testsuite.filterexpr
-
-import org.apache.spark.sql.Row
-import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.BeforeAndAfterAll
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
-
-/**
- * Test Class for filter expression query on String datatypes
- *
- */
-class GrtLtFilterProcessorTestCase extends QueryTest with BeforeAndAfterAll {
-
- override def beforeAll {
- sql("drop table if exists a12_allnull")
-
- sql(
- "create table a12_allnull(empid String,ename String,sal double,deptno int,mgr string,gender" +
- " string," +
- "dob timestamp,comm decimal(4,2),desc string) stored by 'org.apache.carbondata.format'"
- )
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy-MM-dd HH:mm:ss")
- val testData = s"$resourcesPath/filter/emp2allnull.csv"
-
- sql(
- s"""LOAD DATA LOCAL INPATH '$testData' into table a12_allnull OPTIONS('DELIMITER'=',',
- 'QUOTECHAR'='"','FILEHEADER'='empid,ename,sal,deptno,mgr,gender,dob,comm,desc')"""
- .stripMargin
- )
- }
-
- test("In condition With improper format query regarding Null filter") {
- checkAnswer(
- sql("select empid from a12_allnull " + "where empid not in ('china',NULL)"),
- Seq()
- )
- }
-
- override def afterAll {
- sql("drop table if exists a12_allnull")
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/hadooprelation/HadoopFSRelationTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/hadooprelation/HadoopFSRelationTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/hadooprelation/HadoopFSRelationTestCase.scala
deleted file mode 100644
index 1649875..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/hadooprelation/HadoopFSRelationTestCase.scala
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.spark.testsuite.hadooprelation
-
-import org.apache.spark.sql.DataFrame
-import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.BeforeAndAfterAll
-
-/**
- * Test Class for hadoop fs relation
- *
- */
-class HadoopFSRelationTestCase extends QueryTest with BeforeAndAfterAll {
-
- override def beforeAll {
- sql("drop table if exists hadoopfsrelation")
- sql("drop table if exists hadoopfsrelation_hive")
- sql(
- "CREATE TABLE hadoopfsrelation (empno int, empname String, designation " +
- "String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno " +
- "int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate " +
- "Timestamp,attendance int,utilization int,salary int)" +
- "STORED BY 'org.apache.carbondata.format'")
- sql(
- s"LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE hadoopfsrelation " +
- "OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')")
-
- sql("CREATE TABLE hadoopfsrelation_hive (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int)row format delimited fields terminated by ','")
-
- sql(
- s"LOAD DATA local inpath '$resourcesPath/datawithoutheader.csv' INTO TABLE hadoopfsrelation_hive ")
- }
-
- test("hadoopfsrelation select all test") {
- val rdd = sqlContext.read.format("org.apache.spark.sql.CarbonSource")
- .option("tableName", "hadoopfsrelation").load()
- assert(rdd.collect().length > 0)
- }
-
- test("hadoopfsrelation filters test") {
- val rdd: DataFrame = sqlContext.read.format("org.apache.spark.sql.CarbonSource")
- .option("tableName", "hadoopfsrelation").load()
- .select("empno", "empname", "utilization").where("empname in ('arvind','ayushi')")
- checkAnswer(
- rdd,
- sql("select empno,empname,utilization from hadoopfsrelation_hive where empname in ('arvind','ayushi')"))
- }
-
- override def afterAll {
- sql("drop table hadoopfsrelation")
- sql("drop table hadoopfsrelation_hive")
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
deleted file mode 100644
index b5b8261..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.spark.util
-
-import org.apache.spark.sql.test.util.QueryTest
-import org.apache.spark.sql.{CarbonEnv, CarbonRelation}
-import org.scalatest.BeforeAndAfterAll
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.processing.util.TableOptionConstant
-import org.apache.carbondata.processing.loading.model.{CarbonDataLoadSchema, CarbonLoadModel}
-
-/**
- * Test Case for org.apache.carbondata.integration.spark.util.GlobalDictionaryUtil
- */
-class AllDictionaryTestCase extends QueryTest with BeforeAndAfterAll {
-
- var pwd: String = _
- var sampleRelation: CarbonRelation = _
- var complexRelation: CarbonRelation = _
- var sampleAllDictionaryFile: String = _
- var complexAllDictionaryFile: String = _
-
- def buildCarbonLoadModel(relation: CarbonRelation,
- filePath: String,
- header: String,
- allDictFilePath: String): CarbonLoadModel = {
- val carbonLoadModel = new CarbonLoadModel
- carbonLoadModel.setTableName(relation.tableMeta.carbonTableIdentifier.getDatabaseName)
- carbonLoadModel.setDatabaseName(relation.tableMeta.carbonTableIdentifier.getTableName)
- val table = relation.tableMeta.carbonTable
- val carbonSchema = new CarbonDataLoadSchema(table)
- carbonLoadModel.setDatabaseName(table.getDatabaseName)
- carbonLoadModel.setTableName(table.getFactTableName)
- carbonLoadModel.setCarbonDataLoadSchema(carbonSchema)
- carbonLoadModel.setFactFilePath(filePath)
- carbonLoadModel.setCsvHeader(header)
- carbonLoadModel.setCsvDelimiter(",")
- carbonLoadModel.setComplexDelimiterLevel1("\\$")
- carbonLoadModel.setComplexDelimiterLevel2("\\:")
- carbonLoadModel.setAllDictPath(allDictFilePath)
- carbonLoadModel.setSerializationNullFormat(
- TableOptionConstant.SERIALIZATION_NULL_FORMAT.getName + ",\\N")
- carbonLoadModel.setDefaultTimestampFormat(CarbonProperties.getInstance().getProperty(
- CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
- CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT))
- carbonLoadModel.setCsvHeaderColumns(CommonUtil.getCsvHeaderColumns(carbonLoadModel))
- carbonLoadModel
- }
-
- override def beforeAll {
- sql("drop table if exists sample")
- sql("drop table if exists complextypes")
- buildTestData
- // second time comment this line
- buildTable
- buildRelation
- }
-
- def buildTestData() = {
- sampleAllDictionaryFile = s"${resourcesPath}/alldictionary/sample/20160423/1400_1405/*.dictionary"
- complexAllDictionaryFile = s"${resourcesPath}/alldictionary/complex/20160423/1400_1405/*.dictionary"
- }
-
- def buildTable() = {
- try {
- sql(
- "CREATE TABLE IF NOT EXISTS sample (id STRING, name STRING, city STRING, " +
- "age INT) STORED BY 'org.apache.carbondata.format' " +
- "tblproperties('dictionary_include'='city')"
- )
- } catch {
- case ex: Throwable => LOGGER.error(ex.getMessage + "\r\n" + ex.getStackTraceString)
- }
- try {
- sql(
- "create table complextypes (deviceInformationId string, channelsId string, " +
- "ROMSize string, purchasedate string, mobile struct<imei: string, imsi: string>, MAC " +
- "array<string>, locationinfo array<struct<ActiveAreaId: INT, ActiveCountry: string, " +
- "ActiveProvince: string, Activecity: string, ActiveDistrict: string, ActiveStreet: " +
- "string>>, proddate struct<productionDate: string,activeDeactivedate: array<string>>, " +
- "gamePointId INT,contractNumber INT) STORED BY 'org.apache.carbondata.format'" +
- "TBLPROPERTIES('DICTIONARY_EXCLUDE'='ROMSize', 'dictionary_include'='channelsId')"
- )
- } catch {
- case ex: Throwable => LOGGER.error(ex.getMessage + "\r\n" + ex.getStackTraceString)
- }
- }
-
- def buildRelation() = {
- val catalog = CarbonEnv.get.carbonMetastore
- sampleRelation = catalog.lookupRelation1(Option(CarbonCommonConstants.DATABASE_DEFAULT_NAME),
- "sample")(sqlContext).asInstanceOf[CarbonRelation]
- complexRelation = catalog.lookupRelation1(Option(CarbonCommonConstants.DATABASE_DEFAULT_NAME),
- "complextypes")(sqlContext).asInstanceOf[CarbonRelation]
- }
-
- test("Support generate global dictionary from all dictionary files") {
- val header = "id,name,city,age"
- val carbonLoadModel = buildCarbonLoadModel(sampleRelation, null, header, sampleAllDictionaryFile)
- GlobalDictionaryUtil
- .generateGlobalDictionary(sqlContext,
- carbonLoadModel,
- sampleRelation.tableMeta.storePath)
-
- DictionaryTestCaseUtil.
- checkDictionary(sampleRelation, "city", "shenzhen")
- }
-
- test("Support generate global dictionary from all dictionary files for complex type") {
- val header = "deviceInformationId,channelsId,ROMSize,purchasedate,mobile,MAC,locationinfo,proddate,gamePointId,contractNumber"
- val carbonLoadModel = buildCarbonLoadModel(complexRelation, null, header, complexAllDictionaryFile)
- GlobalDictionaryUtil
- .generateGlobalDictionary(sqlContext,
- carbonLoadModel,
- complexRelation.tableMeta.storePath)
-
- DictionaryTestCaseUtil.
- checkDictionary(complexRelation, "channelsId", "1650")
- }
-
- override def afterAll {
- sql("drop table sample")
- sql("drop table complextypes")
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/test/scala/org/apache/carbondata/spark/util/DictionaryTestCaseUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/DictionaryTestCaseUtil.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/DictionaryTestCaseUtil.scala
deleted file mode 100644
index fd2ab2a..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/DictionaryTestCaseUtil.scala
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.spark.util
-
-import org.apache.spark.sql.CarbonRelation
-import org.apache.spark.sql.test.TestQueryExecutor
-
-import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.path.CarbonStorePath
-import org.apache.carbondata.processing.util.CarbonLoaderUtil
-
-/**
- * Utility for global dictionary test cases
- */
-object DictionaryTestCaseUtil {
-
- /**
- * check whether the dictionary of specified column generated
- * @param relation carbon table relation
- * @param columnName name of specified column
- * @param value a value of column
- */
- def checkDictionary(relation: CarbonRelation, columnName: String, value: String) {
- val table = relation.tableMeta.carbonTable
- val dimension = table.getDimensionByName(table.getFactTableName, columnName)
- val tableIdentifier = new CarbonTableIdentifier(table.getDatabaseName, table.getFactTableName, "uniqueid")
- val columnIdentifier = new DictionaryColumnUniqueIdentifier(tableIdentifier,
- dimension.getColumnIdentifier, dimension.getDataType,
- CarbonStorePath.getCarbonTablePath(table.getStorePath, table.getCarbonTableIdentifier)
- )
- val dict = CarbonLoaderUtil.getDictionary(columnIdentifier, TestQueryExecutor.storeLocation)
- assert(dict.getSurrogateKey(value) != CarbonCommonConstants.INVALID_SURROGATE_KEY)
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
deleted file mode 100644
index f0de47d..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
+++ /dev/null
@@ -1,252 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.spark.util
-
-import org.apache.spark.sql.test.util.QueryTest
-import org.apache.spark.sql.{CarbonEnv, CarbonRelation}
-import org.scalatest.BeforeAndAfterAll
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.processing.exception.DataLoadingException
-import org.apache.carbondata.processing.util.TableOptionConstant
-import org.apache.carbondata.processing.loading.model.{CarbonDataLoadSchema, CarbonLoadModel}
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
-
- /**
- * test case for external column dictionary generation
- * also support complicated type
- */
-class ExternalColumnDictionaryTestCase extends QueryTest with BeforeAndAfterAll {
-
- var extComplexRelation: CarbonRelation = _
- var verticalDelimiteRelation: CarbonRelation = _
- var loadSqlRelation: CarbonRelation = _
- var filePath: String = _
- var pwd: String = _
- var complexFilePath1: String = _
- var complexFilePath2: String = _
- var extColDictFilePath1: String = _
- var extColDictFilePath2: String = _
- var extColDictFilePath3: String = _
- var header: String = _
- var header2: String = _
-
- def buildTestData() = {
-
- filePath = s"${resourcesPath}/sample.csv"
- complexFilePath1 = s"${resourcesPath}/complexdata2.csv"
- complexFilePath2 = s"${resourcesPath}/verticalDelimitedData.csv"
- extColDictFilePath1 = s"deviceInformationId:${resourcesPath}/deviceInformationId.csv," +
- s"mobile.imei:${resourcesPath}/mobileimei.csv," +
- s"mac:${resourcesPath}/mac.csv," +
- s"locationInfo.ActiveCountry:${resourcesPath}/locationInfoActiveCountry.csv"
- extColDictFilePath2 = s"deviceInformationId:${resourcesPath}/deviceInformationId2.csv"
- extColDictFilePath3 = s"channelsId:${resourcesPath}/channelsId.csv"
- header = "deviceInformationId,channelsId,ROMSize,purchasedate,mobile,MAC,locationinfo,proddate,gamePointId,contractNumber"
- header2 = "deviceInformationId,channelsId,contractNumber"
- }
-
- def buildTable() = {
- try {
- sql("""CREATE TABLE extComplextypes (deviceInformationId int,
- channelsId string, ROMSize string, purchasedate string,
- mobile struct<imei:string, imsi:string>, MAC array<string>,
- locationinfo array<struct<ActiveAreaId:int, ActiveCountry:string,
- ActiveProvince:string, Activecity:string, ActiveDistrict:string, ActiveStreet:string>>,
- proddate struct<productionDate:string,activeDeactivedate:array<string>>,
- gamePointId double,contractNumber double)
- STORED BY 'org.apache.carbondata.format'
- TBLPROPERTIES('DICTIONARY_INCLUDE' = 'deviceInformationId, channelsId')
- """)
- } catch {
- case ex: Throwable => LOGGER.error(ex.getMessage + "\r\n" + ex.getStackTraceString)
- }
-
- try {
- sql("""CREATE TABLE verticalDelimitedTable (deviceInformationId int,
- channelsId string,contractNumber double)
- STORED BY 'org.apache.carbondata.format'
- TBLPROPERTIES('DICTIONARY_INCLUDE' = 'deviceInformationId, channelsId')
- """)
- } catch {
- case ex: Throwable => LOGGER.error(ex.getMessage + "\r\n" + ex.getStackTraceString)
- }
-
- try {
- sql("""CREATE TABLE loadSqlTest (deviceInformationId int,
- channelsId string, ROMSize string, purchasedate string,
- mobile struct<imei:string, imsi:string>, MAC array<string>,
- locationinfo array<struct<ActiveAreaId:int, ActiveCountry:string,
- ActiveProvince:string, Activecity:string, ActiveDistrict:string, ActiveStreet:string>>,
- proddate struct<productionDate:string,activeDeactivedate:array<string>>,
- gamePointId double,contractNumber double)
- STORED BY 'org.apache.carbondata.format'
- TBLPROPERTIES('DICTIONARY_INCLUDE' = 'deviceInformationId')
- """)
- } catch {
- case ex: Throwable => LOGGER.error(ex.getMessage + "\r\n" + ex.getStackTraceString)
- }
- }
-
- def buildRelation() = {
- val catalog = CarbonEnv.get.carbonMetastore
- extComplexRelation = catalog.lookupRelation1(Option(CarbonCommonConstants.DATABASE_DEFAULT_NAME),
- "extComplextypes")(sqlContext)
- .asInstanceOf[CarbonRelation]
- verticalDelimiteRelation = catalog.lookupRelation1(Option(CarbonCommonConstants.DATABASE_DEFAULT_NAME),
- "verticalDelimitedTable")(sqlContext)
- .asInstanceOf[CarbonRelation]
- loadSqlRelation = catalog.lookupRelation1(Option(CarbonCommonConstants.DATABASE_DEFAULT_NAME),
- "loadSqlTest")(sqlContext)
- .asInstanceOf[CarbonRelation]
- }
-
- def buildCarbonLoadModel(relation: CarbonRelation,
- filePath:String,
- header: String,
- extColFilePath: String,
- csvDelimiter: String = ","): CarbonLoadModel = {
- val carbonLoadModel = new CarbonLoadModel
- carbonLoadModel.setTableName(relation.tableMeta.carbonTableIdentifier.getDatabaseName)
- carbonLoadModel.setDatabaseName(relation.tableMeta.carbonTableIdentifier.getTableName)
- val table = relation.tableMeta.carbonTable
- val carbonSchema = new CarbonDataLoadSchema(table)
- carbonLoadModel.setDatabaseName(table.getDatabaseName)
- carbonLoadModel.setTableName(table.getFactTableName)
- carbonLoadModel.setCarbonDataLoadSchema(carbonSchema)
- carbonLoadModel.setFactFilePath(filePath)
- carbonLoadModel.setCsvHeader(header)
- carbonLoadModel.setCsvDelimiter(csvDelimiter)
- carbonLoadModel.setComplexDelimiterLevel1("\\$")
- carbonLoadModel.setComplexDelimiterLevel2("\\:")
- carbonLoadModel.setColDictFilePath(extColFilePath)
- carbonLoadModel.setQuoteChar("\"");
- carbonLoadModel.setSerializationNullFormat(
- TableOptionConstant.SERIALIZATION_NULL_FORMAT.getName + ",\\N")
- carbonLoadModel.setDefaultTimestampFormat(CarbonProperties.getInstance().getProperty(
- CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
- CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT))
- carbonLoadModel.setDefaultDateFormat(CarbonProperties.getInstance().getProperty(
- CarbonCommonConstants.CARBON_DATE_FORMAT,
- CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT))
- carbonLoadModel.setCsvHeaderColumns(CommonUtil.getCsvHeaderColumns(carbonLoadModel))
- carbonLoadModel.setMaxColumns("100")
- carbonLoadModel
- }
-
- override def beforeAll {
- buildTestData
- buildTable
- buildRelation
- }
-
- test("Generate global dictionary from external column file") {
- // load the first time
- var carbonLoadModel = buildCarbonLoadModel(extComplexRelation, complexFilePath1,
- header, extColDictFilePath1)
- GlobalDictionaryUtil.generateGlobalDictionary(sqlContext, carbonLoadModel,
- extComplexRelation.tableMeta.storePath)
- // check whether the dictionary is generated
- DictionaryTestCaseUtil.checkDictionary(
- extComplexRelation, "deviceInformationId", "10086")
-
- // load the second time
- carbonLoadModel = buildCarbonLoadModel(extComplexRelation, complexFilePath1,
- header, extColDictFilePath2)
- GlobalDictionaryUtil.generateGlobalDictionary(sqlContext, carbonLoadModel,
- extComplexRelation.tableMeta.storePath)
- // check the old dictionary and whether the new distinct value is generated
- DictionaryTestCaseUtil.checkDictionary(
- extComplexRelation, "deviceInformationId", "10086")
- DictionaryTestCaseUtil.checkDictionary(
- extComplexRelation, "deviceInformationId", "10011")
- }
-
- test("When csv delimiter is not comma") {
- // when csv delimiter is comma
- var carbonLoadModel = buildCarbonLoadModel(extComplexRelation, complexFilePath1,
- header, extColDictFilePath3)
- GlobalDictionaryUtil.generateGlobalDictionary(sqlContext, carbonLoadModel,
- extComplexRelation.tableMeta.storePath)
- // check whether the dictionary is generated
- DictionaryTestCaseUtil.checkDictionary(
- extComplexRelation, "channelsId", "1421|")
-
- // when csv delimiter is not comma
- carbonLoadModel = buildCarbonLoadModel(verticalDelimiteRelation, complexFilePath2,
- header2, extColDictFilePath3, "|")
- GlobalDictionaryUtil.generateGlobalDictionary(sqlContext, carbonLoadModel,
- verticalDelimiteRelation.tableMeta.storePath)
- // check whether the dictionary is generated
- DictionaryTestCaseUtil.checkDictionary(
- verticalDelimiteRelation, "channelsId", "1431,")
- }
-
- test("LOAD DML with COLUMNDICT option") {
- try {
- sql(s"""
- LOAD DATA LOCAL INPATH "$complexFilePath1" INTO TABLE loadSqlTest
- OPTIONS('single_pass'='true','FILEHEADER'='$header', 'COLUMNDICT'='$extColDictFilePath1')
- """)
- } catch {
- case ex: Exception =>
- LOGGER.error(ex.getMessage + "\r\n" + ex.getStackTraceString)
- assert(false)
- }
- DictionaryTestCaseUtil.checkDictionary(
- loadSqlRelation, "deviceInformationId", "10086")
- }
-
- test("COLUMNDICT and ALL_DICTIONARY_PATH can not be used together") {
- try {
- sql(s"""
- LOAD DATA LOCAL INPATH "$complexFilePath1" INTO TABLE loadSqlTest
- OPTIONS('COLUMNDICT'='$extColDictFilePath1',"ALL_DICTIONARY_PATH"='$extColDictFilePath1')
- """)
- assert(false)
- } catch {
- case ex: MalformedCarbonCommandException =>
- assertResult(ex.getMessage)("Error: COLUMNDICT and ALL_DICTIONARY_PATH can not be used together " +
- "in options")
- case _: Throwable => assert(false)
- }
- }
-
- test("Measure can not use COLUMNDICT") {
- try {
- sql(s"""
- LOAD DATA LOCAL INPATH "$complexFilePath1" INTO TABLE loadSqlTest
- OPTIONS('single_pass'='true','FILEHEADER'='$header', 'COLUMNDICT'='gamePointId:$filePath')
- """)
- assert(false)
- } catch {
- case ex: DataLoadingException =>
- assertResult(ex.getMessage)("Column gamePointId is not a key column. Only key column can be part " +
- "of dictionary and used in COLUMNDICT option.")
- case _: Throwable => assert(false)
- }
- }
-
- override def afterAll: Unit = {
- sql("DROP TABLE extComplextypes")
- sql("DROP TABLE verticalDelimitedTable")
- sql("DROP TABLE loadSqlTest")
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala
deleted file mode 100644
index d1d31c1..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala
+++ /dev/null
@@ -1,183 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.spark.util
-
-import java.util.concurrent.{Callable, Executors}
-
-import scala.collection.mutable.ListBuffer
-
-import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.spark.sql.test.util.QueryTest
-import org.apache.spark.sql.{CarbonEnv, CarbonRelation}
-import org.scalatest.BeforeAndAfterAll
-
-import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier
-import org.apache.carbondata.core.service.impl.PathFactory
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datastore.impl.FileFactory
-import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.core.util.path.CarbonStorePath
-import org.apache.carbondata.processing.util.TableOptionConstant
-import org.apache.carbondata.processing.loading.model.{CarbonDataLoadSchema, CarbonLoadModel}
-
-class GlobalDictionaryUtilConcurrentTestCase extends QueryTest with BeforeAndAfterAll {
-
- var sampleRelation: CarbonRelation = _
-
- def buildCarbonLoadModel(relation: CarbonRelation,
- filePath: String,
- header: String): CarbonLoadModel = {
- val carbonLoadModel = new CarbonLoadModel
- carbonLoadModel.setTableName(relation.tableMeta.carbonTableIdentifier.getDatabaseName)
- carbonLoadModel.setDatabaseName(relation.tableMeta.carbonTableIdentifier.getTableName)
- // carbonLoadModel.setSchema(relation.cubeMeta.schema)
- val table = relation.tableMeta.carbonTable
- val carbonSchema = new CarbonDataLoadSchema(table)
- carbonLoadModel.setDatabaseName(table.getDatabaseName)
- carbonLoadModel.setTableName(table.getFactTableName)
- carbonLoadModel.setCarbonDataLoadSchema(carbonSchema)
- carbonLoadModel.setFactFilePath(filePath)
- carbonLoadModel.setCsvHeader(header)
- carbonLoadModel.setCsvDelimiter(",")
- carbonLoadModel.setComplexDelimiterLevel1("\\$")
- carbonLoadModel.setComplexDelimiterLevel2("\\:")
- carbonLoadModel.setStorePath(relation.tableMeta.storePath)
- carbonLoadModel.setQuoteChar("\"")
- carbonLoadModel.setSerializationNullFormat(
- TableOptionConstant.SERIALIZATION_NULL_FORMAT.getName + ",\\N")
- carbonLoadModel.setDefaultTimestampFormat(CarbonProperties.getInstance().getProperty(
- CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
- CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT))
- carbonLoadModel.setDefaultDateFormat(CarbonProperties.getInstance().getProperty(
- CarbonCommonConstants.CARBON_DATE_FORMAT,
- CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT))
- carbonLoadModel.setCsvHeaderColumns(CommonUtil.getCsvHeaderColumns(carbonLoadModel))
- carbonLoadModel.setMaxColumns("2000")
- carbonLoadModel
- }
-
- override def beforeAll {
- // second time comment this line
- buildTable
- buildRelation
- }
-
- def buildTable() = {
- try {
- sql(
- "CREATE TABLE IF NOT EXISTS employee (empid STRING) STORED BY 'org.apache.carbondata.format' " +
- "tblproperties('dictionary_include'='empid')")
- } catch {
- case ex: Throwable => LOGGER.error(ex.getMessage + "\r\n" + ex.getStackTraceString)
- }
- }
-
- def buildRelation() = {
- val catalog = CarbonEnv.get.carbonMetastore
- sampleRelation = catalog.lookupRelation1(Option(CarbonCommonConstants.DATABASE_DEFAULT_NAME),
- "employee")(sqlContext)
- .asInstanceOf[CarbonRelation]
- }
-
- def writedummydata(filePath: String, recCount: Int) = {
- var a: Int = 0
- var records: StringBuilder = StringBuilder.newBuilder
- for (a <- 0 to recCount) {
- records.append(a).append("\n")
- }
- val dis = FileFactory.getDataOutputStream(filePath, FileFactory.getFileType(filePath))
- dis.writeBytes(records.toString())
- dis.close()
- }
-
- test("concurrent dictionary generation") {
- CarbonProperties.getInstance.addProperty(CarbonCommonConstants.MAX_QUERY_EXECUTION_TIME, "-1")
- val noOfFiles = 5
- val files = new ListBuffer[String]()
- val loadModels = new ListBuffer[CarbonLoadModel]()
- for (i <- 0 until noOfFiles) {
- val filePath: String = s"${integrationPath}/spark/target/singlecolumn_${10 * (i + 1)}.csv"
- files += filePath
- loadModels += buildCarbonLoadModel(sampleRelation, filePath, "empid")
- writedummydata(filePath, 10 * (i + 1))
- }
- try {
- val dictGenerators = new java.util.ArrayList[Callable[String]](noOfFiles)
- for (i <- 0 until noOfFiles) {
- dictGenerators.add(new DictGenerator(loadModels(i)))
- }
- val executorService = Executors.newFixedThreadPool(10)
- val results = executorService.invokeAll(dictGenerators)
- for (i <- 0 until noOfFiles) {
- val res = results.get(i).get
- assert("Pass".equals(res))
- }
- } catch {
- case ex: Exception =>
- ex.printStackTrace()
- assert(false)
- }
- val carbonTableIdentifier = sampleRelation.tableMeta.carbonTable.getCarbonTableIdentifier
- val columnIdentifier = sampleRelation.tableMeta.carbonTable.getDimensionByName("employee", "empid").getColumnIdentifier
- val dictionaryColumnUniqueIdentifier = new DictionaryColumnUniqueIdentifier(
- carbonTableIdentifier,
- columnIdentifier,
- columnIdentifier.getDataType,
- CarbonStorePath.getCarbonTablePath(storeLocation, carbonTableIdentifier))
- val carbonTablePath = PathFactory.getInstance()
- .getCarbonTablePath(sampleRelation.tableMeta.storePath, carbonTableIdentifier, dictionaryColumnUniqueIdentifier)
- val dictPath = carbonTablePath.getDictionaryFilePath(columnIdentifier.getColumnId)
- val dictFile = FileFactory.getCarbonFile(dictPath, FileFactory.getFileType(dictPath))
- val offSet = dictFile.getSize
- val sortIndexPath = carbonTablePath.getSortIndexFilePath(columnIdentifier.getColumnId, offSet)
- val sortIndexFile = FileFactory.getCarbonFile(sortIndexPath, FileFactory.getFileType(sortIndexPath))
- assert(sortIndexFile.exists())
- val sortIndexFiles = carbonTablePath.getSortIndexFiles(sortIndexFile.getParentFile, columnIdentifier.getColumnId)
- assert(sortIndexFiles.length >= 1)
- deleteFiles(files)
- }
-
- def deleteFiles(files: ListBuffer[String]) {
- for (i <- 0 until files.length) {
- val file = FileFactory.getCarbonFile(files(i), FileFactory.getFileType(files(i)))
- file.delete()
- }
- }
-
- override def afterAll {
- sql("drop table if exists employee")
- CarbonProperties.getInstance.addProperty(CarbonCommonConstants.MAX_QUERY_EXECUTION_TIME,
- Integer.toString(CarbonCommonConstants.DEFAULT_MAX_QUERY_EXECUTION_TIME))
- }
-
- class DictGenerator(loadModel: CarbonLoadModel) extends Callable[String] {
- override def call:String = {
- var result = "Pass"
- try {
- GlobalDictionaryUtil
- .generateGlobalDictionary(sqlContext,
- loadModel,
- sampleRelation.tableMeta.storePath)
- } catch {
- case ex: Exception =>
- result = ex.getMessage
- ex.printStackTrace()
- }
- result
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilTestCase.scala
deleted file mode 100644
index 8864d3e..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilTestCase.scala
+++ /dev/null
@@ -1,210 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.spark.util
-
-import java.io.File
-
-import org.apache.spark.sql.test.util.QueryTest
-import org.apache.spark.sql.{CarbonEnv, CarbonRelation}
-import org.scalatest.BeforeAndAfterAll
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.processing.util.TableOptionConstant
-import org.apache.carbondata.processing.loading.model.{CarbonDataLoadSchema, CarbonLoadModel}
-
-/**
- * Test Case for org.apache.carbondata.spark.util.GlobalDictionaryUtil
- */
-class GlobalDictionaryUtilTestCase extends QueryTest with BeforeAndAfterAll {
-
- var sampleRelation: CarbonRelation = _
- var dimSampleRelation: CarbonRelation = _
- var complexRelation: CarbonRelation = _
- var incrementalLoadTableRelation: CarbonRelation = _
- var filePath: String = _
- var dimFilePath: String = _
- var complexfilePath: String = _
- var complexfilePath1: String = _
- var complexfilePath2: String = _
-
- def buildCarbonLoadModel(relation: CarbonRelation,
- filePath: String,
- header: String): CarbonLoadModel = {
- val carbonLoadModel = new CarbonLoadModel
- carbonLoadModel.setTableName(relation.tableMeta.carbonTableIdentifier.getDatabaseName)
- carbonLoadModel.setDatabaseName(relation.tableMeta.carbonTableIdentifier.getTableName)
- // carbonLoadModel.setSchema(relation.tableMeta.schema)
- val table = relation.tableMeta.carbonTable
- val carbonSchema = new CarbonDataLoadSchema(table)
- carbonLoadModel.setDatabaseName(table.getDatabaseName)
- carbonLoadModel.setTableName(table.getFactTableName)
- carbonLoadModel.setCarbonDataLoadSchema(carbonSchema)
- carbonLoadModel.setFactFilePath(filePath)
- carbonLoadModel.setCsvHeader(header)
- carbonLoadModel.setCsvDelimiter(",")
- carbonLoadModel.setComplexDelimiterLevel1("\\$")
- carbonLoadModel.setComplexDelimiterLevel2("\\:")
- carbonLoadModel.setStorePath(relation.tableMeta.storePath)
- carbonLoadModel.setQuoteChar("\"")
- carbonLoadModel.setSerializationNullFormat(
- TableOptionConstant.SERIALIZATION_NULL_FORMAT.getName + ",\\N")
- carbonLoadModel.setDefaultTimestampFormat(CarbonProperties.getInstance().getProperty(
- CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
- CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT))
- carbonLoadModel.setDefaultDateFormat(CarbonProperties.getInstance().getProperty(
- CarbonCommonConstants.CARBON_DATE_FORMAT,
- CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT))
- carbonLoadModel.setCsvHeaderColumns(CommonUtil.getCsvHeaderColumns(carbonLoadModel))
- carbonLoadModel.setMaxColumns("2000")
- carbonLoadModel
- }
-
- override def beforeAll {
- buildTestData
- // second time comment this line
- buildTable
- buildRelation
- }
-
- def buildTestData() = {
- filePath = s"${resourcesPath}/sample.csv"
- dimFilePath = s"dimTableSample:${resourcesPath}/dimTableSample.csv"
- complexfilePath1 = s"${resourcesPath}/complexdata1.csv"
- complexfilePath2 = s"${resourcesPath}/complexdata2.csv"
- complexfilePath = s"${resourcesPath}/complexdata.csv"
- }
-
- def buildTable() = {
- try {
- sql(
- "CREATE TABLE IF NOT EXISTS sample (id STRING, name STRING, city STRING, " +
- "age INT) STORED BY 'org.apache.carbondata.format'"
- )
- } catch {
- case ex: Throwable => LOGGER.error(ex.getMessage + "\r\n" + ex.getStackTraceString)
- }
- try {
- sql(
- "CREATE TABLE IF NOT EXISTS dimSample (id STRING, name STRING, city STRING, " +
- "age INT) STORED BY 'org.apache.carbondata.format'" +
- "TBLPROPERTIES('DICTIONARY_EXCLUDE'='id,name')"
- )
- } catch {
- case ex: Throwable => LOGGER.error(ex.getMessage + "\r\n" + ex.getStackTraceString)
- }
- try {
- sql(
- "create table complextypes (deviceInformationId INT, channelsId string, " +
- "ROMSize string, purchasedate string, mobile struct<imei: string, imsi: string>, MAC " +
- "array<string>, locationinfo array<struct<ActiveAreaId: INT, ActiveCountry: string, " +
- "ActiveProvince: string, Activecity: string, ActiveDistrict: string, ActiveStreet: " +
- "string>>, proddate struct<productionDate: string,activeDeactivedate: array<string>>, " +
- "gamePointId INT,contractNumber INT) STORED BY 'org.apache.carbondata.format'" +
- "TBLPROPERTIES('DICTIONARY_EXCLUDE'='ROMSize')"
-
- )
- } catch {
- case ex: Throwable => LOGGER.error(ex.getMessage + "\r\n" + ex.getStackTraceString)
- }
-
- try {
- sql(
- "create table incrementalLoadTable (deviceInformationId INT, channelsId " +
- "string, ROMSize string, purchasedate string, mobile struct<imei: string, imsi: string>, " +
- "MAC array<string>, locationinfo array<struct<ActiveAreaId: INT, ActiveCountry: " +
- "string, ActiveProvince: string, Activecity: string, ActiveDistrict: string, ActiveStreet: " +
- "string>>, proddate struct<productionDate: string,activeDeactivedate: array<string>>, " +
- "gamePointId INT,contractNumber INT) STORED BY 'org.apache.carbondata.format'"+
- "TBLPROPERTIES('DICTIONARY_INCLUDE'='deviceInformationId')"
- )
- } catch {
- case ex: Throwable => LOGGER.error(ex.getMessage + "\r\n" + ex.getStackTraceString)
- }
- }
-
- def buildRelation() = {
- val catalog = CarbonEnv.get.carbonMetastore
- sampleRelation = catalog.lookupRelation1(Option(CarbonCommonConstants.DATABASE_DEFAULT_NAME),
- "sample")(sqlContext)
- .asInstanceOf[CarbonRelation]
- dimSampleRelation = catalog
- .lookupRelation1(Option(CarbonCommonConstants.DATABASE_DEFAULT_NAME), "dimSample")(sqlContext)
- .asInstanceOf[CarbonRelation]
- complexRelation = catalog
- .lookupRelation1(Option(CarbonCommonConstants.DATABASE_DEFAULT_NAME), "complextypes")(sqlContext)
- .asInstanceOf[CarbonRelation]
- incrementalLoadTableRelation = catalog
- .lookupRelation1(Option(CarbonCommonConstants.DATABASE_DEFAULT_NAME), "incrementalLoadTable")(sqlContext)
- .asInstanceOf[CarbonRelation]
- }
-
- test("[issue-80]Global Dictionary Generation") {
-
- val carbonLoadModel = buildCarbonLoadModel(sampleRelation, filePath, null)
- GlobalDictionaryUtil
- .generateGlobalDictionary(sqlContext, carbonLoadModel,
- sampleRelation.tableMeta.storePath
- )
-
- // test for dimension table
- // TODO - Need to fill and send the dimension table data as per new DimensionRelation in
- // CarbonDataLoadModel
- // carbonLoadModel = buildCarbonLoadModel(dimSampleRelation, filePath, dimFilePath, null)
- // GlobalDictionaryUtil.generateGlobalDictionary(CarbonHiveContext, carbonLoadModel,
- // dimSampleRelation.tableMeta.dataPath, false)
- }
-
- test("[Issue-190]load csv file without header And support complex type") {
- val header = "deviceInformationId,channelsId,ROMSize,purchasedate,mobile,MAC,locationinfo," +
- "proddate,gamePointId,contractNumber"
- val carbonLoadModel = buildCarbonLoadModel(complexRelation, complexfilePath, header)
- GlobalDictionaryUtil
- .generateGlobalDictionary(sqlContext, carbonLoadModel,
- complexRelation.tableMeta.storePath
- )
- }
-
- test("[Issue-232]Issue in incremental data load for dictionary generation") {
- val header = "deviceInformationId,channelsId,ROMSize,purchasedate,mobile,MAC,locationinfo," +
- "proddate,gamePointId,contractNumber"
- // load 1
- var carbonLoadModel = buildCarbonLoadModel(incrementalLoadTableRelation,
- complexfilePath1,
- header
- )
- GlobalDictionaryUtil
- .generateGlobalDictionary(sqlContext, carbonLoadModel,
- sampleRelation.tableMeta.storePath
- )
- DictionaryTestCaseUtil.
- checkDictionary(incrementalLoadTableRelation, "deviceInformationId", "100010")
-
- // load 2
- carbonLoadModel = buildCarbonLoadModel(incrementalLoadTableRelation,
- complexfilePath2,
- header
- )
- GlobalDictionaryUtil
- .generateGlobalDictionary(sqlContext, carbonLoadModel,
- sampleRelation.tableMeta.storePath
- )
- DictionaryTestCaseUtil.
- checkDictionary(incrementalLoadTableRelation, "deviceInformationId", "100077")
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/test/scala/org/apache/spark/sql/TestCarbonSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/spark/sql/TestCarbonSqlParser.scala b/integration/spark/src/test/scala/org/apache/spark/sql/TestCarbonSqlParser.scala
deleted file mode 100644
index 13d4167..0000000
--- a/integration/spark/src/test/scala/org/apache/spark/sql/TestCarbonSqlParser.scala
+++ /dev/null
@@ -1,327 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.sql
-
-import scala.collection.mutable.Map
-
-import org.apache.spark.sql.test.util.QueryTest
-import org.apache.spark.sql.execution.command.Field
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-
-/**
- * Stub class for calling the CarbonSqlParser
- */
-private class TestCarbonSqlParserStub extends CarbonSqlParser {
-
- def extractDimAndMsrFieldsTest(fields: Seq[Field],
- tableProperties: Map[String, String]): (Seq[Field], Seq[Field], Seq[String], Seq[String]) = {
- extractDimAndMsrFields(fields, tableProperties)
- }
-
-
-}
-
-/**
- * Test class to test Carbon Sql Parser
- */
-class TestCarbonSqlParser extends QueryTest {
-
- /**
- * load all test fields
- * @return
- */
- def loadAllFields: Seq[Field] = {
- var fields: Seq[Field] = Seq[Field]()
-
- var col1 = Field("col1", Option("String"), Option("col1"), None, null, Some("columnar"))
- var col2 = Field("col2", Option("String"), Option("col2"), None, null, Some("columnar"))
- var col3 = Field("col3", Option("String"), Option("col3"), None, null, Some("columnar"))
- var col4 = Field("col4", Option("int"), Option("col4"), None, null, Some("columnar"))
- var col5 = Field("col5", Option("String"), Option("col5"), None, null, Some("columnar"))
- var col6 = Field("col6", Option("String"), Option("col6"), None, null, Some("columnar"))
- var col7 = Field("col7", Option("String"), Option("col7"), None, null, Some("columnar"))
- var col8 = Field("col8", Option("String"), Option("col8"), None, null, Some("columnar"))
-
- fields :+= col1
- fields :+= col2
- fields :+= col3
- fields :+= col4
- fields :+= col5
- fields :+= col6
- fields :+= col7
- fields :+= col8
- fields
- }
-
- // Testing the extracting of Dims and no Dictionary
- test("Test-extractDimColsAndNoDictionaryFields") {
- val tableProperties = Map(CarbonCommonConstants.DICTIONARY_EXCLUDE -> "col2", CarbonCommonConstants.DICTIONARY_INCLUDE -> "col4")
- val fields: Seq[Field] = loadAllFields
-
- val stub = new TestCarbonSqlParserStub()
- val (dimCols, _, noDictionary, _) = stub.extractDimAndMsrFieldsTest(fields, tableProperties)
-
- // testing col
-
- //All dimension fields should be available in dimensions list
- assert(dimCols.size == 8)
- assert(dimCols.lift(0).get.column.equalsIgnoreCase("col1"))
- assert(dimCols.lift(1).get.column.equalsIgnoreCase("col2"))
- assert(dimCols.lift(2).get.column.equalsIgnoreCase("col3"))
- assert(dimCols.lift(3).get.column.equalsIgnoreCase("col4"))
-
- //No dictionary column names will be available in noDictionary list
- assert(noDictionary.size == 7)
- assert(noDictionary.lift(0).get.equalsIgnoreCase("col1"))
- assert(noDictionary.lift(1).get.equalsIgnoreCase("col2"))
- assert(noDictionary.lift(2).get.equalsIgnoreCase("col3"))
- assert(noDictionary.lift(3).get.equalsIgnoreCase("col5"))
- assert(noDictionary.lift(4).get.equalsIgnoreCase("col6"))
- assert(noDictionary.lift(5).get.equalsIgnoreCase("col7"))
- assert(noDictionary.lift(6).get.equalsIgnoreCase("col8"))
-
- }
-
- test("Test-DimAndMsrColsWithNoDictionaryFields1") {
- val tableProperties = Map(CarbonCommonConstants.DICTIONARY_EXCLUDE -> "col1")
- val fields: Seq[Field] = loadAllFields
- val stub = new TestCarbonSqlParserStub()
- val (dimCols, msrCols, noDictionary, _) = stub.extractDimAndMsrFieldsTest(fields, tableProperties)
-
- //below fields should be available in dimensions list
- assert(dimCols.size == 7)
- assert(dimCols.lift(0).get.column.equalsIgnoreCase("col1"))
- assert(dimCols.lift(1).get.column.equalsIgnoreCase("col2"))
- assert(dimCols.lift(2).get.column.equalsIgnoreCase("col3"))
-
- //below column names will be available in noDictionary list
- assert(noDictionary.size == 7)
- assert(noDictionary.lift(0).get.equalsIgnoreCase("col1"))
- assert(noDictionary.lift(1).get.equalsIgnoreCase("col2"))
- assert(noDictionary.lift(2).get.equalsIgnoreCase("col3"))
- assert(noDictionary.lift(3).get.equalsIgnoreCase("col5"))
- assert(noDictionary.lift(4).get.equalsIgnoreCase("col6"))
- assert(noDictionary.lift(5).get.equalsIgnoreCase("col7"))
- assert(noDictionary.lift(6).get.equalsIgnoreCase("col8"))
-
- //check msr
- assert(msrCols.size == 1)
- assert(msrCols.lift(0).get.column.equalsIgnoreCase("col4"))
- }
-
- test("Test-DimAndMsrColsWithNoDictionaryFields2") {
- val tableProperties = Map(CarbonCommonConstants.DICTIONARY_INCLUDE -> "col1")
- val fields: Seq[Field] = loadAllFields
- val stub = new TestCarbonSqlParserStub()
- val (dimCols, msrCols, noDictionary, _) = stub.extractDimAndMsrFieldsTest(fields, tableProperties)
-
- //below dimension fields should be available in dimensions list
- assert(dimCols.size == 7)
- assert(dimCols.lift(0).get.column.equalsIgnoreCase("col1"))
- assert(dimCols.lift(1).get.column.equalsIgnoreCase("col2"))
- assert(dimCols.lift(2).get.column.equalsIgnoreCase("col3"))
-
- //below column names will be available in noDictionary list
- assert(noDictionary.size == 6)
- assert(noDictionary.lift(0).get.equalsIgnoreCase("col2"))
- assert(noDictionary.lift(1).get.equalsIgnoreCase("col3"))
- assert(noDictionary.lift(2).get.equalsIgnoreCase("col5"))
- assert(noDictionary.lift(3).get.equalsIgnoreCase("col6"))
- assert(noDictionary.lift(4).get.equalsIgnoreCase("col7"))
- assert(noDictionary.lift(5).get.equalsIgnoreCase("col8"))
-
- //check msr
- assert(msrCols.size == 1)
- assert(msrCols.lift(0).get.column.equalsIgnoreCase("col4"))
- }
-
- test("Test-DimAndMsrColsWithNoDictionaryFields3") {
- val tableProperties = Map(CarbonCommonConstants.DICTIONARY_EXCLUDE -> "col1", CarbonCommonConstants.DICTIONARY_INCLUDE -> "col4")
- val fields: Seq[Field] = loadAllFields
- val stub = new TestCarbonSqlParserStub()
- val (dimCols, msrCols, noDictionary, _) = stub.extractDimAndMsrFieldsTest(fields,
- tableProperties)
-
- //below dimension fields should be available in dimensions list
- assert(dimCols.size == 8)
- assert(dimCols.lift(0).get.column.equalsIgnoreCase("col1"))
- assert(dimCols.lift(1).get.column.equalsIgnoreCase("col2"))
- assert(dimCols.lift(2).get.column.equalsIgnoreCase("col3"))
- assert(dimCols.lift(3).get.column.equalsIgnoreCase("col4"))
-
- //below column names will be available in noDictionary list
- assert(noDictionary.size == 7)
- assert(noDictionary.lift(0).get.equalsIgnoreCase("col1"))
- assert(noDictionary.lift(1).get.equalsIgnoreCase("col2"))
- assert(noDictionary.lift(2).get.equalsIgnoreCase("col3"))
- assert(noDictionary.lift(3).get.equalsIgnoreCase("col5"))
- assert(noDictionary.lift(4).get.equalsIgnoreCase("col6"))
- assert(noDictionary.lift(5).get.equalsIgnoreCase("col7"))
- assert(noDictionary.lift(6).get.equalsIgnoreCase("col8"))
-
- //check msr
- assert(msrCols.size == 0)
- }
-
- test("Test-DimAndMsrColsWithNoDictionaryFields4") {
- val tableProperties = Map(CarbonCommonConstants.DICTIONARY_EXCLUDE -> "col3", CarbonCommonConstants.DICTIONARY_INCLUDE -> "col2")
- val fields: Seq[Field] = loadAllFields
- val stub = new TestCarbonSqlParserStub()
- val (dimCols, msrCols, noDictionary, _) = stub.extractDimAndMsrFieldsTest(fields, tableProperties)
-
- //below dimension fields should be available in dimensions list
- assert(dimCols.size == 7)
- assert(dimCols.lift(0).get.column.equalsIgnoreCase("col1"))
- assert(dimCols.lift(1).get.column.equalsIgnoreCase("col2"))
- assert(dimCols.lift(2).get.column.equalsIgnoreCase("col3"))
-
- //below column names will be available in noDictionary list
- assert(noDictionary.size == 6)
- assert(noDictionary.lift(0).get.equalsIgnoreCase("col1"))
- assert(noDictionary.lift(1).get.equalsIgnoreCase("col3"))
- assert(noDictionary.lift(2).get.equalsIgnoreCase("col5"))
- assert(noDictionary.lift(3).get.equalsIgnoreCase("col6"))
- assert(noDictionary.lift(4).get.equalsIgnoreCase("col7"))
- assert(noDictionary.lift(5).get.equalsIgnoreCase("col8"))
-
- //check msr
- assert(msrCols.size == 1)
- assert(msrCols.lift(0).get.column.equalsIgnoreCase("col4"))
- }
-
- test("Test-DimAndMsrColsWithNoDictionaryFields5") {
- val tableProperties = Map(CarbonCommonConstants.DICTIONARY_EXCLUDE -> "col1", CarbonCommonConstants.DICTIONARY_INCLUDE -> "col2")
- val fields: Seq[Field] = loadAllFields
- val stub = new TestCarbonSqlParserStub()
- val (dimCols, msrCols, noDictionary, _) = stub.extractDimAndMsrFieldsTest(fields, tableProperties)
-
- //below dimension fields should be available in dimensions list
- assert(dimCols.size == 7)
- assert(dimCols.lift(0).get.column.equalsIgnoreCase("col1"))
- assert(dimCols.lift(1).get.column.equalsIgnoreCase("col2"))
- assert(dimCols.lift(2).get.column.equalsIgnoreCase("col3"))
-
- //below column names will be available in noDictionary list
- assert(noDictionary.size == 6)
- assert(noDictionary.lift(0).get.equalsIgnoreCase("col1"))
- assert(noDictionary.lift(1).get.equalsIgnoreCase("col3"))
- assert(noDictionary.lift(2).get.equalsIgnoreCase("col5"))
- assert(noDictionary.lift(3).get.equalsIgnoreCase("col6"))
- assert(noDictionary.lift(4).get.equalsIgnoreCase("col7"))
- assert(noDictionary.lift(5).get.equalsIgnoreCase("col8"))
-
- //check msr
- assert(msrCols.size == 1)
- assert(msrCols.lift(0).get.column.equalsIgnoreCase("col4"))
- }
-
- test("Test-DimAndMsrColsWithNoDictionaryFields6") {
- val tableProperties = Map(CarbonCommonConstants.DICTIONARY_EXCLUDE -> "col2", CarbonCommonConstants.DICTIONARY_INCLUDE -> "col1")
- val fields: Seq[Field] = loadAllFields
- val stub = new TestCarbonSqlParserStub()
- val (dimCols, msrCols, noDictionary, _) = stub.extractDimAndMsrFieldsTest(fields, tableProperties)
-
- //below dimension fields should be available in dimensions list
- assert(dimCols.size == 7)
- assert(dimCols.lift(0).get.column.equalsIgnoreCase("col1"))
- assert(dimCols.lift(1).get.column.equalsIgnoreCase("col2"))
- assert(dimCols.lift(2).get.column.equalsIgnoreCase("col3"))
-
- //below column names will be available in noDictionary list
- assert(noDictionary.size == 6)
- assert(noDictionary.lift(0).get.equalsIgnoreCase("col2"))
- assert(noDictionary.lift(1).get.equalsIgnoreCase("col3"))
- assert(noDictionary.lift(2).get.equalsIgnoreCase("col5"))
- assert(noDictionary.lift(3).get.equalsIgnoreCase("col6"))
- assert(noDictionary.lift(4).get.equalsIgnoreCase("col7"))
- assert(noDictionary.lift(5).get.equalsIgnoreCase("col8"))
-
- //check msr
- assert(msrCols.size == 1)
- assert(msrCols.lift(0).get.column.equalsIgnoreCase("col4"))
- }
-
- test("Test-DimAndMsrColsWithNoDictionaryFields7") {
- val tableProperties = Map(CarbonCommonConstants.DICTIONARY_EXCLUDE -> "col2 ,col1 ",
- CarbonCommonConstants.DICTIONARY_INCLUDE -> "col3 ,col4 "
- )
- val fields: Seq[Field] = loadAllFields
- val stub = new TestCarbonSqlParserStub()
- val (dimCols, msrCols, noDictionary, _) = stub.extractDimAndMsrFieldsTest(fields, tableProperties)
-
- //below dimension fields should be available in dimensions list
- assert(dimCols.size == 8)
- assert(dimCols.lift(0).get.column.equalsIgnoreCase("col1"))
- assert(dimCols.lift(1).get.column.equalsIgnoreCase("col2"))
- assert(dimCols.lift(2).get.column.equalsIgnoreCase("col3"))
- assert(dimCols.lift(3).get.column.equalsIgnoreCase("col4"))
-
- //below column names will be available in noDictionary list
- assert(noDictionary.size == 6)
- assert(noDictionary.lift(0).get.equalsIgnoreCase("col1"))
- assert(noDictionary.lift(1).get.equalsIgnoreCase("col2"))
- assert(noDictionary.lift(2).get.equalsIgnoreCase("col5"))
- assert(noDictionary.lift(3).get.equalsIgnoreCase("col6"))
- assert(noDictionary.lift(4).get.equalsIgnoreCase("col7"))
- assert(noDictionary.lift(5).get.equalsIgnoreCase("col8"))
-
- //check msr
- assert(msrCols.size == 0)
- }
-
- test("Test-DimAndMsrColsWithNoDictionaryFields8") {
- val tableProperties = Map(CarbonCommonConstants.DICTIONARY_EXCLUDE-> "col2", CarbonCommonConstants.DICTIONARY_INCLUDE -> "col3")
- val fields: Seq[Field] = loadAllFields
- val stub = new TestCarbonSqlParserStub()
- val (dimCols, msrCols, noDictionary, _) = stub.extractDimAndMsrFieldsTest(fields, tableProperties)
-
- //below dimension fields should be available in dimensions list
- assert(dimCols.size == 7)
- assert(dimCols.lift(0).get.column.equalsIgnoreCase("col1"))
- assert(dimCols.lift(1).get.column.equalsIgnoreCase("col2"))
-
- //below column names will be available in noDictionary list
- assert(noDictionary.size == 6)
- assert(noDictionary.lift(0).get.equalsIgnoreCase("col1"))
- assert(noDictionary.lift(1).get.equalsIgnoreCase("col2"))
- assert(noDictionary.lift(2).get.equalsIgnoreCase("col5"))
- assert(noDictionary.lift(3).get.equalsIgnoreCase("col6"))
- assert(noDictionary.lift(4).get.equalsIgnoreCase("col7"))
- assert(noDictionary.lift(5).get.equalsIgnoreCase("col8"))
-
- //check msr
- assert(msrCols.size == 1)
- assert(msrCols.lift(0).get.column.equalsIgnoreCase("col4"))
- }
-
- // Testing the extracting of measures
- test("Test-extractMsrColsFromFields") {
- val tableProperties = Map(CarbonCommonConstants.DICTIONARY_EXCLUDE -> "col2",
- CarbonCommonConstants.DICTIONARY_INCLUDE -> "col1")
- val fields: Seq[Field] = loadAllFields
- val stub = new TestCarbonSqlParserStub()
- val (_, msrCols, _, _) = stub.extractDimAndMsrFieldsTest(fields, tableProperties)
-
- // testing col
- assert(msrCols.lift(0).get.column.equalsIgnoreCase("col4"))
-
- }
-
-}
-
-
[10/16] carbondata git commit: [CARBONDATA-1599] Optimize pull
request template
Posted by ra...@apache.org.
[CARBONDATA-1599] Optimize pull request template
Optimize pull request template for reminding contributors to provide full information.
This closes #1422
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/228ab2fd
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/228ab2fd
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/228ab2fd
Branch: refs/heads/pre-aggregate
Commit: 228ab2fd18514873c930a14c2c039e7efb083020
Parents: 0bf597d
Author: chenliang613 <ch...@apache.org>
Authored: Thu Oct 19 16:55:57 2017 +0530
Committer: lionelcao <wh...@gmail.com>
Committed: Thu Oct 19 20:06:49 2017 +0800
----------------------------------------------------------------------
.github/PULL_REQUEST_TEMPLATE.md | 21 ++++++++++++++-------
1 file changed, 14 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/228ab2fd/.github/PULL_REQUEST_TEMPLATE.md
----------------------------------------------------------------------
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index 02f17e7..0d9f60c 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -3,17 +3,24 @@ quickly and easily:
- [ ] Make sure the PR title is formatted like:
`[CARBONDATA-<Jira issue #>] Description of pull request`
- - [ ] Make sure tests pass via `mvn clean verify`. (Even better, enable
- Travis-CI on your fork and ensure the whole test matrix passes).
- - [ ] Replace `<Jira issue #>` in the title with the actual Jira issue
- number, if there is one.
- - [ ] If this contribution is large, please file an Apache
- [Individual Contributor License Agreement](https://www.apache.org/licenses/icla.txt).
+
+ - [ ] Make sure to add PR description including
+
+ - the root cause/problem statement
+ - What is the implemented solution
+
+ - [ ] Any interfaces changed?
+
+ - [ ] Any backward compatibility impacted?
+
+ - [ ] Document update required?
+
- [ ] Testing done
Please provide details on
- Whether new unit test cases have been added or why no new tests are required?
- - What manual testing you have done?
+ - How it is tested? Please attach test report.
+ - Is it a performance related change? Please attach the performance test report.
- Any additional information to help reviewers in testing this change.
- [ ] For large changes, please consider breaking it into sub-tasks under an umbrella JIRA.
[12/16] carbondata git commit: [CARBONDATA-1444] Support Boolean data
type
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesBaseTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesBaseTest.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesBaseTest.scala
new file mode 100644
index 0000000..c0087a8
--- /dev/null
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesBaseTest.scala
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.spark.testsuite.booleantype
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
+
+class BooleanDataTypesBaseTest extends QueryTest with BeforeAndAfterEach with BeforeAndAfterAll {
+
+ override def beforeEach(): Unit = {
+ sql("drop table if exists carbon_table")
+ sql("drop table if exists boolean_table")
+ }
+
+ override def afterEach(): Unit = {
+ sql("drop table if exists carbon_table")
+ sql("drop table if exists boolean_table")
+ }
+
+ test("Creating table: boolean one column, should support") {
+ try {
+ sql("CREATE TABLE if not exists boolean_table(cc BOOLEAN) STORED BY 'carbondata'")
+ assert(true)
+ } catch {
+ case _: Exception => assert(false)
+ }
+ }
+
+ test("Creating table: boolean and other table, should support") {
+ try {
+ sql(
+ s"""
+ |CREATE TABLE if not exists boolean_table(
+ |aa INT, bb STRING, cc BOOLEAN
+ |) STORED BY 'carbondata'""".stripMargin)
+ assert(true)
+ } catch {
+ case _: Exception => assert(false)
+ }
+ }
+
+ test("Describing table: boolean data type, should support") {
+ sql(
+ s"""
+ |CREATE TABLE if not exists carbon_table(
+ |cc BOOLEAN
+ |) STORED BY 'carbondata'""".stripMargin)
+ checkExistence(sql("describe formatted carbon_table"), true, "boolean")
+ }
+
+ test("Describing table: support boolean data type format and other table ") {
+ sql(
+ s"""
+ | CREATE TABLE carbon_table(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+ checkExistence(sql("describe formatted carbon_table"), true, "boolean")
+ }
+
+ test("Altering table and add column: add boolean type column") {
+ sql(
+ s"""
+ |CREATE TABLE if not exists carbon_table(
+ |aa INT, bb STRING
+ |) STORED BY 'carbondata'""".stripMargin)
+ sql("alter table carbon_table add columns (dd BOOLEAN)")
+ checkExistence(sql("describe formatted carbon_table"), true, "boolean")
+ checkExistence(sql("describe formatted carbon_table"), true, "dd")
+ }
+
+ test("Altering table and add column: exists boolean column, add boolean type column") {
+ sql(
+ s"""
+ |CREATE TABLE if not exists carbon_table(
+ |aa INT, bb STRING, cc BOOLEAN
+ |) STORED BY 'carbondata'""".stripMargin)
+ sql("alter table carbon_table add columns (dd BOOLEAN)")
+ checkExistence(sql("describe formatted carbon_table"), true, "boolean")
+ checkExistence(sql("describe formatted carbon_table"), true, "dd")
+ }
+
+ test("Altering table and add column: exists boolean column, add not boolean type column") {
+ sql(
+ s"""
+ |CREATE TABLE if not exists carbon_table(
+ |aa INT, bb STRING, cc BOOLEAN
+ |) STORED BY 'carbondata'""".stripMargin)
+ sql("alter table carbon_table add columns (dd STRING)")
+ checkExistence(sql("describe formatted carbon_table"), true, "boolean")
+ checkExistence(sql("describe formatted carbon_table"), true, "dd")
+ }
+
+ test("Altering table and add column and insert values: exists boolean column, add boolean type column") {
+ sql(
+ s"""
+ |CREATE TABLE if not exists carbon_table(
+ |aa STRING, bb INT, cc BOOLEAN
+ |) STORED BY 'carbondata'""".stripMargin)
+ sql("alter table carbon_table add columns (dd BOOLEAN)")
+ sql("insert into carbon_table values('adam',11,true,false)")
+ checkAnswer(sql("select * from carbon_table"), Seq(Row("adam", 11, true, false)))
+ }
+
+ test("Altering table and drop column and insert values: exists boolean column, add boolean type column") {
+ sql(
+ s"""
+ |CREATE TABLE if not exists carbon_table(
+ |aa STRING, bb INT, cc BOOLEAN, dd BOOLEAN
+ |) STORED BY 'carbondata'""".stripMargin)
+ sql("alter table carbon_table drop columns (dd)")
+ sql("insert into carbon_table values('adam',11,true)")
+ checkAnswer(sql("select * from carbon_table"), Seq(Row("adam", 11, true)))
+ }
+
+ test("Deleting table and drop column and insert values: exists boolean column, add boolean type column") {
+ sql(
+ s"""
+ |CREATE TABLE if not exists carbon_table(
+ |aa STRING, bb INT, cc BOOLEAN, dd BOOLEAN
+ |) STORED BY 'carbondata'""".stripMargin)
+ sql("alter table carbon_table drop columns (dd)")
+ sql("insert into carbon_table values('adam',11,true)")
+ checkAnswer(sql("select * from carbon_table"), Seq(Row("adam", 11, true)))
+ sql("delete from carbon_table where cc=true")
+ checkAnswer(sql("select COUNT(*) from carbon_table"), Row(0))
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesBigFileTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesBigFileTest.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesBigFileTest.scala
new file mode 100644
index 0000000..53835fb
--- /dev/null
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesBigFileTest.scala
@@ -0,0 +1,729 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.spark.testsuite.booleantype
+
+import java.io.{File, PrintWriter}
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
+
+
+class BooleanDataTypesBigFileTest extends QueryTest with BeforeAndAfterEach with BeforeAndAfterAll {
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+
+ override def beforeEach(): Unit = {
+ sql("drop table if exists boolean_table")
+ sql("drop table if exists boolean_table2")
+ sql("drop table if exists carbon_table")
+ sql("drop table if exists hive_table")
+ }
+
+ override def afterAll(): Unit = {
+ sql("drop table if exists boolean_table")
+ sql("drop table if exists boolean_table2")
+ sql("drop table if exists carbon_table")
+ sql("drop table if exists hive_table")
+ assert(BooleanFile.deleteFile(pathOfManyDataType))
+ assert(BooleanFile.deleteFile(pathOfOnlyBoolean))
+ }
+
+ val pathOfManyDataType = s"$rootPath/integration/spark2/src/test/resources/bool/supportBooleanBigFile.csv"
+ val pathOfOnlyBoolean = s"$rootPath/integration/spark2/src/test/resources/bool/supportBooleanBigFileOnlyBoolean.csv"
+ val trueNum = 10000
+
+ override def beforeAll(): Unit = {
+ assert(BooleanFile.createBooleanFileWithOtherDataType(pathOfManyDataType, trueNum))
+ assert(BooleanFile.createOnlyBooleanFile(pathOfOnlyBoolean, trueNum))
+ }
+
+ test("Loading table: support boolean and other data type, big file") {
+ sql(
+ s"""
+ | CREATE TABLE boolean_table(
+ | intField INT,
+ | booleanField BOOLEAN,
+ | stringField STRING,
+ | doubleField DOUBLE,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ """.stripMargin)
+
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${pathOfManyDataType}'
+ | INTO TABLE boolean_table
+ | options('FILEHEADER'='intField,booleanField,stringField,doubleField,booleanField2')
+ """.stripMargin)
+
+ checkAnswer(
+ sql("select count(*) from boolean_table"),
+ Row(trueNum + trueNum / 10))
+ }
+
+ test("Inserting table: support boolean and other data type, big file") {
+ sql(
+ s"""
+ | CREATE TABLE boolean_table(
+ | intField INT,
+ | booleanField BOOLEAN,
+ | stringField STRING,
+ | doubleField DOUBLE,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ """.stripMargin)
+
+ sql(
+ s"""
+ | CREATE TABLE boolean_table2(
+ | intField INT,
+ | booleanField BOOLEAN,
+ | stringField STRING,
+ | doubleField DOUBLE,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ """.stripMargin)
+
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${pathOfManyDataType}'
+ | INTO TABLE boolean_table
+ | options('FILEHEADER'='intField,booleanField,stringField,doubleField,booleanField2')
+ """.stripMargin)
+
+ sql("insert into boolean_table2 select * from boolean_table")
+
+ checkAnswer(
+ sql("select count(*) from boolean_table2"),
+ Row(trueNum + trueNum / 10))
+ }
+
+ test("Filtering table: support boolean data type, only boolean, big file") {
+ sql(
+ s"""
+ | CREATE TABLE boolean_table(
+ | booleanField BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ """.stripMargin)
+
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${pathOfOnlyBoolean}'
+ | INTO TABLE boolean_table
+ | options('FILEHEADER'='booleanField')
+ """.stripMargin)
+
+ checkAnswer(
+ sql("select count(*) from boolean_table"),
+ Row(trueNum + trueNum / 10))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField is not null"),
+ Row(trueNum + trueNum / 10))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField is null"),
+ Row(0))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField = true"),
+ Row(trueNum))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField >= true"),
+ Row(trueNum))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField > true"),
+ Row(0))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField < true"),
+ Row(trueNum / 10))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField = false"),
+ Row(trueNum / 10))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField <= false"),
+ Row(trueNum / 10))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField > false"),
+ Row(trueNum))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField < false"),
+ Row(0))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField in (false)"),
+ Row(trueNum / 10))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField not in (false)"),
+ Row(trueNum))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField in (true,false)"),
+ Row(trueNum + trueNum / 10))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField not in (true,false)"),
+ Row(0))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField like 'f%'"),
+ Row(trueNum / 10))
+ }
+
+ test("Filtering table: support boolean and other data type, big file") {
+
+ sql(
+ s"""
+ | CREATE TABLE boolean_table(
+ | intField INT,
+ | booleanField BOOLEAN,
+ | stringField STRING,
+ | doubleField DOUBLE,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ """.stripMargin)
+
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${pathOfManyDataType}'
+ | INTO TABLE boolean_table
+ | options('FILEHEADER'='intField,booleanField,stringField,doubleField,booleanField2')
+ """.stripMargin)
+
+ checkAnswer(
+ sql("select booleanField from boolean_table where intField >=1 and intField <11"),
+ Seq(Row(true), Row(true), Row(true), Row(true), Row(true), Row(true), Row(true), Row(true), Row(true), Row(true))
+ )
+
+ checkAnswer(
+ sql(s"select booleanField from boolean_table where intField >='${trueNum - 5}' and intField <=${trueNum + 1}"),
+ Seq(Row(true), Row(true), Row(true), Row(true), Row(true), Row(false), Row(false))
+ )
+
+ checkAnswer(
+ sql(s"select count(*) from boolean_table where intField >='${trueNum - 5}' and doubleField <=${trueNum + 1} and booleanField=false"),
+ Seq(Row(2))
+ )
+
+ checkAnswer(
+ sql(s"select * from boolean_table where intField >4 and doubleField < 6.0"),
+ Seq(Row(5, true, "num5", 5.0, false))
+ )
+
+ checkAnswer(
+ sql("select count(*) from boolean_table"),
+ Row(trueNum + trueNum / 10))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField is not null"),
+ Row(trueNum + trueNum / 10))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField is null"),
+ Row(0))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField = true"),
+ Row(trueNum))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField >= true"),
+ Row(trueNum))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField > true"),
+ Row(0))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField < true"),
+ Row(trueNum / 10))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField = false"),
+ Row(trueNum / 10))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField <= false"),
+ Row(trueNum / 10))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField > false"),
+ Row(trueNum))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField < false"),
+ Row(0))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField in (false)"),
+ Row(trueNum / 10))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField not in (false)"),
+ Row(trueNum))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField in (true,false)"),
+ Row(trueNum + trueNum / 10))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField not in (true,false)"),
+ Row(0))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField like 'f%'"),
+ Row(trueNum / 10))
+ }
+
+ test("Filtering table: support boolean and other data type, big file, load twice") {
+
+ sql(
+ s"""
+ | CREATE TABLE boolean_table(
+ | intField INT,
+ | booleanField BOOLEAN,
+ | stringField STRING,
+ | doubleField DOUBLE,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ """.stripMargin)
+ val repeat: Int = 2
+ for (i <- 0 until repeat) {
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${pathOfManyDataType}'
+ | INTO TABLE boolean_table
+ | options('FILEHEADER'='intField,booleanField,stringField,doubleField,booleanField2')
+ """.stripMargin
+ )
+ }
+
+ checkAnswer(
+ sql("select booleanField from boolean_table where intField >=1 and intField <11"),
+ Seq(Row(true), Row(true), Row(true), Row(true), Row(true), Row(true), Row(true), Row(true), Row(true), Row(true),
+ Row(true), Row(true), Row(true), Row(true), Row(true), Row(true), Row(true), Row(true), Row(true), Row(true))
+ )
+
+ checkAnswer(
+ sql(s"select booleanField from boolean_table where intField >='${trueNum - 5}' and intField <=${trueNum + 1}"),
+ Seq(Row(true), Row(true), Row(true), Row(true), Row(true), Row(false), Row(false),
+ Row(true), Row(true), Row(true), Row(true), Row(true), Row(false), Row(false))
+ )
+
+ checkAnswer(
+ sql(s"select count(*) from boolean_table where intField >='${trueNum - 5}' and doubleField <=${trueNum + 1} and booleanField=false"),
+ Seq(Row(4))
+ )
+
+ checkAnswer(
+ sql(s"select * from boolean_table where intField >4 and doubleField < 6.0"),
+ Seq(Row(5, true, "num5", 5.0, false), Row(5, true, "num5", 5.0, false))
+ )
+
+ checkAnswer(
+ sql("select count(*) from boolean_table"),
+ Row(repeat * (trueNum + trueNum / 10)))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField is not null"),
+ Row(repeat * (trueNum + trueNum / 10)))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField is null"),
+ Row(0))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField = true"),
+ Row(repeat * (trueNum)))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField >= true"),
+ Row(repeat * (trueNum)))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField > true"),
+ Row(0))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField < true"),
+ Row(repeat * (trueNum / 10)))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField = false"),
+ Row(repeat * (trueNum / 10)))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField <= false"),
+ Row(repeat * (trueNum / 10)))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField > false"),
+ Row(repeat * (trueNum)))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField < false"),
+ Row(0))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField in (false)"),
+ Row(repeat * (trueNum / 10)))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField not in (false)"),
+ Row(repeat * (trueNum)))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField in (true,false)"),
+ Row(repeat * (trueNum + trueNum / 10)))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField not in (true,false)"),
+ Row(0))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField like 'f%'"),
+ Row(repeat * (trueNum / 10)))
+ }
+
+ test("Sort_columns: support boolean and other data type, big file") {
+ sql(
+ s"""
+ | CREATE TABLE boolean_table(
+ | intField INT,
+ | booleanField BOOLEAN,
+ | stringField STRING,
+ | doubleField DOUBLE,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='booleanField')
+ """.stripMargin)
+
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${pathOfManyDataType}'
+ | INTO TABLE boolean_table
+ | options('FILEHEADER'='intField,booleanField,stringField,doubleField,booleanField2')
+ """.stripMargin)
+
+ checkAnswer(
+ sql(s"select booleanField from boolean_table where intField >='${trueNum - 5}' and intField <=${trueNum + 1}"),
+ Seq(Row(true), Row(true), Row(true), Row(true), Row(true), Row(false), Row(false))
+ )
+ }
+
+ test("Inserting into Hive table from carbon table: support boolean data type and other format, big file") {
+ sql(
+ s"""
+ | CREATE TABLE carbon_table(
+ | intField INT,
+ | booleanField BOOLEAN,
+ | stringField STRING,
+ | doubleField DOUBLE,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ """.stripMargin)
+
+ sql(
+ s"""
+ | CREATE TABLE hive_table(
+ | intField INT,
+ | booleanField BOOLEAN,
+ | stringField STRING,
+ | doubleField DOUBLE,
+ | booleanField2 BOOLEAN
+ | )
+ | ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ """.stripMargin)
+
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${pathOfManyDataType}'
+ | INTO TABLE carbon_table
+ | options('FILEHEADER'='intField,booleanField,stringField,doubleField,booleanField2')
+ """.stripMargin)
+
+ sql("insert into hive_table select * from carbon_table")
+
+ checkAnswer(
+ sql(s"select booleanField from hive_table where intField >='${trueNum - 5}' and intField <=${trueNum + 1}"),
+ Seq(Row(true), Row(true), Row(true), Row(true), Row(true), Row(false), Row(false))
+ )
+
+ checkAnswer(
+ sql(s"select * from hive_table where intField >4 and doubleField < 6.0"),
+ Seq(Row(5, true, "num5", 5.0, false))
+ )
+
+ checkAnswer(
+ sql("select count(*) from hive_table"),
+ Row(trueNum + trueNum / 10))
+
+ checkAnswer(
+ sql("select count(*) from hive_table where booleanField = true"),
+ Row(trueNum))
+
+ checkAnswer(
+ sql("select count(*) from hive_table where booleanField = false"),
+ Row(trueNum / 10))
+ }
+
+ test("Inserting into carbon table from Hive table: support boolean data type and other format, big file") {
+ sql(
+ s"""
+ | CREATE TABLE hive_table(
+ | intField INT,
+ | booleanField BOOLEAN,
+ | stringField STRING,
+ | doubleField DOUBLE,
+ | booleanField2 BOOLEAN
+ | )
+ | ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ """.stripMargin)
+
+ sql(
+ s"""
+ | CREATE TABLE carbon_table(
+ | intField INT,
+ | booleanField BOOLEAN,
+ | stringField STRING,
+ | doubleField DOUBLE,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ """.stripMargin)
+
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${pathOfManyDataType}'
+ | INTO TABLE hive_table
+ """.stripMargin)
+
+ sql("insert into carbon_table select * from hive_table")
+
+ checkAnswer(
+ sql(s"select booleanField from carbon_table where intField >='${trueNum - 5}' and intField <=${trueNum + 1}"),
+ Seq(Row(true), Row(true), Row(true), Row(true), Row(true), Row(false), Row(false))
+ )
+
+ checkAnswer(
+ sql(s"select * from carbon_table where intField >4 and doubleField < 6.0"),
+ Seq(Row(5, true, "num5", 5.0, false))
+ )
+
+ checkAnswer(
+ sql("select count(*) from carbon_table"),
+ Row(trueNum + trueNum / 10))
+
+ checkAnswer(
+ sql("select count(*) from carbon_table where booleanField = true"),
+ Row(trueNum))
+
+ checkAnswer(
+ sql("select count(*) from carbon_table where booleanField = false"),
+ Row(trueNum / 10))
+ }
+
+ test("Filtering table: unsafe, support boolean and other data type, big file, load twice") {
+ initConf()
+ sql(
+ s"""
+ | CREATE TABLE boolean_table(
+ | intField INT,
+ | booleanField BOOLEAN,
+ | stringField STRING,
+ | doubleField DOUBLE,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ """.stripMargin)
+ val repeat: Int = 2
+ for (i <- 0 until repeat) {
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${pathOfManyDataType}'
+ | INTO TABLE boolean_table
+ | options('FILEHEADER'='intField,booleanField,stringField,doubleField,booleanField2')
+ """.stripMargin
+ )
+ }
+
+ checkAnswer(
+ sql("select booleanField from boolean_table where intField >=1 and intField <11"),
+ Seq(Row(true), Row(true), Row(true), Row(true), Row(true), Row(true), Row(true), Row(true), Row(true), Row(true),
+ Row(true), Row(true), Row(true), Row(true), Row(true), Row(true), Row(true), Row(true), Row(true), Row(true))
+ )
+
+ checkAnswer(
+ sql(s"select booleanField from boolean_table where intField >='${trueNum - 5}' and intField <=${trueNum + 1}"),
+ Seq(Row(true), Row(true), Row(true), Row(true), Row(true), Row(false), Row(false),
+ Row(true), Row(true), Row(true), Row(true), Row(true), Row(false), Row(false))
+ )
+
+ checkAnswer(
+ sql(s"select count(*) from boolean_table where intField >='${trueNum - 5}' and doubleField <=${trueNum + 1} and booleanField=false"),
+ Seq(Row(4))
+ )
+
+ checkAnswer(
+ sql(s"select * from boolean_table where intField >4 and doubleField < 6.0"),
+ Seq(Row(5, true, "num5", 5.0, false), Row(5, true, "num5", 5.0, false))
+ )
+
+ checkAnswer(
+ sql("select count(*) from boolean_table"),
+ Row(repeat * (trueNum + trueNum / 10)))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField is not null"),
+ Row(repeat * (trueNum + trueNum / 10)))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField is null"),
+ Row(0))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField = true"),
+ Row(repeat * (trueNum)))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField >= true"),
+ Row(repeat * (trueNum)))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField > true"),
+ Row(0))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField < true"),
+ Row(repeat * (trueNum / 10)))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField = false"),
+ Row(repeat * (trueNum / 10)))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField <= false"),
+ Row(repeat * (trueNum / 10)))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField > false"),
+ Row(repeat * (trueNum)))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField < false"),
+ Row(0))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField in (false)"),
+ Row(repeat * (trueNum / 10)))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField not in (false)"),
+ Row(repeat * (trueNum)))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField in (true,false)"),
+ Row(repeat * (trueNum + trueNum / 10)))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField not in (true,false)"),
+ Row(0))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField like 'f%'"),
+ Row(repeat * (trueNum / 10)))
+ defaultConf()
+ }
+
+ def initConf(): Unit = {
+ CarbonProperties.getInstance().
+ addProperty(CarbonCommonConstants.ENABLE_UNSAFE_COLUMN_PAGE_LOADING,
+ "true")
+ }
+
+ def defaultConf(): Unit = {
+ CarbonProperties.getInstance().
+ addProperty(CarbonCommonConstants.ENABLE_UNSAFE_COLUMN_PAGE_LOADING,
+ CarbonCommonConstants.ENABLE_DATA_LOADING_STATISTICS_DEFAULT)
+ }
+}
+
+object BooleanFile {
+ def createBooleanFileWithOtherDataType(path: String, trueLines: Int): Boolean = {
+ try {
+ val write = new PrintWriter(path)
+ var d: Double = 0.0
+ for (i <- 0 until trueLines) {
+ write.println(i + "," + true + ",num" + i + "," + d + "," + false)
+ d = d + 1
+ }
+ for (i <- 0 until trueLines / 10) {
+ write.println((trueLines + i) + "," + false + ",num" + (trueLines + i) + "," + d + "," + true)
+ d = d + 1
+ }
+ write.close()
+ } catch {
+ case _: Exception => assert(false)
+ }
+ return true
+ }
+
+ def deleteFile(path: String): Boolean = {
+ try {
+ val file = new File(path)
+ file.delete()
+ } catch {
+ case _: Exception => assert(false)
+ }
+ return true
+ }
+
+ def createOnlyBooleanFile(path: String, num: Int): Boolean = {
+ try {
+ val write = new PrintWriter(path)
+ for (i <- 0 until num) {
+ write.println(true)
+ }
+ for (i <- 0 until num / 10) {
+ write.println(false)
+ }
+ write.close()
+ } catch {
+ case _: Exception => assert(false)
+ }
+ return true
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesFilterTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesFilterTest.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesFilterTest.scala
new file mode 100644
index 0000000..66d74d5
--- /dev/null
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesFilterTest.scala
@@ -0,0 +1,416 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.spark.testsuite.booleantype
+
+import java.io.File
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
+
+class BooleanDataTypesFilterTest extends QueryTest with BeforeAndAfterEach with BeforeAndAfterAll {
+
+ override def beforeAll(): Unit = {
+ sql("drop table if exists carbon_table")
+ sql("drop table if exists boolean_table")
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+ val storeLocation = s"$rootPath/integration/spark2/src/test/resources/bool/supportBooleanOnlyBoolean.csv"
+
+ sql("CREATE TABLE if not exists carbon_table(booleanField BOOLEAN) STORED BY 'carbondata'")
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '$storeLocation'
+ | INTO TABLE carbon_table
+ | OPTIONS('FILEHEADER' = 'booleanField')
+ """.stripMargin)
+
+ val booleanLocation = s"$rootPath/integration/spark2/src/test/resources/bool/supportBoolean.csv"
+ sql(
+ s"""
+ | CREATE TABLE boolean_table(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${booleanLocation}'
+ | INTO TABLE boolean_table
+ | options('FILEHEADER'='shortField,booleanField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData')
+ """.stripMargin)
+ }
+
+ override def afterAll(): Unit = {
+ sql("drop table if exists carbon_table")
+ sql("drop table if exists boolean_table")
+ }
+
+ test("Filtering table: support boolean, Expression: EqualToExpression, IsNotNullExpression, Or") {
+ checkAnswer(sql("select * from carbon_table where booleanField = true"),
+ Seq(Row(true), Row(true), Row(true), Row(true)))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField = true"),
+ Row(4))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField = 'true'"),
+ Row(0))
+
+ checkAnswer(sql(
+ s"""
+ |select count(*)
+ |from carbon_table where booleanField = \"true\"
+ |""".stripMargin),
+ Row(0))
+
+ checkAnswer(sql("select * from carbon_table where booleanField = false"),
+ Seq(Row(false), Row(false), Row(false), Row(false)))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField = false"),
+ Row(4))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField = 'false'"),
+ Row(0))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField = null"),
+ Row(0))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField = false or booleanField = true"),
+ Row(8))
+ }
+
+ test("Filtering table: support boolean, Expression: InExpression") {
+ checkAnswer(sql("select * from carbon_table where booleanField in (true)"),
+ Seq(Row(true), Row(true), Row(true), Row(true)))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField in (true)"),
+ Row(4))
+
+ checkAnswer(sql("select * from carbon_table where booleanField in (false)"),
+ Seq(Row(false), Row(false), Row(false), Row(false)))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField in (false)"),
+ Row(4))
+
+ checkAnswer(sql("select * from carbon_table where booleanField in (true,false)"),
+ Seq(Row(true), Row(true), Row(true), Row(true), Row(false), Row(false), Row(false), Row(false)))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField in (true,false)"),
+ Row(8))
+ }
+
+ test("Filtering table: support boolean, Expression: NotInExpression") {
+ checkAnswer(sql("select * from carbon_table where booleanField not in (false)"),
+ Seq(Row(true), Row(true), Row(true), Row(true)))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField not in (true)"),
+ Row(4))
+
+ checkAnswer(sql("select * from carbon_table where booleanField not in (true)"),
+ Seq(Row(false), Row(false), Row(false), Row(false)))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField not in (true)"),
+ Row(4))
+
+ checkAnswer(sql("select * from carbon_table where booleanField not in (null)"),
+ Seq(Row(true), Row(true), Row(true), Row(true), Row(false), Row(false), Row(false), Row(false)))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField not in (null)"),
+ Row(8))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField not in (true,false)"),
+ Row(0))
+ }
+
+ test("Filtering table: support boolean, Expression: NotEqualsExpression") {
+ checkAnswer(sql("select * from carbon_table where booleanField != false"),
+ Seq(Row(true), Row(true), Row(true), Row(true)))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField != false"),
+ Row(4))
+
+ checkAnswer(sql("select * from carbon_table where booleanField != true"),
+ Seq(Row(false), Row(false), Row(false), Row(false)))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField != true"),
+ Row(4))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField != null"),
+ Row(0))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField != false or booleanField != true"),
+ Row(8))
+ }
+
+ test("Filtering table: support boolean, Expression: GreaterThanEqualToExpression") {
+ checkAnswer(sql("select * from carbon_table where booleanField >= true"),
+ Seq(Row(true), Row(true), Row(true), Row(true)))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField >= true"),
+ Row(4))
+
+ checkAnswer(sql("select * from carbon_table where booleanField >= false"),
+ Seq(Row(true), Row(true), Row(true), Row(true), Row(false), Row(false), Row(false), Row(false)))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField >=false"),
+ Row(8))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField >= null"),
+ Row(0))
+ }
+
+ test("Filtering table: support boolean, Expression: GreaterThanExpression") {
+ checkAnswer(sql("select * from carbon_table where booleanField > false"),
+ Seq(Row(true), Row(true), Row(true), Row(true)))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField > false"),
+ Row(4))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField > false"),
+ Row(4))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField > null"),
+ Row(0))
+ }
+
+ test("Filtering table: support boolean, Expression: LessThanEqualToExpression") {
+ checkAnswer(sql("select * from carbon_table where booleanField <= false"),
+ Seq(Row(false), Row(false), Row(false), Row(false)))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField <= false"),
+ Row(4))
+
+ checkAnswer(sql("select * from carbon_table where booleanField <= true"),
+ Seq(Row(true), Row(true), Row(true), Row(true), Row(false), Row(false), Row(false), Row(false)))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField <= true"),
+ Row(8))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField <= null"),
+ Row(0))
+ }
+
+ test("Filtering table: support boolean, Expression: LessThanExpression") {
+ checkAnswer(sql("select * from carbon_table where booleanField < true"),
+ Seq(Row(false), Row(false), Row(false), Row(false)))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField < true"),
+ Row(4))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField < false"),
+ Row(0))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField < null"),
+ Row(0))
+ }
+
+ test("Filtering table: support boolean, Expression: between") {
+ checkAnswer(sql("select * from carbon_table where booleanField < true and booleanField >= false"),
+ Seq(Row(false), Row(false), Row(false), Row(false)))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField < true and booleanField >= false"),
+ Row(4))
+ }
+
+ test("Filtering table: support boolean, Expression: empty") {
+ checkAnswer(sql("select count(*) from carbon_table where booleanField =''"),
+ Row(0))
+ }
+
+ test("Filtering table: support boolean, Expression: IsNotNull") {
+ checkAnswer(sql("select count(*) from carbon_table where booleanField is not null"),
+ Row(8))
+ checkAnswer(sql("select * from carbon_table where booleanField is not null"),
+ Seq(Row(true), Row(true), Row(true), Row(true), Row(false), Row(false), Row(false), Row(false)))
+ }
+
+ test("Filtering table: support boolean, Expression: IsNull") {
+ checkAnswer(sql("select count(*) from carbon_table where booleanField is null"),
+ Row(3))
+ checkAnswer(sql("select * from carbon_table where booleanField is null"),
+ Seq(Row(null), Row(null), Row(null)))
+ }
+
+ test("Filtering table: support boolean and other data type, and") {
+ checkAnswer(sql("select count(*) from boolean_table where booleanField = false and shortField = 1"),
+ Row(3))
+
+ checkAnswer(sql("select count(*) from boolean_table where booleanField = false and stringField='flink'"),
+ Row(1))
+
+ checkAnswer(sql("select intField,booleanField,stringField from boolean_table where booleanField = false and stringField='flink'"),
+ Row(11, false, "flink"))
+ }
+
+ test("Filtering table: support boolean and other data type, GreaterThanEqualToExpression") {
+ checkAnswer(sql("select count(*) from boolean_table where booleanField <= false and shortField >= 1"),
+ Row(6))
+ }
+
+ test("Filtering table: support boolean, like") {
+ checkAnswer(sql("select * from carbon_table where booleanField like 't%'"),
+ Seq(Row(true), Row(true), Row(true), Row(true)))
+
+ checkAnswer(sql("select * from carbon_table where booleanField like 'tru%'"),
+ Seq(Row(true), Row(true), Row(true), Row(true)))
+
+ checkAnswer(sql("select * from carbon_table where booleanField like '%ue'"),
+ Seq(Row(true), Row(true), Row(true), Row(true)))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField like 't%'"),
+ Row(4))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField like 'T%'"),
+ Row(0))
+
+ checkAnswer(sql("select * from carbon_table where booleanField like 'f%'"),
+ Seq(Row(false), Row(false), Row(false), Row(false)))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField like 'f%'"),
+ Row(4))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField like 'F%'"),
+ Row(0))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField like 'n%'"),
+ Row(0))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField like 'f%' or booleanField like 't%'"),
+ Row(8))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField like '%e'"),
+ Row(8))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField like '%a%'"),
+ Row(4))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField like '%z%'"),
+ Row(0))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField like '%e'"),
+ Row(8))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField like '_rue'"),
+ Row(4))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField like 'f___e'"),
+ Row(4))
+ }
+
+ test("Filtering table: support boolean and other data type, two boolean column") {
+ sql("drop table if exists boolean_table2")
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+ val booleanLocation2 = s"$rootPath/integration/spark2/src/test/resources/bool/supportBooleanTwoBooleanColumns.csv"
+ sql(
+ s"""
+ | CREATE TABLE boolean_table2(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${booleanLocation2}'
+ | INTO TABLE boolean_table2
+ | options('FILEHEADER'='shortField,booleanField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData,booleanField2')
+ """.stripMargin)
+ checkAnswer(sql("select count(*) from boolean_table2 where booleanField = false and booleanField2 = false"),
+ Row(4))
+ checkAnswer(sql("select count(*) from boolean_table2 where booleanField = true and booleanField2 = true"),
+ Row(3))
+ checkAnswer(sql("select count(*) from boolean_table2 where booleanField = false and booleanField2 = true"),
+ Row(2))
+ checkAnswer(sql("select count(*) from boolean_table2 where booleanField = true and booleanField2 = false"),
+ Row(1))
+ sql("drop table if exists boolean_table2")
+ }
+
+ test("Filtering table: support boolean and other data type, load twice") {
+ sql("drop table if exists boolean_table2")
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+ val booleanLocation2 = s"$rootPath/integration/spark2/src/test/resources/bool/supportBooleanTwoBooleanColumns.csv"
+ sql(
+ s"""
+ | CREATE TABLE boolean_table2(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${booleanLocation2}'
+ | INTO TABLE boolean_table2
+ | options('FILEHEADER'='shortField,booleanField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData,booleanField2')
+ """.stripMargin)
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${booleanLocation2}'
+ | INTO TABLE boolean_table2
+ | options('FILEHEADER'='shortField,booleanField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData,booleanField2')
+ """.stripMargin)
+ checkAnswer(sql("select count(*) from boolean_table2 where booleanField = false and booleanField2 = false"),
+ Row(8))
+ checkAnswer(sql("select count(*) from boolean_table2 where booleanField = true and booleanField2 = true"),
+ Row(6))
+ checkAnswer(sql("select count(*) from boolean_table2 where booleanField = false and booleanField2 = true"),
+ Row(4))
+ checkAnswer(sql("select count(*) from boolean_table2 where booleanField = true and booleanField2 = false"),
+ Row(2))
+ sql("drop table if exists boolean_table2")
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesInsertTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesInsertTest.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesInsertTest.scala
new file mode 100644
index 0000000..2f06900
--- /dev/null
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesInsertTest.scala
@@ -0,0 +1,948 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.spark.testsuite.booleantype
+
+import java.io.File
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
+
+class BooleanDataTypesInsertTest extends QueryTest with BeforeAndAfterEach with BeforeAndAfterAll {
+
+ override def beforeEach(): Unit = {
+ sql("drop table if exists boolean_one_column")
+ sql("drop table if exists boolean_table")
+ sql("drop table if exists boolean_table2")
+ sql("drop table if exists boolean_table3")
+ sql("drop table if exists boolean_table4")
+ sql("drop table if exists carbon_table")
+ sql("drop table if exists hive_table")
+ sql("CREATE TABLE if not exists boolean_one_column(booleanField BOOLEAN) STORED BY 'carbondata'")
+ }
+
+ override def afterAll(): Unit = {
+ sql("drop table if exists boolean_one_column")
+ sql("drop table if exists boolean_table")
+ sql("drop table if exists boolean_table2")
+ sql("drop table if exists boolean_table3")
+ sql("drop table if exists boolean_table4")
+ sql("drop table if exists carbon_table")
+ sql("drop table if exists hive_table")
+ }
+
+ test("Inserting and selecting table: one column boolean, should support") {
+ sql("insert into boolean_one_column values(true)")
+ checkAnswer(
+ sql("select * from boolean_one_column"),
+ Seq(Row(true))
+ )
+ }
+
+ test("Inserting and selecting table: one column boolean and many rows, should support") {
+ sql("insert into boolean_one_column values(true)")
+ sql("insert into boolean_one_column values(True)")
+ sql("insert into boolean_one_column values(TRUE)")
+ sql("insert into boolean_one_column values('true')")
+ sql("insert into boolean_one_column values(False)")
+ sql("insert into boolean_one_column values(false)")
+ sql("insert into boolean_one_column values(FALSE)")
+ sql("insert into boolean_one_column values('false')")
+ sql("insert into boolean_one_column values('tr')")
+ sql("insert into boolean_one_column values(null)")
+ sql("insert into boolean_one_column values('truEe')")
+ sql("insert into boolean_one_column values('falsEe')")
+ sql("insert into boolean_one_column values('t')")
+ sql("insert into boolean_one_column values('f')")
+
+ checkAnswer(
+ sql("select * from boolean_one_column"),
+ Seq(Row(true), Row(true), Row(true), Row(true),
+ Row(false), Row(false), Row(false), Row(false),
+ Row(null), Row(null), Row(null), Row(null), Row(null), Row(null))
+ )
+ }
+
+ test("Inserting and selecting table: create one column boolean table and insert two columns") {
+ sql("insert into boolean_one_column values(true,false)")
+ sql("insert into boolean_one_column values(True)")
+ sql("insert into boolean_one_column values(false,true)")
+
+ checkAnswer(
+ sql("select * from boolean_one_column"),
+ Seq(Row(true), Row(true), Row(false))
+ )
+ }
+
+ test("Inserting and selecting table: two columns boolean and many rows, should support") {
+ sql("CREATE TABLE if not exists boolean_table2(col1 BOOLEAN, col2 BOOLEAN) STORED BY 'carbondata'")
+
+ sql("insert into boolean_table2 values(true,true)")
+ sql("insert into boolean_table2 values(True,false)")
+ sql("insert into boolean_table2 values(TRUE,false)")
+ sql("insert into boolean_table2 values(false,true)")
+ sql("insert into boolean_table2 values(FALSE,false)")
+ sql("insert into boolean_table2 values('false',false)")
+ sql("insert into boolean_table2 values(null,true)")
+
+ checkAnswer(
+ sql("select * from boolean_table2"),
+ Seq(Row(true, true), Row(true, false), Row(true, false),
+ Row(false, true), Row(false, false), Row(false, false), Row(null, true))
+ )
+ }
+
+ test("Inserting and selecting table: two columns and other data type, should support") {
+ sql("CREATE TABLE if not exists boolean_table2(col1 INT, col2 BOOLEAN) STORED BY 'carbondata'")
+
+ sql("insert into boolean_table2 values(1,true)")
+ sql("insert into boolean_table2 values(100,true)")
+ sql("insert into boolean_table2 values(1991,false)")
+ sql("insert into boolean_table2 values(906,false)")
+ sql("insert into boolean_table2 values(218,false)")
+ sql("insert into boolean_table2 values(1011,false)")
+
+ checkAnswer(
+ sql("select * from boolean_table2"),
+ Seq(Row(1, true), Row(100, true), Row(1991, false),
+ Row(906, false), Row(218, false), Row(1011, false))
+ )
+ }
+
+ test("Inserting into table with another table: support boolean data type and other format") {
+ sql(
+ s"""
+ | CREATE TABLE boolean_table(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+
+ sql(
+ s"""
+ | CREATE TABLE boolean_table2(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+
+ sql(
+ s"""
+ | CREATE TABLE boolean_table3(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | stringField STRING,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='')
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE TABLE boolean_table4(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | stringField STRING,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='')
+ """.stripMargin)
+
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+ val storeLocation = s"$rootPath/integration/spark2/src/test/resources/bool/supportBooleanTwoBooleanColumns.csv"
+
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${storeLocation}'
+ | INTO TABLE boolean_table
+ | options('FILEHEADER'='shortField,booleanField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData,booleanField2')
+ """.stripMargin)
+
+ sql("insert into boolean_table2 select * from boolean_table")
+ sql("insert into boolean_table3 select shortField,booleanField,intField,stringField,booleanField2 from boolean_table")
+ sql("insert into boolean_table4 select shortField,booleanField,intField,stringField,booleanField2 from boolean_table where shortField > 3")
+
+ checkAnswer(
+ sql("select booleanField,intField from boolean_table2"),
+ Seq(Row(true, 10), Row(false, 17), Row(false, 11),
+ Row(true, 10), Row(true, 10), Row(true, 14),
+ Row(false, 10), Row(false, 10), Row(false, 16), Row(false, 10))
+ )
+
+ checkAnswer(
+ sql("select booleanField,intField from boolean_table3"),
+ Seq(Row(true, 10), Row(false, 17), Row(false, 11),
+ Row(true, 10), Row(true, 10), Row(true, 14),
+ Row(false, 10), Row(false, 10), Row(false, 16), Row(false, 10))
+ )
+
+ checkAnswer(
+ sql("select booleanField,intField from boolean_table4"),
+ Seq(Row(false, 17), Row(false, 16))
+ )
+
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from boolean_table2"),
+ Seq(Row(true, 10, true), Row(false, 17, true), Row(false, 11, true),
+ Row(true, 10, true), Row(true, 10, true), Row(true, 14, false),
+ Row(false, 10, false), Row(false, 10, false), Row(false, 16, false), Row(false, 10, false))
+ )
+
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from boolean_table3"),
+ Seq(Row(true, 10, true), Row(false, 17, true), Row(false, 11, true),
+ Row(true, 10, true), Row(true, 10, true), Row(true, 14, false),
+ Row(false, 10, false), Row(false, 10, false), Row(false, 16, false), Row(false, 10, false))
+ )
+ }
+
+ test("Inserting with the order of data type in source and target table columns being different") {
+ sql(
+ s"""
+ | CREATE TABLE boolean_table(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+
+ sql(
+ s"""
+ | CREATE TABLE boolean_table2(
+ | booleanField BOOLEAN,
+ | shortField SHORT,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+ val storeLocation = s"$rootPath/integration/spark2/src/test/resources/bool/supportBooleanTwoBooleanColumns.csv"
+
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${storeLocation}'
+ | INTO TABLE boolean_table
+ | options('FILEHEADER'='shortField,booleanField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData,booleanField2')
+ """.stripMargin)
+
+ sql("insert into boolean_table2 select * from boolean_table")
+
+ checkAnswer(
+ sql("select booleanField,intField from boolean_table2"),
+ Seq(Row(null, 10), Row(null, 17), Row(null, 11),
+ Row(null, 10), Row(null, 10), Row(null, 14),
+ Row(null, 10), Row(null, 10), Row(null, 16), Row(null, 10))
+ )
+
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from boolean_table2"),
+ Seq(Row(null, 10, true), Row(null, 17, true), Row(null, 11, true),
+ Row(null, 10, true), Row(null, 10, true), Row(null, 14, false),
+ Row(null, 10, false), Row(null, 10, false), Row(null, 16, false), Row(null, 10, false))
+ )
+ }
+
+ test("Inserting with the number of data type in source and target table columns being different, source more than target") {
+ sql(
+ s"""
+ | CREATE TABLE boolean_table(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+
+ sql(
+ s"""
+ | CREATE TABLE boolean_table2(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+ val storeLocation = s"$rootPath/integration/spark2/src/test/resources/bool/supportBooleanTwoBooleanColumns.csv"
+
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${storeLocation}'
+ | INTO TABLE boolean_table
+ | options('FILEHEADER'='shortField,booleanField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData,booleanField2')
+ """.stripMargin)
+
+ sql("insert into boolean_table2 select * from boolean_table")
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from boolean_table2"),
+ Seq(Row(true, 10, null), Row(false, 17, null), Row(false, 11, null),
+ Row(true, 10, null), Row(true, 10, null), Row(true, 14, null),
+ Row(false, 10, null), Row(false, 10, null), Row(false, 16, null), Row(false, 10, null))
+ )
+ }
+
+ test("Inserting with the number of data type in source and target table columns being different, source less than target") {
+ val exception_insert: Exception =intercept[Exception] {
+ sql(
+ s"""
+ | CREATE TABLE boolean_table(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+
+ sql(
+ s"""
+ | CREATE TABLE boolean_table2(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+ val storeLocation = s"$rootPath/integration/spark2/src/test/resources/bool/supportBooleanTwoBooleanColumns.csv"
+
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${storeLocation}'
+ | INTO TABLE boolean_table
+ | options('FILEHEADER'='shortField,booleanField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData,booleanField2')
+ """.stripMargin)
+ sql("insert into boolean_table2 select * from boolean_table")
+ }
+ assert(exception_insert.getMessage.contains("Cannot insert into target table because column number are different"))
+ }
+
+ test("Inserting into Hive table from carbon table: support boolean data type and other format") {
+ sql(
+ s"""
+ | CREATE TABLE carbon_table(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+
+ sql(
+ s"""
+ | CREATE TABLE hive_table(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ """.stripMargin)
+
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+ val storeLocation = s"$rootPath/integration/spark2/src/test/resources/bool/supportBooleanTwoBooleanColumns.csv"
+
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${storeLocation}'
+ | INTO TABLE carbon_table
+ | options('FILEHEADER'='shortField,booleanField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData,booleanField2')
+ """.stripMargin)
+
+ sql("insert into hive_table select * from carbon_table where shortField = 1 and booleanField = true")
+
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from carbon_table"),
+ Seq(Row(true, 10, true), Row(false, 17, true), Row(false, 11, true),
+ Row(true, 10, true), Row(true, 10, true), Row(true, 14, false),
+ Row(false, 10, false), Row(false, 10, false), Row(false, 16, false), Row(false, 10, false))
+ )
+
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from hive_table"),
+ Seq(Row(true, 10, true), Row(true, 10, true), Row(true, 10, true))
+ )
+
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from carbon_table where exists (select booleanField,intField,booleanField2 " +
+ "from hive_table where carbon_table.intField=hive_table.intField)"),
+ Seq(Row(true, 10, true), Row(true, 10, true), Row(true, 10, true), Row(false, 10, false), Row(false, 10, false), Row(false, 10, false))
+ )
+ }
+
+ test("Inserting into carbon table from Hive table: support boolean data type and other format") {
+ sql(
+ s"""
+ | CREATE TABLE hive_table(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ | ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ """.stripMargin)
+
+ sql(
+ s"""
+ | CREATE TABLE carbon_table(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+ val storeLocation = s"$rootPath/integration/spark2/src/test/resources/bool/supportBooleanTwoBooleanColumns.csv"
+
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${storeLocation}'
+ | INTO TABLE hive_table
+ """.stripMargin)
+
+ sql("insert into carbon_table select * from hive_table where shortField = 1 and booleanField = true")
+
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from hive_table"),
+ Seq(Row(true, 10, true), Row(false, 17, true), Row(false, 11, true),
+ Row(true, 10, true), Row(true, 10, true), Row(true, 14, false),
+ Row(false, 10, false), Row(false, 10, false), Row(false, 16, false), Row(false, 10, false))
+ )
+
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from carbon_table"),
+ Seq(Row(true, 10, true), Row(true, 10, true), Row(true, 10, true))
+ )
+
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from hive_table where exists (select booleanField,intField,booleanField2 " +
+ "from carbon_table where hive_table.intField=carbon_table.intField)"),
+ Seq(Row(true, 10, true), Row(true, 10, true), Row(true, 10, true), Row(false, 10, false), Row(false, 10, false), Row(false, 10, false))
+ )
+ }
+
+ test("Inserting overwrite: one column boolean and many rows, should support") {
+ sql("insert into boolean_one_column values(True)")
+
+ sql("insert overwrite table boolean_one_column values(false)")
+ checkAnswer(
+ sql("select * from boolean_one_column"),
+ Seq(Row(false))
+ )
+
+ sql("insert overwrite table boolean_one_column values(true)")
+ checkAnswer(
+ sql("select * from boolean_one_column"),
+ Seq(Row(true))
+ )
+
+ sql("insert overwrite table boolean_one_column values(null)")
+ checkAnswer(
+ sql("select * from boolean_one_column"),
+ Seq(Row(null))
+ )
+
+ sql("insert overwrite table boolean_one_column values(true)")
+ checkAnswer(
+ sql("select * from boolean_one_column"),
+ Seq(Row(true))
+ )
+
+ sql("insert overwrite table boolean_one_column values('t')")
+ checkAnswer(
+ sql("select * from boolean_one_column"),
+ Seq(Row(null))
+ )
+ }
+
+ test("Inserting overwrite: create one column boolean table and insert two columns") {
+ sql("insert overwrite table boolean_one_column values(true,false)")
+ checkAnswer(
+ sql("select * from boolean_one_column"),
+ Seq(Row(true))
+ )
+ sql("insert overwrite table boolean_one_column values(True)")
+ sql("insert overwrite table boolean_one_column values(false,true)")
+ checkAnswer(
+ sql("select * from boolean_one_column"),
+ Seq(Row(false))
+ )
+ }
+
+ test("Inserting overwrite: two columns boolean and many rows, should support") {
+ sql("CREATE TABLE if not exists boolean_table2(col1 BOOLEAN, col2 BOOLEAN) STORED BY 'carbondata'")
+
+ sql("insert overwrite table boolean_table2 values(true,true)")
+ checkAnswer(
+ sql("select * from boolean_table2"),
+ Seq(Row(true, true))
+ )
+
+ sql("insert overwrite table boolean_table2 values(True,false)")
+ checkAnswer(
+ sql("select * from boolean_table2"),
+ Seq(Row(true, false))
+ )
+ sql("insert overwrite table boolean_table2 values(FALSE,false)")
+ sql("insert overwrite table boolean_table2 values('false',false)")
+ checkAnswer(
+ sql("select * from boolean_table2"),
+ Seq(Row(false, false))
+ )
+ sql("insert overwrite table boolean_table2 values(null,true)")
+ checkAnswer(
+ sql("select * from boolean_table2"),
+ Seq(Row(null, true))
+ )
+ }
+
+ test("Inserting overwrite: two columns and other data type, should support") {
+ sql("CREATE TABLE if not exists boolean_table2(col1 INT, col2 BOOLEAN) STORED BY 'carbondata'")
+ sql("insert overwrite table boolean_table2 values(1,true)")
+ checkAnswer(
+ sql("select * from boolean_table2"),
+ Seq(Row(1, true))
+ )
+ sql("insert overwrite table boolean_table2 values(100,true)")
+ sql("insert overwrite table boolean_table2 values(1991,false)")
+ checkAnswer(
+ sql("select * from boolean_table2"),
+ Seq(Row(1991, false))
+ )
+ sql("insert overwrite table boolean_table2 values(906,false)")
+ checkAnswer(
+ sql("select * from boolean_table2"),
+ Seq(Row(906, false))
+ )
+ sql("insert overwrite table boolean_table2 values(218,false)")
+ checkAnswer(
+ sql("select * from boolean_table2"),
+ Seq(Row(218, false))
+ )
+ sql("insert overwrite table boolean_table2 values(1011,true)")
+ checkAnswer(
+ sql("select * from boolean_table2"),
+ Seq(Row(1011, true))
+ )
+ }
+
+ test("Inserting overwrite: overwrite table with another table: support boolean data type and other format") {
+ sql(
+ s"""
+ | CREATE TABLE boolean_table(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+
+ sql(
+ s"""
+ | CREATE TABLE boolean_table2(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+
+ sql(
+ s"""
+ | CREATE TABLE boolean_table3(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | stringField STRING,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='')
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE TABLE boolean_table4(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | stringField STRING,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='')
+ """.stripMargin)
+
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+ val storeLocation = s"$rootPath/integration/spark2/src/test/resources/bool/supportBooleanTwoBooleanColumns.csv"
+
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${storeLocation}'
+ | INTO TABLE boolean_table
+ | options('FILEHEADER'='shortField,booleanField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData,booleanField2')
+ """.stripMargin)
+
+ sql("insert overwrite table boolean_table2 select * from boolean_table")
+ sql("insert overwrite table boolean_table3 select shortField,booleanField,intField,stringField,booleanField2 from boolean_table")
+ sql("insert overwrite table boolean_table4 select shortField,booleanField,intField,stringField,booleanField2 from boolean_table where shortField > 3")
+
+ checkAnswer(
+ sql("select booleanField,intField from boolean_table2"),
+ Seq(Row(true, 10), Row(false, 17), Row(false, 11),
+ Row(true, 10), Row(true, 10), Row(true, 14),
+ Row(false, 10), Row(false, 10), Row(false, 16), Row(false, 10))
+ )
+
+ checkAnswer(
+ sql("select booleanField,intField from boolean_table3"),
+ Seq(Row(true, 10), Row(false, 17), Row(false, 11),
+ Row(true, 10), Row(true, 10), Row(true, 14),
+ Row(false, 10), Row(false, 10), Row(false, 16), Row(false, 10))
+ )
+
+ checkAnswer(
+ sql("select booleanField,intField from boolean_table4"),
+ Seq(Row(false, 17), Row(false, 16))
+ )
+
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from boolean_table2"),
+ Seq(Row(true, 10, true), Row(false, 17, true), Row(false, 11, true),
+ Row(true, 10, true), Row(true, 10, true), Row(true, 14, false),
+ Row(false, 10, false), Row(false, 10, false), Row(false, 16, false), Row(false, 10, false))
+ )
+
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from boolean_table3"),
+ Seq(Row(true, 10, true), Row(false, 17, true), Row(false, 11, true),
+ Row(true, 10, true), Row(true, 10, true), Row(true, 14, false),
+ Row(false, 10, false), Row(false, 10, false), Row(false, 16, false), Row(false, 10, false))
+ )
+ }
+
+ test("Inserting overwrite: overwrite table Hive table from carbon table: support boolean data type and other format") {
+ sql(
+ s"""
+ | CREATE TABLE carbon_table(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+
+ sql(
+ s"""
+ | CREATE TABLE hive_table(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ """.stripMargin)
+
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+ val storeLocation = s"$rootPath/integration/spark2/src/test/resources/bool/supportBooleanTwoBooleanColumns.csv"
+
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${storeLocation}'
+ | INTO TABLE carbon_table
+ | options('FILEHEADER'='shortField,booleanField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData,booleanField2')
+ """.stripMargin)
+
+ sql("insert overwrite table hive_table select * from carbon_table where shortField = 1 and booleanField = true")
+
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from carbon_table"),
+ Seq(Row(true, 10, true), Row(false, 17, true), Row(false, 11, true),
+ Row(true, 10, true), Row(true, 10, true), Row(true, 14, false),
+ Row(false, 10, false), Row(false, 10, false), Row(false, 16, false), Row(false, 10, false))
+ )
+
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from hive_table"),
+ Seq(Row(true, 10, true), Row(true, 10, true), Row(true, 10, true))
+ )
+
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from carbon_table where exists (select booleanField,intField,booleanField2 " +
+ "from hive_table where carbon_table.intField=hive_table.intField)"),
+ Seq(Row(true, 10, true), Row(true, 10, true), Row(true, 10, true), Row(false, 10, false), Row(false, 10, false), Row(false, 10, false))
+ )
+ }
+
+ test("Inserting overwrite: overwrite table carbon table from Hive table: support boolean data type and other format") {
+ sql(
+ s"""
+ | CREATE TABLE hive_table(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ | ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ """.stripMargin)
+
+ sql(
+ s"""
+ | CREATE TABLE carbon_table(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+ val storeLocation = s"$rootPath/integration/spark2/src/test/resources/bool/supportBooleanTwoBooleanColumns.csv"
+
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${storeLocation}'
+ | INTO TABLE hive_table
+ """.stripMargin)
+
+ sql("insert overwrite table carbon_table select * from hive_table where shortField = 1 and booleanField = true")
+
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from hive_table"),
+ Seq(Row(true, 10, true), Row(false, 17, true), Row(false, 11, true),
+ Row(true, 10, true), Row(true, 10, true), Row(true, 14, false),
+ Row(false, 10, false), Row(false, 10, false), Row(false, 16, false), Row(false, 10, false))
+ )
+
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from carbon_table"),
+ Seq(Row(true, 10, true), Row(true, 10, true), Row(true, 10, true))
+ )
+
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from hive_table where exists (select booleanField,intField,booleanField2 " +
+ "from carbon_table where hive_table.intField=carbon_table.intField)"),
+ Seq(Row(true, 10, true), Row(true, 10, true), Row(true, 10, true), Row(false, 10, false), Row(false, 10, false), Row(false, 10, false))
+ )
+ }
+
+}
[13/16] carbondata git commit: [CARBONDATA-1444] Support Boolean data
type
Posted by ra...@apache.org.
[CARBONDATA-1444] Support Boolean data type
Spark/Hive table support Boolean data type, the internal table also should support Boolean data type.
Boolean data type Range: TRUE or FALSE. Do not use quotation marks around the TRUE and FALSE literal values. You can write the literal values in uppercase, lowercase, or mixed case. The values queried from a table are always returned in lowercase, true or false.
This closes #1362
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/6abdd97a
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/6abdd97a
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/6abdd97a
Branch: refs/heads/pre-aggregate
Commit: 6abdd97a42e945a0df61573aa477c9b3ddb7af02
Parents: 228ab2f
Author: xubo245 <60...@qq.com>
Authored: Mon Oct 16 19:59:50 2017 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Fri Oct 20 08:36:09 2017 +0800
----------------------------------------------------------------------
.../core/datastore/page/ColumnPage.java | 43 +-
.../core/datastore/page/LazyColumnPage.java | 4 +-
.../page/UnsafeFixLengthColumnPage.java | 3 +-
.../page/encoding/ColumnPageEncoderMeta.java | 6 +-
.../page/encoding/DefaultEncodingFactory.java | 5 +-
.../page/encoding/EncodingFactory.java | 5 +
.../page/encoding/bool/BooleanConvert.java | 63 ++
.../datastore/page/encoding/rle/RLECodec.java | 5 +-
.../statistics/PrimitivePageStatsCollector.java | 24 +-
.../ThriftWrapperSchemaConverterImpl.java | 6 +-
.../impl/AbstractScannedResultCollector.java | 4 +-
.../conditional/EqualToExpression.java | 4 +-
.../GreaterThanEqualToExpression.java | 4 +-
.../conditional/GreaterThanExpression.java | 4 +-
.../expression/conditional/InExpression.java | 4 +-
.../conditional/LessThanEqualToExpression.java | 4 +-
.../conditional/LessThanExpression.java | 4 +-
.../conditional/NotEqualsExpression.java | 4 +-
.../expression/conditional/NotInExpression.java | 4 +-
.../carbondata/core/scan/filter/FilterUtil.java | 5 +-
.../executer/ExcludeFilterExecuterImpl.java | 4 +-
.../executer/IncludeFilterExecuterImpl.java | 4 +-
.../executer/RowLevelFilterExecuterImpl.java | 8 +-
.../vector/MeasureDataVectorProcessor.java | 61 +-
.../util/AbstractDataFileFooterConverter.java | 2 +
.../core/util/CarbonMetadataUtil.java | 4 +-
.../apache/carbondata/core/util/CarbonUtil.java | 9 +-
.../carbondata/core/util/DataTypeUtil.java | 31 +-
.../core/util/comparator/Comparator.java | 28 +-
.../ThriftWrapperSchemaConverterImplTest.java | 12 +-
.../conditional/EqualToExpressionUnitTest.java | 2 +-
.../GreaterThanEqualToExpressionUnitTest.java | 2 +-
.../GreaterThanExpressionUnitTest.java | 2 +-
.../conditional/InExpressionUnitTest.java | 2 +-
.../LessThanEqualToExpressionUnitTest.java | 2 +-
.../conditional/LessThanExpressionUnitTest.java | 2 +-
.../NotEqualsExpressionUnitTest.java | 2 +-
.../conditional/NotInExpressionUnitTest.java | 2 +-
.../DataCompactionBoundaryConditionsTest.scala | 3 +
.../spark/util/DataTypeConverterUtil.scala | 4 +
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 3 +-
.../VectorizedCarbonRecordReader.java | 3 +-
.../src/test/resources/bool/supportBoolean.csv | 10 +
.../resources/bool/supportBooleanBadRecords.csv | 10 +
.../bool/supportBooleanDifferentFormat.csv | 20 +
.../bool/supportBooleanOnlyBoolean.csv | 11 +
.../bool/supportBooleanTwoBooleanColumns.csv | 10 +
.../bool/supportBooleanWithFileHeader.csv | 11 +
.../booleantype/BooleanDataTypesBaseTest.scala | 157 +++
.../BooleanDataTypesBigFileTest.scala | 729 ++++++++++++++
.../BooleanDataTypesFilterTest.scala | 416 ++++++++
.../BooleanDataTypesInsertTest.scala | 948 +++++++++++++++++++
.../booleantype/BooleanDataTypesLoadTest.scala | 744 +++++++++++++++
.../BooleanDataTypesParameterTest.scala | 288 ++++++
.../booleantype/BooleanDataTypesSortTest.scala | 145 +++
.../sort/unsafe/UnsafeCarbonRowPage.java | 12 +-
.../sort/sortdata/IntermediateFileMerger.java | 4 +-
.../processing/sort/sortdata/SortDataRows.java | 4 +-
.../sort/sortdata/SortTempFileChunkHolder.java | 4 +-
59 files changed, 3848 insertions(+), 72 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
index 012413b..35bc560 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
@@ -26,6 +26,7 @@ import org.apache.carbondata.core.datastore.TableSpec;
import org.apache.carbondata.core.datastore.compression.Compressor;
import org.apache.carbondata.core.datastore.compression.CompressorFactory;
import org.apache.carbondata.core.datastore.page.encoding.ColumnPageEncoderMeta;
+import org.apache.carbondata.core.datastore.page.encoding.bool.BooleanConvert;
import org.apache.carbondata.core.datastore.page.statistics.ColumnPageStatsCollector;
import org.apache.carbondata.core.datastore.page.statistics.SimpleStatsResult;
import org.apache.carbondata.core.memory.MemoryException;
@@ -184,7 +185,9 @@ public abstract class ColumnPage {
int pageSize) throws MemoryException {
ColumnPage instance;
if (unsafe) {
- if (dataType == DataTypes.BYTE ||
+ if (dataType == DataTypes.BOOLEAN) {
+ instance = new UnsafeFixLengthColumnPage(columnSpec, BYTE, pageSize);
+ } else if (dataType == DataTypes.BYTE ||
dataType == DataTypes.SHORT ||
dataType == DataTypes.SHORT_INT ||
dataType == DataTypes.INT ||
@@ -200,7 +203,7 @@ public abstract class ColumnPage {
throw new RuntimeException("Unsupported data dataType: " + dataType);
}
} else {
- if (dataType == DataTypes.BYTE) {
+ if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.BYTE) {
instance = newBytePage(columnSpec, new byte[pageSize]);
} else if (dataType == DataTypes.SHORT) {
instance = newShortPage(columnSpec, new short[pageSize]);
@@ -344,7 +347,10 @@ public abstract class ColumnPage {
nullBitSet.set(rowId);
return;
}
- if (dataType == DataTypes.BYTE) {
+ if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.BYTE) {
+ if (columnSpec.getSchemaDataType() == DataTypes.BOOLEAN) {
+ value = BooleanConvert.boolean2Byte((Boolean) value);
+ }
putByte(rowId, (byte) value);
statsCollector.update((byte) value);
} else if (dataType == DataTypes.SHORT) {
@@ -411,6 +417,13 @@ public abstract class ColumnPage {
public abstract void putShortInt(int rowId, int value);
/**
+ * Set boolean value at rowId
+ */
+ public void putBoolean(int rowId, boolean value) {
+ putByte(rowId, BooleanConvert.boolean2Byte(value));
+ }
+
+ /**
* Set byte array from offset to length at rowId
*/
public abstract void putBytes(int rowId, byte[] bytes, int offset, int length);
@@ -420,7 +433,9 @@ public abstract class ColumnPage {
* Set null at rowId
*/
private void putNull(int rowId) {
- if (dataType == DataTypes.BYTE) {
+ if (dataType == DataTypes.BOOLEAN) {
+ putBoolean(rowId, false);
+ } else if (dataType == DataTypes.BYTE) {
putByte(rowId, (byte) 0);
} else if (dataType == DataTypes.SHORT) {
putShort(rowId, (short) 0);
@@ -453,6 +468,13 @@ public abstract class ColumnPage {
public abstract int getShortInt(int rowId);
/**
+ * Get boolean value at rowId
+ */
+ public boolean getBoolean(int rowId) {
+ return BooleanConvert.byte2Boolean(getByte(rowId));
+ }
+
+ /**
* Get int value at rowId
*/
public abstract int getInt(int rowId);
@@ -498,6 +520,13 @@ public abstract class ColumnPage {
public abstract byte[] getShortIntPage();
/**
+ * Get boolean value page
+ */
+ public byte[] getBooleanPage() {
+ return getBytePage();
+ }
+
+ /**
* Get int value page
*/
public abstract int[] getIntPage();
@@ -541,7 +570,9 @@ public abstract class ColumnPage {
* Compress page data using specified compressor
*/
public byte[] compress(Compressor compressor) throws MemoryException, IOException {
- if (dataType == DataTypes.BYTE) {
+ if (dataType == DataTypes.BOOLEAN) {
+ return compressor.compressByte(getBooleanPage());
+ } else if (dataType == DataTypes.BYTE) {
return compressor.compressByte(getBytePage());
} else if (dataType == DataTypes.SHORT) {
return compressor.compressShort(getShortPage());
@@ -574,7 +605,7 @@ public abstract class ColumnPage {
Compressor compressor = CompressorFactory.getInstance().getCompressor(meta.getCompressorName());
TableSpec.ColumnSpec columnSpec = meta.getColumnSpec();
DataType storeDataType = meta.getStoreDataType();
- if (storeDataType == DataTypes.BYTE) {
+ if (storeDataType == DataTypes.BOOLEAN || storeDataType == DataTypes.BYTE) {
byte[] byteData = compressor.unCompressByte(compressedData, offset, length);
return newBytePage(columnSpec, byteData);
} else if (storeDataType == DataTypes.SHORT) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/datastore/page/LazyColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/LazyColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/LazyColumnPage.java
index cebb3c0..11bdaca 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/LazyColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/LazyColumnPage.java
@@ -53,7 +53,7 @@ public class LazyColumnPage extends ColumnPage {
@Override
public long getLong(int rowId) {
DataType dataType = columnPage.getDataType();
- if (dataType == DataTypes.BYTE) {
+ if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.BYTE) {
return converter.decodeLong(columnPage.getByte(rowId));
} else if (dataType == DataTypes.SHORT) {
return converter.decodeLong(columnPage.getShort(rowId));
@@ -71,7 +71,7 @@ public class LazyColumnPage extends ColumnPage {
@Override
public double getDouble(int rowId) {
DataType dataType = columnPage.getDataType();
- if (dataType == DataTypes.BYTE) {
+ if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.BYTE) {
return converter.decodeDouble(columnPage.getByte(rowId));
} else if (dataType == DataTypes.SHORT) {
return converter.decodeDouble(columnPage.getShort(rowId));
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java
index 055c27a..5695b70 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java
@@ -56,7 +56,8 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
UnsafeFixLengthColumnPage(TableSpec.ColumnSpec columnSpec, DataType dataType, int pageSize)
throws MemoryException {
super(columnSpec, dataType, pageSize);
- if (dataType == DataTypes.BYTE ||
+ if (dataType == DataTypes.BOOLEAN ||
+ dataType == DataTypes.BYTE ||
dataType == DataTypes.SHORT ||
dataType == DataTypes.INT ||
dataType == DataTypes.LONG ||
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/ColumnPageEncoderMeta.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/ColumnPageEncoderMeta.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/ColumnPageEncoderMeta.java
index daccc32..a38da84 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/ColumnPageEncoderMeta.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/ColumnPageEncoderMeta.java
@@ -79,8 +79,6 @@ public class ColumnPageEncoderMeta extends ValueEncoderMeta implements Writable
}
}
-
-
public DataType getStoreDataType() {
return storeDataType;
}
@@ -108,7 +106,7 @@ public class ColumnPageEncoderMeta extends ValueEncoderMeta implements Writable
private void writeMinMax(DataOutput out) throws IOException {
DataType dataType = columnSpec.getSchemaDataType();
- if (dataType == DataTypes.BYTE) {
+ if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.BYTE) {
out.writeByte((byte) getMaxValue());
out.writeByte((byte) getMinValue());
out.writeLong(0L); // unique value is obsoleted, maintain for compatibility
@@ -151,7 +149,7 @@ public class ColumnPageEncoderMeta extends ValueEncoderMeta implements Writable
private void readMinMax(DataInput in) throws IOException {
DataType dataType = columnSpec.getSchemaDataType();
- if (dataType == DataTypes.BYTE) {
+ if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.BYTE) {
this.setMaxValue(in.readByte());
this.setMinValue(in.readByte());
in.readLong(); // for non exist value which is obsoleted, it is backward compatibility;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DefaultEncodingFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DefaultEncodingFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DefaultEncodingFactory.java
index d78d144..03af657 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DefaultEncodingFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DefaultEncodingFactory.java
@@ -32,6 +32,7 @@ import org.apache.carbondata.core.datastore.page.encoding.dimension.legacy.Compl
import org.apache.carbondata.core.datastore.page.encoding.dimension.legacy.DictDimensionIndexCodec;
import org.apache.carbondata.core.datastore.page.encoding.dimension.legacy.DirectDictDimensionIndexCodec;
import org.apache.carbondata.core.datastore.page.encoding.dimension.legacy.HighCardDictDimensionIndexCodec;
+import org.apache.carbondata.core.datastore.page.encoding.rle.RLECodec;
import org.apache.carbondata.core.datastore.page.statistics.SimpleStatsResult;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
@@ -113,7 +114,9 @@ public class DefaultEncodingFactory extends EncodingFactory {
private ColumnPageEncoder createEncoderForMeasure(ColumnPage columnPage) {
SimpleStatsResult stats = columnPage.getStatistics();
DataType dataType = stats.getDataType();
- if (dataType == DataTypes.BYTE ||
+ if (dataType == DataTypes.BOOLEAN) {
+ return new RLECodec().createEncoder(null);
+ } else if (dataType == DataTypes.BYTE ||
dataType == DataTypes.SHORT ||
dataType == DataTypes.INT ||
dataType == DataTypes.LONG) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/EncodingFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/EncodingFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/EncodingFactory.java
index 180228a..c298525 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/EncodingFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/EncodingFactory.java
@@ -43,6 +43,7 @@ import org.apache.carbondata.format.Encoding;
import static org.apache.carbondata.format.Encoding.ADAPTIVE_DELTA_INTEGRAL;
import static org.apache.carbondata.format.Encoding.ADAPTIVE_FLOATING;
import static org.apache.carbondata.format.Encoding.ADAPTIVE_INTEGRAL;
+import static org.apache.carbondata.format.Encoding.BOOL_BYTE;
import static org.apache.carbondata.format.Encoding.DIRECT_COMPRESS;
import static org.apache.carbondata.format.Encoding.RLE_INTEGRAL;
@@ -94,6 +95,10 @@ public abstract class EncodingFactory {
RLEEncoderMeta metadata = new RLEEncoderMeta();
metadata.readFields(in);
return new RLECodec().createDecoder(metadata);
+ } else if (encoding == BOOL_BYTE) {
+ RLEEncoderMeta metadata = new RLEEncoderMeta();
+ metadata.readFields(in);
+ return new RLECodec().createDecoder(metadata);
} else {
// for backward compatibility
ValueEncoderMeta metadata = CarbonUtil.deserializeEncoderMetaV3(encoderMeta);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/bool/BooleanConvert.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/bool/BooleanConvert.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/bool/BooleanConvert.java
new file mode 100644
index 0000000..b373adf
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/bool/BooleanConvert.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.datastore.page.encoding.bool;
+
+/**
+ * convert tools for boolean data type
+ */
+public class BooleanConvert {
+
+ public static final byte TRUE_VALUE = 1;
+ public static final byte FALSE_VALUE = 0;
+ /**
+ * convert boolean to byte
+ *
+ * @param data data of boolean data type
+ * @return byte type data by convert
+ */
+ public static byte boolean2Byte(boolean data) {
+ return data ? TRUE_VALUE : FALSE_VALUE;
+ }
+
+ /**
+ * convert byte to boolean
+ *
+ * @param data byte type data
+ * @return boolean type data
+ */
+ public static boolean byte2Boolean(int data) {
+ return data == TRUE_VALUE;
+ }
+
+ /**
+ * parse boolean, true and false to boolean
+ *
+ * @param input string type data
+ * @return Boolean type data
+ */
+ public static Boolean parseBoolean(String input) {
+ String value = input.toLowerCase();
+ if (("false").equals(value)) {
+ return Boolean.FALSE;
+ } else if (("true").equals(value)) {
+ return Boolean.TRUE;
+ } else {
+ throw new NumberFormatException("Not a valid Boolean type");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/rle/RLECodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/rle/RLECodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/rle/RLECodec.java
index 809bac0..7007084 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/rle/RLECodec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/rle/RLECodec.java
@@ -71,7 +71,8 @@ public class RLECodec implements ColumnPageCodec {
// This codec supports integral type only
private void validateDataType(DataType dataType) {
- if (! (dataType == DataTypes.BYTE || dataType == DataTypes.SHORT || dataType == DataTypes.INT ||
+ if (! (dataType == DataTypes.BOOLEAN || dataType == DataTypes.BYTE ||
+ dataType == DataTypes.SHORT || dataType == DataTypes.INT ||
dataType == DataTypes.LONG)) {
throw new UnsupportedOperationException(dataType + " is not supported for RLE");
}
@@ -293,7 +294,7 @@ public class RLECodec implements ColumnPageCodec {
DataType dataType = columnSpec.getSchemaDataType();
DataInputStream in = new DataInputStream(new ByteArrayInputStream(input, offset, length));
ColumnPage resultPage = ColumnPage.newPage(columnSpec, dataType, pageSize);
- if (dataType == DataTypes.BYTE) {
+ if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.BYTE) {
decodeBytePage(in, resultPage);
} else if (dataType == DataTypes.SHORT) {
decodeShortPage(in, resultPage);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java
index 304d998..ed92622 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java
@@ -20,10 +20,14 @@ package org.apache.carbondata.core.datastore.page.statistics;
import java.math.BigDecimal;
import org.apache.carbondata.core.datastore.page.encoding.ColumnPageEncoderMeta;
+import org.apache.carbondata.core.datastore.page.encoding.bool.BooleanConvert;
import org.apache.carbondata.core.metadata.ValueEncoderMeta;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import static org.apache.carbondata.core.datastore.page.encoding.bool.BooleanConvert.FALSE_VALUE;
+import static org.apache.carbondata.core.datastore.page.encoding.bool.BooleanConvert.TRUE_VALUE;
+
/** statics for primitive column page */
public class PrimitivePageStatsCollector implements ColumnPageStatsCollector, SimpleStatsResult {
private DataType dataType;
@@ -53,7 +57,7 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector, Si
meta.getScale(), meta.getPrecision());
// set min max from meta
DataType dataType = meta.getSchemaDataType();
- if (dataType == DataTypes.BYTE) {
+ if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.BYTE) {
instance.minByte = (byte) meta.getMinValue();
instance.maxByte = (byte) meta.getMaxValue();
} else if (dataType == DataTypes.SHORT) {
@@ -87,7 +91,7 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector, Si
new PrimitivePageStatsCollector(DataType.getDataType(meta.getType()), -1, -1);
// set min max from meta
DataType dataType = DataType.getDataType(meta.getType());
- if (dataType == DataTypes.BYTE) {
+ if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.BYTE) {
instance.minByte = (byte) meta.getMinValue();
instance.maxByte = (byte) meta.getMaxValue();
} else if (dataType == DataTypes.SHORT) {
@@ -118,7 +122,10 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector, Si
private PrimitivePageStatsCollector(DataType dataType, int scale, int precision) {
this.dataType = dataType;
- if (dataType == DataTypes.BYTE) {
+ if (dataType == DataTypes.BOOLEAN) {
+ minByte = TRUE_VALUE;
+ maxByte = FALSE_VALUE;
+ } else if (dataType == DataTypes.BYTE) {
minByte = Byte.MAX_VALUE;
maxByte = Byte.MIN_VALUE;
} else if (dataType == DataTypes.SHORT) {
@@ -148,7 +155,7 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector, Si
@Override
public void updateNull(int rowId) {
long value = 0;
- if (dataType == DataTypes.BYTE) {
+ if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.BYTE) {
update((byte) value);
} else if (dataType == DataTypes.SHORT) {
update((short) value);
@@ -270,7 +277,10 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector, Si
@Override
public String toString() {
- if (dataType == DataTypes.BYTE) {
+ if (dataType == DataTypes.BOOLEAN) {
+ return String.format("min: %s, max: %s ", BooleanConvert.byte2Boolean(minByte),
+ BooleanConvert.byte2Boolean(minByte));
+ } else if (dataType == DataTypes.BYTE) {
return String.format("min: %s, max: %s, decimal: %s ", minByte, maxByte, decimal);
} else if (dataType == DataTypes.SHORT) {
return String.format("min: %s, max: %s, decimal: %s ", minShort, maxShort, decimal);
@@ -286,7 +296,7 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector, Si
@Override
public Object getMin() {
- if (dataType == DataTypes.BYTE) {
+ if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.BYTE) {
return minByte;
} else if (dataType == DataTypes.SHORT) {
return minShort;
@@ -304,7 +314,7 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector, Si
@Override
public Object getMax() {
- if (dataType == DataTypes.BYTE) {
+ if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.BYTE) {
return maxByte;
} else if (dataType == DataTypes.SHORT) {
return maxShort;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
index 6b2cb90..7faa7e6 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
@@ -127,7 +127,9 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
return null;
}
// data type object maybe created by GSON, use id to compare the type instead of object address
- if (dataType.getId() == DataTypes.STRING.getId()) {
+ if (dataType.getId() == DataTypes.BOOLEAN.getId()) {
+ return org.apache.carbondata.format.DataType.BOOLEAN;
+ } else if (dataType.getId() == DataTypes.STRING.getId()) {
return org.apache.carbondata.format.DataType.STRING;
} else if (dataType.getId() == DataTypes.INT.getId()) {
return org.apache.carbondata.format.DataType.INT;
@@ -361,6 +363,8 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
return null;
}
switch (dataType) {
+ case BOOLEAN:
+ return DataTypes.BOOLEAN;
case STRING:
return DataTypes.STRING;
case INT:
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/AbstractScannedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/AbstractScannedResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/AbstractScannedResultCollector.java
index 1fa42dc..7e24413 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/AbstractScannedResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/AbstractScannedResultCollector.java
@@ -91,7 +91,9 @@ public abstract class AbstractScannedResultCollector implements ScannedResultCol
CarbonMeasure carbonMeasure) {
if (!dataChunk.getNullBits().get(index)) {
DataType dataType = carbonMeasure.getDataType();
- if (dataType == DataTypes.SHORT) {
+ if (dataType == DataTypes.BOOLEAN) {
+ return dataChunk.getBoolean(index);
+ } else if (dataType == DataTypes.SHORT) {
return (short) dataChunk.getLong(index);
} else if (dataType == DataTypes.INT) {
return (int) dataChunk.getLong(index);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpression.java
index 780ca89..f143189 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpression.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpression.java
@@ -67,7 +67,9 @@ public class EqualToExpression extends BinaryConditionalExpression {
}
DataType dataType = val1.getDataType();
- if (dataType == DataTypes.STRING) {
+ if (dataType == DataTypes.BOOLEAN) {
+ result = val1.getBoolean().equals(val2.getBoolean());
+ } else if (dataType == DataTypes.STRING) {
result = val1.getString().equals(val2.getString());
} else if (dataType == DataTypes.SHORT) {
result = val1.getShort().equals(val2.getShort());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanEqualToExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanEqualToExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanEqualToExpression.java
index 469672e..1472959 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanEqualToExpression.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanEqualToExpression.java
@@ -50,7 +50,9 @@ public class GreaterThanEqualToExpression extends BinaryConditionalExpression {
}
boolean result = false;
DataType dataType = exprResVal1.getDataType();
- if (dataType == DataTypes.STRING) {
+ if (dataType == DataTypes.BOOLEAN) {
+ result = elRes.getBoolean().compareTo(erRes.getBoolean()) >= 0;
+ } else if (dataType == DataTypes.STRING) {
result = elRes.getString().compareTo(erRes.getString()) >= 0;
} else if (dataType == DataTypes.SHORT) {
result = elRes.getShort() >= (erRes.getShort());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpression.java
index 931de6c..b8a8a7c 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpression.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpression.java
@@ -52,7 +52,9 @@ public class GreaterThanExpression extends BinaryConditionalExpression {
}
boolean result = false;
DataType dataType = val1.getDataType();
- if (dataType == DataTypes.STRING) {
+ if (dataType == DataTypes.BOOLEAN) {
+ result = exprLeftRes.getBoolean().compareTo(exprRightRes.getBoolean()) > 0;
+ } else if (dataType == DataTypes.STRING) {
result = exprLeftRes.getString().compareTo(exprRightRes.getString()) > 0;
} else if (dataType == DataTypes.DOUBLE) {
result = exprLeftRes.getDouble() > (exprRightRes.getDouble());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/InExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/InExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/InExpression.java
index bc714f7..7243741 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/InExpression.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/InExpression.java
@@ -54,7 +54,9 @@ public class InExpression extends BinaryConditionalExpression {
val = expressionResVal;
}
DataType dataType = val.getDataType();
- if (dataType == DataTypes.STRING) {
+ if (dataType == DataTypes.BOOLEAN) {
+ val = new ExpressionResult(val.getDataType(), expressionResVal.getBoolean());
+ } else if (dataType == DataTypes.STRING) {
val = new ExpressionResult(val.getDataType(), expressionResVal.getString());
} else if (dataType == DataTypes.SHORT) {
val = new ExpressionResult(val.getDataType(), expressionResVal.getShort());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/LessThanEqualToExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/LessThanEqualToExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/LessThanEqualToExpression.java
index f416a6b..6a9fc3c 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/LessThanEqualToExpression.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/LessThanEqualToExpression.java
@@ -50,7 +50,9 @@ public class LessThanEqualToExpression extends BinaryConditionalExpression {
}
boolean result = false;
DataType dataType = exprResValue1.getDataType();
- if (dataType == DataTypes.STRING) {
+ if (dataType == DataTypes.BOOLEAN) {
+ result = elRes.getBoolean().compareTo(erRes.getBoolean()) <= 0;
+ } else if (dataType == DataTypes.STRING) {
result = elRes.getString().compareTo(erRes.getString()) <= 0;
} else if (dataType == DataTypes.SHORT) {
result = elRes.getShort() <= (erRes.getShort());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/LessThanExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/LessThanExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/LessThanExpression.java
index c0d7c10..4283d83 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/LessThanExpression.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/LessThanExpression.java
@@ -54,7 +54,9 @@ public class LessThanExpression extends BinaryConditionalExpression {
}
DataType dataType = val1.getDataType();
- if (dataType == DataTypes.STRING) {
+ if (dataType == DataTypes.BOOLEAN) {
+ result = elRes.getBoolean().compareTo(erRes.getBoolean()) < 0;
+ } else if (dataType == DataTypes.STRING) {
result = elRes.getString().compareTo(erRes.getString()) < 0;
} else if (dataType == DataTypes.SHORT) {
result = elRes.getShort() < (erRes.getShort());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/NotEqualsExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/NotEqualsExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/NotEqualsExpression.java
index 8930c94..5055caf 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/NotEqualsExpression.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/NotEqualsExpression.java
@@ -63,7 +63,9 @@ public class NotEqualsExpression extends BinaryConditionalExpression {
}
}
DataType dataType = val1.getDataType();
- if (dataType == DataTypes.STRING) {
+ if (dataType == DataTypes.BOOLEAN) {
+ result = !val1.getBoolean().equals(val2.getBoolean());
+ } else if (dataType == DataTypes.STRING) {
result = !val1.getString().equals(val2.getString());
} else if (dataType == DataTypes.SHORT) {
result = val1.getShort().shortValue() != val2.getShort().shortValue();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpression.java
index 5f6359b..89a0374 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpression.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpression.java
@@ -75,7 +75,9 @@ public class NotInExpression extends BinaryConditionalExpression {
val = exprResVal;
}
DataType dataType = val.getDataType();
- if (dataType == DataTypes.STRING) {
+ if (dataType == DataTypes.BOOLEAN) {
+ val = new ExpressionResult(val.getDataType(), exprResVal.getBoolean());
+ } else if (dataType == DataTypes.STRING) {
val = new ExpressionResult(val.getDataType(), exprResVal.getString());
} else if (dataType == DataTypes.SHORT) {
val = new ExpressionResult(val.getDataType(), exprResVal.getShort());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
index cfe2cc8..ecac617 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
@@ -1296,7 +1296,10 @@ public final class FilterUtil {
public static int compareFilterKeyBasedOnDataType(String dictionaryVal, String memberVal,
DataType dataType) {
try {
- if (dataType == DataTypes.SHORT) {
+ if (dataType == DataTypes.BOOLEAN) {
+ return Boolean.compare((Boolean.parseBoolean(dictionaryVal)),
+ (Boolean.parseBoolean(memberVal)));
+ } else if (dataType == DataTypes.SHORT) {
return Short.compare((Short.parseShort(dictionaryVal)), (Short.parseShort(memberVal)));
} else if (dataType == DataTypes.INT) {
return Integer.compare((Integer.parseInt(dictionaryVal)), (Integer.parseInt(memberVal)));
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
index 4013578..7a80c88 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
@@ -124,7 +124,9 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
}
private DataType getMeasureDataType(MeasureColumnResolvedFilterInfo msrColumnEvaluatorInfo) {
- if (msrColumnEvaluatorInfo.getType() == DataTypes.SHORT) {
+ if (msrColumnEvaluatorInfo.getType() == DataTypes.BOOLEAN) {
+ return DataTypes.BOOLEAN;
+ } else if (msrColumnEvaluatorInfo.getType() == DataTypes.SHORT) {
return DataTypes.SHORT;
} else if (msrColumnEvaluatorInfo.getType() == DataTypes.INT) {
return DataTypes.INT;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
index 04fce4a..843da71 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
@@ -148,7 +148,9 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
}
private DataType getMeasureDataType(MeasureColumnResolvedFilterInfo msrColumnEvaluatorInfo) {
- if (msrColumnEvaluatorInfo.getType() == DataTypes.SHORT) {
+ if (msrColumnEvaluatorInfo.getType() == DataTypes.BOOLEAN) {
+ return DataTypes.BOOLEAN;
+ } else if (msrColumnEvaluatorInfo.getType() == DataTypes.SHORT) {
return DataTypes.SHORT;
} else if (msrColumnEvaluatorInfo.getType() == DataTypes.INT) {
return DataTypes.INT;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
index c243368..dbf8d4c 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
@@ -351,7 +351,9 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
for (int i = 0; i < msrColEvalutorInfoList.size(); i++) {
MeasureColumnResolvedFilterInfo msrColumnEvalutorInfo = msrColEvalutorInfoList.get(i);
DataType dataType = msrColumnEvalutorInfo.getType();
- if (dataType == DataTypes.SHORT) {
+ if (dataType == DataTypes.BOOLEAN) {
+ msrType = DataTypes.BOOLEAN;
+ } else if (dataType == DataTypes.SHORT) {
msrType = DataTypes.SHORT;
} else if (dataType == DataTypes.INT) {
msrType = DataTypes.INT;
@@ -376,7 +378,9 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
ColumnPage columnPage =
blockChunkHolder.getMeasureRawDataChunk()[measureBlocksIndex[0]]
.convertToColumnPage(pageIndex);
- if (msrType == DataTypes.SHORT) {
+ if (msrType == DataTypes.BOOLEAN) {
+ msrValue = columnPage.getBoolean(index);
+ } else if (msrType == DataTypes.SHORT) {
msrValue = (short) columnPage.getLong(index);
} else if (msrType == DataTypes.INT) {
msrValue = (int) columnPage.getLong(index);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/MeasureDataVectorProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/MeasureDataVectorProcessor.java b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/MeasureDataVectorProcessor.java
index 9b01e1f..cf6c88b 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/MeasureDataVectorProcessor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/MeasureDataVectorProcessor.java
@@ -89,6 +89,63 @@ public class MeasureDataVectorProcessor {
}
}
+ /**
+ * Fill Measure Vector For Boolean data type
+ */
+ public static class BooleanMeasureVectorFiller implements MeasureVectorFiller {
+
+ @Override
+ public void fillMeasureVector(ColumnPage dataChunk, ColumnVectorInfo info) {
+ int offset = info.offset;
+ int len = offset + info.size;
+ int vectorOffset = info.vectorOffset;
+ CarbonColumnVector vector = info.vector;
+ BitSet nullBitSet = dataChunk.getNullBits();
+ if (nullBitSet.isEmpty()) {
+ for (int i = offset; i < len; i++) {
+ vector.putBoolean(vectorOffset, dataChunk.getBoolean(i));
+ vectorOffset++;
+ }
+ } else {
+ for (int i = offset; i < len; i++) {
+ if (nullBitSet.get(i)) {
+ vector.putNull(vectorOffset);
+ } else {
+ vector.putBoolean(vectorOffset, dataChunk.getBoolean(i));
+ }
+ vectorOffset++;
+ }
+ }
+ }
+
+ @Override
+ public void fillMeasureVectorForFilter(int[] rowMapping,
+ ColumnPage dataChunk, ColumnVectorInfo info) {
+ int offset = info.offset;
+ int len = offset + info.size;
+ int vectorOffset = info.vectorOffset;
+ CarbonColumnVector vector = info.vector;
+ BitSet nullBitSet = dataChunk.getNullBits();
+ if (nullBitSet.isEmpty()) {
+ for (int i = offset; i < len; i++) {
+ int currentRow = rowMapping[i];
+ vector.putBoolean(vectorOffset, dataChunk.getBoolean(currentRow));
+ vectorOffset++;
+ }
+ } else {
+ for (int i = offset; i < len; i++) {
+ int currentRow = rowMapping[i];
+ if (nullBitSet.get(currentRow)) {
+ vector.putNull(vectorOffset);
+ } else {
+ vector.putBoolean(vectorOffset, dataChunk.getBoolean(currentRow));
+ }
+ vectorOffset++;
+ }
+ }
+ }
+ }
+
public static class ShortMeasureVectorFiller implements MeasureVectorFiller {
@Override
@@ -307,7 +364,9 @@ public class MeasureDataVectorProcessor {
public static class MeasureVectorFillerFactory {
public static MeasureVectorFiller getMeasureVectorFiller(DataType dataType) {
- if (dataType == DataTypes.SHORT) {
+ if (dataType == DataTypes.BOOLEAN) {
+ return new BooleanMeasureVectorFiller();
+ } else if (dataType == DataTypes.SHORT) {
return new ShortMeasureVectorFiller();
} else if (dataType == DataTypes.INT) {
return new IntegralMeasureVectorFiller();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
index 1020e5b..15f85d6 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
@@ -352,6 +352,8 @@ public abstract class AbstractDataFileFooterConverter {
protected DataType thriftDataTyopeToWrapperDataType(
org.apache.carbondata.format.DataType dataTypeThrift) {
switch (dataTypeThrift) {
+ case BOOLEAN:
+ return DataTypes.BOOLEAN;
case STRING:
return DataTypes.STRING;
case SHORT:
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
index 2c9ef66..d232d61 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
@@ -344,7 +344,9 @@ public class CarbonMetadataUtil {
private static int compareMeasureData(byte[] first, byte[] second, DataType dataType) {
ByteBuffer firstBuffer = null;
ByteBuffer secondBuffer = null;
- if (dataType == DataTypes.DOUBLE) {
+ if (dataType == DataTypes.BOOLEAN) {
+ return first[0] - second[0];
+ } else if (dataType == DataTypes.DOUBLE) {
firstBuffer = ByteBuffer.allocate(8);
firstBuffer.put(first);
secondBuffer = ByteBuffer.allocate(8);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 2924c09..ef3e71d 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -1938,11 +1938,10 @@ public final class CarbonUtil {
*/
public static byte[] getValueAsBytes(DataType dataType, Object value) {
ByteBuffer b;
- if (dataType == DataTypes.BYTE) {
- b = ByteBuffer.allocate(8);
- b.putLong((byte) value);
- b.flip();
- return b.array();
+ if (dataType == DataTypes.BYTE || dataType == DataTypes.BOOLEAN) {
+ byte[] bytes = new byte[1];
+ bytes[0] = (byte) value;
+ return bytes;
} else if (dataType == DataTypes.SHORT) {
b = ByteBuffer.allocate(8);
b.putLong((short) value);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index b8cd59d..01e34a7 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -34,6 +34,7 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.page.ColumnPage;
+import org.apache.carbondata.core.datastore.page.encoding.bool.BooleanConvert;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
import org.apache.carbondata.core.metadata.datatype.DataType;
@@ -102,7 +103,9 @@ public final class DataTypeUtil {
*/
public static Object getMeasureValueBasedOnDataType(String msrValue, DataType dataType,
CarbonMeasure carbonMeasure) {
- if (dataType == DataTypes.DECIMAL) {
+ if (dataType == DataTypes.BOOLEAN) {
+ return BooleanConvert.parseBoolean(msrValue);
+ } else if (dataType == DataTypes.DECIMAL) {
BigDecimal bigDecimal =
new BigDecimal(msrValue).setScale(carbonMeasure.getScale(), RoundingMode.HALF_UP);
return normalizeDecimalValue(bigDecimal, carbonMeasure.getPrecision());
@@ -126,7 +129,9 @@ public final class DataTypeUtil {
return null;
}
ByteBuffer bb = ByteBuffer.wrap(data);
- if (dataType == DataTypes.SHORT) {
+ if (dataType == DataTypes.BOOLEAN) {
+ return BooleanConvert.byte2Boolean(bb.get());
+ } else if (dataType == DataTypes.SHORT) {
return (short) bb.getLong();
} else if (dataType == DataTypes.INT) {
return (int) bb.getLong();
@@ -141,7 +146,9 @@ public final class DataTypeUtil {
public static Object getMeasureObjectBasedOnDataType(ColumnPage measurePage, int index,
DataType dataType, CarbonMeasure carbonMeasure) {
- if (dataType == DataTypes.SHORT) {
+ if (dataType == DataTypes.BOOLEAN) {
+ return measurePage.getBoolean(index);
+ } else if (dataType == DataTypes.SHORT) {
return (short) measurePage.getLong(index);
} else if (dataType == DataTypes.INT) {
return (int) measurePage.getLong(index);
@@ -240,6 +247,9 @@ public final class DataTypeUtil {
public static DataType getDataType(String dataTypeStr) {
DataType dataType = null;
switch (dataTypeStr) {
+ case "BOOLEAN":
+ dataType = DataTypes.BOOLEAN;
+ break;
case "DATE":
dataType = DataTypes.DATE;
break;
@@ -303,7 +313,12 @@ public final class DataTypeUtil {
return null;
}
try {
- if (actualDataType == DataTypes.INT) {
+ if (actualDataType == DataTypes.BOOLEAN) {
+ if (data.isEmpty()) {
+ return null;
+ }
+ return BooleanConvert.parseBoolean(data);
+ } else if (actualDataType == DataTypes.INT) {
if (data.isEmpty()) {
return null;
}
@@ -366,7 +381,9 @@ public final class DataTypeUtil {
public static byte[] getBytesBasedOnDataTypeForNoDictionaryColumn(String dimensionValue,
DataType actualDataType, String dateFormat) {
- if (actualDataType == DataTypes.STRING) {
+ if (actualDataType == DataTypes.BOOLEAN) {
+ return ByteUtil.toBytes(BooleanConvert.parseBoolean(dimensionValue));
+ } else if (actualDataType == DataTypes.STRING) {
return ByteUtil.toBytes(dimensionValue);
} else if (actualDataType == DataTypes.BOOLEAN) {
return ByteUtil.toBytes(Boolean.parseBoolean(dimensionValue));
@@ -411,7 +428,9 @@ public final class DataTypeUtil {
return null;
}
try {
- if (actualDataType == DataTypes.STRING) {
+ if (actualDataType == DataTypes.BOOLEAN) {
+ return ByteUtil.toBoolean(dataInBytes);
+ } else if (actualDataType == DataTypes.STRING) {
return getDataTypeConverter().convertFromByteToUTF8String(dataInBytes);
} else if (actualDataType == DataTypes.BOOLEAN) {
return ByteUtil.toBoolean(dataInBytes);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/util/comparator/Comparator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/comparator/Comparator.java b/core/src/main/java/org/apache/carbondata/core/util/comparator/Comparator.java
index a43ed0f..d1beb16 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/comparator/Comparator.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/comparator/Comparator.java
@@ -26,7 +26,9 @@ import org.apache.carbondata.core.util.ByteUtil;
public final class Comparator {
public static SerializableComparator getComparator(DataType dataType) {
- if (dataType == DataTypes.INT) {
+ if (dataType == DataTypes.BOOLEAN) {
+ return new BooleanSerializableComparator();
+ } else if (dataType == DataTypes.INT) {
return new IntSerializableComparator();
} else if (dataType == DataTypes.SHORT) {
return new ShortSerializableComparator();
@@ -49,7 +51,9 @@ public final class Comparator {
* @return
*/
public static SerializableComparator getComparatorByDataTypeForMeasure(DataType dataType) {
- if (dataType == DataTypes.INT) {
+ if (dataType == DataTypes.BOOLEAN) {
+ return new BooleanSerializableComparator();
+ } else if (dataType == DataTypes.INT) {
return new IntSerializableComparator();
} else if (dataType == DataTypes.SHORT) {
return new ShortSerializableComparator();
@@ -71,6 +75,26 @@ class ByteArraySerializableComparator implements SerializableComparator {
}
}
+class BooleanSerializableComparator implements SerializableComparator {
+ @Override
+ public int compare(Object key1, Object key2) {
+ if (key1 == null && key2 == null) {
+ return 0;
+ } else if (key1 == null) {
+ return -1;
+ } else if (key2 == null) {
+ return 1;
+ }
+ if (Boolean.compare((boolean) key1, (boolean) key2) < 0) {
+ return -1;
+ } else if (Boolean.compare((boolean) key1, (boolean) key2) > 0) {
+ return 1;
+ } else {
+ return 0;
+ }
+ }
+}
+
class IntSerializableComparator implements SerializableComparator {
@Override public int compare(Object key1, Object key2) {
if (key1 == null && key2 == null) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java b/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
index 8bce684..4a3ef32 100644
--- a/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
@@ -414,7 +414,11 @@ public class ThriftWrapperSchemaConverterImplTest {
assertEquals(expectedResult, actualResult);
}
- @Test public void testFromWrapperToExternalColumnSchema() {
+ @Test public void testFromWrapperToExternalColumnSchemaForBooleanDataType() {
+ org.apache.carbondata.format.ColumnSchema thriftColumnSchema =
+ new org.apache.carbondata.format.ColumnSchema(org.apache.carbondata.format.DataType.BOOLEAN,
+ "columnName", "1", true, encoders, true);
+ thriftColumnSchema.setSchemaOrdinal(1);
ColumnSchema wrapperColumnSchema = new ColumnSchema();
new MockUp<ColumnSchema>() {
@@ -1304,7 +1308,7 @@ public class ThriftWrapperSchemaConverterImplTest {
encoders.add(null);
org.apache.carbondata.format.ColumnSchema thriftColumnSchema = null;
thriftColumnSchema =
- new org.apache.carbondata.format.ColumnSchema(org.apache.carbondata.format.DataType.STRING,
+ new org.apache.carbondata.format.ColumnSchema(org.apache.carbondata.format.DataType.BOOLEAN,
"columnName", "1", true, encoders, true);
thriftColumnSchema.setSchemaOrdinal(1);
@@ -1327,7 +1331,7 @@ public class ThriftWrapperSchemaConverterImplTest {
}
@Mock public DataType getDataType() {
- return DataTypes.BOOLEAN;
+ return DataTypes.STRING;
}
@Mock public String getColumnName() {
@@ -1445,7 +1449,7 @@ public class ThriftWrapperSchemaConverterImplTest {
}
@Mock public DataType getDataType() {
- return DataTypes.BOOLEAN;
+ return DataTypes.STRING;
}
@Mock public String getColumnName() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpressionUnitTest.java
index 0abfb18..51bb0fe 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpressionUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpressionUnitTest.java
@@ -182,7 +182,7 @@ public class EqualToExpressionUnitTest {
}
}
- @Test(expected = FilterUnsupportedException.class) public void testForEqualToExpressionForDefaultCase()
+ @Test public void testForEqualToExpressionForDefaultCase()
throws FilterUnsupportedException, FilterIllegalMemberException {
ColumnExpression right = new ColumnExpression("contact", DataTypes.BOOLEAN);
right.setColIndex(0);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanEqualToExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanEqualToExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanEqualToExpressionUnitTest.java
index 35d8886..0375a6d 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanEqualToExpressionUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanEqualToExpressionUnitTest.java
@@ -189,7 +189,7 @@ public class GreaterThanEqualToExpressionUnitTest {
}
}
- @Test(expected = FilterUnsupportedException.class) public void testForGreaterThanEqualToExpressionWithDefaultCase()
+ @Test public void testForGreaterThanEqualToExpressionWithDefaultCase()
throws FilterUnsupportedException, FilterIllegalMemberException {
ColumnExpression right = new ColumnExpression("contact", DataTypes.BOOLEAN);
right.setColIndex(0);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpressionUnitTest.java
index 7d6a168..1940069 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpressionUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpressionUnitTest.java
@@ -279,7 +279,7 @@ public class GreaterThanExpressionUnitTest {
assertTrue(result.getBoolean());
}
- @Test(expected = FilterUnsupportedException.class) public void testForGreaterThanExpressionWithDefaultCase()
+ @Test public void testForGreaterThanExpressionWithDefaultCase()
throws FilterUnsupportedException, FilterIllegalMemberException {
ColumnExpression right = new ColumnExpression("contact", DataTypes.BOOLEAN);
right.setColIndex(0);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/InExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/InExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/InExpressionUnitTest.java
index 2fd6585..d31ec31 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/InExpressionUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/InExpressionUnitTest.java
@@ -224,7 +224,7 @@ public class InExpressionUnitTest {
assertTrue(result.getBoolean());
}
- @Test(expected = FilterUnsupportedException.class) public void testForInExpressionWithDefaultCase()
+ @Test public void testForInExpressionWithDefaultCase()
throws FilterUnsupportedException, FilterIllegalMemberException {
ColumnExpression left = new ColumnExpression("contact", DataTypes.BOOLEAN);
left.setColIndex(0);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanEqualToExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanEqualToExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanEqualToExpressionUnitTest.java
index 7000ceb..a2a5d2a 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanEqualToExpressionUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanEqualToExpressionUnitTest.java
@@ -282,7 +282,7 @@ public class LessThanEqualToExpressionUnitTest {
assertTrue(result.getBoolean());
}
- @Test(expected = FilterUnsupportedException.class) public void testForLessThanEqualToExpressionWithDefaultCase()
+ @Test public void testForLessThanEqualToExpressionWithDefaultCase()
throws FilterUnsupportedException, FilterIllegalMemberException {
ColumnExpression right = new ColumnExpression("contact", DataTypes.BOOLEAN);
right.setColIndex(0);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanExpressionUnitTest.java
index 889ebbd..d385dc1 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanExpressionUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanExpressionUnitTest.java
@@ -279,7 +279,7 @@ public class LessThanExpressionUnitTest {
assertTrue(result.getBoolean());
}
- @Test(expected = FilterUnsupportedException.class) public void testForLessThanExpressionWithDefaultCase()
+ @Test public void testForLessThanExpressionWithDefaultCase()
throws FilterUnsupportedException, FilterIllegalMemberException {
ColumnExpression right = new ColumnExpression("contact", DataTypes.BOOLEAN);
right.setColIndex(0);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotEqualsExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotEqualsExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotEqualsExpressionUnitTest.java
index 1cb008c..e7fa544 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotEqualsExpressionUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotEqualsExpressionUnitTest.java
@@ -247,7 +247,7 @@ public class NotEqualsExpressionUnitTest {
}
}
- @Test(expected = FilterUnsupportedException.class) public void testForNotEqualsExpressionWithDefaultCase()
+ @Test public void testForNotEqualsExpressionWithDefaultCase()
throws FilterUnsupportedException, FilterIllegalMemberException {
ColumnExpression right = new ColumnExpression("contact", DataTypes.BOOLEAN);
right.setColIndex(0);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpressionUnitTest.java
index 230e305..5758625 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpressionUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpressionUnitTest.java
@@ -223,7 +223,7 @@ public class NotInExpressionUnitTest {
assertTrue(result.getBoolean());
}
- @Test(expected = FilterUnsupportedException.class) public void testDefaultCaseForNotInExpression()
+ @Test public void testDefaultCaseForNotInExpression()
throws FilterUnsupportedException, FilterIllegalMemberException {
ColumnExpression left = new ColumnExpression("contact", DataTypes.BOOLEAN);
left.setColIndex(0);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionBoundaryConditionsTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionBoundaryConditionsTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionBoundaryConditionsTest.scala
index 5fadfb7..31cb4d0 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionBoundaryConditionsTest.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionBoundaryConditionsTest.scala
@@ -99,6 +99,9 @@ class DataCompactionBoundaryConditionsTest extends QueryTest with BeforeAndAfter
sql("drop table if exists boundarytest")
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
+ CarbonProperties.getInstance().
+ addProperty(CarbonCommonConstants.COMPACTION_SEGMENT_LEVEL_THRESHOLD,
+ CarbonCommonConstants.DEFAULT_SEGMENT_LEVEL_THRESHOLD)
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
index 027c654..89a3ac3 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
@@ -26,6 +26,7 @@ object DataTypeConverterUtil {
def convertToCarbonType(dataType: String): DataType = {
dataType.toLowerCase match {
+ case "boolean" => DataTypes.BOOLEAN
case "string" => DataTypes.STRING
case "int" => DataTypes.INT
case "integer" => DataTypes.INT
@@ -48,6 +49,7 @@ object DataTypeConverterUtil {
def convertToCarbonTypeForSpark2(dataType: String): DataType = {
dataType.toLowerCase match {
+ case "booleantype" => DataTypes.BOOLEAN
case "stringtype" => DataTypes.STRING
case "inttype" => DataTypes.INT
case "integertype" => DataTypes.INT
@@ -79,6 +81,7 @@ object DataTypeConverterUtil {
def convertToString(dataType: DataType): String = {
dataType match {
+ case DataTypes.BOOLEAN => "boolean"
case DataTypes.STRING => "string"
case DataTypes.SHORT => "smallint"
case DataTypes.INT => "int"
@@ -106,6 +109,7 @@ object DataTypeConverterUtil {
dataType match {
case "string" => ThriftDataType.STRING
case "int" => ThriftDataType.INT
+ case "boolean" => ThriftDataType.BOOLEAN
case "short" => ThriftDataType.SHORT
case "long" | "bigint" => ThriftDataType.LONG
case "double" => ThriftDataType.DOUBLE
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index 16301d6..4649082 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -163,6 +163,7 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
protected val FLOAT = carbonKeyWord("FLOAT")
protected val SHORT = carbonKeyWord("SHORT")
protected val INT = carbonKeyWord("INT")
+ protected val BOOLEAN = carbonKeyWord("BOOLEAN")
protected val BIGINT = carbonKeyWord("BIGINT")
protected val ARRAY = carbonKeyWord("ARRAY")
protected val STRUCT = carbonKeyWord("STRUCT")
@@ -954,7 +955,7 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
protected lazy val dimCol: Parser[Field] = anyFieldDef
protected lazy val primitiveTypes =
- STRING ^^^ "string" | INTEGER ^^^ "integer" |
+ STRING ^^^ "string" |BOOLEAN ^^^ "boolean" | INTEGER ^^^ "integer" |
TIMESTAMP ^^^ "timestamp" | NUMERIC ^^^ "numeric" |
BIGINT ^^^ "bigint" | (SHORT | SMALLINT) ^^^ "smallint" |
INT ^^^ "int" | DOUBLE ^^^ "double" | FLOAT ^^^ "double" | decimalType |
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java b/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java
index 1183d94..1d3783e 100644
--- a/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java
+++ b/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java
@@ -206,7 +206,8 @@ class VectorizedCarbonRecordReader extends AbstractRecordReader<Object> {
for (int i = 0; i < queryMeasures.size(); i++) {
QueryMeasure msr = queryMeasures.get(i);
DataType dataType = msr.getMeasure().getDataType();
- if (dataType == DataTypes.SHORT || dataType == DataTypes.INT || dataType == DataTypes.LONG) {
+ if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.SHORT ||
+ dataType == DataTypes.INT || dataType == DataTypes.LONG) {
fields[msr.getQueryOrder()] = new StructField(msr.getColumnName(),
CarbonScalaUtil.convertCarbonToSparkDataType(msr.getMeasure().getDataType()), true,
null);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/integration/spark2/src/test/resources/bool/supportBoolean.csv
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/resources/bool/supportBoolean.csv b/integration/spark2/src/test/resources/bool/supportBoolean.csv
new file mode 100644
index 0000000..25e57d7
--- /dev/null
+++ b/integration/spark2/src/test/resources/bool/supportBoolean.csv
@@ -0,0 +1,10 @@
+1,true,10,1100,48.4,spark,2015/4/23 12:01:01,1.23,2015/4/23,aaa,2.5,'foo'#'bar'#'world'
+5,false,17,1140,43.4,spark,2015/7/27 12:01:02,3.45,2015/7/27,bbb,2.5,'foo'#'bar'#'world'
+1,false,11,1100,44.4,flink,2015/5/23 12:01:03,23.23,2015/5/23,ccc,2.5,'foo'#'bar'#'world'
+1,true,10,1150,43.4,spark,2015/7/24 12:01:04,254.12,2015/7/24,ddd,2.5,'foo'#'bar'#'world'
+1,true,10,1100,47.4,spark,2015/7/23 12:01:05,876.14,2015/7/23,eeee,3.5,'foo'#'bar'#'world'
+3,true,14,1160,43.4,hive,2015/7/26 12:01:06,3454.32,2015/7/26,ff,2.5,'foo'#'bar'#'world'
+2,false,10,1100,43.4,impala,2015/7/23 12:01:07,456.98,2015/7/23,ggg,2.5,'foo'#'bar'#'world'
+1,false,10,1100,43.4,spark,2015/5/23 12:01:08,32.53,2015/5/23,hhh,2.5,'foo'#'bar'#'world'
+4,false,16,1130,42.4,impala,2015/7/23 12:01:09,67.23,2015/7/23,iii,2.5,'foo'#'bar'#'world'
+1,false,10,1100,43.4,spark,2015/7/23 12:01:10,832.23,2015/7/23,jjj,2.5,'foo'#'bar'#'world'
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/integration/spark2/src/test/resources/bool/supportBooleanBadRecords.csv
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/resources/bool/supportBooleanBadRecords.csv b/integration/spark2/src/test/resources/bool/supportBooleanBadRecords.csv
new file mode 100644
index 0000000..c9c769f
--- /dev/null
+++ b/integration/spark2/src/test/resources/bool/supportBooleanBadRecords.csv
@@ -0,0 +1,10 @@
+1,true,10,1100,48.4,spark,2015/4/23 12:01:01,1.23,2015/4/23,aaa,2.5,'foo'#'bar'#'world',true
+5,falsee,17,1140,43.4,spark,2015/7/27 12:01:02,3.45,2015/7/27,bbb,2.5,'foo'#'bar'#'world',true
+1,f,11,1100,44.4,flink,2015/5/23 12:01:03,23.23,2015/5/23,ccc,2.5,'foo'#'bar'#'world',true
+1,truee,10,1150,43.4,spark,2015/7/24 12:01:04,254.12,2015/7/24,ddd,2.5,'foo'#'bar'#'world',true
+1,truea,10,1100,47.4,spark,2015/7/23 12:01:05,876.14,2015/7/23,eeee,3.5,'foo'#'bar'#'world',true
+3,true,14,1160,43.4,hive,2015/7/26 12:01:06,3454.32,2015/7/26,ff,2.5,'foo'#'bar'#'world',falsee
+2,false,10,1100,43.4,impala,2015/7/23 12:01:07,456.98,2015/7/23,ggg,2.5,'foo'#'bar'#'world',falsea
+1,false,10,1100,43.4,spark,2015/5/23 12:01:08,32.53,2015/5/23,hhh,2.5,'foo'#'bar'#'world',falsef
+4,false,16,1130,42.4,impala,2015/7/23 12:01:09,67.23,2015/7/23,iii,2.5,'foo'#'bar'#'world',falsea
+1,false,10,1100,43.4,spark,2015/7/23 12:01:10,832.23,2015/7/23,jjj,2.5,'foo'#'bar'#'world',false
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/integration/spark2/src/test/resources/bool/supportBooleanDifferentFormat.csv
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/resources/bool/supportBooleanDifferentFormat.csv b/integration/spark2/src/test/resources/bool/supportBooleanDifferentFormat.csv
new file mode 100644
index 0000000..032e92b
--- /dev/null
+++ b/integration/spark2/src/test/resources/bool/supportBooleanDifferentFormat.csv
@@ -0,0 +1,20 @@
+True
+TRUE
+true
+"true"
+false
+False
+FALSE
+"FALSE"
+
+null
+NULL
+TRUEA
+true"
+'true'
+truee
+'false'
+falsee
+FFALSE
+f
+t
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/integration/spark2/src/test/resources/bool/supportBooleanOnlyBoolean.csv
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/resources/bool/supportBooleanOnlyBoolean.csv b/integration/spark2/src/test/resources/bool/supportBooleanOnlyBoolean.csv
new file mode 100644
index 0000000..3ce21d8
--- /dev/null
+++ b/integration/spark2/src/test/resources/bool/supportBooleanOnlyBoolean.csv
@@ -0,0 +1,11 @@
+True
+TRUE
+true
+"true"
+false
+False
+FALSE
+"FALSE"
+
+null
+NULL
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/integration/spark2/src/test/resources/bool/supportBooleanTwoBooleanColumns.csv
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/resources/bool/supportBooleanTwoBooleanColumns.csv b/integration/spark2/src/test/resources/bool/supportBooleanTwoBooleanColumns.csv
new file mode 100644
index 0000000..696befb
--- /dev/null
+++ b/integration/spark2/src/test/resources/bool/supportBooleanTwoBooleanColumns.csv
@@ -0,0 +1,10 @@
+1,true,10,1100,48.4,spark,2015/4/23 12:01:01,1.23,2015/4/23,aaa,2.5,'foo'#'bar'#'world',true
+5,false,17,1140,43.4,spark,2015/7/27 12:01:02,3.45,2015/7/27,bbb,2.5,'foo'#'bar'#'world',true
+1,false,11,1100,44.4,flink,2015/5/23 12:01:03,23.23,2015/5/23,ccc,2.5,'foo'#'bar'#'world',true
+1,true,10,1150,43.4,spark,2015/7/24 12:01:04,254.12,2015/7/24,ddd,2.5,'foo'#'bar'#'world',true
+1,true,10,1100,47.4,spark,2015/7/23 12:01:05,876.14,2015/7/23,eeee,3.5,'foo'#'bar'#'world',true
+3,true,14,1160,43.4,hive,2015/7/26 12:01:06,3454.32,2015/7/26,ff,2.5,'foo'#'bar'#'world',false
+2,false,10,1100,43.4,impala,2015/7/23 12:01:07,456.98,2015/7/23,ggg,2.5,'foo'#'bar'#'world',false
+1,false,10,1100,43.4,spark,2015/5/23 12:01:08,32.53,2015/5/23,hhh,2.5,'foo'#'bar'#'world',false
+4,false,16,1130,42.4,impala,2015/7/23 12:01:09,67.23,2015/7/23,iii,2.5,'foo'#'bar'#'world',false
+1,false,10,1100,43.4,spark,2015/7/23 12:01:10,832.23,2015/7/23,jjj,2.5,'foo'#'bar'#'world',false
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/integration/spark2/src/test/resources/bool/supportBooleanWithFileHeader.csv
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/resources/bool/supportBooleanWithFileHeader.csv b/integration/spark2/src/test/resources/bool/supportBooleanWithFileHeader.csv
new file mode 100644
index 0000000..237eade
--- /dev/null
+++ b/integration/spark2/src/test/resources/bool/supportBooleanWithFileHeader.csv
@@ -0,0 +1,11 @@
+shortField,booleanField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData
+1,true,10,1100,48.4,spark,2015/4/23 12:01:01,1.23,2015/4/23,aaa,2.5,'foo'#'bar'#'world'
+5,false,17,1140,43.4,spark,2015/7/27 12:01:02,3.45,2015/7/27,bbb,2.5,'foo'#'bar'#'world'
+1,false,11,1100,44.4,flink,2015/5/23 12:01:03,23.23,2015/5/23,ccc,2.5,'foo'#'bar'#'world'
+1,true,10,1150,43.4,spark,2015/7/24 12:01:04,254.12,2015/7/24,ddd,2.5,'foo'#'bar'#'world'
+1,true,10,1100,47.4,spark,2015/7/23 12:01:05,876.14,2015/7/23,eeee,3.5,'foo'#'bar'#'world'
+3,true,14,1160,43.4,hive,2015/7/26 12:01:06,3454.32,2015/7/26,ff,2.5,'foo'#'bar'#'world'
+2,false,10,1100,43.4,impala,2015/7/23 12:01:07,456.98,2015/7/23,ggg,2.5,'foo'#'bar'#'world'
+1,false,10,1100,43.4,spark,2015/5/23 12:01:08,32.53,2015/5/23,hhh,2.5,'foo'#'bar'#'world'
+4,false,16,1130,42.4,impala,2015/7/23 12:01:09,67.23,2015/7/23,iii,2.5,'foo'#'bar'#'world'
+1,false,10,1100,43.4,spark,2015/7/23 12:01:10,832.23,2015/7/23,jjj,2.5,'foo'#'bar'#'world'
[03/16] carbondata git commit: [CARBONDATA-1597] Remove spark1
integration
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonOptimizer.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonOptimizer.scala b/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonOptimizer.scala
deleted file mode 100644
index 914203f..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonOptimizer.scala
+++ /dev/null
@@ -1,862 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.optimizer
-
-import java.util
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable.ArrayBuffer
-
-import org.apache.spark.sql._
-import org.apache.spark.sql.catalyst.CarbonTableIdentifierImplicit._
-import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.catalyst.expressions.aggregate._
-import org.apache.spark.sql.catalyst.optimizer.Optimizer
-import org.apache.spark.sql.catalyst.plans.logical._
-import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.execution.command.ProjectForUpdateCommand
-import org.apache.spark.sql.execution.RunnableCommand
-import org.apache.spark.sql.execution.datasources.LogicalRelation
-import org.apache.spark.sql.types.{IntegerType, StringType}
-
-import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.stats.QueryStatistic
-import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory
-import org.apache.carbondata.spark.{CarbonAliasDecoderRelation, CarbonFilters}
-
-/**
- * Carbon Optimizer to add dictionary decoder.
- */
-object CarbonOptimizer {
-
- def optimizer(optimizer: Optimizer, conf: CarbonSQLConf, version: String): Optimizer = {
- CodeGenerateFactory.getInstance().optimizerFactory.createOptimizer(optimizer, conf)
- }
-
- def execute(plan: LogicalPlan, optimizer: Optimizer): LogicalPlan = {
- val executedPlan: LogicalPlan = optimizer.execute(plan)
- val relations = CarbonOptimizer.collectCarbonRelation(plan)
- if (relations.nonEmpty) {
- new ResolveCarbonFunctions(relations).apply(executedPlan)
- } else {
- executedPlan
- }
- }
-
- // get the carbon relation from plan.
- def collectCarbonRelation(plan: LogicalPlan): Seq[CarbonDecoderRelation] = {
- plan collect {
- case l: LogicalRelation if l.relation.isInstanceOf[CarbonDatasourceRelation] =>
- CarbonDecoderRelation(l.attributeMap, l.relation.asInstanceOf[CarbonDatasourceRelation])
- }
- }
-}
-
-/**
- * It does two jobs. 1. Change the datatype for dictionary encoded column 2. Add the dictionary
- * decoder plan.
- */
-class ResolveCarbonFunctions(relations: Seq[CarbonDecoderRelation])
- extends Rule[LogicalPlan] with PredicateHelper {
- val LOGGER = LogServiceFactory.getLogService(this.getClass.getName)
- def apply(logicalPlan: LogicalPlan): LogicalPlan = {
- if (relations.nonEmpty && !isOptimized(logicalPlan)) {
- val plan = processPlan(logicalPlan)
- val udfTransformedPlan = pushDownUDFToJoinLeftRelation(plan)
- LOGGER.info("Starting to optimize plan")
- val recorder = CarbonTimeStatisticsFactory.createExecutorRecorder("")
- val queryStatistic = new QueryStatistic()
- val result = transformCarbonPlan(udfTransformedPlan, relations)
- queryStatistic.addStatistics("Time taken for Carbon Optimizer to optimize: ",
- System.currentTimeMillis)
- recorder.recordStatistics(queryStatistic)
- recorder.logStatistics()
- result
- } else {
- LOGGER.info("Skip CarbonOptimizer")
- logicalPlan
- }
- }
-
- private def processPlan(plan: LogicalPlan): LogicalPlan = {
- plan transform {
- case ProjectForUpdate(table, cols, Seq(updatePlan)) =>
- var isTransformed = false
- val newPlan = updatePlan transform {
- case Project(pList, child) if (!isTransformed) =>
- val (dest: Seq[NamedExpression], source: Seq[NamedExpression]) = pList
- .splitAt(pList.size - cols.size)
- val diff = cols.diff(dest.map(_.name))
- if (diff.size > 0) {
- sys.error(s"Unknown column(s) ${diff.mkString(",")} in table ${table.tableName}")
- }
- isTransformed = true
- Project(dest.filter(a => !cols.contains(a.name)) ++ source, child)
- }
- ProjectForUpdateCommand(newPlan, table.tableIdentifier)
- }
- }
- private def pushDownUDFToJoinLeftRelation(plan: LogicalPlan): LogicalPlan = {
- val output = plan match {
- case proj@Project(cols, Join(
- left, right, jointype: org.apache.spark.sql.catalyst.plans.JoinType, condition)) =>
- var projectionToBeAdded: Seq[org.apache.spark.sql.catalyst.expressions.Alias] = Seq.empty
- val newCols = cols.map { col =>
- col match {
- case a@Alias(s: ScalaUDF, name)
- if (name.equalsIgnoreCase(CarbonCommonConstants.POSITION_ID) ||
- name.equalsIgnoreCase(
- CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID)) =>
- projectionToBeAdded :+= a
- AttributeReference(name, StringType, true)().withExprId(a.exprId)
- case other => other
- }
- }
- val newLeft = left match {
- case Project(columns, logicalPlan) =>
- Project(columns ++ projectionToBeAdded, logicalPlan)
- case filter: Filter =>
- Project(filter.output ++ projectionToBeAdded, filter)
- case other => other
- }
- Project(newCols, Join(newLeft, right, jointype, condition))
- case other => other
- }
- output
- }
- def isOptimized(plan: LogicalPlan): Boolean = {
- plan find {
- case cd: CarbonDictionaryCatalystDecoder => true
- case ic: InsertIntoCarbonTable => true
- case other => false
- } isDefined
- }
-
- case class ExtraNodeInfo(var hasCarbonRelation: Boolean)
-
- def fillNodeInfo(
- plan: LogicalPlan,
- extraNodeInfos: java.util.HashMap[LogicalPlan, ExtraNodeInfo]): ExtraNodeInfo = {
- plan match {
- case l: LogicalRelation if l.relation.isInstanceOf[CarbonDatasourceRelation] =>
- val extraNodeInfo = ExtraNodeInfo(true)
- extraNodeInfo
- case others =>
- val extraNodeInfo = ExtraNodeInfo(false)
- others.children.foreach { childPlan =>
- val childExtraNodeInfo = fillNodeInfo(childPlan, extraNodeInfos)
- if (childExtraNodeInfo.hasCarbonRelation) {
- extraNodeInfo.hasCarbonRelation = true
- }
- }
- // only put no carbon realtion plan
- if (!extraNodeInfo.hasCarbonRelation) {
- extraNodeInfos.put(plan, extraNodeInfo)
- }
- extraNodeInfo
- }
- }
-
- /**
- * Steps for changing the plan.
- * 1. It finds out the join condition columns and dimension aggregate columns which are need to
- * be decoded just before that plan executes.
- * 2. Plan starts transform by adding the decoder to the plan where it needs the decoded data
- * like dimension aggregate columns decoder under aggregator and join condition decoder under
- * join children.
- */
- def transformCarbonPlan(plan: LogicalPlan,
- relations: Seq[CarbonDecoderRelation]): LogicalPlan = {
- if (plan.isInstanceOf[RunnableCommand]) {
- return plan
- }
- var decoder = false
- val mapOfNonCarbonPlanNodes = new java.util.HashMap[LogicalPlan, ExtraNodeInfo]
- fillNodeInfo(plan, mapOfNonCarbonPlanNodes)
- val aliasMap = CarbonAliasDecoderRelation()
- // collect alias information before hand.
- collectInformationOnAttributes(plan, aliasMap)
-
- def hasCarbonRelation(currentPlan: LogicalPlan): Boolean = {
- val extraNodeInfo = mapOfNonCarbonPlanNodes.get(currentPlan)
- if (extraNodeInfo == null) {
- true
- } else {
- extraNodeInfo.hasCarbonRelation
- }
- }
-
- val attrMap = new util.HashMap[AttributeReferenceWrapper, CarbonDecoderRelation]()
- relations.foreach(_.fillAttributeMap(attrMap))
-
- def addTempDecoder(currentPlan: LogicalPlan): LogicalPlan = {
-
- def transformAggregateExpression(agg: Aggregate,
- aggonGroups: util.HashSet[AttributeReferenceWrapper] = null): LogicalPlan = {
- val attrsOndimAggs = new util.HashSet[AttributeReferenceWrapper]
- if (aggonGroups != null) {
- attrsOndimAggs.addAll(aggonGroups)
- }
- agg.aggregateExpressions.map {
- case attr: AttributeReference =>
- case a@Alias(attr: AttributeReference, name) =>
- case aggExp: AggregateExpression =>
- aggExp.transform {
- case aggExp: AggregateExpression =>
- collectDimensionAggregates(aggExp, attrsOndimAggs, aliasMap, attrMap)
- aggExp
- }
- case others =>
- others.collect {
- case attr: AttributeReference
- if isDictionaryEncoded(attr, attrMap, aliasMap) =>
- attrsOndimAggs.add(AttributeReferenceWrapper(aliasMap.getOrElse(attr, attr)))
- }
- }
- var child = agg.child
- // Incase if the child also aggregate then push down decoder to child
- if (attrsOndimAggs.size() > 0 && !child.equals(agg)) {
- child = CarbonDictionaryTempDecoder(attrsOndimAggs,
- new util.HashSet[AttributeReferenceWrapper](),
- agg.child)
- }
- if (!decoder && aggonGroups == null) {
- decoder = true
- CarbonDictionaryTempDecoder(new util.HashSet[AttributeReferenceWrapper](),
- new util.HashSet[AttributeReferenceWrapper](),
- Aggregate(agg.groupingExpressions, agg.aggregateExpressions, child),
- isOuter = true)
- } else {
- Aggregate(agg.groupingExpressions, agg.aggregateExpressions, child)
- }
- }
-
- currentPlan match {
- case limit@Limit(_, child: Sort) =>
- if (!decoder) {
- decoder = true
- CarbonDictionaryTempDecoder(new util.HashSet[AttributeReferenceWrapper](),
- new util.HashSet[AttributeReferenceWrapper](),
- limit,
- isOuter = true)
- } else {
- limit
- }
- case sort: Sort if !sort.child.isInstanceOf[CarbonDictionaryTempDecoder] =>
- val attrsOnSort = new util.HashSet[AttributeReferenceWrapper]()
- sort.order.map { s =>
- s.collect {
- case attr: AttributeReference
- if isDictionaryEncoded(attr, attrMap, aliasMap) =>
- attrsOnSort.add(AttributeReferenceWrapper(aliasMap.getOrElse(attr, attr)))
- }
- }
- var child = sort.child
- if (attrsOnSort.size() > 0 && !child.isInstanceOf[Sort]) {
- child = CarbonDictionaryTempDecoder(attrsOnSort,
- new util.HashSet[AttributeReferenceWrapper](), sort.child)
- }
- if (!decoder) {
- decoder = true
- CarbonDictionaryTempDecoder(new util.HashSet[AttributeReferenceWrapper](),
- new util.HashSet[AttributeReferenceWrapper](),
- Sort(sort.order, sort.global, child),
- isOuter = true)
- } else {
- Sort(sort.order, sort.global, child)
- }
-
- case union: Union
- if !(union.left.isInstanceOf[CarbonDictionaryTempDecoder] ||
- union.right.isInstanceOf[CarbonDictionaryTempDecoder]) =>
- val leftCondAttrs = new util.HashSet[AttributeReferenceWrapper]
- val rightCondAttrs = new util.HashSet[AttributeReferenceWrapper]
- val leftLocalAliasMap = CarbonAliasDecoderRelation()
- val rightLocalAliasMap = CarbonAliasDecoderRelation()
- // collect alias information for the child plan again. It is required as global alias
- // may have duplicated in case of aliases
- collectInformationOnAttributes(union.left, leftLocalAliasMap)
- collectInformationOnAttributes(union.right, rightLocalAliasMap)
- union.left.output.foreach { attr =>
- if (isDictionaryEncoded(attr, attrMap, leftLocalAliasMap)) {
- leftCondAttrs.add(AttributeReferenceWrapper(leftLocalAliasMap.getOrElse(attr, attr)))
- }
- }
- union.right.output.foreach { attr =>
- if (isDictionaryEncoded(attr, attrMap, rightLocalAliasMap)) {
- rightCondAttrs.add(
- AttributeReferenceWrapper(rightLocalAliasMap.getOrElse(attr, attr)))
- }
- }
- var leftPlan = union.left
- var rightPlan = union.right
- if (hasCarbonRelation(leftPlan) && leftCondAttrs.size() > 0 &&
- !leftPlan.isInstanceOf[CarbonDictionaryCatalystDecoder]) {
- leftPlan = CarbonDictionaryTempDecoder(leftCondAttrs,
- new util.HashSet[AttributeReferenceWrapper](),
- union.left, isOuter = false, Some(leftLocalAliasMap))
- }
- if (hasCarbonRelation(rightPlan) && rightCondAttrs.size() > 0 &&
- !rightPlan.isInstanceOf[CarbonDictionaryCatalystDecoder]) {
- rightPlan = CarbonDictionaryTempDecoder(rightCondAttrs,
- new util.HashSet[AttributeReferenceWrapper](),
- union.right, isOuter = false, Some(rightLocalAliasMap))
- }
- if (!decoder) {
- decoder = true
- CarbonDictionaryTempDecoder(new util.HashSet[AttributeReferenceWrapper](),
- new util.HashSet[AttributeReferenceWrapper](),
- Union(leftPlan, rightPlan),
- isOuter = true)
- } else {
- Union(leftPlan, rightPlan)
- }
-
- case agg: Aggregate if !agg.child.isInstanceOf[CarbonDictionaryTempDecoder] =>
- transformAggregateExpression(agg)
- case expand: Expand if !expand.child.isInstanceOf[CarbonDictionaryTempDecoder] =>
- val attrsOnExpand = new util.HashSet[AttributeReferenceWrapper]
- expand.projections.map {s =>
- s.map {
- case attr: AttributeReference =>
- case a@Alias(attr: AttributeReference, name) =>
- case others =>
- others.collect {
- case attr: AttributeReference
- if isDictionaryEncoded(attr, attrMap, aliasMap) =>
- attrsOnExpand.add(AttributeReferenceWrapper(aliasMap.getOrElse(attr, attr)))
- }
- }
- }
- var child = expand.child
- if (attrsOnExpand.size() > 0 && !child.isInstanceOf[Expand]) {
- child = CarbonDictionaryTempDecoder(attrsOnExpand,
- new util.HashSet[AttributeReferenceWrapper](),
- expand.child)
- }
- if (!decoder) {
- decoder = true
- CarbonDictionaryTempDecoder(new util.HashSet[AttributeReferenceWrapper](),
- new util.HashSet[AttributeReferenceWrapper](),
- CodeGenerateFactory.getInstance().expandFactory.createExpand(expand, child),
- isOuter = true)
- } else {
- CodeGenerateFactory.getInstance().expandFactory.createExpand(expand, child)
- }
- case filter: Filter if !filter.child.isInstanceOf[CarbonDictionaryTempDecoder] =>
- val attrsOnConds = new util.HashSet[AttributeReferenceWrapper]
- // In case the child is join then we cannot push down the filters so decode them earlier
- if (filter.child.isInstanceOf[Join] || filter.child.isInstanceOf[Sort]) {
- filter.condition.collect {
- case attr: AttributeReference =>
- attrsOnConds.add(AttributeReferenceWrapper(aliasMap.getOrElse(attr, attr)))
- }
- } else {
- CarbonFilters
- .selectFilters(splitConjunctivePredicates(filter.condition), attrsOnConds, aliasMap)
- }
-
- var child = filter.child
- if (attrsOnConds.size() > 0 && !child.isInstanceOf[Filter]) {
- child = CarbonDictionaryTempDecoder(attrsOnConds,
- new util.HashSet[AttributeReferenceWrapper](),
- filter.child)
- }
-
- if (!decoder) {
- decoder = true
- CarbonDictionaryTempDecoder(new util.HashSet[AttributeReferenceWrapper](),
- new util.HashSet[AttributeReferenceWrapper](),
- Filter(filter.condition, child),
- isOuter = true)
- } else {
- Filter(filter.condition, child)
- }
-
- case j: Join
- if !(j.left.isInstanceOf[CarbonDictionaryTempDecoder] ||
- j.right.isInstanceOf[CarbonDictionaryTempDecoder]) =>
- val attrsOnJoin = new util.HashSet[Attribute]
- j.condition match {
- case Some(expression) =>
- expression.collect {
- case attr: AttributeReference
- if isDictionaryEncoded(attr, attrMap, aliasMap) =>
- attrsOnJoin.add(aliasMap.getOrElse(attr, attr))
- }
- case _ =>
- }
-
- val leftCondAttrs = new util.HashSet[AttributeReferenceWrapper]
- val rightCondAttrs = new util.HashSet[AttributeReferenceWrapper]
- if (attrsOnJoin.size() > 0) {
-
- attrsOnJoin.asScala.map { attr =>
- if (qualifierPresence(j.left, attr)) {
- leftCondAttrs.add(AttributeReferenceWrapper(attr))
- }
- if (qualifierPresence(j.right, attr)) {
- rightCondAttrs.add(AttributeReferenceWrapper(attr))
- }
- }
- var leftPlan = j.left
- var rightPlan = j.right
- if (leftCondAttrs.size() > 0 &&
- !leftPlan.isInstanceOf[CarbonDictionaryCatalystDecoder]) {
- leftPlan = leftPlan match {
- case agg: Aggregate =>
- CarbonDictionaryTempDecoder(leftCondAttrs,
- new util.HashSet[AttributeReferenceWrapper](),
- transformAggregateExpression(agg, leftCondAttrs))
- case _ =>
- CarbonDictionaryTempDecoder(leftCondAttrs,
- new util.HashSet[AttributeReferenceWrapper](),
- j.left)
- }
- }
- if (rightCondAttrs.size() > 0 &&
- !rightPlan.isInstanceOf[CarbonDictionaryCatalystDecoder]) {
- rightPlan = rightPlan match {
- case agg: Aggregate =>
- CarbonDictionaryTempDecoder(rightCondAttrs,
- new util.HashSet[AttributeReferenceWrapper](),
- transformAggregateExpression(agg, rightCondAttrs))
- case _ =>
- CarbonDictionaryTempDecoder(rightCondAttrs,
- new util.HashSet[AttributeReferenceWrapper](),
- j.right)
- }
- }
- if (!decoder) {
- decoder = true
- CarbonDictionaryTempDecoder(new util.HashSet[AttributeReferenceWrapper](),
- new util.HashSet[AttributeReferenceWrapper](),
- Join(leftPlan, rightPlan, j.joinType, j.condition),
- isOuter = true)
- } else {
- Join(leftPlan, rightPlan, j.joinType, j.condition)
- }
- } else {
- j
- }
-
- case p: Project
- if relations.nonEmpty && !p.child.isInstanceOf[CarbonDictionaryTempDecoder] =>
- val attrsOnProjects = new util.HashSet[AttributeReferenceWrapper]
- p.projectList.map {
- case attr: AttributeReference =>
- case a@Alias(attr: AttributeReference, name) =>
- case others =>
- others.collect {
- case attr: AttributeReference
- if isDictionaryEncoded(attr, attrMap, aliasMap) =>
- attrsOnProjects.add(AttributeReferenceWrapper(aliasMap.getOrElse(attr, attr)))
- }
- }
- var child = p.child
- if (attrsOnProjects.size() > 0 && !child.isInstanceOf[Project]) {
- child = CarbonDictionaryTempDecoder(attrsOnProjects,
- new util.HashSet[AttributeReferenceWrapper](),
- p.child)
- }
- if (!decoder) {
- decoder = true
- CarbonDictionaryTempDecoder(new util.HashSet[AttributeReferenceWrapper](),
- new util.HashSet[AttributeReferenceWrapper](),
- Project(p.projectList, child),
- isOuter = true)
- } else {
- Project(p.projectList, child)
- }
-
- case wd: Window if !wd.child.isInstanceOf[CarbonDictionaryTempDecoder] =>
- val attrsOnProjects = new util.HashSet[AttributeReferenceWrapper]
- wd.projectList.map {
- case attr: AttributeReference =>
- case others =>
- others.collect {
- case attr: AttributeReference
- if isDictionaryEncoded(attr, attrMap, aliasMap) =>
- attrsOnProjects.add(AttributeReferenceWrapper(aliasMap.getOrElse(attr, attr)))
- }
- }
- wd.windowExpressions.map { others =>
- others.collect {
- case attr: AttributeReference
- if isDictionaryEncoded(attr, attrMap, aliasMap) =>
- attrsOnProjects.add(AttributeReferenceWrapper(aliasMap.getOrElse(attr, attr)))
- }
- }
- wd.partitionSpec.map{
- case attr: AttributeReference =>
- case others =>
- others.collect {
- case attr: AttributeReference
- if isDictionaryEncoded(attr, attrMap, aliasMap) =>
- attrsOnProjects.add(AttributeReferenceWrapper(aliasMap.getOrElse(attr, attr)))
- }
- }
- wd.orderSpec.map { s =>
- s.collect {
- case attr: AttributeReference
- if isDictionaryEncoded(attr, attrMap, aliasMap) =>
- attrsOnProjects.add(AttributeReferenceWrapper(aliasMap.getOrElse(attr, attr)))
- }
- }
- wd.partitionSpec.map { s =>
- s.collect {
- case attr: AttributeReference
- if isDictionaryEncoded(attr, attrMap, aliasMap) =>
- attrsOnProjects.add(AttributeReferenceWrapper(aliasMap.getOrElse(attr, attr)))
- }
- }
- var child = wd.child
- if (attrsOnProjects.size() > 0 && !child.isInstanceOf[Project]) {
- child = CarbonDictionaryTempDecoder(attrsOnProjects,
- new util.HashSet[AttributeReferenceWrapper](),
- wd.child)
- }
- if (!decoder) {
- decoder = true
- CarbonDictionaryTempDecoder(new util.HashSet[AttributeReferenceWrapper](),
- new util.HashSet[AttributeReferenceWrapper](),
- Window(wd.projectList, wd.windowExpressions, wd.partitionSpec, wd.orderSpec, child),
- isOuter = true)
- } else {
- Window(wd.projectList, wd.windowExpressions, wd.partitionSpec, wd.orderSpec, child)
- }
-
- case l: LogicalRelation if l.relation.isInstanceOf[CarbonDatasourceRelation] =>
- if (!decoder) {
- decoder = true
- CarbonDictionaryTempDecoder(new util.HashSet[AttributeReferenceWrapper](),
- new util.HashSet[AttributeReferenceWrapper](), l, isOuter = true)
- } else {
- l
- }
-
- case others => others
- }
-
- }
-
- val transFormedPlan =
- plan transformDown {
- case cd: CarbonDictionaryTempDecoder if cd.isOuter =>
- decoder = true
- cd
- case currentPlan =>
- if (hasCarbonRelation(currentPlan)) {
- addTempDecoder(currentPlan)
- } else {
- currentPlan
- }
- }
-
- val processor = new CarbonDecoderProcessor
- processor.updateDecoders(processor.getDecoderList(transFormedPlan))
- updateProjection(updateTempDecoder(transFormedPlan, aliasMap, attrMap))
- }
-
- private def updateTempDecoder(plan: LogicalPlan,
- aliasMapOriginal: CarbonAliasDecoderRelation,
- attrMap: java.util.HashMap[AttributeReferenceWrapper, CarbonDecoderRelation]):
- LogicalPlan = {
- var allAttrsNotDecode: util.Set[AttributeReferenceWrapper] =
- new util.HashSet[AttributeReferenceWrapper]()
- val marker = new CarbonPlanMarker
- var aliasMap = aliasMapOriginal
- plan transformDown {
- case cd: CarbonDictionaryTempDecoder if !cd.processed =>
- cd.processed = true
- allAttrsNotDecode = cd.attrsNotDecode
- aliasMap = cd.aliasMap.getOrElse(aliasMap)
- marker.pushMarker(allAttrsNotDecode)
- if (cd.isOuter) {
- CarbonDictionaryCatalystDecoder(relations,
- ExcludeProfile(cd.getAttrsNotDecode.asScala.toSeq),
- aliasMap,
- isOuter = true,
- cd.child)
- } else {
- CarbonDictionaryCatalystDecoder(relations,
- IncludeProfile(cd.getAttrList.asScala.toSeq),
- aliasMap,
- isOuter = false,
- cd.child)
- }
- case cd: CarbonDictionaryCatalystDecoder =>
- cd
- case sort: Sort =>
- val sortExprs = sort.order.map { s =>
- s.transform {
- case attr: AttributeReference =>
- updateDataType(attr, attrMap, allAttrsNotDecode, aliasMap)
- }.asInstanceOf[SortOrder]
- }
- Sort(sortExprs, sort.global, sort.child)
- case agg: Aggregate if !agg.child.isInstanceOf[CarbonDictionaryCatalystDecoder] =>
- val aggExps = agg.aggregateExpressions.map { aggExp =>
- aggExp.transform {
- case attr: AttributeReference =>
- updateDataType(attr, attrMap, allAttrsNotDecode, aliasMap)
- }
- }.asInstanceOf[Seq[NamedExpression]]
-
- val grpExps = agg.groupingExpressions.map { gexp =>
- gexp.transform {
- case attr: AttributeReference =>
- updateDataType(attr, attrMap, allAttrsNotDecode, aliasMap)
- }
- }
- Aggregate(grpExps, aggExps, agg.child)
- case expand: Expand =>
- expand.transformExpressions {
- case attr: AttributeReference =>
- updateDataType(attr, attrMap, allAttrsNotDecode, aliasMap)
- }
- case filter: Filter =>
- val filterExps = filter.condition transform {
- case attr: AttributeReference =>
- updateDataType(attr, attrMap, allAttrsNotDecode, aliasMap)
- }
- Filter(filterExps, filter.child)
- case j: Join =>
- marker.pushBinaryMarker(allAttrsNotDecode)
- j
- case u: Union =>
- marker.pushBinaryMarker(allAttrsNotDecode)
- u
- case p: Project if relations.nonEmpty =>
- val prExps = p.projectList.map { prExp =>
- prExp.transform {
- case attr: AttributeReference =>
- updateDataType(attr, attrMap, allAttrsNotDecode, aliasMap)
- }
- }.asInstanceOf[Seq[NamedExpression]]
- Project(prExps, p.child)
- case wd: Window if relations.nonEmpty =>
- val prExps = wd.projectList.map { prExp =>
- prExp.transform {
- case attr: AttributeReference =>
- updateDataType(attr, attrMap, allAttrsNotDecode, aliasMap)
- }
- }.asInstanceOf[Seq[Attribute]]
- val wdExps = wd.windowExpressions.map { gexp =>
- gexp.transform {
- case attr: AttributeReference =>
- updateDataType(attr, attrMap, allAttrsNotDecode, aliasMap)
- }
- }.asInstanceOf[Seq[NamedExpression]]
- val partitionSpec = wd.partitionSpec.map{ exp =>
- exp.transform {
- case attr: AttributeReference =>
- updateDataType(attr, attrMap, allAttrsNotDecode, aliasMap)
- }
- }
- val orderSpec = wd.orderSpec.map { exp =>
- exp.transform {
- case attr: AttributeReference =>
- updateDataType(attr, attrMap, allAttrsNotDecode, aliasMap)
- }
- }.asInstanceOf[Seq[SortOrder]]
- Window(prExps, wdExps, partitionSpec, orderSpec, wd.child)
-
- case l: LogicalRelation if l.relation.isInstanceOf[CarbonDatasourceRelation] =>
- allAttrsNotDecode = marker.revokeJoin()
- l
- case others => others
- }
- }
-
- private def updateProjection(plan: LogicalPlan): LogicalPlan = {
- val transFormedPlan = plan transform {
- case p@Project(projectList: Seq[NamedExpression], cd: CarbonDictionaryCatalystDecoder) =>
- if (cd.child.isInstanceOf[Filter] || cd.child.isInstanceOf[LogicalRelation]) {
- Project(projectList: Seq[NamedExpression], cd.child)
- } else {
- p
- }
- case f@Filter(condition: Expression, cd: CarbonDictionaryCatalystDecoder) =>
- if (cd.child.isInstanceOf[Project] || cd.child.isInstanceOf[LogicalRelation]) {
- Filter(condition, cd.child)
- } else {
- f
- }
- }
- // Remove unnecessary decoders
- val finalPlan = transFormedPlan transform {
- case CarbonDictionaryCatalystDecoder(_, profile, _, false, child)
- if profile.isInstanceOf[IncludeProfile] && profile.isEmpty =>
- child
- }
- val updateDtrFn = finalPlan transform {
- case p@Project(projectList: Seq[NamedExpression], cd) =>
- if (cd.isInstanceOf[Filter] || cd.isInstanceOf[LogicalRelation]) {
- p.transformAllExpressions {
- case a@Alias(exp, _)
- if !exp.deterministic && !exp.isInstanceOf[CustomDeterministicExpression] =>
- Alias(CustomDeterministicExpression(exp), a.name)(a.exprId, a.qualifiers,
- a.explicitMetadata)
- case exp: NamedExpression
- if !exp.deterministic && !exp.isInstanceOf[CustomDeterministicExpression] =>
- CustomDeterministicExpression(exp)
- }
- } else {
- p
- }
- case f@Filter(condition: Expression, cd) =>
- if (cd.isInstanceOf[Project] || cd.isInstanceOf[LogicalRelation]) {
- f.transformAllExpressions {
- case a@Alias(exp, _)
- if !exp.deterministic && !exp.isInstanceOf[CustomDeterministicExpression] =>
- Alias(CustomDeterministicExpression(exp), a.name)(a.exprId, a.qualifiers,
- a.explicitMetadata)
- case exp: NamedExpression
- if !exp.deterministic && !exp.isInstanceOf[CustomDeterministicExpression] =>
- CustomDeterministicExpression(exp)
- }
- } else {
- f
- }
- }
-
- updateDtrFn
- }
-
- private def collectInformationOnAttributes(plan: LogicalPlan,
- aliasMap: CarbonAliasDecoderRelation) {
- plan transformAllExpressions {
- case a@Alias(exp, name) =>
- exp match {
- case attr: Attribute => aliasMap.put(a.toAttribute, attr)
- case _ => aliasMap.put(a.toAttribute, AttributeReference("", StringType)())
- }
- a
- }
- // collect the output of expand and add projections attributes as alias to it.
- plan.collect {
- case expand: Expand =>
- expand.projections.foreach {s =>
- s.zipWithIndex.foreach { f =>
- f._1 match {
- case attr: AttributeReference =>
- aliasMap.put(expand.output(f._2).toAttribute, attr)
- case a@Alias(attr: AttributeReference, name) =>
- aliasMap.put(expand.output(f._2).toAttribute, attr)
- case others =>
- }
- }
- }
- }
- }
-
- // Collect aggregates on dimensions so that we can add decoder to it.
- private def collectDimensionAggregates(aggExp: AggregateExpression,
- attrsOndimAggs: util.HashSet[AttributeReferenceWrapper],
- aliasMap: CarbonAliasDecoderRelation,
- attrMap: java.util.HashMap[AttributeReferenceWrapper, CarbonDecoderRelation]) {
- aggExp collect {
- case attr: AttributeReference if isDictionaryEncoded(attr, attrMap, aliasMap) =>
- attrsOndimAggs.add(AttributeReferenceWrapper(aliasMap.getOrElse(attr, attr)))
- }
- }
-
- /**
- * Update the attribute datatype with [IntegerType] if the carbon column is encoded with
- * dictionary.
- *
- */
- private def updateDataType(attr: Attribute,
- attrMap: java.util.HashMap[AttributeReferenceWrapper, CarbonDecoderRelation],
- allAttrsNotDecode: java.util.Set[AttributeReferenceWrapper],
- aliasMap: CarbonAliasDecoderRelation): Attribute = {
- val uAttr = aliasMap.getOrElse(attr, attr)
- val relation = Option(attrMap.get(AttributeReferenceWrapper(uAttr)))
- if (relation.isDefined) {
- relation.get.dictionaryMap.get(uAttr.name) match {
- case Some(true)
- if !allAttrsNotDecode.contains(AttributeReferenceWrapper(uAttr)) =>
- val newAttr = AttributeReference(attr.name,
- IntegerType,
- attr.nullable,
- attr.metadata)(attr.exprId, attr.qualifiers)
- newAttr
- case _ => attr
- }
- } else {
- attr
- }
- }
-
- private def isDictionaryEncoded(attr: Attribute,
- attrMap: java.util.HashMap[AttributeReferenceWrapper, CarbonDecoderRelation],
- aliasMap: CarbonAliasDecoderRelation): Boolean = {
- val uAttr = aliasMap.getOrElse(attr, attr)
- val relation = Option(attrMap.get(AttributeReferenceWrapper(uAttr)))
- if (relation.isDefined) {
- relation.get.dictionaryMap.get(uAttr.name) match {
- case Some(true) => true
- case _ => false
- }
- } else {
- false
- }
- }
-
- def qualifierPresence(plan: LogicalPlan, attr: Attribute): Boolean = {
- var present = false
- plan collect {
- case l: LogicalRelation if l.attributeMap.contains(attr) =>
- present = true
- }
- present
- }
-}
-
-case class CarbonDecoderRelation(
- attributeMap: AttributeMap[AttributeReference],
- carbonRelation: CarbonDatasourceRelation) {
-
- val extraAttrs = new ArrayBuffer[Attribute]()
-
- def addAttribute(attr: Attribute): Unit = {
- extraAttrs += attr
- }
-
- def contains(attr: Attribute): Boolean = {
- val exists =
- attributeMap.exists(entry => entry._1.name.equalsIgnoreCase(attr.name) &&
- entry._1.exprId.equals(attr.exprId)) ||
- extraAttrs.exists(entry => entry.name.equalsIgnoreCase(attr.name) &&
- entry.exprId.equals(attr.exprId))
- exists
- }
-
- def fillAttributeMap(attrMap: java.util.HashMap[AttributeReferenceWrapper,
- CarbonDecoderRelation]): Unit = {
- attributeMap.foreach { attr =>
- attrMap.put(AttributeReferenceWrapper(attr._1), this)
- }
- }
-
- lazy val dictionaryMap = carbonRelation.carbonRelation.metaData.dictionaryMap
-}
-
-
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/main/scala/org/apache/spark/sql/test/SparkTestQueryExecutor.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/test/SparkTestQueryExecutor.scala b/integration/spark/src/main/scala/org/apache/spark/sql/test/SparkTestQueryExecutor.scala
deleted file mode 100644
index bb00126..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/test/SparkTestQueryExecutor.scala
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.test
-
-import org.apache.spark.{SparkConf, SparkContext}
-import org.apache.spark.sql.{CarbonContext, DataFrame, SQLContext}
-
-import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
-
-/**
- * This class is a sql executor of unit test case for spark version 1.x.
- */
-
-class SparkTestQueryExecutor extends TestQueryExecutorRegister {
- override def sql(sqlText: String): DataFrame = SparkTestQueryExecutor.cc.sql(sqlText)
-
- override def sqlContext: SQLContext = SparkTestQueryExecutor.cc
-
- override def stop(): Unit = SparkTestQueryExecutor.cc.sparkContext.stop()
-}
-
-object SparkTestQueryExecutor {
- private val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
- LOGGER.info("use TestQueryExecutorImplV1")
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.STORE_LOCATION_TEMP_PATH,
- System.getProperty("java.io.tmpdir"))
- .addProperty(CarbonCommonConstants.LOCK_TYPE, CarbonCommonConstants.CARBON_LOCK_TYPE_LOCAL)
- .addProperty(CarbonCommonConstants.STORE_LOCATION, TestQueryExecutor.storeLocation)
-
- val sc = new SparkContext(new SparkConf()
- .setAppName("CarbonSpark")
- .setMaster("local[2]")
- .set("spark.sql.shuffle.partitions", "20"))
- sc.setLogLevel("ERROR")
-
- val cc = new CarbonContext(sc, TestQueryExecutor.storeLocation, TestQueryExecutor.metastoredb)
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/main/scala/org/apache/spark/util/TaskContextUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/util/TaskContextUtil.scala b/integration/spark/src/main/scala/org/apache/spark/util/TaskContextUtil.scala
deleted file mode 100644
index e73f78c..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/util/TaskContextUtil.scala
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.util
-
-import org.apache.spark.TaskContext
-
-
-object TaskContextUtil {
- def setTaskContext(context: TaskContext): Unit = {
- val localThreadContext = TaskContext.get()
- if (localThreadContext == null) {
- TaskContext.setTaskContext(context)
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
----------------------------------------------------------------------
diff --git a/integration/spark/src/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister b/integration/spark/src/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
deleted file mode 100644
index d09c9b5..0000000
--- a/integration/spark/src/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
+++ /dev/null
@@ -1,17 +0,0 @@
-## ------------------------------------------------------------------------
-## Licensed to the Apache Software Foundation (ASF) under one or more
-## contributor license agreements. See the NOTICE file distributed with
-## this work for additional information regarding copyright ownership.
-## The ASF licenses this file to You under the Apache License, Version 2.0
-## (the "License"); you may not use this file except in compliance with
-## the License. You may obtain a copy of the License at
-##
-## http://www.apache.org/licenses/LICENSE-2.0
-##
-## Unless required by applicable law or agreed to in writing, software
-## distributed under the License is distributed on an "AS IS" BASIS,
-## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-## See the License for the specific language governing permissions and
-## limitations under the License.
-## ------------------------------------------------------------------------
-org.apache.spark.sql.CarbonSource
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/resources/META-INF/services/org.apache.spark.sql.test.TestQueryExecutorRegister
----------------------------------------------------------------------
diff --git a/integration/spark/src/resources/META-INF/services/org.apache.spark.sql.test.TestQueryExecutorRegister b/integration/spark/src/resources/META-INF/services/org.apache.spark.sql.test.TestQueryExecutorRegister
deleted file mode 100644
index fc96db4..0000000
--- a/integration/spark/src/resources/META-INF/services/org.apache.spark.sql.test.TestQueryExecutorRegister
+++ /dev/null
@@ -1,17 +0,0 @@
-## ------------------------------------------------------------------------
-## Licensed to the Apache Software Foundation (ASF) under one or more
-## contributor license agreements. See the NOTICE file distributed with
-## this work for additional information regarding copyright ownership.
-## The ASF licenses this file to You under the Apache License, Version 2.0
-## (the "License"); you may not use this file except in compliance with
-## the License. You may obtain a copy of the License at
-##
-## http://www.apache.org/licenses/LICENSE-2.0
-##
-## Unless required by applicable law or agreed to in writing, software
-## distributed under the License is distributed on an "AS IS" BASIS,
-## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-## See the License for the specific language governing permissions and
-## limitations under the License.
-## ------------------------------------------------------------------------
-org.apache.spark.sql.test.SparkTestQueryExecutor
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/test/resources/badrecords/test2.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/badrecords/test2.csv b/integration/spark/src/test/resources/badrecords/test2.csv
deleted file mode 100644
index 51d25b2..0000000
--- a/integration/spark/src/test/resources/badrecords/test2.csv
+++ /dev/null
@@ -1,4 +0,0 @@
-0,569,silo
-1,843658743265874365874365874365584376547598375987,hello
-2,87436587349436587568784658743065874376,priyal
-3,56985,simple
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexPrimitiveTimestampDirectDictionary.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexPrimitiveTimestampDirectDictionary.scala b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexPrimitiveTimestampDirectDictionary.scala
deleted file mode 100644
index aaaf66b..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexPrimitiveTimestampDirectDictionary.scala
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.integration.spark.testsuite.complexType
-
-import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.BeforeAndAfterAll
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
-
-/**
- * Test class of creating and loading for carbon table with double
- *
- */
-class TestComplexPrimitiveTimestampDirectDictionary extends QueryTest with BeforeAndAfterAll {
-
- override def beforeAll: Unit = {
- sql("drop table if exists complexcarbontimestamptable")
- sql("drop table if exists complexhivetimestamptable")
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy-MM-dd HH:mm:ss.SSS")
- sql("CREATE TABLE complexcarbontimestamptable (empno string,workdate Timestamp,punchinout array<Timestamp>, worktime struct<begintime:Timestamp, endtime:Timestamp>, salary double) STORED BY 'org.apache.carbondata.format'")
- sql(s"LOAD DATA local inpath '$resourcesPath/datasamplecomplex.csv' INTO TABLE complexcarbontimestamptable OPTIONS" +
- "('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'FILEHEADER'='empno,workdate,punchinout,worktime,salary')");
- sql("CREATE TABLE complexhivetimestamptable (empno string,workdate Timestamp,punchinout array<Timestamp>, worktime struct<begintime:Timestamp, endtime:Timestamp>, salary double)row format delimited fields terminated by ',' collection items terminated by '$'")
- sql(s"LOAD DATA local inpath '$resourcesPath/datasamplecomplex.csv' INTO TABLE complexhivetimestamptable")
- }
-
- test("select * query") {
- checkAnswer(sql("select * from complexcarbontimestamptable"),
- sql("select * from complexhivetimestamptable"))
- }
-
- test("timestamp complex type in the middle of complex types") {
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy-MM-dd HH:mm:ss.SSS")
- sql("CREATE TABLE testtimestampcarbon(imei string,rat array<string>, sid array<int>, end_time array<Timestamp>, probeid array<double>, contact struct<name:string, id:string>)STORED BY 'org.apache.carbondata.format'")
- sql("LOAD DATA local inpath '" + resourcesPath + "/timestampdata.csv' INTO TABLE testtimestampcarbon options('DELIMITER'=',', 'QUOTECHAR'='\"','COMPLEX_DELIMITER_LEVEL_1'='$', 'FILEHEADER'='imei,rat,sid,end_time,probeid,contact')")
- sql("CREATE TABLE testtimestamphive(imei string,rat array<string>, sid array<int>, end_time array<Timestamp>, probeid array<double>, contact struct<name:string, id:string>)row format delimited fields terminated by ',' collection items terminated by '$'")
- sql(s"LOAD DATA local inpath '$resourcesPath/timestampdata.csv' INTO TABLE testtimestamphive")
- checkAnswer(sql("select * from testtimestampcarbon"), sql("select * from testtimestamphive"))
- sql("drop table if exists testtimestampcarbon")
- sql("drop table if exists testtimestamphive")
- }
-
- override def afterAll {
- sql("drop table if exists complexcarbontimestamptable")
- sql("drop table if exists complexhivetimestamptable")
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/SparkDatasourceSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/SparkDatasourceSuite.scala b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/SparkDatasourceSuite.scala
deleted file mode 100644
index 98e4f18..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/SparkDatasourceSuite.scala
+++ /dev/null
@@ -1,192 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.integration.spark.testsuite.dataload
-
-
-import org.apache.spark.sql.test.util.QueryTest
-import org.apache.spark.sql.{DataFrame, Row, SQLContext, SaveMode}
-import org.scalatest.BeforeAndAfterAll
-
-class SparkDatasourceSuite extends QueryTest with BeforeAndAfterAll {
-
- var df: DataFrame = _
-
- override def beforeAll {
- sql("DROP TABLE IF EXISTS carbon1")
-
- import sqlContext.implicits._
- df = sqlContext.sparkContext.parallelize(1 to 1000)
- .map(x => ("a", "b", x))
- .toDF("c1", "c2", "c3")
-
- // save dataframe to carbon file
- df.write
- .format("carbondata")
- .option("tableName", "carbon1")
- .mode(SaveMode.Overwrite)
- .save()
- }
-
- test("read and write using CarbonContext") {
- val in = sqlContext.read
- .format("carbondata")
- .option("tableName", "carbon1")
- .load()
-
- assert(in.where("c3 > 500").count() == 500)
- }
-
- test("read and write using CarbonContext with compression") {
- val in = sqlContext.read
- .format("carbondata")
- .option("tableName", "carbon1")
- .option("compress", "true")
- .load()
-
- assert(in.where("c3 > 500").count() == 500)
- }
-
- test("test overwrite") {
- sql("DROP TABLE IF EXISTS carbon4")
- df.write
- .format("carbondata")
- .option("tableName", "carbon4")
- .mode(SaveMode.Overwrite)
- .save()
- df.write
- .format("carbondata")
- .option("tableName", "carbon4")
- .mode(SaveMode.Overwrite)
- .save()
- val in = sqlContext.read
- .format("carbondata")
- .option("tableName", "carbon4")
- .load()
- assert(in.where("c3 > 500").count() == 500)
- sql("DROP TABLE IF EXISTS carbon4")
- }
-
- test("read and write using CarbonContext, multiple load") {
- sql("DROP TABLE IF EXISTS carbon4")
- df.write
- .format("carbondata")
- .option("tableName", "carbon4")
- .mode(SaveMode.Overwrite)
- .save()
- df.write
- .format("carbondata")
- .option("tableName", "carbon4")
- .mode(SaveMode.Append)
- .save()
- val in = sqlContext.read
- .format("carbondata")
- .option("tableName", "carbon4")
- .load()
- assert(in.where("c3 > 500").count() == 1000)
- sql("DROP TABLE IF EXISTS carbon4")
- }
-
- test("query using SQLContext") {
- val newSQLContext = new SQLContext(sqlContext.sparkContext)
- newSQLContext.sql(
- s"""
- | CREATE TEMPORARY TABLE temp
- | (c1 string, c2 string, c3 int)
- | USING org.apache.spark.sql.CarbonSource
- | OPTIONS (path '$storeLocation/default/carbon1')
- """.stripMargin)
- checkAnswer(newSQLContext.sql(
- """
- | SELECT c1, c2, count(*)
- | FROM temp
- | WHERE c3 > 100
- | GROUP BY c1, c2
- """.stripMargin), Seq(Row("a", "b", 900)))
- newSQLContext.dropTempTable("temp")
- }
-
- test("query using SQLContext without providing schema") {
- val newSQLContext = new SQLContext(sqlContext.sparkContext)
- newSQLContext.sql(
- s"""
- | CREATE TEMPORARY TABLE temp
- | USING org.apache.spark.sql.CarbonSource
- | OPTIONS (path '$storeLocation/default/carbon1')
- """.stripMargin)
- checkAnswer(newSQLContext.sql(
- """
- | SELECT c1, c2, count(*)
- | FROM temp
- | WHERE c3 > 100
- | GROUP BY c1, c2
- """.stripMargin), Seq(Row("a", "b", 900)))
- newSQLContext.dropTempTable("temp")
- }
-
- test("query using SQLContext, multiple load") {
- sql("DROP TABLE IF EXISTS test")
- sql(
- """
- | CREATE TABLE test(id int, name string, city string, age int)
- | STORED BY 'org.apache.carbondata.format'
- """.stripMargin)
- val testData = s"${resourcesPath}/sample.csv"
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table test")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table test")
-
- val newSQLContext = new SQLContext(sqlContext.sparkContext)
- newSQLContext.sql(
- s"""
- | CREATE TEMPORARY TABLE temp
- | (id int, name string, city string, age int)
- | USING org.apache.spark.sql.CarbonSource
- | OPTIONS (path '$storeLocation/default/test')
- """.stripMargin)
- checkAnswer(newSQLContext.sql(
- """
- | SELECT count(id)
- | FROM temp
- """.stripMargin), Seq(Row(8)))
- newSQLContext.dropTempTable("temp")
- sql("DROP TABLE test")
- }
-
- test("json data with long datatype issue CARBONDATA-405") {
- val jsonDF = sqlContext.read.format("json").load(s"$resourcesPath/test.json")
- jsonDF.write
- .format("carbondata")
- .option("tableName", "dftesttable")
- .option("compress", "true")
- .mode(SaveMode.Overwrite)
- .save()
- val carbonDF = sqlContext
- .read
- .format("carbondata")
- .option("tableName", "dftesttable")
- .load()
- checkAnswer(
- carbonDF.select("age", "name"),
- jsonDF.select("age", "name"))
- sql("drop table dftesttable")
- }
-
- override def afterAll {
- sql("DROP TABLE IF EXISTS carbon1")
- sql("DROP TABLE IF EXISTS carbon2")
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithSingleQuotechar.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithSingleQuotechar.scala b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithSingleQuotechar.scala
deleted file mode 100644
index b61ecce..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithSingleQuotechar.scala
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.integration.spark.testsuite.dataload
-
-import org.apache.spark.sql.Row
-import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.BeforeAndAfterAll
-/**
- * Test Class for data loading when there is single quote in fact data
- *
- */
-class TestLoadDataWithSingleQuotechar extends QueryTest with BeforeAndAfterAll {
- override def beforeAll {
- sql("DROP TABLE IF EXISTS carbontable")
- sql(
- "CREATE TABLE carbontable (id Int, name String) STORED BY 'carbondata'")
- }
-
- test("test data loading with single quote char") {
- try {
- sql(
- s"LOAD DATA LOCAL INPATH '$resourcesPath/dataWithSingleQuote.csv' INTO TABLE " +
- "carbontable OPTIONS('DELIMITER'= ',')")
- sql("SELECT * from carbontable")
- checkAnswer(
- sql("SELECT * from carbontable"),
- Seq(Row(1,"Tom"),
- Row(2,"Tony\n3,Lily"),
- Row(4,"Games\""),
- Row(5,"prival\"\n6,\"hello\"")
- )
- )
- } catch {
- case e: Throwable =>
- assert(false)
- }
- }
-
- override def afterAll {
- sql("DROP TABLE carbontable")
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/AllQueriesSpark1TestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/AllQueriesSpark1TestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/AllQueriesSpark1TestCase.scala
deleted file mode 100644
index 1c003c7..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/AllQueriesSpark1TestCase.scala
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.spark.testsuite.allqueries
-
-import org.apache.spark.sql.test.util.QueryTest
-import org.apache.spark.sql.{Row, SaveMode}
-import org.scalatest.BeforeAndAfterAll
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
-
-/**
- * Test Class for all query on multiple datatypes
- *
- */
-class AllQueriesSpark1TestCase extends QueryTest with BeforeAndAfterAll {
-
- override def beforeAll {
- clean
-
- sql("create table if not exists Carbon_automation_test (imei string,deviceInformationId int,MAC string,deviceColor string,device_backColor string,modelId string,marketName string,AMSize string,ROMSize string,CUPAudit string,CPIClocked string,series string,productionDate timestamp,bomCode string,internalModels string, deliveryTime string, channelsId string, channelsName string , deliveryAreaId string, deliveryCountry string, deliveryProvince string, deliveryCity string,deliveryDistrict string, deliveryStreet string, oxSingleNumber string, ActiveCheckTime string, ActiveAreaId string, ActiveCountry string, ActiveProvince string, Activecity string, ActiveDistrict string, ActiveStreet string, ActiveOperatorId string, Active_releaseId string, Active_EMUIVersion string, Active_operaSysVersion string, Active_BacVerNumber string, Active_BacFlashVer string, Active_webUIVersion string, Active_webUITypeCarrVer string,Active_webTypeDataVerNumber string, Active_operatorsVersion string, Active
_phonePADPartitionedVersions string, Latest_YEAR int, Latest_MONTH int, Latest_DAY int, Latest_HOUR string, Latest_areaId string, Latest_country string, Latest_province string, Latest_city string, Latest_district string, Latest_street string, Latest_releaseId string, Latest_EMUIVersion string, Latest_operaSysVersion string, Latest_BacVerNumber string, Latest_BacFlashVer string, Latest_webUIVersion string, Latest_webUITypeCarrVer string, Latest_webTypeDataVerNumber string, Latest_operatorsVersion string, Latest_phonePADPartitionedVersions string, Latest_operatorId string, gamePointDescription string, gamePointId int,contractNumber int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('DICTIONARY_INCLUDE'='Latest_MONTH,Latest_DAY,deviceInformationId')")
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
- sql("LOAD DATA LOCAL INPATH '" + resourcesPath + "/100_olap.csv' INTO table Carbon_automation_test options('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'FILEHEADER'= 'imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Lat
est_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')")
- }
-
- def clean {
- sql("drop table if exists Carbon_automation_test")
- }
-
- override def afterAll {
- clean
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
- }
-
-
- //TC_113
- test("select percentile_approx(deviceInformationId,0.2) as a from Carbon_automation_test")({
- checkAnswer(
- sql("select percentile_approx(deviceInformationId,0.2) as a from Carbon_automation_test"),
- Seq(Row(100005.8)))
- })
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableSpark1TestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableSpark1TestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableSpark1TestCase.scala
deleted file mode 100644
index d762ec6..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableSpark1TestCase.scala
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.spark.testsuite.allqueries
-
-import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.BeforeAndAfterAll
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
-
-class InsertIntoCarbonTableSpark1TestCase extends QueryTest with BeforeAndAfterAll {
- override def beforeAll {
- sql("drop table if exists THive")
- sql("create table THive (imei string,deviceInformationId int,MAC string,deviceColor string,device_backColor string,modelId string,marketName string,AMSize string,ROMSize string,CUPAudit string,CPIClocked string,series string,productionDate timestamp,bomCode string,internalModels string, deliveryTime string, channelsId string, channelsName string , deliveryAreaId string, deliveryCountry string, deliveryProvince string, deliveryCity string,deliveryDistrict string, deliveryStreet string, oxSingleNumber string, ActiveCheckTime string, ActiveAreaId string, ActiveCountry string, ActiveProvince string, Activecity string, ActiveDistrict string, ActiveStreet string, ActiveOperatorId string, Active_releaseId string, Active_EMUIVersion string, Active_operaSysVersion string, Active_BacVerNumber string, Active_BacFlashVer string, Active_webUIVersion string, Active_webUITypeCarrVer string,Active_webTypeDataVerNumber string, Active_operatorsVersion string, Active_phonePADPartitionedVersions st
ring, Latest_YEAR int, Latest_MONTH int, Latest_DAY Decimal(30,10), Latest_HOUR string, Latest_areaId string, Latest_country string, Latest_province string, Latest_city string, Latest_district string, Latest_street string, Latest_releaseId string, Latest_EMUIVersion string, Latest_operaSysVersion string, Latest_BacVerNumber string, Latest_BacFlashVer string, Latest_webUIVersion string, Latest_webUITypeCarrVer string, Latest_webTypeDataVerNumber string, Latest_operatorsVersion string, Latest_phonePADPartitionedVersions string, Latest_operatorId string, gamePointDescription string,gamePointId double,contractNumber BigInt) ROW FORMAT DELIMITED FIELDS TERMINATED BY ','")
- sql(s"LOAD DATA local INPATH '$resourcesPath/100_olap.csv' INTO TABLE THive")
- }
-
-
- test("insert from carbon-select columns-source table has more column then target column") {
- val timeStampPropOrig = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT)
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
-
- sql("drop table if exists load")
- sql("drop table if exists inser")
- sql("CREATE TABLE load(imei string,age int,task bigint,num double,level decimal(10,3),productdate timestamp,name string,point int)STORED BY 'org.apache.carbondata.format'")
- sql("LOAD DATA INPATH '" + resourcesPath + "/shortolap.csv' INTO TABLE load options ('DELIMITER'=',', 'QUOTECHAR'='\"','FILEHEADER' = 'imei,age,task,num,level,productdate,name,point')")
- sql("CREATE TABLE inser(imei string,age int,task bigint,num double,level decimal(10,3),productdate timestamp)STORED BY 'org.apache.carbondata.format'")
- sql("insert into inser select * from load")
- checkAnswer(
- sql("select * from inser"),
- sql("select imei,age,task,num,level,productdate from load")
- )
- sql("drop table if exists load")
- sql("drop table if exists inser")
- CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, timeStampPropOrig)
- }
-
- test("insert->hive column more than carbon column->success") {
- sql("drop table if exists TCarbon")
- sql("create table TCarbon (imei string,deviceInformationId int,MAC string,deviceColor string,gamePointId double,contractNumber BigInt) STORED BY 'org.apache.carbondata.format'")
-
- sql("insert into TCarbon select imei,deviceInformationId,MAC,deviceColor,gamePointId,contractNumber,device_backColor,modelId,CUPAudit,CPIClocked from THive")
- checkAnswer(
- sql("select imei,deviceInformationId,MAC,deviceColor,gamePointId,contractNumber from THive"),
- sql("select imei,deviceInformationId,MAC,deviceColor,gamePointId,contractNumber from TCarbon")
- )
- }
-
-// test("insert->insert empty data -pass") {
-// sql("drop table if exists TCarbon")
-// sql("create table TCarbon (imei string,deviceInformationId int,MAC string) STORED BY 'org.apache.carbondata.format'")
-// sql("insert into TCarbon select imei,deviceInformationId,MAC from THive where MAC='wrongdata'")
-// val result = sql("select imei,deviceInformationId,MAC from TCarbon where MAC='wrongdata'").collect()
-// checkAnswer(
-// sql("select imei,deviceInformationId,MAC from THive where MAC='wrongdata'"),
-// sql("select imei,deviceInformationId,MAC from TCarbon where MAC='wrongdata'")
-// )
-// }
-
- override def afterAll {
- sql("drop table if exists load")
- sql("drop table if exists inser")
- sql("DROP TABLE IF EXISTS THive")
- sql("DROP TABLE IF EXISTS TCarbon")
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/badrecordloger/BadRecordLoggerSharedDictionaryTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/badrecordloger/BadRecordLoggerSharedDictionaryTest.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/badrecordloger/BadRecordLoggerSharedDictionaryTest.scala
deleted file mode 100644
index 7aee00d..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/badrecordloger/BadRecordLoggerSharedDictionaryTest.scala
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.spark.testsuite.badrecordloger
-
-import java.io.File
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.spark.sql.test.util.QueryTest
-import org.apache.spark.sql.hive.HiveContext
-import org.scalatest.BeforeAndAfterAll
-
-
-/**
- * Test Class for detailed query on timestamp datatypes
- *
- *
- */
-class BadRecordLoggerSharedDictionaryTest extends QueryTest with BeforeAndAfterAll {
- var hiveContext: HiveContext = _
- var csvFilePath : String = null
- var timestamp_format: String = null
-
- override def beforeAll {
- sql("drop table IF EXISTS testdrive")
- sql(
- """create table testdrive (ID int,CUST_ID int,cust_name string)
- STORED BY 'org.apache.carbondata.format'
- TBLPROPERTIES("columnproperties.cust_name.shared_column"="shared.cust_name",
- "columnproperties.ID.shared_column"="shared.ID",
- "columnproperties.CUST_ID.shared_column"="shared.CUST_ID",
- 'DICTIONARY_INCLUDE'='ID,CUST_ID')"""
- )
-
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC,
- new File("./target/test/badRecords")
- .getCanonicalPath
- )
-
- val carbonProp = CarbonProperties.getInstance()
- timestamp_format = carbonProp.getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
- CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
- carbonProp.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/mm/dd")
- val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../")
- .getCanonicalPath
- csvFilePath = currentDirectory + "/src/test/resources/badrecords/test2.csv"
-
- }
- test("dataload with bad record test") {
- try {
- sql(
- s"""LOAD DATA INPATH '$csvFilePath' INTO TABLE testdrive OPTIONS('DELIMITER'=',',
- |'QUOTECHAR'= '"', 'BAD_RECORDS_LOGGER_ENABLE'='TRUE', 'BAD_RECORDS_ACTION'='FAIL',
- |'FILEHEADER'= 'ID,CUST_ID,cust_name')""".stripMargin)
- } catch {
- case e: Throwable =>
- assert(e.getMessage.contains("Data load failed due to bad record"))
- }
- }
-
- override def afterAll {
- sql("drop table IF EXISTS testdrive")
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, timestamp_format)
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/createtable/TestCreateTableSyntax.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/createtable/TestCreateTableSyntax.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/createtable/TestCreateTableSyntax.scala
deleted file mode 100644
index 7400839..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/createtable/TestCreateTableSyntax.scala
+++ /dev/null
@@ -1,186 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.spark.testsuite.createtable
-
-import org.apache.spark.sql.Row
-import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.BeforeAndAfterAll
-
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
-
-/**
- * Test Class for validating create table syntax for carbontable
- *
- */
-class TestCreateTableSyntax extends QueryTest with BeforeAndAfterAll {
-
- override def beforeAll {
- }
-
- test("Struct field with underscore and struct<struct> syntax check") {
- sql("drop table if exists carbontable")
- sql("create table carbontable(id int, username struct<sur_name:string," +
- "actual_name:struct<first_name:string,last_name:string>>, country string, salary double)" +
- "STORED BY 'org.apache.carbondata.format'")
- sql("describe carbontable")
- }
-
- test("Test table rename operation on carbon table and on hive table") {
- sql("drop table if exists hivetable")
- sql("drop table if exists carbontable")
- sql("drop table if exists hiveRenamedTable")
- sql("drop table if exists carbonRenamedTable")
- sql("create table hivetable(test1 int, test2 array<String>,test3 array<bigint>,"+
- "test4 array<int>,test5 array<decimal>,test6 array<timestamp>,test7 array<double>)"+
- "row format delimited fields terminated by ',' collection items terminated by '$' map keys terminated by ':'")
- sql("alter table hivetable rename To hiveRenamedTable")
- sql("create table carbontable(test1 int, test2 array<String>,test3 array<bigint>,"+
- "test4 array<int>,test5 array<decimal>,test6 array<timestamp>,test7 array<double>)"+
- "STORED BY 'org.apache.carbondata.format'")
- sql("alter table carbontable compact 'minor'")
- try {
- sql("alter table carbontable rename To carbonRenamedTable")
- assert(false)
- } catch {
- case e : MalformedCarbonCommandException => {
- assert(e.getMessage.equals("Unsupported alter operation on carbon table"))
- }
- }
- }
-
-
- test("test carbon table create with complex datatype as dictionary exclude") {
- try {
- sql("drop table if exists carbontable")
- sql("create table carbontable(id int, name string, dept string, mobile array<string>, "+
- "country string, salary double) STORED BY 'org.apache.carbondata.format' " +
- "TBLPROPERTIES('DICTIONARY_EXCLUDE'='dept,mobile')")
- assert(false)
- } catch {
- case e : MalformedCarbonCommandException => {
- assert(e.getMessage.equals("DICTIONARY_EXCLUDE is unsupported for complex datatype column: mobile"))
- }
- }
- }
-
- test("test carbon table create with double datatype as dictionary exclude") {
- try {
- sql("drop table if exists carbontable")
- sql("create table carbontable(id int, name string, dept string, mobile array<string>, "+
- "country string, salary double) STORED BY 'org.apache.carbondata.format' " +
- "TBLPROPERTIES('DICTIONARY_EXCLUDE'='salary')")
- assert(false)
- } catch {
- case e : MalformedCarbonCommandException => {
- assert(e.getMessage.equals("DICTIONARY_EXCLUDE is unsupported for double " +
- "data type column: salary"))
- }
- }
- }
- test("test carbon table create with int datatype as dictionary exclude") {
- sql("drop table if exists carbontable")
- sql("create table carbontable(id int, name string, dept string, mobile array<string>, " +
- "country string, salary double) STORED BY 'org.apache.carbondata.format' " +
- "TBLPROPERTIES('DICTIONARY_EXCLUDE'='id')")
- assert(true)
- }
-
- test("test carbon table create with decimal datatype as dictionary exclude") {
- try {
- sql("drop table if exists carbontable")
- sql("create table carbontable(id int, name string, dept string, mobile array<string>, "+
- "country string, salary decimal) STORED BY 'org.apache.carbondata.format' " +
- "TBLPROPERTIES('DICTIONARY_EXCLUDE'='salary')")
- assert(false)
- } catch {
- case e : MalformedCarbonCommandException => {
- assert(e.getMessage.equals("DICTIONARY_EXCLUDE is unsupported for decimal " +
- "data type column: salary"))
- }
- }
- }
-
- test("describe formatted on hive table and carbon table") {
- sql("drop table if exists hivetable")
- sql("drop table if exists carbontable")
- sql("create table carbontable(id int, username struct<sur_name:string," +
- "actual_name:struct<first_name:string,last_name:string>>, country string, salary double)" +
- "STORED BY 'org.apache.carbondata.format'")
- sql("describe formatted carbontable")
- sql("create table hivetable(id int, username struct<sur_name:string," +
- "actual_name:struct<first_name:string,last_name:string>>, country string, salary double)")
- sql("describe formatted hivetable")
- }
-
- test("describe command carbon table for decimal scale and precision test") {
- sql("drop table if exists carbontablePrecision")
- sql("create table carbontablePrecision(id int, name string, dept string, mobile array<string>, "+
- "country string, salary decimal(10,6)) STORED BY 'org.apache.carbondata.format' " +
- "TBLPROPERTIES('DICTIONARY_INCLUDE'='salary,id')")
- checkAnswer(
- sql("describe carbontablePrecision"),
- Seq(Row("country","string",""),
- Row("dept","string",""),Row("id","int",""),Row("mobile","array<string>",""),Row("name","string",""),
- Row("salary","decimal(10,6)","")
- )
- )
- }
-
- test("create carbon table without dimensions") {
- try {
- sql("drop table if exists carbontable")
- sql("create table carbontable(msr1 int, msr2 double, msr3 bigint, msr4 decimal)" +
- " stored by 'org.apache.carbondata.format'")
- assert(true)
- } catch {
- case e : MalformedCarbonCommandException => {
- assert(e.getMessage.equals("Table default.carbontable can not be created without " +
- "key columns. Please use DICTIONARY_INCLUDE or DICTIONARY_EXCLUDE to " +
- "set at least one key column if all specified columns are numeric types"))
- }
- }
- }
-
- test("create carbon table with repeated table properties") {
- try {
- sql("drop table if exists carbontable")
- sql(
- """
- CREATE TABLE IF NOT EXISTS carbontable
- (ID Int, date Timestamp, country String,
- name String, phonetype String, serialname String, salary Int)
- STORED BY 'carbondata'
- TBLPROPERTIES('DICTIONARY_EXCLUDE'='country','DICTIONARY_INCLUDE'='ID',
- 'DICTIONARY_EXCLUDE'='phonetype', 'DICTIONARY_INCLUDE'='salary')
- """)
- assert(false)
- } catch {
- case e : MalformedCarbonCommandException => {
- assert(e.getMessage.equals("Table properties is repeated: dictionary_include,dictionary_exclude"))
- }
- }
- }
-
- override def afterAll {
- sql("drop table if exists hivetable")
- sql("drop table if exists carbontable")
- sql("drop table if exists hiveRenamedTable")
- sql("drop table if exists carbonRenamedTable")
- sql("drop table if exists carbontablePrecision")
- }
-}
\ No newline at end of file
[16/16] carbondata git commit: [CARBONDATA-1598] Remove all spark 1.x
info(CI, README, documents)
Posted by ra...@apache.org.
[CARBONDATA-1598] Remove all spark 1.x info(CI, README, documents)
This closes #1423
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/5a67c98f
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/5a67c98f
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/5a67c98f
Branch: refs/heads/pre-aggregate
Commit: 5a67c98f1072c656acebf1b853e45c1f1b5d203f
Parents: 7036696
Author: chenliang613 <ch...@apache.org>
Authored: Thu Oct 19 18:10:29 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Mon Oct 23 10:20:16 2017 +0530
----------------------------------------------------------------------
README.md | 2 -
assembly/pom.xml | 21 ----------
build/README.md | 15 ++++----
docs/quick-start-guide.md | 62 ------------------------------
integration/spark-common-test/pom.xml | 22 -----------
5 files changed, 8 insertions(+), 114 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5a67c98f/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 06f1ce5..297cece 100644
--- a/README.md
+++ b/README.md
@@ -29,8 +29,6 @@ You can find the latest CarbonData document and learn more at:
## Status
Spark2.1:
[](https://builds.apache.org/view/CarbonData/job/carbondata-master-spark-2.1/)
-Spark1.6:
-[](https://builds.apache.org/view/CarbonData/job/carbondata-master-spark-1.6/)
## Features
CarbonData file format is a columnar store in HDFS, it has many features that a modern columnar format has, such as splittable, compression schema ,complex data type etc, and CarbonData has following unique features:
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5a67c98f/assembly/pom.xml
----------------------------------------------------------------------
diff --git a/assembly/pom.xml b/assembly/pom.xml
index d705b66..b5652a5 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -124,29 +124,8 @@
</plugin>
</plugins>
</build>
-
<profiles>
<profile>
- <id>spark-1.5</id>
- <dependencies>
- <dependency>
- <groupId>org.apache.carbondata</groupId>
- <artifactId>carbondata-spark</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
- </profile>
- <profile>
- <id>spark-1.6</id>
- <dependencies>
- <dependency>
- <groupId>org.apache.carbondata</groupId>
- <artifactId>carbondata-spark</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
- </profile>
- <profile>
<id>spark-2.1</id>
<!-- default -->
<activation>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5a67c98f/build/README.md
----------------------------------------------------------------------
diff --git a/build/README.md b/build/README.md
index 5fa6814..50f6ce2 100644
--- a/build/README.md
+++ b/build/README.md
@@ -27,12 +27,17 @@
* [Apache Thrift 0.9.3](http://archive.apache.org/dist/thrift/0.9.3/)
## Build command
-Build without test,by default carbondata takes Spark 1.6.2 to build the project
+From 1.3.0 onwards, CarbonData supports spark 2.x, build without test,by default carbondata takes Spark 2.1.0 to build the project
```
mvn -DskipTests clean package
```
-Build with different supported versions of Spark.
+Build with test
+```
+mvn clean package
+```
+
+Before 1.3.0, build with different supported versions of Spark
```
mvn -DskipTests -Pspark-1.5 -Dspark.version=1.5.1 clean package
mvn -DskipTests -Pspark-1.5 -Dspark.version=1.5.2 clean package
@@ -44,13 +49,9 @@ mvn -DskipTests -Pspark-1.6 -Dspark.version=1.6.3 clean package
mvn -DskipTests -Pspark-2.1 -Dspark.version=2.1.0 clean package
```
-Build with test
-```
-mvn clean package
-```
## For contributors : To build the format code after any changes, please follow the below command.
Note:Need install Apache Thrift 0.9.3
```
-mvn clean -DskipTests -Pbuild-with-format -Pspark-1.6 package
+mvn clean -DskipTests -Pbuild-with-format -Pspark-2.1 package
```
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5a67c98f/docs/quick-start-guide.md
----------------------------------------------------------------------
diff --git a/docs/quick-start-guide.md b/docs/quick-start-guide.md
index 5b39b9e..d833679 100644
--- a/docs/quick-start-guide.md
+++ b/docs/quick-start-guide.md
@@ -98,65 +98,3 @@ scala>carbon.sql("SELECT city, avg(age), sum(age)
FROM test_table
GROUP BY city").show()
```
-
-## Interactive Analysis with Spark Shell Version 1.6
-
-#### Basics
-
-Start Spark shell by running the following command in the Spark directory:
-
-```
-./bin/spark-shell --jars <carbondata assembly jar path>
-```
-**NOTE**: Assembly jar will be available after [building CarbonData](https://github.com/apache/carbondata/blob/master/build/README.md) and can be copied from `./assembly/target/scala-2.1x/carbondata_xxx.jar`
-
-**NOTE**: In this shell, SparkContext is readily available as `sc`.
-
-* In order to execute the Queries we need to import CarbonContext:
-
-```
-import org.apache.spark.sql.CarbonContext
-```
-
-* Create an instance of CarbonContext in the following manner :
-
-```
-val cc = new CarbonContext(sc, "<hdfs store path>")
-```
-**NOTE**: If running on local machine without hdfs, configure the local machine's store path instead of hdfs store path
-
-#### Executing Queries
-
-###### Creating a Table
-
-```
-scala>cc.sql("CREATE TABLE
- IF NOT EXISTS test_table (
- id string,
- name string,
- city string,
- age Int)
- STORED BY 'carbondata'")
-```
-To see the table created :
-
-```
-scala>cc.sql("SHOW TABLES").show()
-```
-
-###### Loading Data to a Table
-
-```
-scala>cc.sql("LOAD DATA INPATH 'sample.csv file path'
- INTO TABLE test_table")
-```
-**NOTE**: Please provide the real file path of `sample.csv` for the above script.
-
-###### Querying Data from a Table
-
-```
-scala>cc.sql("SELECT * FROM test_table").show()
-scala>cc.sql("SELECT city, avg(age), sum(age)
- FROM test_table
- GROUP BY city").show()
-```
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5a67c98f/integration/spark-common-test/pom.xml
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/pom.xml b/integration/spark-common-test/pom.xml
index b2ee316..8806c0a 100644
--- a/integration/spark-common-test/pom.xml
+++ b/integration/spark-common-test/pom.xml
@@ -328,28 +328,6 @@
</build>
<profiles>
<profile>
- <id>spark-1.5</id>
- <dependencies>
- <dependency>
- <groupId>org.apache.carbondata</groupId>
- <artifactId>carbondata-spark</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
- </profile>
- <profile>
- <id>spark-1.6</id>
- <dependencies>
- <dependency>
- <groupId>org.apache.carbondata</groupId>
- <artifactId>carbondata-spark</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
- </profile>
- <profile>
<id>spark-2.1</id>
<activation>
<activeByDefault>true</activeByDefault>
[04/16] carbondata git commit: [CARBONDATA-1597] Remove spark1
integration
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetadataUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetadataUtil.scala b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetadataUtil.scala
deleted file mode 100644
index cd42fba..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetadataUtil.scala
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.sql.hive
-
-import org.apache.spark.sql.SQLContext
-import org.apache.spark.sql.catalyst.SqlParser
-
-import org.apache.carbondata.common.logging.LogServiceFactory
-
-
-/**
- * This class contains all carbon hive metadata related utilities
- */
-object CarbonHiveMetadataUtil {
-
- @transient
- val LOGGER = LogServiceFactory.getLogService(CarbonHiveMetadataUtil.getClass.getName)
-
-
- /**
- * This method invalidates the table from HiveMetastoreCatalog before dropping table
- *
- * @param databaseName
- * @param tableName
- * @param sqlContext
- */
- def invalidateAndDropTable(databaseName: String,
- tableName: String,
- sqlContext: SQLContext): Unit = {
- val hiveContext = sqlContext.asInstanceOf[HiveContext]
- val tableWithDb = databaseName + "." + tableName
- val tableIdent = SqlParser.parseTableIdentifier(tableWithDb)
- try {
- hiveContext.catalog.invalidateTable(tableIdent)
- hiveContext.runSqlHive(s"DROP TABLE IF EXISTS $databaseName.$tableName")
- } catch {
- case e: Exception =>
- LOGGER.audit(
- s"Error While deleting the table $databaseName.$tableName during drop carbon table" +
- e.getMessage)
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
deleted file mode 100644
index 4d5d39a..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
+++ /dev/null
@@ -1,562 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.hive
-
-import java.io._
-import java.util.UUID
-
-import scala.Array.canBuildFrom
-import scala.collection.mutable.ArrayBuffer
-import scala.language.implicitConversions
-import scala.util.parsing.combinator.RegexParsers
-
-import org.apache.spark.sql._
-import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.analysis.NoSuchTableException
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.execution.command.Partitioner
-import org.apache.spark.sql.hive.client.ClientInterface
-import org.apache.spark.sql.types._
-
-import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datamap.DataMapStoreManager
-import org.apache.carbondata.core.datastore.impl.FileFactory
-import org.apache.carbondata.core.datastore.impl.FileFactory.FileType
-import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonMetadata, CarbonTableIdentifier}
-import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.reader.ThriftReader
-import org.apache.carbondata.core.stats.{QueryStatistic, QueryStatisticsConstants}
-import org.apache.carbondata.core.util.{CarbonProperties, CarbonTimeStatisticsFactory, CarbonUtil}
-import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
-import org.apache.carbondata.core.writer.ThriftWriter
-import org.apache.carbondata.format.{SchemaEvolutionEntry, TableInfo}
-import org.apache.carbondata.processing.merger.TableMeta
-
-case class MetaData(var tablesMeta: ArrayBuffer[TableMeta])
-
-case class CarbonMetaData(dims: Seq[String],
- msrs: Seq[String],
- carbonTable: CarbonTable,
- dictionaryMap: DictionaryMap)
-
-object CarbonMetastore {
-
- def readSchemaFileToThriftTable(schemaFilePath: String): TableInfo = {
- val createTBase = new ThriftReader.TBaseCreator() {
- override def create(): org.apache.thrift.TBase[TableInfo, TableInfo._Fields] = {
- new TableInfo()
- }
- }
- val thriftReader = new ThriftReader(schemaFilePath, createTBase)
- var tableInfo: TableInfo = null
- try {
- thriftReader.open()
- tableInfo = thriftReader.read().asInstanceOf[TableInfo]
- } finally {
- thriftReader.close()
- }
- tableInfo
- }
-
- def writeThriftTableToSchemaFile(schemaFilePath: String, tableInfo: TableInfo): Unit = {
- val thriftWriter = new ThriftWriter(schemaFilePath, false)
- try {
- thriftWriter.open()
- thriftWriter.write(tableInfo);
- } finally {
- thriftWriter.close()
- }
- }
-
-}
-
-case class DictionaryMap(dictionaryMap: Map[String, Boolean]) {
- def get(name: String): Option[Boolean] = {
- dictionaryMap.get(name.toLowerCase)
- }
-}
-
-class CarbonMetastore(hiveContext: HiveContext, val storePath: String,
- client: ClientInterface, queryId: String) extends HiveMetastoreCatalog(client, hiveContext) {
-
- @transient
- val LOGGER = LogServiceFactory.getLogService("org.apache.spark.sql.CarbonMetastoreCatalog")
-
- val tableModifiedTimeStore = new java.util.HashMap[String, Long]()
- tableModifiedTimeStore
- .put(CarbonCommonConstants.DATABASE_DEFAULT_NAME, System.currentTimeMillis())
-
- val metadata = loadMetadata(storePath)
-
- def getTableCreationTime(databaseName: String, tableName: String): Long = {
- val tableMeta = metadata.tablesMeta.filter(
- c => c.carbonTableIdentifier.getDatabaseName.equalsIgnoreCase(databaseName) &&
- c.carbonTableIdentifier.getTableName.equalsIgnoreCase(tableName))
- val tableCreationTime = tableMeta.head.carbonTable.getTableLastUpdatedTime
- tableCreationTime
- }
-
- def lookupRelation1(dbName: Option[String],
- tableName: String)(sqlContext: SQLContext): LogicalPlan = {
- lookupRelation1(TableIdentifier(tableName, dbName))(sqlContext)
- }
-
- def lookupRelation1(tableIdentifier: TableIdentifier,
- alias: Option[String] = None)(sqlContext: SQLContext): LogicalPlan = {
- checkSchemasModifiedTimeAndReloadTables()
- val database = tableIdentifier.database.getOrElse(getDB.getDatabaseName(None, sqlContext))
- val tables = getTableFromMetadata(database, tableIdentifier.table)
- tables match {
- case Some(t) =>
- CarbonRelation(database, tableIdentifier.table,
- CarbonSparkUtil.createSparkMeta(tables.head.carbonTable), tables.head, alias)(sqlContext)
- case None =>
- LOGGER.audit(s"Table Not Found: ${tableIdentifier.table}")
- throw new NoSuchTableException
- }
- }
-
- /**
- * This method will search for a table in the catalog metadata
- *
- * @param database
- * @param tableName
- * @return
- */
- def getTableFromMetadata(database: String,
- tableName: String): Option[TableMeta] = {
- metadata.tablesMeta
- .find(c => c.carbonTableIdentifier.getDatabaseName.equalsIgnoreCase(database) &&
- c.carbonTableIdentifier.getTableName.equalsIgnoreCase(tableName))
- }
-
- def tableExists(identifier: TableIdentifier)(sqlContext: SQLContext): Boolean = {
- checkSchemasModifiedTimeAndReloadTables()
- val database = identifier.database.getOrElse(getDB.getDatabaseName(None, sqlContext))
- val tables = metadata.tablesMeta.filter(
- c => c.carbonTableIdentifier.getDatabaseName.equalsIgnoreCase(database) &&
- c.carbonTableIdentifier.getTableName.equalsIgnoreCase(identifier.table))
- tables.nonEmpty
- }
-
- def loadMetadata(metadataPath: String): MetaData = {
- val recorder = CarbonTimeStatisticsFactory.createDriverRecorder()
- val statistic = new QueryStatistic()
- // creating zookeeper instance once.
- // if zookeeper is configured as carbon lock type.
- val zookeeperurl = hiveContext.getConf(CarbonCommonConstants.ZOOKEEPER_URL, null)
- if (null != zookeeperurl) {
- CarbonProperties.getInstance
- .addProperty(CarbonCommonConstants.ZOOKEEPER_URL, zookeeperurl)
- }
- if (metadataPath == null) {
- return null
- }
- // if no locktype is configured and store type is HDFS set HDFS lock as default
- if (null == CarbonProperties.getInstance
- .getProperty(CarbonCommonConstants.LOCK_TYPE) &&
- FileType.HDFS == FileFactory.getFileType(metadataPath)) {
- CarbonProperties.getInstance
- .addProperty(CarbonCommonConstants.LOCK_TYPE,
- CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS
- )
- LOGGER.info("Default lock type HDFSLOCK is configured")
- }
- val fileType = FileFactory.getFileType(metadataPath)
- val metaDataBuffer = new ArrayBuffer[TableMeta]
- fillMetaData(metadataPath, fileType, metaDataBuffer)
- updateSchemasUpdatedTime(readSchemaFileSystemTime("", ""))
- statistic.addStatistics(QueryStatisticsConstants.LOAD_META,
- System.currentTimeMillis())
- recorder.recordStatisticsForDriver(statistic, queryId)
- MetaData(metaDataBuffer)
- }
-
- private def fillMetaData(basePath: String, fileType: FileType,
- metaDataBuffer: ArrayBuffer[TableMeta]): Unit = {
- val databasePath = basePath // + "/schemas"
- try {
- if (FileFactory.isFileExist(databasePath, fileType)) {
- val file = FileFactory.getCarbonFile(databasePath, fileType)
- val databaseFolders = file.listFiles()
-
- databaseFolders.foreach(databaseFolder => {
- if (databaseFolder.isDirectory) {
- val dbName = databaseFolder.getName
- val tableFolders = databaseFolder.listFiles()
-
- tableFolders.foreach(tableFolder => {
- if (tableFolder.isDirectory) {
- val carbonTableIdentifier = new CarbonTableIdentifier(databaseFolder.getName,
- tableFolder.getName, UUID.randomUUID().toString)
- val carbonTablePath = CarbonStorePath.getCarbonTablePath(basePath,
- carbonTableIdentifier)
- val tableMetadataFile = carbonTablePath.getSchemaFilePath
-
- if (FileFactory.isFileExist(tableMetadataFile, fileType)) {
- val tableName = tableFolder.getName
- val tableUniqueName = databaseFolder.getName + "_" + tableFolder.getName
-
-
- val createTBase = new ThriftReader.TBaseCreator() {
- override def create(): org.apache.thrift.TBase[TableInfo, TableInfo._Fields] = {
- new TableInfo()
- }
- }
- val thriftReader = new ThriftReader(tableMetadataFile, createTBase)
- thriftReader.open()
- val tableInfo: TableInfo = thriftReader.read().asInstanceOf[TableInfo]
- thriftReader.close()
-
- val schemaConverter = new ThriftWrapperSchemaConverterImpl
- val wrapperTableInfo = schemaConverter
- .fromExternalToWrapperTableInfo(tableInfo, dbName, tableName, basePath)
- val schemaFilePath = CarbonStorePath
- .getCarbonTablePath(storePath, carbonTableIdentifier).getSchemaFilePath
- wrapperTableInfo.setStorePath(storePath)
- wrapperTableInfo
- .setMetaDataFilepath(CarbonTablePath.getFolderContainingFile(schemaFilePath))
- CarbonMetadata.getInstance().loadTableMetadata(wrapperTableInfo)
- val carbonTable = CarbonMetadata.getInstance().getCarbonTable(tableUniqueName)
- metaDataBuffer += new TableMeta(carbonTable.getCarbonTableIdentifier, storePath,
- null, carbonTable)
- }
- }
- })
- }
- })
- } else {
- // Create folders and files.
- FileFactory.mkdirs(databasePath, fileType)
- }
- } catch {
- case s: java.io.FileNotFoundException =>
- // Create folders and files.
- FileFactory.mkdirs(databasePath, fileType)
- }
- }
-
- /**
- *
- * Prepare Thrift Schema from wrapper TableInfo and write to Schema file.
- * Load CarbonTable from wrapper tableinfo
- *
- */
- def createTableFromThrift(
- tableInfo: org.apache.carbondata.core.metadata.schema.table.TableInfo,
- dbName: String, tableName: String, partitioner: Partitioner)
- (sqlContext: SQLContext): String = {
- if (tableExists(TableIdentifier(tableName, Some(dbName)))(sqlContext)) {
- sys.error(s"Table [$tableName] already exists under Database [$dbName]")
- }
- val schemaConverter = new ThriftWrapperSchemaConverterImpl
- val thriftTableInfo = schemaConverter
- .fromWrapperToExternalTableInfo(tableInfo, dbName, tableName)
- val schemaEvolutionEntry = new SchemaEvolutionEntry(tableInfo.getLastUpdatedTime)
- thriftTableInfo.getFact_table.getSchema_evolution.getSchema_evolution_history
- .add(schemaEvolutionEntry)
-
- val carbonTableIdentifier = new CarbonTableIdentifier(dbName, tableName,
- tableInfo.getFactTable.getTableId)
- val carbonTablePath = CarbonStorePath.getCarbonTablePath(storePath, carbonTableIdentifier)
- val schemaFilePath = carbonTablePath.getSchemaFilePath
- val schemaMetadataPath = CarbonTablePath.getFolderContainingFile(schemaFilePath)
- tableInfo.setMetaDataFilepath(schemaMetadataPath)
- tableInfo.setStorePath(storePath)
- CarbonMetadata.getInstance().loadTableMetadata(tableInfo)
- val tableMeta = new TableMeta(carbonTableIdentifier, storePath, null,
- CarbonMetadata.getInstance().getCarbonTable(dbName + "_" + tableName))
-
- val fileType = FileFactory.getFileType(schemaMetadataPath)
- if (!FileFactory.isFileExist(schemaMetadataPath, fileType)) {
- FileFactory.mkdirs(schemaMetadataPath, fileType)
- }
-
- val thriftWriter = new ThriftWriter(schemaFilePath, false)
- thriftWriter.open()
- thriftWriter.write(thriftTableInfo)
- thriftWriter.close()
-
- metadata.tablesMeta += tableMeta
- logInfo(s"Table $tableName for Database $dbName created successfully.")
- LOGGER.info(s"Table $tableName for Database $dbName created successfully.")
- updateSchemasUpdatedTime(touchSchemaFileSystemTime(dbName, tableName))
- carbonTablePath.getPath
- }
-
- private def updateMetadataByWrapperTable(
- wrapperTableInfo: org.apache.carbondata.core.metadata.schema.table.TableInfo): Unit = {
-
- CarbonMetadata.getInstance().loadTableMetadata(wrapperTableInfo)
- val carbonTable = CarbonMetadata.getInstance().getCarbonTable(
- wrapperTableInfo.getTableUniqueName)
- for (i <- metadata.tablesMeta.indices) {
- if (wrapperTableInfo.getTableUniqueName.equals(
- metadata.tablesMeta(i).carbonTableIdentifier.getTableUniqueName)) {
- metadata.tablesMeta(i).carbonTable = carbonTable
- }
- }
- }
-
- def updateMetadataByThriftTable(schemaFilePath: String,
- tableInfo: TableInfo, dbName: String, tableName: String, storePath: String): Unit = {
-
- tableInfo.getFact_table.getSchema_evolution.getSchema_evolution_history.get(0)
- .setTime_stamp(System.currentTimeMillis())
- val schemaConverter = new ThriftWrapperSchemaConverterImpl
- val wrapperTableInfo = schemaConverter
- .fromExternalToWrapperTableInfo(tableInfo, dbName, tableName, storePath)
- wrapperTableInfo
- .setMetaDataFilepath(CarbonTablePath.getFolderContainingFile(schemaFilePath))
- wrapperTableInfo.setStorePath(storePath)
- updateMetadataByWrapperTable(wrapperTableInfo)
- }
-
- /**
- * Shows all tables for given schema.
- */
- def getTables(databaseName: Option[String])(sqlContext: SQLContext): Seq[(String, Boolean)] = {
-
- val dbName =
- databaseName.getOrElse(sqlContext.asInstanceOf[HiveContext].catalog.client.currentDatabase)
- checkSchemasModifiedTimeAndReloadTables()
- metadata.tablesMeta.filter { c =>
- c.carbonTableIdentifier.getDatabaseName.equalsIgnoreCase(dbName)
- }.map { c => (c.carbonTableIdentifier.getTableName, false) }
- }
-
- def isTablePathExists(tableIdentifier: TableIdentifier)(sqlContext: SQLContext): Boolean = {
- val dbName = tableIdentifier.database.getOrElse(getDB.getDatabaseName(None, sqlContext))
- val tableName = tableIdentifier.table
-
- val tablePath = CarbonStorePath.getCarbonTablePath(this.storePath,
- new CarbonTableIdentifier(dbName, tableName, "")).getPath
-
- val fileType = FileFactory.getFileType(tablePath)
- FileFactory.isFileExist(tablePath, fileType)
- }
-
- def dropTable(tableStorePath: String, tableIdentifier: TableIdentifier)
- (sqlContext: SQLContext) {
- val dbName = tableIdentifier.database.get
- val tableName = tableIdentifier.table
-
- val metadataFilePath = CarbonStorePath.getCarbonTablePath(tableStorePath,
- new CarbonTableIdentifier(dbName, tableName, "")).getMetadataDirectoryPath
-
- val fileType = FileFactory.getFileType(metadataFilePath)
-
- if (FileFactory.isFileExist(metadataFilePath, fileType)) {
- // while drop we should refresh the schema modified time so that if any thing has changed
- // in the other beeline need to update.
- checkSchemasModifiedTimeAndReloadTables
- val file = FileFactory.getCarbonFile(metadataFilePath, fileType)
- CarbonUtil.deleteFoldersAndFilesSilent(file.getParentFile)
- val metadataToBeRemoved: Option[TableMeta] = getTableFromMetadata(dbName,
- tableIdentifier.table)
- metadataToBeRemoved match {
- case Some(tableMeta) =>
- metadata.tablesMeta -= tableMeta
- CarbonMetadata.getInstance.removeTable(dbName + "_" + tableName)
- CarbonMetadata.getInstance.removeTable(dbName + "_" + tableName)
- updateSchemasUpdatedTime(touchSchemaFileSystemTime(dbName, tableName))
- case None =>
- logInfo(s"Metadata does not contain entry for table $tableName in database $dbName")
- }
- CarbonHiveMetadataUtil.invalidateAndDropTable(dbName, tableName, sqlContext)
- // discard cached table info in cachedDataSourceTables
- sqlContext.catalog.refreshTable(tableIdentifier)
- DataMapStoreManager.getInstance().
- clearDataMap(AbsoluteTableIdentifier.from(storePath, dbName, tableName))
- }
- }
-
- private def getTimestampFileAndType(databaseName: String, tableName: String) = {
- val timestampFile = storePath + "/" + CarbonCommonConstants.SCHEMAS_MODIFIED_TIME_FILE
- val timestampFileType = FileFactory.getFileType(timestampFile)
- (timestampFile, timestampFileType)
- }
-
- /**
- * This method will put the updated timestamp of schema file in the table modified time store map
- *
- * @param timeStamp
- */
- def updateSchemasUpdatedTime(timeStamp: Long) {
- tableModifiedTimeStore.put("default", timeStamp)
- }
-
- /**
- * This method will read the timestamp of empty schema file
- *
- * @param databaseName
- * @param tableName
- * @return
- */
- def readSchemaFileSystemTime(databaseName: String, tableName: String): Long = {
- val (timestampFile, timestampFileType) = getTimestampFileAndType(databaseName, tableName)
- if (FileFactory.isFileExist(timestampFile, timestampFileType)) {
- FileFactory.getCarbonFile(timestampFile, timestampFileType).getLastModifiedTime
- } else {
- System.currentTimeMillis()
- }
- }
-
- /**
- * This method will check and create an empty schema timestamp file
- *
- * @param databaseName
- * @param tableName
- * @return
- */
- def touchSchemaFileSystemTime(databaseName: String, tableName: String): Long = {
- val (timestampFile, timestampFileType) = getTimestampFileAndType(databaseName, tableName)
- if (!FileFactory.isFileExist(timestampFile, timestampFileType)) {
- LOGGER.audit(s"Creating timestamp file for $databaseName.$tableName")
- FileFactory.createNewFile(timestampFile, timestampFileType)
- }
- val systemTime = System.currentTimeMillis()
- FileFactory.getCarbonFile(timestampFile, timestampFileType)
- .setLastModifiedTime(systemTime)
- systemTime
- }
-
- def checkSchemasModifiedTimeAndReloadTables() {
- val (timestampFile, timestampFileType) = getTimestampFileAndType("", "")
- if (FileFactory.isFileExist(timestampFile, timestampFileType)) {
- if (!(FileFactory.getCarbonFile(timestampFile, timestampFileType).
- getLastModifiedTime ==
- tableModifiedTimeStore.get(CarbonCommonConstants.DATABASE_DEFAULT_NAME))) {
- refreshCache()
- }
- }
- }
-
- def refreshCache() {
- metadata.tablesMeta = loadMetadata(storePath).tablesMeta
- }
-
- def getSchemaLastUpdatedTime(databaseName: String, tableName: String): Long = {
- var schemaLastUpdatedTime = System.currentTimeMillis
- val (timestampFile, timestampFileType) = getTimestampFileAndType(databaseName, tableName)
- if (FileFactory.isFileExist(timestampFile, timestampFileType)) {
- schemaLastUpdatedTime = FileFactory.getCarbonFile(timestampFile, timestampFileType)
- .getLastModifiedTime
- }
- schemaLastUpdatedTime
- }
-
- def createDatabaseDirectory(dbName: String) {
- val databasePath = storePath + File.separator + dbName
- val fileType = FileFactory.getFileType(databasePath)
- FileFactory.mkdirs(databasePath, fileType)
- }
-
- def dropDatabaseDirectory(dbName: String) {
- val databasePath = storePath + File.separator + dbName
- val fileType = FileFactory.getFileType(databasePath)
- if (FileFactory.isFileExist(databasePath, fileType)) {
- val dbPath = FileFactory.getCarbonFile(databasePath, fileType)
- CarbonUtil.deleteFoldersAndFiles(dbPath)
- }
- }
-
-}
-
-
-object CarbonMetastoreTypes extends RegexParsers {
- protected lazy val primitiveType: Parser[DataType] =
- "string" ^^^ StringType |
- "float" ^^^ FloatType |
- "int" ^^^ IntegerType |
- "tinyint" ^^^ ShortType |
- "short" ^^^ ShortType |
- "double" ^^^ DoubleType |
- "long" ^^^ LongType |
- "binary" ^^^ BinaryType |
- "boolean" ^^^ BooleanType |
- fixedDecimalType |
- "decimal" ^^^ "decimal" ^^^ DecimalType(18, 2) |
- "varchar\\((\\d+)\\)".r ^^^ StringType |
- "timestamp" ^^^ TimestampType |
- "date" ^^^ DateType |
- "char\\((\\d+)\\)".r ^^^ StringType
-
- protected lazy val fixedDecimalType: Parser[DataType] =
- "decimal" ~> "(" ~> "^[1-9]\\d*".r ~ ("," ~> "^[0-9]\\d*".r <~ ")") ^^ {
- case precision ~ scale =>
- DecimalType(precision.toInt, scale.toInt)
- }
-
- protected lazy val arrayType: Parser[DataType] =
- "array" ~> "<" ~> dataType <~ ">" ^^ {
- case tpe => ArrayType(tpe)
- }
-
- protected lazy val mapType: Parser[DataType] =
- "map" ~> "<" ~> dataType ~ "," ~ dataType <~ ">" ^^ {
- case t1 ~ _ ~ t2 => MapType(t1, t2)
- }
-
- protected lazy val structField: Parser[StructField] =
- "[a-zA-Z0-9_]*".r ~ ":" ~ dataType ^^ {
- case name ~ _ ~ tpe => StructField(name, tpe, nullable = true)
- }
-
- protected lazy val structType: Parser[DataType] =
- "struct" ~> "<" ~> repsep(structField, ",") <~ ">" ^^ {
- case fields => StructType(fields)
- }
-
- protected lazy val dataType: Parser[DataType] =
- arrayType |
- mapType |
- structType |
- primitiveType
-
- def toDataType(metastoreType: String): DataType = {
- parseAll(dataType, metastoreType) match {
- case Success(result, _) => result
- case failure: NoSuccess => sys.error(s"Unsupported dataType: $metastoreType")
- }
- }
-
- def toMetastoreType(dt: DataType): String = {
- dt match {
- case ArrayType(elementType, _) => s"array<${ toMetastoreType(elementType) }>"
- case StructType(fields) =>
- s"struct<${
- fields.map(f => s"${ f.name }:${ toMetastoreType(f.dataType) }")
- .mkString(",")
- }>"
- case StringType => "string"
- case FloatType => "float"
- case IntegerType => "int"
- case ShortType => "tinyint"
- case DoubleType => "double"
- case LongType => "bigint"
- case BinaryType => "binary"
- case BooleanType => "boolean"
- case DecimalType() => "decimal"
- case DateType => "date"
- case TimestampType => "timestamp"
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonSQLDialect.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonSQLDialect.scala b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonSQLDialect.scala
deleted file mode 100644
index 82c5f7f..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonSQLDialect.scala
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.hive
-
-import org.apache.spark.sql.CarbonSqlParser
-import org.apache.spark.sql.catalyst.ParserDialect
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
-
-private[spark] class CarbonSQLDialect(hiveContext: HiveContext) extends ParserDialect {
-
- @transient
- protected val sqlParser = new CarbonSqlParser
-
- override def parse(sqlText: String): LogicalPlan = {
-
- try {
- sqlParser.parse(sqlText)
- } catch {
- // MalformedCarbonCommandException need to throw directly
- // because hive can no parse carbon command
- case ce: MalformedCarbonCommandException =>
- throw ce
- case _: Throwable =>
- HiveQl.parseSql(sqlText)
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonStrategies.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonStrategies.scala b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonStrategies.scala
deleted file mode 100644
index d3d699a..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonStrategies.scala
+++ /dev/null
@@ -1,370 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.hive
-
-import java.util
-
-import scala.collection.JavaConverters._
-
-import org.apache.spark.sql._
-import org.apache.spark.sql.catalyst.CarbonTableIdentifierImplicit._
-import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation
-import org.apache.spark.sql.catalyst.expressions
-import org.apache.spark.sql.catalyst.expressions.{Alias, AttributeReference, AttributeSet, ScalaUDF, _}
-import org.apache.spark.sql.catalyst.planning.{PhysicalOperation, QueryPlanner}
-import org.apache.spark.sql.catalyst.plans.logical.{Filter => LogicalFilter, LogicalPlan}
-import org.apache.spark.sql.execution.{ExecutedCommand, Filter, Project, SparkPlan}
-import org.apache.spark.sql.execution.command._
-import org.apache.spark.sql.execution.datasources.{DescribeCommand => LogicalDescribeCommand, LogicalRelation}
-import org.apache.spark.sql.hive.execution.{DropTable, HiveNativeCommand}
-import org.apache.spark.sql.hive.execution.command._
-import org.apache.spark.sql.optimizer.CarbonDecoderRelation
-import org.apache.spark.sql.types.{IntegerType, StringType}
-
-import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.spark.CarbonAliasDecoderRelation
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
-
-class CarbonStrategies(sqlContext: SQLContext) extends QueryPlanner[SparkPlan] {
-
- override def strategies: Seq[Strategy] = getStrategies
-
- val LOGGER = LogServiceFactory.getLogService("CarbonStrategies")
-
- def getStrategies: Seq[Strategy] = {
- val total = sqlContext.planner.strategies :+ CarbonTableScan
- total
- }
-
- /**
- * Carbon strategies for performing late materizlization (decoding dictionary key
- * as late as possbile)
- */
- private[sql] object CarbonTableScan extends Strategy {
-
- def apply(plan: LogicalPlan): Seq[SparkPlan] = {
- plan match {
- case PhysicalOperation(projectList, predicates, l: LogicalRelation)
- if l.relation.isInstanceOf[CarbonDatasourceRelation] =>
- if (isStarQuery(plan)) {
- carbonRawScanForStarQuery(projectList, predicates, l)(sqlContext) :: Nil
- } else {
- carbonRawScan(projectList, predicates, l)(sqlContext) :: Nil
- }
- case InsertIntoCarbonTable(relation: CarbonDatasourceRelation,
- _, child: LogicalPlan, overwrite, _) =>
- ExecutedCommand(LoadTableByInsert(relation, child, overwrite)) :: Nil
- case CarbonDictionaryCatalystDecoder(relations, profile, aliasMap, _, child) =>
- CarbonDictionaryDecoder(relations,
- profile,
- aliasMap,
- planLater(child))(sqlContext) :: Nil
- case _ =>
- Nil
- }
- }
-
- /**
- * Create carbon scan
- */
- private def carbonRawScan(projectListRaw: Seq[NamedExpression],
- predicates: Seq[Expression],
- logicalRelation: LogicalRelation)(sc: SQLContext): SparkPlan = {
-
- val relation = logicalRelation.relation.asInstanceOf[CarbonDatasourceRelation]
- val tableName: String =
- relation.carbonRelation.metaData.carbonTable.getFactTableName.toLowerCase
- // Check out any expressions are there in project list. if they are present then we need to
- // decode them as well.
-
- val projectList = projectListRaw.map {p =>
- p.transform {
- case CustomDeterministicExpression(exp) => exp
- }
- }.asInstanceOf[Seq[NamedExpression]]
- val newProjectList = projectList.map {
- case a@Alias(s: ScalaUDF, name)
- if name.equalsIgnoreCase(CarbonCommonConstants.POSITION_ID) ||
- name.equalsIgnoreCase(
- CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID) =>
- AttributeReference(name, StringType, true)().withExprId(a.exprId)
- case a@Alias(s: ScalaUDF, name)
- if name.equalsIgnoreCase(CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_SEGMENTID) =>
- val reference =
- AttributeReference(CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID,
- StringType, true)().withExprId(a.exprId)
- val alias = a.transform {
- case s: ScalaUDF =>
- ScalaUDF(s.function, s.dataType, Seq(reference), s.inputTypes)
- }.asInstanceOf[Alias]
- Alias(alias.child, alias.name)(alias.exprId, alias.qualifiers, alias.explicitMetadata)
- case other => other
- }
- val projectSet = AttributeSet(newProjectList.flatMap(_.references))
- val filterSet = AttributeSet(predicates.flatMap(_.references))
- val scan = CarbonScan(projectSet.toSeq,
- relation.carbonRelation,
- predicates)(sqlContext)
- newProjectList.map {
- case attr: AttributeReference =>
- case Alias(attr: AttributeReference, _) =>
- case others =>
- others.references.map { f =>
- val dictionary = relation.carbonRelation.metaData.dictionaryMap.get(f.name)
- if (dictionary.isDefined && dictionary.get) {
- scan.attributesNeedToDecode.add(f.asInstanceOf[AttributeReference])
- }
- }
- }
- val scanWithDecoder =
- if (scan.attributesNeedToDecode.size() > 0) {
- val decoder = getCarbonDecoder(logicalRelation,
- sc,
- tableName,
- scan.attributesNeedToDecode.asScala.toSeq,
- scan)
- if (scan.unprocessedExprs.nonEmpty) {
- val filterCondToAdd = scan.unprocessedExprs.reduceLeftOption(expressions.And)
- filterCondToAdd.map(Filter(_, decoder)).getOrElse(decoder)
- } else {
- decoder
- }
- } else {
- scan
- }
-
- if (projectList.map(_.toAttribute) == scan.columnProjection &&
- projectSet.size == projectList.size &&
- filterSet.subsetOf(projectSet)) {
- // copied from spark pruneFilterProjectRaw
- // When it is possible to just use column pruning to get the right projection and
- // when the columns of this projection are enough to evaluate all filter conditions,
- // just do a scan with no extra project.
- scanWithDecoder
- } else {
- Project(newProjectList, scanWithDecoder)
- }
- }
-
- /**
- * Create carbon scan for star query
- */
- private def carbonRawScanForStarQuery(projectList: Seq[NamedExpression],
- predicates: Seq[Expression],
- logicalRelation: LogicalRelation)(sc: SQLContext): SparkPlan = {
- val relation = logicalRelation.relation.asInstanceOf[CarbonDatasourceRelation]
- val tableName: String =
- relation.carbonRelation.metaData.carbonTable.getFactTableName.toLowerCase
- // Check out any expressions are there in project list. if they are present then we need to
- // decode them as well.
- val projectExprsNeedToDecode = new java.util.HashSet[Attribute]()
- val scan = CarbonScan(projectList.map(_.toAttribute),
- relation.carbonRelation,
- predicates,
- useUnsafeCoversion = false)(sqlContext)
- projectExprsNeedToDecode.addAll(scan.attributesNeedToDecode)
- val updatedAttrs = scan.columnProjection.map(attr =>
- updateDataType(attr.asInstanceOf[AttributeReference], relation, projectExprsNeedToDecode))
- scan.columnProjection = updatedAttrs
- if (projectExprsNeedToDecode.size() > 0
- && isDictionaryEncoded(projectExprsNeedToDecode.asScala.toSeq, relation)) {
- val decoder = getCarbonDecoder(logicalRelation,
- sc,
- tableName,
- projectExprsNeedToDecode.asScala.toSeq,
- scan)
- if (scan.unprocessedExprs.nonEmpty) {
- val filterCondToAdd = scan.unprocessedExprs.reduceLeftOption(expressions.And)
- filterCondToAdd.map(Filter(_, decoder)).getOrElse(decoder)
- } else {
- decoder
- }
- } else {
- if (scan.unprocessedExprs.nonEmpty) {
- val filterCondToAdd = scan.unprocessedExprs.reduceLeftOption(expressions.And)
- filterCondToAdd.map(Filter(_, scan)).getOrElse(scan)
- } else {
- scan
- }
- }
- }
-
- def getCarbonDecoder(logicalRelation: LogicalRelation,
- sc: SQLContext,
- tableName: String,
- projectExprsNeedToDecode: Seq[Attribute],
- scan: CarbonScan): CarbonDictionaryDecoder = {
- val relation = CarbonDecoderRelation(logicalRelation.attributeMap,
- logicalRelation.relation.asInstanceOf[CarbonDatasourceRelation])
- val attrs = projectExprsNeedToDecode.map { attr =>
- val newAttr = AttributeReference(attr.name,
- attr.dataType,
- attr.nullable,
- attr.metadata)(attr.exprId, Seq(tableName))
- relation.addAttribute(newAttr)
- newAttr
- }
- CarbonDictionaryDecoder(Seq(relation), IncludeProfile(attrs),
- CarbonAliasDecoderRelation(), scan)(sc)
- }
-
- def isDictionaryEncoded(projectExprsNeedToDecode: Seq[Attribute],
- relation: CarbonDatasourceRelation): Boolean = {
- var isEncoded = false
- projectExprsNeedToDecode.foreach { attr =>
- if (relation.carbonRelation.metaData.dictionaryMap.get(attr.name).getOrElse(false)) {
- isEncoded = true
- }
- }
- isEncoded
- }
-
- def updateDataType(attr: AttributeReference,
- relation: CarbonDatasourceRelation,
- allAttrsNotDecode: util.Set[Attribute]): AttributeReference = {
- if (relation.carbonRelation.metaData.dictionaryMap.get(attr.name).getOrElse(false) &&
- !allAttrsNotDecode.asScala.exists(p => p.name.equals(attr.name))) {
- AttributeReference(attr.name,
- IntegerType,
- attr.nullable,
- attr.metadata)(attr.exprId, attr.qualifiers)
- } else {
- attr
- }
- }
-
- private def isStarQuery(plan: LogicalPlan) = {
- plan match {
- case LogicalFilter(condition, l: LogicalRelation)
- if l.relation.isInstanceOf[CarbonDatasourceRelation] =>
- true
- case l: LogicalRelation if l.relation.isInstanceOf[CarbonDatasourceRelation] => true
- case _ => false
- }
- }
- }
-
- object DDLStrategies extends Strategy {
- def apply(plan: LogicalPlan): Seq[SparkPlan] = plan match {
- case DropTable(tableName, ifNotExists)
- if CarbonEnv.get.carbonMetastore
- .isTablePathExists(toTableIdentifier(tableName.toLowerCase))(sqlContext) =>
- val identifier = toTableIdentifier(tableName.toLowerCase)
- ExecutedCommand(DropTableCommand(ifNotExists, identifier.database, identifier.table)) :: Nil
- case ShowLoadsCommand(databaseName, table, limit) =>
- ExecutedCommand(ShowLoads(databaseName, table, limit, plan.output)) :: Nil
- case LoadTable(databaseNameOp, tableName, factPathFromUser, dimFilesPath,
- options, isOverwriteExist, inputSqlString, dataFrame, _) =>
- val isCarbonTable = CarbonEnv.get.carbonMetastore
- .tableExists(TableIdentifier(tableName, databaseNameOp))(sqlContext)
- if (isCarbonTable || options.nonEmpty) {
- ExecutedCommand(LoadTable(databaseNameOp, tableName, factPathFromUser, dimFilesPath,
- options, isOverwriteExist, inputSqlString, dataFrame)) :: Nil
- } else {
- ExecutedCommand(HiveNativeCommand(inputSqlString)) :: Nil
- }
- case alterTable@AlterTableCompaction(altertablemodel) =>
- val isCarbonTable = CarbonEnv.get.carbonMetastore
- .tableExists(TableIdentifier(altertablemodel.tableName,
- altertablemodel.dbName))(sqlContext)
- if (isCarbonTable) {
- if (altertablemodel.compactionType.equalsIgnoreCase("minor") ||
- altertablemodel.compactionType.equalsIgnoreCase("major")) {
- ExecutedCommand(alterTable) :: Nil
- } else {
- throw new MalformedCarbonCommandException(
- "Unsupported alter operation on carbon table")
- }
- } else {
- ExecutedCommand(HiveNativeCommand(altertablemodel.alterSql)) :: Nil
- }
- case CreateDatabase(dbName, sql) =>
- ExecutedCommand(CreateDatabaseCommand(dbName, HiveNativeCommand(sql))) :: Nil
- case DropDatabase(dbName, isCascade, sql) =>
- if (isCascade) {
- ExecutedCommand(DropDatabaseCascadeCommand(dbName, HiveNativeCommand(sql))) :: Nil
- } else {
- ExecutedCommand(DropDatabaseCommand(dbName, HiveNativeCommand(sql))) :: Nil
- }
- case UseDatabase(sql) =>
- ExecutedCommand(HiveNativeCommand(sql)) :: Nil
- case d: HiveNativeCommand =>
- try {
- val resolvedTable = sqlContext.executePlan(CarbonHiveSyntax.parse(d.sql)).optimizedPlan
- planLater(resolvedTable) :: Nil
- } catch {
- case ce: MalformedCarbonCommandException =>
- throw ce
- case ae: AnalysisException =>
- throw ae
- case e: Exception => ExecutedCommand(d) :: Nil
- }
- case DescribeFormattedCommand(sql, tblIdentifier) =>
- val isTable = CarbonEnv.get.carbonMetastore
- .tableExists(tblIdentifier)(sqlContext)
- if (isTable) {
- val describe =
- LogicalDescribeCommand(UnresolvedRelation(tblIdentifier, None), isExtended = false)
- val resolvedTable = sqlContext.executePlan(describe.table).analyzed
- val resultPlan = sqlContext.executePlan(resolvedTable).executedPlan
- ExecutedCommand(DescribeCommandFormatted(resultPlan, plan.output, tblIdentifier)) :: Nil
- } else {
- ExecutedCommand(HiveNativeCommand(sql)) :: Nil
- }
- case ShowPartitions(t) =>
- val isCarbonTable = CarbonEnv.get.carbonMetastore
- .tableExists(t)(sqlContext)
- if (isCarbonTable) {
- ExecutedCommand(ShowCarbonPartitionsCommand(t)) :: Nil
- } else {
- var tableName = t.table
- var database = t.database
- var sql: String = null
- if (database.isEmpty) {
- sql = s"show partitions $tableName"
- } else {
- sql = s"show partitions $database.$tableName"
- }
- ExecutedCommand(HiveNativeCommand(sql)) :: Nil
- }
- case _ =>
- Nil
- }
-
- def toTableIdentifier(name: String): TableIdentifier = {
- val identifier = name.split("\\.")
- identifier match {
- case Array(tableName) => TableIdentifier(tableName, None)
- case Array(dbName, tableName) => TableIdentifier(tableName, Some(dbName))
- }
- }
- }
-
-}
-
-object CarbonHiveSyntax {
-
- @transient
- protected val sqlParser = new CarbonSqlParser
-
- def parse(sqlText: String): LogicalPlan = {
- sqlParser.parse(sqlText)
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/main/scala/org/apache/spark/sql/hive/HiveQlWrapper.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/hive/HiveQlWrapper.scala b/integration/spark/src/main/scala/org/apache/spark/sql/hive/HiveQlWrapper.scala
deleted file mode 100644
index 6b244f9..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/hive/HiveQlWrapper.scala
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.hive
-
-import org.apache.hadoop.hive.ql.parse.ASTNode
-
-/**
- * Wrapper class for using the hiveQl class of hive.
- */
-object HiveQlWrapper {
-
- def getAst(sql: String): ASTNode = {
-
- HiveQl.getAst(sql)
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/main/scala/org/apache/spark/sql/hive/cli/CarbonSQLCLIDriver.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/hive/cli/CarbonSQLCLIDriver.scala b/integration/spark/src/main/scala/org/apache/spark/sql/hive/cli/CarbonSQLCLIDriver.scala
deleted file mode 100644
index ad70027..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/hive/cli/CarbonSQLCLIDriver.scala
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.sql.hive.cli
-
-import java.io.File
-
-import scala.collection.JavaConverters._
-
-import org.apache.spark.{SparkConf, SparkContext}
-import org.apache.spark.scheduler.StatsReportListener
-import org.apache.spark.sql.CarbonContext
-import org.apache.spark.sql.hive.HiveContext
-import org.apache.spark.sql.hive.thriftserver.{SparkSQLCLIDriver, SparkSQLEnv}
-import org.apache.spark.util.Utils
-
-import org.apache.carbondata.common.logging.LogServiceFactory
-
-object CarbonSQLCLIDriver {
-
- private val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
- var hiveContext: HiveContext = _
- var sparkContext: SparkContext = _
-
- def main(args: Array[String]): Unit = {
- init()
- SparkSQLEnv.sparkContext = sparkContext
- SparkSQLEnv.hiveContext = hiveContext
- SparkSQLCLIDriver.installSignalHandler()
- SparkSQLCLIDriver.main(args)
- }
-
- def init() {
- if (hiveContext == null) {
- val sparkConf = new SparkConf(loadDefaults = true)
- val maybeSerializer = sparkConf.getOption("spark.serializer")
- val maybeKryoReferenceTracking = sparkConf.getOption("spark.kryo.referenceTracking")
- // If user doesn't specify the appName, we want to get [SparkSQL::localHostName] instead of
- // the default appName [CarbonSQLCLIDriver] in cli or beeline.
- val maybeAppName = sparkConf
- .getOption("spark.app.name")
- .filterNot(_ == classOf[SparkSQLCLIDriver].getName)
- val maybeStorePath = sparkConf.getOption("spark.carbon.storepath")
-
- sparkConf
- .setAppName(maybeAppName.getOrElse(s"CarbonSparkSQL::${ Utils.localHostName() }"))
- .set(
- "spark.serializer",
- maybeSerializer.getOrElse("org.apache.spark.serializer.KryoSerializer"))
- .set(
- "spark.kryo.referenceTracking",
- maybeKryoReferenceTracking.getOrElse("false"))
-
- sparkContext = new SparkContext(sparkConf)
- sparkContext.addSparkListener(new StatsReportListener())
- val path = System.getenv("CARBON_HOME") + "/bin/carbonsqlclistore"
- val store = new File(path)
- store.mkdirs()
- hiveContext = new CarbonContext(sparkContext,
- maybeStorePath.getOrElse(store.getCanonicalPath),
- store.getCanonicalPath)
-
- hiveContext.setConf("spark.sql.hive.version", HiveContext.hiveExecutionVersion)
- hiveContext.hiveconf.getAllProperties.asScala.toSeq.sorted.foreach { case (k, v) =>
- LOGGER.debug(s"HiveConf var: $k=$v")
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/main/scala/org/apache/spark/sql/hive/execution/command/CarbonHiveCommands.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/hive/execution/command/CarbonHiveCommands.scala b/integration/spark/src/main/scala/org/apache/spark/sql/hive/execution/command/CarbonHiveCommands.scala
deleted file mode 100644
index 0f42940..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/hive/execution/command/CarbonHiveCommands.scala
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.hive.execution.command
-
-import org.apache.spark.sql._
-import org.apache.spark.sql.execution.RunnableCommand
-import org.apache.spark.sql.execution.command.DropTableCommand
-import org.apache.spark.sql.hive.execution.HiveNativeCommand
-
-private[hive] case class CreateDatabaseCommand(dbName: String,
- command: HiveNativeCommand) extends RunnableCommand {
- def run(sqlContext: SQLContext): Seq[Row] = {
- val rows = command.run(sqlContext)
- CarbonEnv.get.carbonMetastore.createDatabaseDirectory(dbName)
- rows
- }
-}
-
-private[hive] case class DropDatabaseCommand(dbName: String,
- command: HiveNativeCommand) extends RunnableCommand {
- def run(sqlContext: SQLContext): Seq[Row] = {
- val rows = command.run(sqlContext)
- CarbonEnv.get.carbonMetastore.dropDatabaseDirectory(dbName)
- rows
- }
-}
-
-private[hive] case class DropDatabaseCascadeCommand(dbName: String,
- command: HiveNativeCommand) extends RunnableCommand {
- def run(sqlContext: SQLContext): Seq[Row] = {
- val tablesInDB = CarbonEnv.get.carbonMetastore
- .getTables(Some(dbName))(sqlContext).map(x => x._1)
- val rows = command.run(sqlContext)
- tablesInDB.foreach{tableName =>
- DropTableCommand(true, Some(dbName), tableName).run(sqlContext)
- }
- CarbonEnv.get.carbonMetastore.dropDatabaseDirectory(dbName)
- rows
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0bf597d9/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala b/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
deleted file mode 100644
index 2e45954..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
+++ /dev/null
@@ -1,431 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.spark
-
-import scala.collection.mutable.ArrayBuffer
-
-import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression, Literal, StartsWith, _}
-import org.apache.spark.sql.optimizer.AttributeReferenceWrapper
-import org.apache.spark.sql.sources
-import org.apache.spark.sql.types._
-
-import org.apache.carbondata.core.metadata.datatype.{DataTypes => CarbonDataTypes}
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn
-import org.apache.carbondata.core.scan.expression.{ColumnExpression => CarbonColumnExpression, Expression => CarbonExpression, LiteralExpression => CarbonLiteralExpression}
-import org.apache.carbondata.core.scan.expression.conditional.{GreaterThanEqualToExpression, LessThanExpression, _}
-import org.apache.carbondata.core.scan.expression.logical.{AndExpression, FalseExpression, OrExpression}
-import org.apache.carbondata.spark.util.CarbonScalaUtil
-
-/**
- * All filter conversions are done here.
- */
-object CarbonFilters {
-
- /**
- * Converts data sources filters to carbon filter predicates.
- */
- def createCarbonFilter(schema: StructType,
- predicate: sources.Filter): Option[CarbonExpression] = {
- val dataTypeOf = schema.map(f => f.name -> f.dataType).toMap
-
- def createFilter(predicate: sources.Filter): Option[CarbonExpression] = {
- predicate match {
-
- case sources.EqualTo(name, value) =>
- Some(new EqualToExpression(getCarbonExpression(name),
- getCarbonLiteralExpression(name, value)))
- case sources.Not(sources.EqualTo(name, value)) =>
- Some(new NotEqualsExpression(getCarbonExpression(name),
- getCarbonLiteralExpression(name, value)))
-
- case sources.EqualNullSafe(name, value) =>
- Some(new EqualToExpression(getCarbonExpression(name),
- getCarbonLiteralExpression(name, value)))
- case sources.Not(sources.EqualNullSafe(name, value)) =>
- Some(new NotEqualsExpression(getCarbonExpression(name),
- getCarbonLiteralExpression(name, value)))
-
- case sources.GreaterThan(name, value) =>
- Some(new GreaterThanExpression(getCarbonExpression(name),
- getCarbonLiteralExpression(name, value)))
- case sources.LessThan(name, value) =>
- Some(new LessThanExpression(getCarbonExpression(name),
- getCarbonLiteralExpression(name, value)))
- case sources.GreaterThanOrEqual(name, value) =>
- Some(new GreaterThanEqualToExpression(getCarbonExpression(name),
- getCarbonLiteralExpression(name, value)))
- case sources.LessThanOrEqual(name, value) =>
- Some(new LessThanEqualToExpression(getCarbonExpression(name),
- getCarbonLiteralExpression(name, value)))
-
- case sources.In(name, values) =>
- Some(new InExpression(getCarbonExpression(name),
- new ListExpression(
- convertToJavaList(values.map(f => getCarbonLiteralExpression(name, f)).toList))))
- case sources.Not(sources.In(name, values)) =>
- Some(new NotInExpression(getCarbonExpression(name),
- new ListExpression(
- convertToJavaList(values.map(f => getCarbonLiteralExpression(name, f)).toList))))
-
- case sources.IsNull(name) =>
- Some(new EqualToExpression(getCarbonExpression(name),
- getCarbonLiteralExpression(name, null), true))
- case sources.IsNotNull(name) =>
- Some(new NotEqualsExpression(getCarbonExpression(name),
- getCarbonLiteralExpression(name, null), true))
-
- case sources.And(lhs, rhs) =>
- (createFilter(lhs) ++ createFilter(rhs)).reduceOption(new AndExpression(_, _))
-
- case sources.Or(lhs, rhs) =>
- for {
- lhsFilter <- createFilter(lhs)
- rhsFilter <- createFilter(rhs)
- } yield {
- new OrExpression(lhsFilter, rhsFilter)
- }
- case sources.StringStartsWith(name, value) if value.length > 0 =>
- val l = new GreaterThanEqualToExpression(getCarbonExpression(name),
- getCarbonLiteralExpression(name, value))
- val maxValueLimit = value.substring(0, value.length - 1) +
- (value.charAt(value.length - 1).toInt + 1).toChar
- val r = new LessThanExpression(
- getCarbonExpression(name), getCarbonLiteralExpression(name, maxValueLimit))
- Some(new AndExpression(l, r))
- case _ => None
- }
- }
-
- def getCarbonExpression(name: String) = {
- new CarbonColumnExpression(name,
- CarbonScalaUtil.convertSparkToCarbonDataType(dataTypeOf(name)))
- }
-
- def getCarbonLiteralExpression(name: String, value: Any): CarbonExpression = {
- val dataTypeOfAttribute = CarbonScalaUtil.convertSparkToCarbonDataType(dataTypeOf(name))
- val dataType = if (Option(value).isDefined
- && dataTypeOfAttribute == CarbonDataTypes.STRING
- && value.isInstanceOf[Double]) {
- CarbonDataTypes.DOUBLE
- } else {
- dataTypeOfAttribute
- }
- new CarbonLiteralExpression(value, dataType)
- }
-
- createFilter(predicate)
- }
-
-
- // Check out which filters can be pushed down to carbon, remaining can be handled in spark layer.
- // Mostly dimension filters are only pushed down since it is faster in carbon.
- // TODO - The Filters are first converted Intermediate sources filters expression and then these
- // expressions are again converted back to CarbonExpression. Instead of two step process of
- // evaluating the filters it can be merged into a single one.
- def selectFilters(filters: Seq[Expression],
- attrList: java.util.HashSet[AttributeReferenceWrapper],
- aliasMap: CarbonAliasDecoderRelation): Unit = {
- def translate(expr: Expression, or: Boolean = false): Option[sources.Filter] = {
- expr match {
- case or@ Or(left, right) =>
-
- val leftFilter = translate(left, or = true)
- val rightFilter = translate(right, or = true)
- if (leftFilter.isDefined && rightFilter.isDefined) {
- Some( sources.Or(leftFilter.get, rightFilter.get))
- } else {
- or.collect {
- case attr: AttributeReference =>
- attrList.add(AttributeReferenceWrapper(aliasMap.getOrElse(attr, attr)))
- }
- None
- }
-
- case And(left, right) =>
- val leftFilter = translate(left, or)
- val rightFilter = translate(right, or)
- if (or) {
- if (leftFilter.isDefined && rightFilter.isDefined) {
- (leftFilter ++ rightFilter).reduceOption(sources.And)
- } else {
- None
- }
- } else {
- (leftFilter ++ rightFilter).reduceOption(sources.And)
- }
-
- case EqualTo(a: Attribute, Literal(v, t)) =>
- Some(sources.EqualTo(a.name, v))
- case EqualTo(l@Literal(v, t), a: Attribute) =>
- Some(sources.EqualTo(a.name, v))
- case Not(EqualTo(a: Attribute, Literal(v, t))) =>
- Some(sources.Not(sources.EqualTo(a.name, v)))
- case Not(EqualTo(Literal(v, t), a: Attribute)) =>
- Some(sources.Not(sources.EqualTo(a.name, v)))
- case IsNotNull(a: Attribute) =>
- Some(sources.IsNotNull(a.name))
- case IsNull(a: Attribute) =>
- Some(sources.IsNull(a.name))
- case Not(In(a: Attribute, list)) if !list.exists(!_.isInstanceOf[Literal]) =>
- val hSet = list.map(e => e.eval(EmptyRow))
- Some(sources.Not(sources.In(a.name, hSet.toArray)))
- case In(a: Attribute, list) if !list.exists(!_.isInstanceOf[Literal]) =>
- val hSet = list.map(e => e.eval(EmptyRow))
- Some(sources.In(a.name, hSet.toArray))
- case GreaterThan(a: Attribute, Literal(v, t)) =>
- Some(sources.GreaterThan(a.name, v))
- case GreaterThan(Literal(v, t), a: Attribute) =>
- Some(sources.LessThan(a.name, v))
- case LessThan(a: Attribute, Literal(v, t)) =>
- Some(sources.LessThan(a.name, v))
- case LessThan(Literal(v, t), a: Attribute) =>
- Some(sources.GreaterThan(a.name, v))
- case GreaterThanOrEqual(a: Attribute, Literal(v, t)) =>
- Some(sources.GreaterThanOrEqual(a.name, v))
- case GreaterThanOrEqual(Literal(v, t), a: Attribute) =>
- Some(sources.LessThanOrEqual(a.name, v))
- case LessThanOrEqual(a: Attribute, Literal(v, t)) =>
- Some(sources.LessThanOrEqual(a.name, v))
- case LessThanOrEqual(Literal(v, t), a: Attribute) =>
- Some(sources.GreaterThanOrEqual(a.name, v))
- case StartsWith(a: Attribute, Literal(v, t)) =>
- Some(sources.StringStartsWith(a.name, v.toString))
-
- case others =>
- if (!or) {
- others.collect {
- case attr: AttributeReference =>
- attrList.add(AttributeReferenceWrapper(aliasMap.getOrElse(attr, attr)))
- }
- }
- None
- }
- }
- filters.flatMap(translate(_, false)).toArray
- }
-
- def isCarbonSupportedDataTypes(expr: Expression): Boolean = {
- expr.dataType match {
- case StringType => true
- case IntegerType => true
- case LongType => true
- case DoubleType => true
- case FloatType => true
- case BooleanType => true
- case TimestampType => true
- case ArrayType(_, _) => true
- case StructType(_) => true
- case DecimalType() => true
- case _ => false
- }
- }
-
- def processExpression(exprs: Seq[Expression],
- attributesNeedToDecode: java.util.HashSet[AttributeReference],
- unprocessedExprs: ArrayBuffer[Expression],
- carbonTable: CarbonTable): Option[CarbonExpression] = {
- def transformExpression(expr: Expression, or: Boolean = false): Option[CarbonExpression] = {
- expr match {
- case orFilter@ Or(left, right)
- if (isCarbonSupportedDataTypes(left) && isCarbonSupportedDataTypes(right)) =>
- val leftFilter = transformExpression(left, or = true)
- val rightFilter = transformExpression(right, or = true)
- if (leftFilter.isDefined && rightFilter.isDefined) {
- Some(new OrExpression(leftFilter.get, rightFilter.get))
- } else {
- if (!or) {
- orFilter.collect {
- case attr: AttributeReference => attributesNeedToDecode.add(attr)
- }
- unprocessedExprs += orFilter
- }
- None
- }
-
- case And(left, right) if (isCarbonSupportedDataTypes(left) &&
- isCarbonSupportedDataTypes(right)) =>
- val leftFilter = transformExpression(left, or)
- val rightFilter = transformExpression(right, or)
- if (or) {
- if (leftFilter.isDefined && rightFilter.isDefined) {
- (leftFilter ++ rightFilter).reduceOption(new AndExpression(_, _))
- } else {
- None
- }
- } else {
- (leftFilter ++ rightFilter).reduceOption(new AndExpression(_, _))
- }
-
-
- case EqualTo(a: Attribute, l@Literal(v, t)) if (isCarbonSupportedDataTypes(a) &&
- isCarbonSupportedDataTypes(l)) =>
- Some(
- new EqualToExpression(
- transformExpression(a).get,
- transformExpression(l).get
- )
- )
- case EqualTo(l@Literal(v, t), a: Attribute) if (isCarbonSupportedDataTypes(l) &&
- isCarbonSupportedDataTypes(a)) =>
- Some(
- new EqualToExpression(
- transformExpression(a).get,
- transformExpression(l).get
- )
- )
-
- case Not(EqualTo(a: Attribute, l@Literal(v, t))) if (isCarbonSupportedDataTypes(a) &&
- isCarbonSupportedDataTypes(l)) =>
- Some(
- new NotEqualsExpression(
- transformExpression(a).get,
- transformExpression(l).get
- )
- )
- case Not(EqualTo(l@Literal(v, t), a: Attribute)) if (isCarbonSupportedDataTypes(l) &&
- isCarbonSupportedDataTypes(a)) =>
- Some(
- new NotEqualsExpression(
- transformExpression(a).get,
- transformExpression(l).get
- )
- )
- case IsNotNull(child: Attribute) if (isCarbonSupportedDataTypes(child)) =>
- Some(new NotEqualsExpression(transformExpression(child).get,
- transformExpression(Literal(null)).get, true))
- case IsNull(child: Attribute) if (isCarbonSupportedDataTypes(child)) =>
- Some(new EqualToExpression(transformExpression(child).get,
- transformExpression(Literal(null)).get, true))
- case Not(In(a: Attribute, list))
- if !list.exists(!_.isInstanceOf[Literal]) && isCarbonSupportedDataTypes(a) =>
- if (list.exists(x => isNullLiteral(x.asInstanceOf[Literal]))) {
- Some(new FalseExpression(transformExpression(a).get))
- } else {
- Some(new NotInExpression(transformExpression(a).get,
- new ListExpression(convertToJavaList(list.map(transformExpression(_).get)))))
- }
- case In(a: Attribute, list) if !list.exists(!_.isInstanceOf[Literal]) &&
- isCarbonSupportedDataTypes(a) =>
- Some(new InExpression(transformExpression(a).get,
- new ListExpression(convertToJavaList(list
- .map(transformExpression(_).get)))))
-
- case GreaterThan(a: Attribute, l@Literal(v, t))
- if (isCarbonSupportedDataTypes(a) && isCarbonSupportedDataTypes(l)) =>
- Some(new GreaterThanExpression(transformExpression(a).get, transformExpression(l).get))
- case GreaterThan(l@Literal(v, t), a: Attribute)
- if (isCarbonSupportedDataTypes(l) && isCarbonSupportedDataTypes(a)) =>
- Some(new LessThanExpression(transformExpression(a).get, transformExpression(l).get))
-
- case LessThan(a: Attribute, l@Literal(v, t))
- if (isCarbonSupportedDataTypes(a) && isCarbonSupportedDataTypes(l)) =>
- Some(new LessThanExpression(transformExpression(a).get, transformExpression(l).get))
- case LessThan(l@Literal(v, t), a: Attribute)
- if (isCarbonSupportedDataTypes(l) && isCarbonSupportedDataTypes(a)) =>
- Some(new GreaterThanExpression(transformExpression(a).get, transformExpression(l).get))
-
- case GreaterThanOrEqual(a: Attribute, l@Literal(v, t))
- if (isCarbonSupportedDataTypes(a) && isCarbonSupportedDataTypes(l)) =>
- Some(new GreaterThanEqualToExpression(transformExpression(a).get,
- transformExpression(l).get))
- case GreaterThanOrEqual(l@Literal(v, t), a: Attribute)
- if (isCarbonSupportedDataTypes(l) && isCarbonSupportedDataTypes(a)) =>
- Some(new LessThanEqualToExpression(transformExpression(a).get,
- transformExpression(l).get))
-
- case LessThanOrEqual(a: Attribute, l@Literal(v, t))
- if (isCarbonSupportedDataTypes(a) && isCarbonSupportedDataTypes(l)) =>
- Some(new LessThanEqualToExpression(transformExpression(a).get,
- transformExpression(l).get))
- case LessThanOrEqual(l@Literal(v, t), a: Attribute)
- if (isCarbonSupportedDataTypes(l) && isCarbonSupportedDataTypes(a)) =>
- Some(new GreaterThanEqualToExpression(transformExpression(a).get,
- transformExpression(l).get))
-
- case AttributeReference(name, dataType, _, _) =>
- Some(new CarbonColumnExpression(name,
- CarbonScalaUtil.convertSparkToCarbonDataType(
- getActualCarbonDataType(name, carbonTable))))
- case Literal(name, dataType) => Some(new
- CarbonLiteralExpression(name, CarbonScalaUtil.convertSparkToCarbonDataType(dataType)))
- case StartsWith(left : Attribute, right@Literal(pattern, dataType)) if
- pattern.toString.size > 0 &&
- isCarbonSupportedDataTypes
- (left) &&
- isCarbonSupportedDataTypes
- (right) =>
- val l = new GreaterThanEqualToExpression(transformExpression(left).get,
- transformExpression(right).get)
- val maxValueLimit = pattern.toString.substring(0, pattern.toString.length - 1) +
- (pattern.toString.charAt(pattern.toString.length - 1).toInt + 1)
- .toChar
- val r = new LessThanExpression(
- transformExpression(left).get,
- new CarbonLiteralExpression(maxValueLimit,
- CarbonScalaUtil.convertSparkToCarbonDataType(dataType)))
- Some(new AndExpression(l, r))
- case others =>
- if (!or) {
- others.collect {
- case attr: AttributeReference => attributesNeedToDecode.add(attr)
- }
- unprocessedExprs += others
- }
- None
- }
- }
- exprs.flatMap(transformExpression(_, false)).reduceOption(new AndExpression(_, _))
- }
- private def isNullLiteral(exp: Expression): Boolean = {
- if (null != exp
- && exp.isInstanceOf[Literal]
- && (exp.asInstanceOf[Literal].dataType == org.apache.spark.sql.types.DataTypes.NullType)
- || (exp.asInstanceOf[Literal].value == null)) {
- true
- } else {
- false
- }
- }
- private def getActualCarbonDataType(column: String, carbonTable: CarbonTable) = {
- var carbonColumn: CarbonColumn =
- carbonTable.getDimensionByName(carbonTable.getFactTableName, column)
- val dataType = if (carbonColumn != null) {
- carbonColumn.getDataType
- } else {
- carbonColumn = carbonTable.getMeasureByName(carbonTable.getFactTableName, column)
- carbonColumn.getDataType match {
- case CarbonDataTypes.INT => CarbonDataTypes.INT
- case CarbonDataTypes.SHORT => CarbonDataTypes.SHORT
- case CarbonDataTypes.LONG => CarbonDataTypes.LONG
- case CarbonDataTypes.DECIMAL => CarbonDataTypes.DECIMAL
- case _ => CarbonDataTypes.DOUBLE
- }
- }
- CarbonScalaUtil.convertCarbonToSparkDataType(dataType)
- }
-
- // Convert scala list to java list, Cannot use scalaList.asJava as while deserializing it is
- // not able find the classes inside scala list and gives ClassNotFoundException.
- private def convertToJavaList(
- scalaList: Seq[CarbonExpression]): java.util.List[CarbonExpression] = {
- val javaList = new java.util.ArrayList[CarbonExpression]()
- scalaList.foreach(javaList.add)
- javaList
- }
-}