You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2017/10/31 07:00:10 UTC
[12/22] carbondata git commit: [CARBONDATA-1444] Support Boolean data
type
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesBaseTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesBaseTest.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesBaseTest.scala
new file mode 100644
index 0000000..c0087a8
--- /dev/null
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesBaseTest.scala
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.spark.testsuite.booleantype
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
+
+class BooleanDataTypesBaseTest extends QueryTest with BeforeAndAfterEach with BeforeAndAfterAll {
+
+ override def beforeEach(): Unit = {
+ sql("drop table if exists carbon_table")
+ sql("drop table if exists boolean_table")
+ }
+
+ override def afterEach(): Unit = {
+ sql("drop table if exists carbon_table")
+ sql("drop table if exists boolean_table")
+ }
+
+ test("Creating table: boolean one column, should support") {
+ try {
+ sql("CREATE TABLE if not exists boolean_table(cc BOOLEAN) STORED BY 'carbondata'")
+ assert(true)
+ } catch {
+ case _: Exception => assert(false)
+ }
+ }
+
+ test("Creating table: boolean and other table, should support") {
+ try {
+ sql(
+ s"""
+ |CREATE TABLE if not exists boolean_table(
+ |aa INT, bb STRING, cc BOOLEAN
+ |) STORED BY 'carbondata'""".stripMargin)
+ assert(true)
+ } catch {
+ case _: Exception => assert(false)
+ }
+ }
+
+ test("Describing table: boolean data type, should support") {
+ sql(
+ s"""
+ |CREATE TABLE if not exists carbon_table(
+ |cc BOOLEAN
+ |) STORED BY 'carbondata'""".stripMargin)
+ checkExistence(sql("describe formatted carbon_table"), true, "boolean")
+ }
+
+ test("Describing table: support boolean data type format and other table ") {
+ sql(
+ s"""
+ | CREATE TABLE carbon_table(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+ checkExistence(sql("describe formatted carbon_table"), true, "boolean")
+ }
+
+ test("Altering table and add column: add boolean type column") {
+ sql(
+ s"""
+ |CREATE TABLE if not exists carbon_table(
+ |aa INT, bb STRING
+ |) STORED BY 'carbondata'""".stripMargin)
+ sql("alter table carbon_table add columns (dd BOOLEAN)")
+ checkExistence(sql("describe formatted carbon_table"), true, "boolean")
+ checkExistence(sql("describe formatted carbon_table"), true, "dd")
+ }
+
+ test("Altering table and add column: exists boolean column, add boolean type column") {
+ sql(
+ s"""
+ |CREATE TABLE if not exists carbon_table(
+ |aa INT, bb STRING, cc BOOLEAN
+ |) STORED BY 'carbondata'""".stripMargin)
+ sql("alter table carbon_table add columns (dd BOOLEAN)")
+ checkExistence(sql("describe formatted carbon_table"), true, "boolean")
+ checkExistence(sql("describe formatted carbon_table"), true, "dd")
+ }
+
+ test("Altering table and add column: exists boolean column, add not boolean type column") {
+ sql(
+ s"""
+ |CREATE TABLE if not exists carbon_table(
+ |aa INT, bb STRING, cc BOOLEAN
+ |) STORED BY 'carbondata'""".stripMargin)
+ sql("alter table carbon_table add columns (dd STRING)")
+ checkExistence(sql("describe formatted carbon_table"), true, "boolean")
+ checkExistence(sql("describe formatted carbon_table"), true, "dd")
+ }
+
+ test("Altering table and add column and insert values: exists boolean column, add boolean type column") {
+ sql(
+ s"""
+ |CREATE TABLE if not exists carbon_table(
+ |aa STRING, bb INT, cc BOOLEAN
+ |) STORED BY 'carbondata'""".stripMargin)
+ sql("alter table carbon_table add columns (dd BOOLEAN)")
+ sql("insert into carbon_table values('adam',11,true,false)")
+ checkAnswer(sql("select * from carbon_table"), Seq(Row("adam", 11, true, false)))
+ }
+
+ test("Altering table and drop column and insert values: exists boolean column, add boolean type column") {
+ sql(
+ s"""
+ |CREATE TABLE if not exists carbon_table(
+ |aa STRING, bb INT, cc BOOLEAN, dd BOOLEAN
+ |) STORED BY 'carbondata'""".stripMargin)
+ sql("alter table carbon_table drop columns (dd)")
+ sql("insert into carbon_table values('adam',11,true)")
+ checkAnswer(sql("select * from carbon_table"), Seq(Row("adam", 11, true)))
+ }
+
+ test("Deleting table and drop column and insert values: exists boolean column, add boolean type column") {
+ sql(
+ s"""
+ |CREATE TABLE if not exists carbon_table(
+ |aa STRING, bb INT, cc BOOLEAN, dd BOOLEAN
+ |) STORED BY 'carbondata'""".stripMargin)
+ sql("alter table carbon_table drop columns (dd)")
+ sql("insert into carbon_table values('adam',11,true)")
+ checkAnswer(sql("select * from carbon_table"), Seq(Row("adam", 11, true)))
+ sql("delete from carbon_table where cc=true")
+ checkAnswer(sql("select COUNT(*) from carbon_table"), Row(0))
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesBigFileTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesBigFileTest.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesBigFileTest.scala
new file mode 100644
index 0000000..53835fb
--- /dev/null
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesBigFileTest.scala
@@ -0,0 +1,729 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.spark.testsuite.booleantype
+
+import java.io.{File, PrintWriter}
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
+
+
+class BooleanDataTypesBigFileTest extends QueryTest with BeforeAndAfterEach with BeforeAndAfterAll {
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+
+ override def beforeEach(): Unit = {
+ sql("drop table if exists boolean_table")
+ sql("drop table if exists boolean_table2")
+ sql("drop table if exists carbon_table")
+ sql("drop table if exists hive_table")
+ }
+
+ override def afterAll(): Unit = {
+ sql("drop table if exists boolean_table")
+ sql("drop table if exists boolean_table2")
+ sql("drop table if exists carbon_table")
+ sql("drop table if exists hive_table")
+ assert(BooleanFile.deleteFile(pathOfManyDataType))
+ assert(BooleanFile.deleteFile(pathOfOnlyBoolean))
+ }
+
+ val pathOfManyDataType = s"$rootPath/integration/spark2/src/test/resources/bool/supportBooleanBigFile.csv"
+ val pathOfOnlyBoolean = s"$rootPath/integration/spark2/src/test/resources/bool/supportBooleanBigFileOnlyBoolean.csv"
+ val trueNum = 10000
+
+ override def beforeAll(): Unit = {
+ assert(BooleanFile.createBooleanFileWithOtherDataType(pathOfManyDataType, trueNum))
+ assert(BooleanFile.createOnlyBooleanFile(pathOfOnlyBoolean, trueNum))
+ }
+
+ test("Loading table: support boolean and other data type, big file") {
+ sql(
+ s"""
+ | CREATE TABLE boolean_table(
+ | intField INT,
+ | booleanField BOOLEAN,
+ | stringField STRING,
+ | doubleField DOUBLE,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ """.stripMargin)
+
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${pathOfManyDataType}'
+ | INTO TABLE boolean_table
+ | options('FILEHEADER'='intField,booleanField,stringField,doubleField,booleanField2')
+ """.stripMargin)
+
+ checkAnswer(
+ sql("select count(*) from boolean_table"),
+ Row(trueNum + trueNum / 10))
+ }
+
+ test("Inserting table: support boolean and other data type, big file") {
+ sql(
+ s"""
+ | CREATE TABLE boolean_table(
+ | intField INT,
+ | booleanField BOOLEAN,
+ | stringField STRING,
+ | doubleField DOUBLE,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ """.stripMargin)
+
+ sql(
+ s"""
+ | CREATE TABLE boolean_table2(
+ | intField INT,
+ | booleanField BOOLEAN,
+ | stringField STRING,
+ | doubleField DOUBLE,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ """.stripMargin)
+
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${pathOfManyDataType}'
+ | INTO TABLE boolean_table
+ | options('FILEHEADER'='intField,booleanField,stringField,doubleField,booleanField2')
+ """.stripMargin)
+
+ sql("insert into boolean_table2 select * from boolean_table")
+
+ checkAnswer(
+ sql("select count(*) from boolean_table2"),
+ Row(trueNum + trueNum / 10))
+ }
+
+ test("Filtering table: support boolean data type, only boolean, big file") {
+ sql(
+ s"""
+ | CREATE TABLE boolean_table(
+ | booleanField BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ """.stripMargin)
+
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${pathOfOnlyBoolean}'
+ | INTO TABLE boolean_table
+ | options('FILEHEADER'='booleanField')
+ """.stripMargin)
+
+ checkAnswer(
+ sql("select count(*) from boolean_table"),
+ Row(trueNum + trueNum / 10))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField is not null"),
+ Row(trueNum + trueNum / 10))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField is null"),
+ Row(0))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField = true"),
+ Row(trueNum))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField >= true"),
+ Row(trueNum))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField > true"),
+ Row(0))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField < true"),
+ Row(trueNum / 10))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField = false"),
+ Row(trueNum / 10))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField <= false"),
+ Row(trueNum / 10))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField > false"),
+ Row(trueNum))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField < false"),
+ Row(0))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField in (false)"),
+ Row(trueNum / 10))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField not in (false)"),
+ Row(trueNum))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField in (true,false)"),
+ Row(trueNum + trueNum / 10))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField not in (true,false)"),
+ Row(0))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField like 'f%'"),
+ Row(trueNum / 10))
+ }
+
+ test("Filtering table: support boolean and other data type, big file") {
+
+ sql(
+ s"""
+ | CREATE TABLE boolean_table(
+ | intField INT,
+ | booleanField BOOLEAN,
+ | stringField STRING,
+ | doubleField DOUBLE,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ """.stripMargin)
+
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${pathOfManyDataType}'
+ | INTO TABLE boolean_table
+ | options('FILEHEADER'='intField,booleanField,stringField,doubleField,booleanField2')
+ """.stripMargin)
+
+ checkAnswer(
+ sql("select booleanField from boolean_table where intField >=1 and intField <11"),
+ Seq(Row(true), Row(true), Row(true), Row(true), Row(true), Row(true), Row(true), Row(true), Row(true), Row(true))
+ )
+
+ checkAnswer(
+ sql(s"select booleanField from boolean_table where intField >='${trueNum - 5}' and intField <=${trueNum + 1}"),
+ Seq(Row(true), Row(true), Row(true), Row(true), Row(true), Row(false), Row(false))
+ )
+
+ checkAnswer(
+ sql(s"select count(*) from boolean_table where intField >='${trueNum - 5}' and doubleField <=${trueNum + 1} and booleanField=false"),
+ Seq(Row(2))
+ )
+
+ checkAnswer(
+ sql(s"select * from boolean_table where intField >4 and doubleField < 6.0"),
+ Seq(Row(5, true, "num5", 5.0, false))
+ )
+
+ checkAnswer(
+ sql("select count(*) from boolean_table"),
+ Row(trueNum + trueNum / 10))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField is not null"),
+ Row(trueNum + trueNum / 10))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField is null"),
+ Row(0))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField = true"),
+ Row(trueNum))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField >= true"),
+ Row(trueNum))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField > true"),
+ Row(0))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField < true"),
+ Row(trueNum / 10))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField = false"),
+ Row(trueNum / 10))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField <= false"),
+ Row(trueNum / 10))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField > false"),
+ Row(trueNum))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField < false"),
+ Row(0))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField in (false)"),
+ Row(trueNum / 10))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField not in (false)"),
+ Row(trueNum))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField in (true,false)"),
+ Row(trueNum + trueNum / 10))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField not in (true,false)"),
+ Row(0))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField like 'f%'"),
+ Row(trueNum / 10))
+ }
+
+ test("Filtering table: support boolean and other data type, big file, load twice") {
+
+ sql(
+ s"""
+ | CREATE TABLE boolean_table(
+ | intField INT,
+ | booleanField BOOLEAN,
+ | stringField STRING,
+ | doubleField DOUBLE,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ """.stripMargin)
+ val repeat: Int = 2
+ for (i <- 0 until repeat) {
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${pathOfManyDataType}'
+ | INTO TABLE boolean_table
+ | options('FILEHEADER'='intField,booleanField,stringField,doubleField,booleanField2')
+ """.stripMargin
+ )
+ }
+
+ checkAnswer(
+ sql("select booleanField from boolean_table where intField >=1 and intField <11"),
+ Seq(Row(true), Row(true), Row(true), Row(true), Row(true), Row(true), Row(true), Row(true), Row(true), Row(true),
+ Row(true), Row(true), Row(true), Row(true), Row(true), Row(true), Row(true), Row(true), Row(true), Row(true))
+ )
+
+ checkAnswer(
+ sql(s"select booleanField from boolean_table where intField >='${trueNum - 5}' and intField <=${trueNum + 1}"),
+ Seq(Row(true), Row(true), Row(true), Row(true), Row(true), Row(false), Row(false),
+ Row(true), Row(true), Row(true), Row(true), Row(true), Row(false), Row(false))
+ )
+
+ checkAnswer(
+ sql(s"select count(*) from boolean_table where intField >='${trueNum - 5}' and doubleField <=${trueNum + 1} and booleanField=false"),
+ Seq(Row(4))
+ )
+
+ checkAnswer(
+ sql(s"select * from boolean_table where intField >4 and doubleField < 6.0"),
+ Seq(Row(5, true, "num5", 5.0, false), Row(5, true, "num5", 5.0, false))
+ )
+
+ checkAnswer(
+ sql("select count(*) from boolean_table"),
+ Row(repeat * (trueNum + trueNum / 10)))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField is not null"),
+ Row(repeat * (trueNum + trueNum / 10)))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField is null"),
+ Row(0))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField = true"),
+ Row(repeat * (trueNum)))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField >= true"),
+ Row(repeat * (trueNum)))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField > true"),
+ Row(0))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField < true"),
+ Row(repeat * (trueNum / 10)))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField = false"),
+ Row(repeat * (trueNum / 10)))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField <= false"),
+ Row(repeat * (trueNum / 10)))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField > false"),
+ Row(repeat * (trueNum)))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField < false"),
+ Row(0))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField in (false)"),
+ Row(repeat * (trueNum / 10)))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField not in (false)"),
+ Row(repeat * (trueNum)))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField in (true,false)"),
+ Row(repeat * (trueNum + trueNum / 10)))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField not in (true,false)"),
+ Row(0))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField like 'f%'"),
+ Row(repeat * (trueNum / 10)))
+ }
+
+ test("Sort_columns: support boolean and other data type, big file") {
+ sql(
+ s"""
+ | CREATE TABLE boolean_table(
+ | intField INT,
+ | booleanField BOOLEAN,
+ | stringField STRING,
+ | doubleField DOUBLE,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='booleanField')
+ """.stripMargin)
+
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${pathOfManyDataType}'
+ | INTO TABLE boolean_table
+ | options('FILEHEADER'='intField,booleanField,stringField,doubleField,booleanField2')
+ """.stripMargin)
+
+ checkAnswer(
+ sql(s"select booleanField from boolean_table where intField >='${trueNum - 5}' and intField <=${trueNum + 1}"),
+ Seq(Row(true), Row(true), Row(true), Row(true), Row(true), Row(false), Row(false))
+ )
+ }
+
+ test("Inserting into Hive table from carbon table: support boolean data type and other format, big file") {
+ sql(
+ s"""
+ | CREATE TABLE carbon_table(
+ | intField INT,
+ | booleanField BOOLEAN,
+ | stringField STRING,
+ | doubleField DOUBLE,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ """.stripMargin)
+
+ sql(
+ s"""
+ | CREATE TABLE hive_table(
+ | intField INT,
+ | booleanField BOOLEAN,
+ | stringField STRING,
+ | doubleField DOUBLE,
+ | booleanField2 BOOLEAN
+ | )
+ | ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ """.stripMargin)
+
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${pathOfManyDataType}'
+ | INTO TABLE carbon_table
+ | options('FILEHEADER'='intField,booleanField,stringField,doubleField,booleanField2')
+ """.stripMargin)
+
+ sql("insert into hive_table select * from carbon_table")
+
+ checkAnswer(
+ sql(s"select booleanField from hive_table where intField >='${trueNum - 5}' and intField <=${trueNum + 1}"),
+ Seq(Row(true), Row(true), Row(true), Row(true), Row(true), Row(false), Row(false))
+ )
+
+ checkAnswer(
+ sql(s"select * from hive_table where intField >4 and doubleField < 6.0"),
+ Seq(Row(5, true, "num5", 5.0, false))
+ )
+
+ checkAnswer(
+ sql("select count(*) from hive_table"),
+ Row(trueNum + trueNum / 10))
+
+ checkAnswer(
+ sql("select count(*) from hive_table where booleanField = true"),
+ Row(trueNum))
+
+ checkAnswer(
+ sql("select count(*) from hive_table where booleanField = false"),
+ Row(trueNum / 10))
+ }
+
+ test("Inserting into carbon table from Hive table: support boolean data type and other format, big file") {
+ sql(
+ s"""
+ | CREATE TABLE hive_table(
+ | intField INT,
+ | booleanField BOOLEAN,
+ | stringField STRING,
+ | doubleField DOUBLE,
+ | booleanField2 BOOLEAN
+ | )
+ | ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ """.stripMargin)
+
+ sql(
+ s"""
+ | CREATE TABLE carbon_table(
+ | intField INT,
+ | booleanField BOOLEAN,
+ | stringField STRING,
+ | doubleField DOUBLE,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ """.stripMargin)
+
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${pathOfManyDataType}'
+ | INTO TABLE hive_table
+ """.stripMargin)
+
+ sql("insert into carbon_table select * from hive_table")
+
+ checkAnswer(
+ sql(s"select booleanField from carbon_table where intField >='${trueNum - 5}' and intField <=${trueNum + 1}"),
+ Seq(Row(true), Row(true), Row(true), Row(true), Row(true), Row(false), Row(false))
+ )
+
+ checkAnswer(
+ sql(s"select * from carbon_table where intField >4 and doubleField < 6.0"),
+ Seq(Row(5, true, "num5", 5.0, false))
+ )
+
+ checkAnswer(
+ sql("select count(*) from carbon_table"),
+ Row(trueNum + trueNum / 10))
+
+ checkAnswer(
+ sql("select count(*) from carbon_table where booleanField = true"),
+ Row(trueNum))
+
+ checkAnswer(
+ sql("select count(*) from carbon_table where booleanField = false"),
+ Row(trueNum / 10))
+ }
+
+ test("Filtering table: unsafe, support boolean and other data type, big file, load twice") {
+ initConf()
+ sql(
+ s"""
+ | CREATE TABLE boolean_table(
+ | intField INT,
+ | booleanField BOOLEAN,
+ | stringField STRING,
+ | doubleField DOUBLE,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ """.stripMargin)
+ val repeat: Int = 2
+ for (i <- 0 until repeat) {
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${pathOfManyDataType}'
+ | INTO TABLE boolean_table
+ | options('FILEHEADER'='intField,booleanField,stringField,doubleField,booleanField2')
+ """.stripMargin
+ )
+ }
+
+ checkAnswer(
+ sql("select booleanField from boolean_table where intField >=1 and intField <11"),
+ Seq(Row(true), Row(true), Row(true), Row(true), Row(true), Row(true), Row(true), Row(true), Row(true), Row(true),
+ Row(true), Row(true), Row(true), Row(true), Row(true), Row(true), Row(true), Row(true), Row(true), Row(true))
+ )
+
+ checkAnswer(
+ sql(s"select booleanField from boolean_table where intField >='${trueNum - 5}' and intField <=${trueNum + 1}"),
+ Seq(Row(true), Row(true), Row(true), Row(true), Row(true), Row(false), Row(false),
+ Row(true), Row(true), Row(true), Row(true), Row(true), Row(false), Row(false))
+ )
+
+ checkAnswer(
+ sql(s"select count(*) from boolean_table where intField >='${trueNum - 5}' and doubleField <=${trueNum + 1} and booleanField=false"),
+ Seq(Row(4))
+ )
+
+ checkAnswer(
+ sql(s"select * from boolean_table where intField >4 and doubleField < 6.0"),
+ Seq(Row(5, true, "num5", 5.0, false), Row(5, true, "num5", 5.0, false))
+ )
+
+ checkAnswer(
+ sql("select count(*) from boolean_table"),
+ Row(repeat * (trueNum + trueNum / 10)))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField is not null"),
+ Row(repeat * (trueNum + trueNum / 10)))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField is null"),
+ Row(0))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField = true"),
+ Row(repeat * (trueNum)))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField >= true"),
+ Row(repeat * (trueNum)))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField > true"),
+ Row(0))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField < true"),
+ Row(repeat * (trueNum / 10)))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField = false"),
+ Row(repeat * (trueNum / 10)))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField <= false"),
+ Row(repeat * (trueNum / 10)))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField > false"),
+ Row(repeat * (trueNum)))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField < false"),
+ Row(0))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField in (false)"),
+ Row(repeat * (trueNum / 10)))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField not in (false)"),
+ Row(repeat * (trueNum)))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField in (true,false)"),
+ Row(repeat * (trueNum + trueNum / 10)))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField not in (true,false)"),
+ Row(0))
+
+ checkAnswer(
+ sql("select count(*) from boolean_table where booleanField like 'f%'"),
+ Row(repeat * (trueNum / 10)))
+ defaultConf()
+ }
+
+ def initConf(): Unit = {
+ CarbonProperties.getInstance().
+ addProperty(CarbonCommonConstants.ENABLE_UNSAFE_COLUMN_PAGE_LOADING,
+ "true")
+ }
+
+ def defaultConf(): Unit = {
+ CarbonProperties.getInstance().
+ addProperty(CarbonCommonConstants.ENABLE_UNSAFE_COLUMN_PAGE_LOADING,
+ CarbonCommonConstants.ENABLE_DATA_LOADING_STATISTICS_DEFAULT)
+ }
+}
+
+object BooleanFile {
+ def createBooleanFileWithOtherDataType(path: String, trueLines: Int): Boolean = {
+ try {
+ val write = new PrintWriter(path)
+ var d: Double = 0.0
+ for (i <- 0 until trueLines) {
+ write.println(i + "," + true + ",num" + i + "," + d + "," + false)
+ d = d + 1
+ }
+ for (i <- 0 until trueLines / 10) {
+ write.println((trueLines + i) + "," + false + ",num" + (trueLines + i) + "," + d + "," + true)
+ d = d + 1
+ }
+ write.close()
+ } catch {
+ case _: Exception => assert(false)
+ }
+ return true
+ }
+
+ def deleteFile(path: String): Boolean = {
+ try {
+ val file = new File(path)
+ file.delete()
+ } catch {
+ case _: Exception => assert(false)
+ }
+ return true
+ }
+
+ def createOnlyBooleanFile(path: String, num: Int): Boolean = {
+ try {
+ val write = new PrintWriter(path)
+ for (i <- 0 until num) {
+ write.println(true)
+ }
+ for (i <- 0 until num / 10) {
+ write.println(false)
+ }
+ write.close()
+ } catch {
+ case _: Exception => assert(false)
+ }
+ return true
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesFilterTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesFilterTest.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesFilterTest.scala
new file mode 100644
index 0000000..66d74d5
--- /dev/null
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesFilterTest.scala
@@ -0,0 +1,416 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.spark.testsuite.booleantype
+
+import java.io.File
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
+
+class BooleanDataTypesFilterTest extends QueryTest with BeforeAndAfterEach with BeforeAndAfterAll {
+
+ override def beforeAll(): Unit = {
+ sql("drop table if exists carbon_table")
+ sql("drop table if exists boolean_table")
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+ val storeLocation = s"$rootPath/integration/spark2/src/test/resources/bool/supportBooleanOnlyBoolean.csv"
+
+ sql("CREATE TABLE if not exists carbon_table(booleanField BOOLEAN) STORED BY 'carbondata'")
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '$storeLocation'
+ | INTO TABLE carbon_table
+ | OPTIONS('FILEHEADER' = 'booleanField')
+ """.stripMargin)
+
+ val booleanLocation = s"$rootPath/integration/spark2/src/test/resources/bool/supportBoolean.csv"
+ sql(
+ s"""
+ | CREATE TABLE boolean_table(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${booleanLocation}'
+ | INTO TABLE boolean_table
+ | options('FILEHEADER'='shortField,booleanField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData')
+ """.stripMargin)
+ }
+
+ override def afterAll(): Unit = {
+ sql("drop table if exists carbon_table")
+ sql("drop table if exists boolean_table")
+ }
+
+ test("Filtering table: support boolean, Expression: EqualToExpression, IsNotNullExpression, Or") {
+ checkAnswer(sql("select * from carbon_table where booleanField = true"),
+ Seq(Row(true), Row(true), Row(true), Row(true)))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField = true"),
+ Row(4))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField = 'true'"),
+ Row(0))
+
+ checkAnswer(sql(
+ s"""
+ |select count(*)
+ |from carbon_table where booleanField = \"true\"
+ |""".stripMargin),
+ Row(0))
+
+ checkAnswer(sql("select * from carbon_table where booleanField = false"),
+ Seq(Row(false), Row(false), Row(false), Row(false)))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField = false"),
+ Row(4))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField = 'false'"),
+ Row(0))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField = null"),
+ Row(0))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField = false or booleanField = true"),
+ Row(8))
+ }
+
+ test("Filtering table: support boolean, Expression: InExpression") {
+ checkAnswer(sql("select * from carbon_table where booleanField in (true)"),
+ Seq(Row(true), Row(true), Row(true), Row(true)))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField in (true)"),
+ Row(4))
+
+ checkAnswer(sql("select * from carbon_table where booleanField in (false)"),
+ Seq(Row(false), Row(false), Row(false), Row(false)))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField in (false)"),
+ Row(4))
+
+ checkAnswer(sql("select * from carbon_table where booleanField in (true,false)"),
+ Seq(Row(true), Row(true), Row(true), Row(true), Row(false), Row(false), Row(false), Row(false)))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField in (true,false)"),
+ Row(8))
+ }
+
+ test("Filtering table: support boolean, Expression: NotInExpression") {
+ checkAnswer(sql("select * from carbon_table where booleanField not in (false)"),
+ Seq(Row(true), Row(true), Row(true), Row(true)))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField not in (true)"),
+ Row(4))
+
+ checkAnswer(sql("select * from carbon_table where booleanField not in (true)"),
+ Seq(Row(false), Row(false), Row(false), Row(false)))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField not in (true)"),
+ Row(4))
+
+ checkAnswer(sql("select * from carbon_table where booleanField not in (null)"),
+ Seq(Row(true), Row(true), Row(true), Row(true), Row(false), Row(false), Row(false), Row(false)))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField not in (null)"),
+ Row(8))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField not in (true,false)"),
+ Row(0))
+ }
+
+ test("Filtering table: support boolean, Expression: NotEqualsExpression") {
+ checkAnswer(sql("select * from carbon_table where booleanField != false"),
+ Seq(Row(true), Row(true), Row(true), Row(true)))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField != false"),
+ Row(4))
+
+ checkAnswer(sql("select * from carbon_table where booleanField != true"),
+ Seq(Row(false), Row(false), Row(false), Row(false)))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField != true"),
+ Row(4))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField != null"),
+ Row(0))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField != false or booleanField != true"),
+ Row(8))
+ }
+
+ test("Filtering table: support boolean, Expression: GreaterThanEqualToExpression") {
+ checkAnswer(sql("select * from carbon_table where booleanField >= true"),
+ Seq(Row(true), Row(true), Row(true), Row(true)))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField >= true"),
+ Row(4))
+
+ checkAnswer(sql("select * from carbon_table where booleanField >= false"),
+ Seq(Row(true), Row(true), Row(true), Row(true), Row(false), Row(false), Row(false), Row(false)))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField >=false"),
+ Row(8))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField >= null"),
+ Row(0))
+ }
+
+ test("Filtering table: support boolean, Expression: GreaterThanExpression") {
+ checkAnswer(sql("select * from carbon_table where booleanField > false"),
+ Seq(Row(true), Row(true), Row(true), Row(true)))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField > false"),
+ Row(4))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField > false"),
+ Row(4))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField > null"),
+ Row(0))
+ }
+
+ test("Filtering table: support boolean, Expression: LessThanEqualToExpression") {
+ checkAnswer(sql("select * from carbon_table where booleanField <= false"),
+ Seq(Row(false), Row(false), Row(false), Row(false)))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField <= false"),
+ Row(4))
+
+ checkAnswer(sql("select * from carbon_table where booleanField <= true"),
+ Seq(Row(true), Row(true), Row(true), Row(true), Row(false), Row(false), Row(false), Row(false)))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField <= true"),
+ Row(8))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField <= null"),
+ Row(0))
+ }
+
+ test("Filtering table: support boolean, Expression: LessThanExpression") {
+ checkAnswer(sql("select * from carbon_table where booleanField < true"),
+ Seq(Row(false), Row(false), Row(false), Row(false)))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField < true"),
+ Row(4))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField < false"),
+ Row(0))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField < null"),
+ Row(0))
+ }
+
+ test("Filtering table: support boolean, Expression: between") {
+ checkAnswer(sql("select * from carbon_table where booleanField < true and booleanField >= false"),
+ Seq(Row(false), Row(false), Row(false), Row(false)))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField < true and booleanField >= false"),
+ Row(4))
+ }
+
+ test("Filtering table: support boolean, Expression: empty") {
+ checkAnswer(sql("select count(*) from carbon_table where booleanField =''"),
+ Row(0))
+ }
+
+ test("Filtering table: support boolean, Expression: IsNotNull") {
+ checkAnswer(sql("select count(*) from carbon_table where booleanField is not null"),
+ Row(8))
+ checkAnswer(sql("select * from carbon_table where booleanField is not null"),
+ Seq(Row(true), Row(true), Row(true), Row(true), Row(false), Row(false), Row(false), Row(false)))
+ }
+
+ test("Filtering table: support boolean, Expression: IsNull") {
+ checkAnswer(sql("select count(*) from carbon_table where booleanField is null"),
+ Row(3))
+ checkAnswer(sql("select * from carbon_table where booleanField is null"),
+ Seq(Row(null), Row(null), Row(null)))
+ }
+
+ test("Filtering table: support boolean and other data type, and") {
+ checkAnswer(sql("select count(*) from boolean_table where booleanField = false and shortField = 1"),
+ Row(3))
+
+ checkAnswer(sql("select count(*) from boolean_table where booleanField = false and stringField='flink'"),
+ Row(1))
+
+ checkAnswer(sql("select intField,booleanField,stringField from boolean_table where booleanField = false and stringField='flink'"),
+ Row(11, false, "flink"))
+ }
+
+ test("Filtering table: support boolean and other data type, GreaterThanEqualToExpression") {
+ checkAnswer(sql("select count(*) from boolean_table where booleanField <= false and shortField >= 1"),
+ Row(6))
+ }
+
+ test("Filtering table: support boolean, like") {
+ checkAnswer(sql("select * from carbon_table where booleanField like 't%'"),
+ Seq(Row(true), Row(true), Row(true), Row(true)))
+
+ checkAnswer(sql("select * from carbon_table where booleanField like 'tru%'"),
+ Seq(Row(true), Row(true), Row(true), Row(true)))
+
+ checkAnswer(sql("select * from carbon_table where booleanField like '%ue'"),
+ Seq(Row(true), Row(true), Row(true), Row(true)))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField like 't%'"),
+ Row(4))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField like 'T%'"),
+ Row(0))
+
+ checkAnswer(sql("select * from carbon_table where booleanField like 'f%'"),
+ Seq(Row(false), Row(false), Row(false), Row(false)))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField like 'f%'"),
+ Row(4))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField like 'F%'"),
+ Row(0))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField like 'n%'"),
+ Row(0))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField like 'f%' or booleanField like 't%'"),
+ Row(8))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField like '%e'"),
+ Row(8))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField like '%a%'"),
+ Row(4))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField like '%z%'"),
+ Row(0))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField like '%e'"),
+ Row(8))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField like '_rue'"),
+ Row(4))
+
+ checkAnswer(sql("select count(*) from carbon_table where booleanField like 'f___e'"),
+ Row(4))
+ }
+
+ test("Filtering table: support boolean and other data type, two boolean column") {
+ sql("drop table if exists boolean_table2")
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+ val booleanLocation2 = s"$rootPath/integration/spark2/src/test/resources/bool/supportBooleanTwoBooleanColumns.csv"
+ sql(
+ s"""
+ | CREATE TABLE boolean_table2(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${booleanLocation2}'
+ | INTO TABLE boolean_table2
+ | options('FILEHEADER'='shortField,booleanField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData,booleanField2')
+ """.stripMargin)
+ checkAnswer(sql("select count(*) from boolean_table2 where booleanField = false and booleanField2 = false"),
+ Row(4))
+ checkAnswer(sql("select count(*) from boolean_table2 where booleanField = true and booleanField2 = true"),
+ Row(3))
+ checkAnswer(sql("select count(*) from boolean_table2 where booleanField = false and booleanField2 = true"),
+ Row(2))
+ checkAnswer(sql("select count(*) from boolean_table2 where booleanField = true and booleanField2 = false"),
+ Row(1))
+ sql("drop table if exists boolean_table2")
+ }
+
+ test("Filtering table: support boolean and other data type, load twice") {
+ sql("drop table if exists boolean_table2")
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+ val booleanLocation2 = s"$rootPath/integration/spark2/src/test/resources/bool/supportBooleanTwoBooleanColumns.csv"
+ sql(
+ s"""
+ | CREATE TABLE boolean_table2(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${booleanLocation2}'
+ | INTO TABLE boolean_table2
+ | options('FILEHEADER'='shortField,booleanField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData,booleanField2')
+ """.stripMargin)
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${booleanLocation2}'
+ | INTO TABLE boolean_table2
+ | options('FILEHEADER'='shortField,booleanField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData,booleanField2')
+ """.stripMargin)
+ checkAnswer(sql("select count(*) from boolean_table2 where booleanField = false and booleanField2 = false"),
+ Row(8))
+ checkAnswer(sql("select count(*) from boolean_table2 where booleanField = true and booleanField2 = true"),
+ Row(6))
+ checkAnswer(sql("select count(*) from boolean_table2 where booleanField = false and booleanField2 = true"),
+ Row(4))
+ checkAnswer(sql("select count(*) from boolean_table2 where booleanField = true and booleanField2 = false"),
+ Row(2))
+ sql("drop table if exists boolean_table2")
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesInsertTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesInsertTest.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesInsertTest.scala
new file mode 100644
index 0000000..2f06900
--- /dev/null
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesInsertTest.scala
@@ -0,0 +1,948 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.spark.testsuite.booleantype
+
+import java.io.File
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
+
+class BooleanDataTypesInsertTest extends QueryTest with BeforeAndAfterEach with BeforeAndAfterAll {
+
+ override def beforeEach(): Unit = {
+ sql("drop table if exists boolean_one_column")
+ sql("drop table if exists boolean_table")
+ sql("drop table if exists boolean_table2")
+ sql("drop table if exists boolean_table3")
+ sql("drop table if exists boolean_table4")
+ sql("drop table if exists carbon_table")
+ sql("drop table if exists hive_table")
+ sql("CREATE TABLE if not exists boolean_one_column(booleanField BOOLEAN) STORED BY 'carbondata'")
+ }
+
+ override def afterAll(): Unit = {
+ sql("drop table if exists boolean_one_column")
+ sql("drop table if exists boolean_table")
+ sql("drop table if exists boolean_table2")
+ sql("drop table if exists boolean_table3")
+ sql("drop table if exists boolean_table4")
+ sql("drop table if exists carbon_table")
+ sql("drop table if exists hive_table")
+ }
+
+ test("Inserting and selecting table: one column boolean, should support") {
+ sql("insert into boolean_one_column values(true)")
+ checkAnswer(
+ sql("select * from boolean_one_column"),
+ Seq(Row(true))
+ )
+ }
+
+ test("Inserting and selecting table: one column boolean and many rows, should support") {
+ sql("insert into boolean_one_column values(true)")
+ sql("insert into boolean_one_column values(True)")
+ sql("insert into boolean_one_column values(TRUE)")
+ sql("insert into boolean_one_column values('true')")
+ sql("insert into boolean_one_column values(False)")
+ sql("insert into boolean_one_column values(false)")
+ sql("insert into boolean_one_column values(FALSE)")
+ sql("insert into boolean_one_column values('false')")
+ sql("insert into boolean_one_column values('tr')")
+ sql("insert into boolean_one_column values(null)")
+ sql("insert into boolean_one_column values('truEe')")
+ sql("insert into boolean_one_column values('falsEe')")
+ sql("insert into boolean_one_column values('t')")
+ sql("insert into boolean_one_column values('f')")
+
+ checkAnswer(
+ sql("select * from boolean_one_column"),
+ Seq(Row(true), Row(true), Row(true), Row(true),
+ Row(false), Row(false), Row(false), Row(false),
+ Row(null), Row(null), Row(null), Row(null), Row(null), Row(null))
+ )
+ }
+
+ test("Inserting and selecting table: create one column boolean table and insert two columns") {
+ sql("insert into boolean_one_column values(true,false)")
+ sql("insert into boolean_one_column values(True)")
+ sql("insert into boolean_one_column values(false,true)")
+
+ checkAnswer(
+ sql("select * from boolean_one_column"),
+ Seq(Row(true), Row(true), Row(false))
+ )
+ }
+
+ test("Inserting and selecting table: two columns boolean and many rows, should support") {
+ sql("CREATE TABLE if not exists boolean_table2(col1 BOOLEAN, col2 BOOLEAN) STORED BY 'carbondata'")
+
+ sql("insert into boolean_table2 values(true,true)")
+ sql("insert into boolean_table2 values(True,false)")
+ sql("insert into boolean_table2 values(TRUE,false)")
+ sql("insert into boolean_table2 values(false,true)")
+ sql("insert into boolean_table2 values(FALSE,false)")
+ sql("insert into boolean_table2 values('false',false)")
+ sql("insert into boolean_table2 values(null,true)")
+
+ checkAnswer(
+ sql("select * from boolean_table2"),
+ Seq(Row(true, true), Row(true, false), Row(true, false),
+ Row(false, true), Row(false, false), Row(false, false), Row(null, true))
+ )
+ }
+
+ test("Inserting and selecting table: two columns and other data type, should support") {
+ sql("CREATE TABLE if not exists boolean_table2(col1 INT, col2 BOOLEAN) STORED BY 'carbondata'")
+
+ sql("insert into boolean_table2 values(1,true)")
+ sql("insert into boolean_table2 values(100,true)")
+ sql("insert into boolean_table2 values(1991,false)")
+ sql("insert into boolean_table2 values(906,false)")
+ sql("insert into boolean_table2 values(218,false)")
+ sql("insert into boolean_table2 values(1011,false)")
+
+ checkAnswer(
+ sql("select * from boolean_table2"),
+ Seq(Row(1, true), Row(100, true), Row(1991, false),
+ Row(906, false), Row(218, false), Row(1011, false))
+ )
+ }
+
+ test("Inserting into table with another table: support boolean data type and other format") {
+ sql(
+ s"""
+ | CREATE TABLE boolean_table(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+
+ sql(
+ s"""
+ | CREATE TABLE boolean_table2(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+
+ sql(
+ s"""
+ | CREATE TABLE boolean_table3(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | stringField STRING,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='')
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE TABLE boolean_table4(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | stringField STRING,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='')
+ """.stripMargin)
+
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+ val storeLocation = s"$rootPath/integration/spark2/src/test/resources/bool/supportBooleanTwoBooleanColumns.csv"
+
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${storeLocation}'
+ | INTO TABLE boolean_table
+ | options('FILEHEADER'='shortField,booleanField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData,booleanField2')
+ """.stripMargin)
+
+ sql("insert into boolean_table2 select * from boolean_table")
+ sql("insert into boolean_table3 select shortField,booleanField,intField,stringField,booleanField2 from boolean_table")
+ sql("insert into boolean_table4 select shortField,booleanField,intField,stringField,booleanField2 from boolean_table where shortField > 3")
+
+ checkAnswer(
+ sql("select booleanField,intField from boolean_table2"),
+ Seq(Row(true, 10), Row(false, 17), Row(false, 11),
+ Row(true, 10), Row(true, 10), Row(true, 14),
+ Row(false, 10), Row(false, 10), Row(false, 16), Row(false, 10))
+ )
+
+ checkAnswer(
+ sql("select booleanField,intField from boolean_table3"),
+ Seq(Row(true, 10), Row(false, 17), Row(false, 11),
+ Row(true, 10), Row(true, 10), Row(true, 14),
+ Row(false, 10), Row(false, 10), Row(false, 16), Row(false, 10))
+ )
+
+ checkAnswer(
+ sql("select booleanField,intField from boolean_table4"),
+ Seq(Row(false, 17), Row(false, 16))
+ )
+
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from boolean_table2"),
+ Seq(Row(true, 10, true), Row(false, 17, true), Row(false, 11, true),
+ Row(true, 10, true), Row(true, 10, true), Row(true, 14, false),
+ Row(false, 10, false), Row(false, 10, false), Row(false, 16, false), Row(false, 10, false))
+ )
+
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from boolean_table3"),
+ Seq(Row(true, 10, true), Row(false, 17, true), Row(false, 11, true),
+ Row(true, 10, true), Row(true, 10, true), Row(true, 14, false),
+ Row(false, 10, false), Row(false, 10, false), Row(false, 16, false), Row(false, 10, false))
+ )
+ }
+
+ test("Inserting with the order of data type in source and target table columns being different") {
+ sql(
+ s"""
+ | CREATE TABLE boolean_table(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+
+ sql(
+ s"""
+ | CREATE TABLE boolean_table2(
+ | booleanField BOOLEAN,
+ | shortField SHORT,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+ val storeLocation = s"$rootPath/integration/spark2/src/test/resources/bool/supportBooleanTwoBooleanColumns.csv"
+
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${storeLocation}'
+ | INTO TABLE boolean_table
+ | options('FILEHEADER'='shortField,booleanField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData,booleanField2')
+ """.stripMargin)
+
+ sql("insert into boolean_table2 select * from boolean_table")
+
+ checkAnswer(
+ sql("select booleanField,intField from boolean_table2"),
+ Seq(Row(null, 10), Row(null, 17), Row(null, 11),
+ Row(null, 10), Row(null, 10), Row(null, 14),
+ Row(null, 10), Row(null, 10), Row(null, 16), Row(null, 10))
+ )
+
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from boolean_table2"),
+ Seq(Row(null, 10, true), Row(null, 17, true), Row(null, 11, true),
+ Row(null, 10, true), Row(null, 10, true), Row(null, 14, false),
+ Row(null, 10, false), Row(null, 10, false), Row(null, 16, false), Row(null, 10, false))
+ )
+ }
+
+ test("Inserting with the number of data type in source and target table columns being different, source more than target") {
+ sql(
+ s"""
+ | CREATE TABLE boolean_table(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+
+ sql(
+ s"""
+ | CREATE TABLE boolean_table2(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+ val storeLocation = s"$rootPath/integration/spark2/src/test/resources/bool/supportBooleanTwoBooleanColumns.csv"
+
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${storeLocation}'
+ | INTO TABLE boolean_table
+ | options('FILEHEADER'='shortField,booleanField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData,booleanField2')
+ """.stripMargin)
+
+ sql("insert into boolean_table2 select * from boolean_table")
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from boolean_table2"),
+ Seq(Row(true, 10, null), Row(false, 17, null), Row(false, 11, null),
+ Row(true, 10, null), Row(true, 10, null), Row(true, 14, null),
+ Row(false, 10, null), Row(false, 10, null), Row(false, 16, null), Row(false, 10, null))
+ )
+ }
+
+ test("Inserting with the number of data type in source and target table columns being different, source less than target") {
+ val exception_insert: Exception =intercept[Exception] {
+ sql(
+ s"""
+ | CREATE TABLE boolean_table(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+
+ sql(
+ s"""
+ | CREATE TABLE boolean_table2(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+ val storeLocation = s"$rootPath/integration/spark2/src/test/resources/bool/supportBooleanTwoBooleanColumns.csv"
+
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${storeLocation}'
+ | INTO TABLE boolean_table
+ | options('FILEHEADER'='shortField,booleanField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData,booleanField2')
+ """.stripMargin)
+ sql("insert into boolean_table2 select * from boolean_table")
+ }
+ assert(exception_insert.getMessage.contains("Cannot insert into target table because column number are different"))
+ }
+
+ test("Inserting into Hive table from carbon table: support boolean data type and other format") {
+ sql(
+ s"""
+ | CREATE TABLE carbon_table(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+
+ sql(
+ s"""
+ | CREATE TABLE hive_table(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ """.stripMargin)
+
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+ val storeLocation = s"$rootPath/integration/spark2/src/test/resources/bool/supportBooleanTwoBooleanColumns.csv"
+
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${storeLocation}'
+ | INTO TABLE carbon_table
+ | options('FILEHEADER'='shortField,booleanField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData,booleanField2')
+ """.stripMargin)
+
+ sql("insert into hive_table select * from carbon_table where shortField = 1 and booleanField = true")
+
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from carbon_table"),
+ Seq(Row(true, 10, true), Row(false, 17, true), Row(false, 11, true),
+ Row(true, 10, true), Row(true, 10, true), Row(true, 14, false),
+ Row(false, 10, false), Row(false, 10, false), Row(false, 16, false), Row(false, 10, false))
+ )
+
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from hive_table"),
+ Seq(Row(true, 10, true), Row(true, 10, true), Row(true, 10, true))
+ )
+
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from carbon_table where exists (select booleanField,intField,booleanField2 " +
+ "from hive_table where carbon_table.intField=hive_table.intField)"),
+ Seq(Row(true, 10, true), Row(true, 10, true), Row(true, 10, true), Row(false, 10, false), Row(false, 10, false), Row(false, 10, false))
+ )
+ }
+
+ test("Inserting into carbon table from Hive table: support boolean data type and other format") {
+ sql(
+ s"""
+ | CREATE TABLE hive_table(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ | ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ """.stripMargin)
+
+ sql(
+ s"""
+ | CREATE TABLE carbon_table(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+ val storeLocation = s"$rootPath/integration/spark2/src/test/resources/bool/supportBooleanTwoBooleanColumns.csv"
+
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${storeLocation}'
+ | INTO TABLE hive_table
+ """.stripMargin)
+
+ sql("insert into carbon_table select * from hive_table where shortField = 1 and booleanField = true")
+
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from hive_table"),
+ Seq(Row(true, 10, true), Row(false, 17, true), Row(false, 11, true),
+ Row(true, 10, true), Row(true, 10, true), Row(true, 14, false),
+ Row(false, 10, false), Row(false, 10, false), Row(false, 16, false), Row(false, 10, false))
+ )
+
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from carbon_table"),
+ Seq(Row(true, 10, true), Row(true, 10, true), Row(true, 10, true))
+ )
+
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from hive_table where exists (select booleanField,intField,booleanField2 " +
+ "from carbon_table where hive_table.intField=carbon_table.intField)"),
+ Seq(Row(true, 10, true), Row(true, 10, true), Row(true, 10, true), Row(false, 10, false), Row(false, 10, false), Row(false, 10, false))
+ )
+ }
+
+ test("Inserting overwrite: one column boolean and many rows, should support") {
+ sql("insert into boolean_one_column values(True)")
+
+ sql("insert overwrite table boolean_one_column values(false)")
+ checkAnswer(
+ sql("select * from boolean_one_column"),
+ Seq(Row(false))
+ )
+
+ sql("insert overwrite table boolean_one_column values(true)")
+ checkAnswer(
+ sql("select * from boolean_one_column"),
+ Seq(Row(true))
+ )
+
+ sql("insert overwrite table boolean_one_column values(null)")
+ checkAnswer(
+ sql("select * from boolean_one_column"),
+ Seq(Row(null))
+ )
+
+ sql("insert overwrite table boolean_one_column values(true)")
+ checkAnswer(
+ sql("select * from boolean_one_column"),
+ Seq(Row(true))
+ )
+
+ sql("insert overwrite table boolean_one_column values('t')")
+ checkAnswer(
+ sql("select * from boolean_one_column"),
+ Seq(Row(null))
+ )
+ }
+
+ test("Inserting overwrite: create one column boolean table and insert two columns") {
+ sql("insert overwrite table boolean_one_column values(true,false)")
+ checkAnswer(
+ sql("select * from boolean_one_column"),
+ Seq(Row(true))
+ )
+ sql("insert overwrite table boolean_one_column values(True)")
+ sql("insert overwrite table boolean_one_column values(false,true)")
+ checkAnswer(
+ sql("select * from boolean_one_column"),
+ Seq(Row(false))
+ )
+ }
+
+ test("Inserting overwrite: two columns boolean and many rows, should support") {
+ sql("CREATE TABLE if not exists boolean_table2(col1 BOOLEAN, col2 BOOLEAN) STORED BY 'carbondata'")
+
+ sql("insert overwrite table boolean_table2 values(true,true)")
+ checkAnswer(
+ sql("select * from boolean_table2"),
+ Seq(Row(true, true))
+ )
+
+ sql("insert overwrite table boolean_table2 values(True,false)")
+ checkAnswer(
+ sql("select * from boolean_table2"),
+ Seq(Row(true, false))
+ )
+ sql("insert overwrite table boolean_table2 values(FALSE,false)")
+ sql("insert overwrite table boolean_table2 values('false',false)")
+ checkAnswer(
+ sql("select * from boolean_table2"),
+ Seq(Row(false, false))
+ )
+ sql("insert overwrite table boolean_table2 values(null,true)")
+ checkAnswer(
+ sql("select * from boolean_table2"),
+ Seq(Row(null, true))
+ )
+ }
+
+ test("Inserting overwrite: two columns and other data type, should support") {
+ sql("CREATE TABLE if not exists boolean_table2(col1 INT, col2 BOOLEAN) STORED BY 'carbondata'")
+ sql("insert overwrite table boolean_table2 values(1,true)")
+ checkAnswer(
+ sql("select * from boolean_table2"),
+ Seq(Row(1, true))
+ )
+ sql("insert overwrite table boolean_table2 values(100,true)")
+ sql("insert overwrite table boolean_table2 values(1991,false)")
+ checkAnswer(
+ sql("select * from boolean_table2"),
+ Seq(Row(1991, false))
+ )
+ sql("insert overwrite table boolean_table2 values(906,false)")
+ checkAnswer(
+ sql("select * from boolean_table2"),
+ Seq(Row(906, false))
+ )
+ sql("insert overwrite table boolean_table2 values(218,false)")
+ checkAnswer(
+ sql("select * from boolean_table2"),
+ Seq(Row(218, false))
+ )
+ sql("insert overwrite table boolean_table2 values(1011,true)")
+ checkAnswer(
+ sql("select * from boolean_table2"),
+ Seq(Row(1011, true))
+ )
+ }
+
+ test("Inserting overwrite: overwrite table with another table: support boolean data type and other format") {
+ sql(
+ s"""
+ | CREATE TABLE boolean_table(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+
+ sql(
+ s"""
+ | CREATE TABLE boolean_table2(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+
+ sql(
+ s"""
+ | CREATE TABLE boolean_table3(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | stringField STRING,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='')
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE TABLE boolean_table4(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | stringField STRING,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='')
+ """.stripMargin)
+
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+ val storeLocation = s"$rootPath/integration/spark2/src/test/resources/bool/supportBooleanTwoBooleanColumns.csv"
+
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${storeLocation}'
+ | INTO TABLE boolean_table
+ | options('FILEHEADER'='shortField,booleanField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData,booleanField2')
+ """.stripMargin)
+
+ sql("insert overwrite table boolean_table2 select * from boolean_table")
+ sql("insert overwrite table boolean_table3 select shortField,booleanField,intField,stringField,booleanField2 from boolean_table")
+ sql("insert overwrite table boolean_table4 select shortField,booleanField,intField,stringField,booleanField2 from boolean_table where shortField > 3")
+
+ checkAnswer(
+ sql("select booleanField,intField from boolean_table2"),
+ Seq(Row(true, 10), Row(false, 17), Row(false, 11),
+ Row(true, 10), Row(true, 10), Row(true, 14),
+ Row(false, 10), Row(false, 10), Row(false, 16), Row(false, 10))
+ )
+
+ checkAnswer(
+ sql("select booleanField,intField from boolean_table3"),
+ Seq(Row(true, 10), Row(false, 17), Row(false, 11),
+ Row(true, 10), Row(true, 10), Row(true, 14),
+ Row(false, 10), Row(false, 10), Row(false, 16), Row(false, 10))
+ )
+
+ checkAnswer(
+ sql("select booleanField,intField from boolean_table4"),
+ Seq(Row(false, 17), Row(false, 16))
+ )
+
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from boolean_table2"),
+ Seq(Row(true, 10, true), Row(false, 17, true), Row(false, 11, true),
+ Row(true, 10, true), Row(true, 10, true), Row(true, 14, false),
+ Row(false, 10, false), Row(false, 10, false), Row(false, 16, false), Row(false, 10, false))
+ )
+
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from boolean_table3"),
+ Seq(Row(true, 10, true), Row(false, 17, true), Row(false, 11, true),
+ Row(true, 10, true), Row(true, 10, true), Row(true, 14, false),
+ Row(false, 10, false), Row(false, 10, false), Row(false, 16, false), Row(false, 10, false))
+ )
+ }
+
+ test("Inserting overwrite: overwrite table Hive table from carbon table: support boolean data type and other format") {
+ sql(
+ s"""
+ | CREATE TABLE carbon_table(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+
+ sql(
+ s"""
+ | CREATE TABLE hive_table(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ """.stripMargin)
+
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+ val storeLocation = s"$rootPath/integration/spark2/src/test/resources/bool/supportBooleanTwoBooleanColumns.csv"
+
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${storeLocation}'
+ | INTO TABLE carbon_table
+ | options('FILEHEADER'='shortField,booleanField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData,booleanField2')
+ """.stripMargin)
+
+ sql("insert overwrite table hive_table select * from carbon_table where shortField = 1 and booleanField = true")
+
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from carbon_table"),
+ Seq(Row(true, 10, true), Row(false, 17, true), Row(false, 11, true),
+ Row(true, 10, true), Row(true, 10, true), Row(true, 14, false),
+ Row(false, 10, false), Row(false, 10, false), Row(false, 16, false), Row(false, 10, false))
+ )
+
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from hive_table"),
+ Seq(Row(true, 10, true), Row(true, 10, true), Row(true, 10, true))
+ )
+
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from carbon_table where exists (select booleanField,intField,booleanField2 " +
+ "from hive_table where carbon_table.intField=hive_table.intField)"),
+ Seq(Row(true, 10, true), Row(true, 10, true), Row(true, 10, true), Row(false, 10, false), Row(false, 10, false), Row(false, 10, false))
+ )
+ }
+
+ test("Inserting overwrite: overwrite table carbon table from Hive table: support boolean data type and other format") {
+ sql(
+ s"""
+ | CREATE TABLE hive_table(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ | ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ """.stripMargin)
+
+ sql(
+ s"""
+ | CREATE TABLE carbon_table(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('sort_columns'='','DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+ val storeLocation = s"$rootPath/integration/spark2/src/test/resources/bool/supportBooleanTwoBooleanColumns.csv"
+
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${storeLocation}'
+ | INTO TABLE hive_table
+ """.stripMargin)
+
+ sql("insert overwrite table carbon_table select * from hive_table where shortField = 1 and booleanField = true")
+
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from hive_table"),
+ Seq(Row(true, 10, true), Row(false, 17, true), Row(false, 11, true),
+ Row(true, 10, true), Row(true, 10, true), Row(true, 14, false),
+ Row(false, 10, false), Row(false, 10, false), Row(false, 16, false), Row(false, 10, false))
+ )
+
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from carbon_table"),
+ Seq(Row(true, 10, true), Row(true, 10, true), Row(true, 10, true))
+ )
+
+ checkAnswer(
+ sql("select booleanField,intField,booleanField2 from hive_table where exists (select booleanField,intField,booleanField2 " +
+ "from carbon_table where hive_table.intField=carbon_table.intField)"),
+ Seq(Row(true, 10, true), Row(true, 10, true), Row(true, 10, true), Row(false, 10, false), Row(false, 10, false), Row(false, 10, false))
+ )
+ }
+
+}